import os from pathlib import Path import torch from diffusers import AutoencoderKL, LMSDiscreteScheduler, UNet2DConditionModel from tqdm.auto import tqdm from transformers import CLIPTextModel, CLIPTokenizer, logging from utils import load_embedding_bin, set_timesteps, latents_to_pil from loss import blue_loss, cosine_loss torch.manual_seed(11) logging.set_verbosity_error() # Set device torch_device = ( "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" ) if "mps" == torch_device: os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" # Style embeddings STYLE_EMBEDDINGS = { "illustration-style": "illustration_style.bin", "line-art": "line-art.bin", "hitokomoru-style": "hitokomoru-style.bin", "midjourney-style": "midjourney-style.bin", "hanfu-anime-style": "hanfu-anime-style.bin", "birb-style": "birb-style.bin", "style-of-marc-allante": "Marc Allante.bin", } LOSS = {"blue_loss": blue_loss, "cosine_loss": cosine_loss} STYLE_SEEDS = [11, 56, 110, 65, 5, 29, 47] # Load the autoencoder model which will be used to decode the latents into image space. vae = AutoencoderKL.from_pretrained( "CompVis/stable-diffusion-v1-4", subfolder="vae" ).to(torch_device) # # # Load the tokenizer and text encoder to tokenize and encode the text. tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") text_encoder = CLIPTextModel.from_pretrained( "openai/clip-vit-large-patch14").to(torch_device) # # The UNet model for generating the latents. unet = UNet2DConditionModel.from_pretrained( "CompVis/stable-diffusion-v1-4", subfolder="unet" ).to(torch_device) # # # The noise scheduler scheduler = LMSDiscreteScheduler( beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, ) token_emb_layer = text_encoder.text_model.embeddings.token_embedding pos_emb_layer = text_encoder.text_model.embeddings.position_embedding position_ids = text_encoder.text_model.embeddings.position_ids[:, :77] position_embeddings = pos_emb_layer(position_ids) def build_causal_attention_mask(bsz, seq_len, dtype): # lazily create causal attention mask, with full attention between the vision tokens # pytorch uses additive attention mask; fill with -inf mask = torch.empty(bsz, seq_len, seq_len, dtype=dtype) mask.fill_(torch.tensor(torch.finfo(dtype).min)) mask.triu_(1) # zero out the lower diagonal mask = mask.unsqueeze(1) # expand mask return mask def get_output_embeds(input_embeddings): # CLIP's text model uses causal mask, so we prepare it here: bsz, seq_len = input_embeddings.shape[:2] causal_attention_mask = build_causal_attention_mask( bsz, seq_len, dtype=input_embeddings.dtype ) # Getting the output embeddings involves calling the model with passing output_hidden_states=True # so that it doesn't just return the pooled final predictions: encoder_outputs = text_encoder.text_model.encoder( inputs_embeds=input_embeddings, attention_mask=None, # We aren't using an attention mask so that can be None causal_attention_mask=causal_attention_mask.to(torch_device), output_attentions=None, output_hidden_states=True, # We want the output embs not the final output return_dict=None, ) # We're interested in the output hidden state only output = encoder_outputs[0] # There is a final layer norm we need to pass these through output = text_encoder.text_model.final_layer_norm(output) # And now they're ready! return output # Generating an image with these modified embeddings def generate_with_embs(text_embeddings, seed, max_length): height = 512 # default height of Stable Diffusion width = 512 # default width of Stable Diffusion num_inference_steps = 30 # Number of denoising steps guidance_scale = 7.5 # Scale for classifier-free guidance generator = torch.manual_seed(seed) batch_size = 1 # tokenizer uncond_input = tokenizer( [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="pt", ) with torch.no_grad(): uncond_embeddings = text_encoder(uncond_input.input_ids.to(torch_device))[0] text_embeddings = torch.cat([uncond_embeddings, text_embeddings]) # Prep Scheduler set_timesteps(scheduler, num_inference_steps) # Prep latents # step = " prep_latents " latents = torch.randn( (batch_size, unet.in_channels, height // 8, width // 8), generator=generator, ) latents = latents.to(torch_device) latents = latents * scheduler.init_noise_sigma # Loop for i, t in tqdm(enumerate(scheduler.timesteps), total=len(scheduler.timesteps)): # expand the latents if we are doing classifier-free guidance to avoid doing two forward passes. latent_model_input = torch.cat([latents] * 2) latent_model_input = scheduler.scale_model_input(latent_model_input, t) # predict the noise residual with torch.no_grad(): noise_pred = unet( latent_model_input, t, encoder_hidden_states=text_embeddings )["sample"] # perform guidance noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) noise_pred = noise_pred_uncond + guidance_scale * ( noise_pred_text - noise_pred_uncond ) # compute the previous noisy sample x_t -> x_t-1 latents = scheduler.step(noise_pred, t, latents).prev_sample return latents_to_pil(latents)[0] def generate_image_from_embeddings( mod_output_embeddings, seed, max_length, loss_selection, additional_prompt): height = 512 width = 512 num_inference_steps = 50 guidance_scale = 8 generator = torch.manual_seed(seed) batch_size = 1 if loss_selection == "blue_loss": loss_fn = LOSS["blue_loss"] loss_scale = 120 else: loss_fn = LOSS["cosine_loss"](additional_prompt) loss_scale = 20 # Use the modified_output_embeddings directly text_embeddings = mod_output_embeddings uncond_input = tokenizer( [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="pt", ) with torch.no_grad(): uncond_embeddings = text_encoder( uncond_input.input_ids.to(torch_device))[0] text_embeddings = torch.cat([uncond_embeddings, text_embeddings]) # Prep Scheduler set_timesteps(scheduler, num_inference_steps) # Prep latents latents = torch.randn( (batch_size, unet.config.in_channels, height // 8, width // 8), generator=generator, ) latents = latents.to(torch_device) latents = latents * scheduler.init_noise_sigma # Loop for i, t in tqdm(enumerate(scheduler.timesteps), total=len(scheduler.timesteps)): # expand the latents if we are doing classifier-free guidance to avoid doing two forward passes. latent_model_input = torch.cat([latents] * 2) sigma = scheduler.sigmas[i] latent_model_input = scheduler.scale_model_input(latent_model_input, t) # predict the noise residual with torch.no_grad(): noise_pred = unet( latent_model_input, t, encoder_hidden_states=text_embeddings )["sample"] # perform CFG noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) noise_pred = noise_pred_uncond + guidance_scale * ( noise_pred_text - noise_pred_uncond ) #### ADDITIONAL GUIDANCE ### if i % 5 == 0: # Requires grad on the latents latents = latents.detach().requires_grad_() # Get the predicted x0: # latents_x0 = latents - sigma * noise_pred latents_x0 = scheduler.step(noise_pred, t, latents).pred_original_sample scheduler._step_index -= 1 # Decode to image space denoised_images = ( vae.decode((1 / 0.18215) * latents_x0).sample / 2 + 0.5 ) # range (0, 1) # Calculate loss loss = loss_fn(denoised_images) * loss_scale # Occasionally print it out if i % 10 == 0: print(i, "loss:", loss.item()) # Get gradient cond_grad = torch.autograd.grad(loss, latents)[0] # Modify the latents based on this gradient latents = latents.detach() - cond_grad * sigma**2 # Now step with scheduler latents = scheduler.step(noise_pred, t, latents).prev_sample return latents_to_pil(latents)[0] def generate_image_per_style(prompt, style_embed, style_seed, style_embedding_key): modified_output_embeddings = None gen_out_style_image = None max_length = 0 # Tokenize text_input = tokenizer( prompt, padding="max_length", max_length=tokenizer.model_max_length, truncation=True, return_tensors="pt", ) input_ids = text_input.input_ids.to(torch_device) # Get token embeddings token_embeddings = token_emb_layer(input_ids) replacement_token_embedding = style_embed[style_embedding_key] # Insert this into the token embeddings token_embeddings[ 0, torch.where(input_ids[0] == 6829)[0] ] = replacement_token_embedding.to(torch_device) # Combine with pos embs input_embeddings = token_embeddings + position_embeddings # Feed through to get final output embs modified_output_embeddings = get_output_embeds(input_embeddings) # And generate an image with this: max_length = text_input.input_ids.shape[-1] gen_out_style_image = generate_with_embs( modified_output_embeddings, style_seed, max_length ) return gen_out_style_image def generate_image_per_loss( prompt, style_embed, style_seed, style_embedding_key, loss, additional_prompt ): prompt = f"{prompt} <{style_embedding_key}>" gen_out_loss_image = None # Tokenize text_input = tokenizer( prompt, padding="max_length", max_length=tokenizer.model_max_length, truncation=True, return_tensors="pt", ) input_ids = text_input.input_ids.to(torch_device) # Get token embeddings token_embeddings = token_emb_layer(input_ids) replacement_token_embedding = style_embed[style_embedding_key].to(torch_device) # Insert this into the token embeddings token_embeddings[ 0, torch.where(input_ids[0] == 6829)[0] ] = replacement_token_embedding # Combine with pos embs input_embeddings = token_embeddings + position_embeddings modified_output_embeddings = get_output_embeds(input_embeddings) # max_length = tokenizer.model_max_length max_length = text_input.input_ids.shape[-1] gen_out_loss_image = generate_image_from_embeddings( modified_output_embeddings, style_seed, max_length, loss, additional_prompt ) return gen_out_loss_image def generate_image_per_prompt_style(text_in, style_in, loss, additional_prompt): gen_style_image = None gen_loss_image = None STYLE_KEYS = [] style_key = "" if style_in not in STYLE_EMBEDDINGS: raise ValueError( f"Unknown style: {style_in}. Available styles are: {', '.join(STYLE_EMBEDDINGS.keys())}" ) STYLE_SEEDS = [32, 64, 128, 16, 8, 96] STYLE_KEYS = list(STYLE_EMBEDDINGS.keys()) print(f"prompt: {text_in}") print(f"style: {style_in}") idx = STYLE_KEYS.index(style_in) style_file = STYLE_EMBEDDINGS[style_in] print(f"style_file: {style_file}") style_seed = STYLE_SEEDS[idx] style_key = Path(style_file).stem style_key = style_key.replace("_", "-") print(style_key, STYLE_KEYS, style_file) file_path = os.path.join(os.getcwd(), style_file) embedding = load_embedding_bin(file_path) style_key = f"<{style_key}>" prompt = f"{text_in} {style_key}" gen_style_image = generate_image_per_style(prompt, embedding, style_seed, style_key) gen_loss_image = generate_image_per_loss(prompt, embedding, style_seed, style_key, loss, additional_prompt) return [gen_style_image, gen_loss_image]