# Authors: Hui Ren (rhfeiyang.github.io) import torch from PIL import Image import argparse import os, json, random import matplotlib.pyplot as plt import glob, re from tqdm import tqdm import numpy as np import sys import gc from transformers import CLIPTextModel, CLIPTokenizer, BertModel, BertTokenizer # import train_util from utils.train_util import get_noisy_image, encode_prompts from diffusers import AutoencoderKL, DDPMScheduler, DiffusionPipeline, UNet2DConditionModel, LMSDiscreteScheduler, DDIMScheduler, PNDMScheduler from typing import Any, Dict, List, Optional, Tuple, Union from utils.lora import LoRANetwork, DEFAULT_TARGET_REPLACE, UNET_TARGET_REPLACE_MODULE_CONV import argparse # from diffusers.training_utils import EMAModel import shutil import yaml from easydict import EasyDict from utils.metrics import StyleContentMetric from torchvision import transforms from custom_datasets.coco import CustomCocoCaptions from custom_datasets.imagepair import ImageSet from custom_datasets import get_dataset # from stable_diffusion.utils.modules import get_diffusion_modules # from diffusers import StableDiffusionImg2ImgPipeline from diffusers.utils.torch_utils import randn_tensor import pickle import time from datetime import datetime def flush(): torch.cuda.empty_cache() gc.collect() def get_train_method(lora_weight): if lora_weight is None: return 'None' if 'full' in lora_weight: train_method = 'full' elif "down_1_up_2_attn" in lora_weight: train_method = 'up_2_attn' print(f"Using up_2_attn for {lora_weight}") elif "down_2_up_1_up_2_attn" in lora_weight: train_method = 'down_2_up_2_attn' elif "down_2_up_2_attn" in lora_weight: train_method = 'down_2_up_2_attn' elif "down_2_attn" in lora_weight: train_method = 'down_2_attn' elif 'noxattn' in lora_weight: train_method = 'noxattn' elif "xattn" in lora_weight: train_method = 'xattn' elif "attn" in lora_weight: train_method = 'attn' elif "all_up" in lora_weight: train_method = 'all_up' else: train_method = 'None' return train_method def get_validation_dataloader(infer_prompts:list[str]=None, infer_images :list[str]=None,resolution=512, batch_size=10, num_workers=4, val_set="laion_pop500"): data_transforms = transforms.Compose( [ transforms.Resize(resolution), transforms.CenterCrop(resolution), ] ) def preprocess(example): ret={} ret["image"] = data_transforms(example["image"]) if "image" in example else None if "caption" in example: if isinstance(example["caption"][0], list): ret["caption"] = example["caption"][0][0] else: ret["caption"] = example["caption"][0] if "seed" in example: ret["seed"] = example["seed"] if "id" in example: ret["id"] = example["id"] if "path" in example: ret["path"] = example["path"] return ret def collate_fn(examples): out = {} if "image" in examples[0]: pixel_values = [example["image"] for example in examples] out["pixel_values"] = pixel_values # notice: only take the first prompt for each image if "caption" in examples[0]: prompts = [example["caption"] for example in examples] out["prompts"] = prompts if "seed" in examples[0]: seeds = [example["seed"] for example in examples] out["seed"] = seeds if "path" in examples[0]: paths = [example["path"] for example in examples] out["path"] = paths return out if infer_prompts is None: if val_set == "lhq500": dataset = get_dataset("lhq_sub500", get_val=False)["train"] elif val_set == "custom_coco100": dataset = get_dataset("custom_coco100", get_val=False)["train"] elif val_set == "custom_coco500": dataset = get_dataset("custom_coco500", get_val=False)["train"] elif os.path.isdir(val_set): image_folder = os.path.join(val_set, "paintings") caption_folder = os.path.join(val_set, "captions") dataset = ImageSet(folder=image_folder, caption=caption_folder, keep_in_mem=True) elif "custom_caption" in val_set: from custom_datasets.custom_caption import Caption_set name = val_set.replace("custom_caption_", "") dataset = Caption_set(set_name = name) elif val_set == "laion_pop500": dataset = get_dataset("laion_pop500", get_val=False)["train"] elif val_set == "laion_pop500_first_sentence": dataset = get_dataset("laion_pop500_first_sentence", get_val=False)["train"] else: raise ValueError("Unknown dataset") dataset.with_transform(preprocess) elif isinstance(infer_prompts, torch.utils.data.Dataset): dataset = infer_prompts try: dataset.with_transform(preprocess) except: pass else: class Dataset(torch.utils.data.Dataset): def __init__(self, prompts, images=None): self.prompts = prompts self.images = images self.get_img = False if images is not None: assert len(prompts) == len(images) self.get_img = True if isinstance(images[0], str): self.images = [Image.open(image).convert("RGB") for image in images] else: self.images = [None] * len(prompts) def __len__(self): return len(self.prompts) def __getitem__(self, idx): img = self.images[idx] if self.get_img and img is not None: img = data_transforms(img) return {"caption": self.prompts[idx], "image":img} dataset = Dataset(infer_prompts, infer_images) dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_fn, drop_last=False, num_workers=num_workers, pin_memory=True) return dataloader def get_lora_network(unet , lora_path, train_method="None", rank=1, alpha=1.0, device="cuda", weight_dtype=torch.float32): if train_method in [None, "None"]: train_method = get_train_method(lora_path) print(f"Train method: {train_method}") network_type = "c3lier" if train_method == 'xattn': network_type = 'lierla' modules = DEFAULT_TARGET_REPLACE if network_type == "c3lier": modules += UNET_TARGET_REPLACE_MODULE_CONV alpha = 1 if "rank" in lora_path: rank = int(re.search(r'rank(\d+)', lora_path).group(1)) if 'alpha1' in lora_path: alpha = 1.0 print(f"Rank: {rank}, Alpha: {alpha}") network = LoRANetwork( unet, rank=rank, multiplier=1.0, alpha=alpha, train_method=train_method, ).to(device, dtype=weight_dtype) if lora_path not in [None, "None"]: lora_state_dict = torch.load(lora_path) miss = network.load_state_dict(lora_state_dict, strict=False) print(f"Missing: {miss}") ret = {"network": network, "train_method": train_method} return ret def get_model(pretrained_ckpt_path, unet_ckpt=None,revision=None, variant=None, lora_path=None, weight_dtype=torch.float32, device="cuda"): modules = {} pipe = DiffusionPipeline.from_pretrained(pretrained_ckpt_path, revision=revision, variant=variant) if unet_ckpt is not None: pipe.unet.from_pretrained(unet_ckpt, subfolder="unet_ema", revision=revision, variant=variant) unet = pipe.unet vae = pipe.vae text_encoder = pipe.text_encoder tokenizer = pipe.tokenizer modules["unet"] = unet modules["vae"] = vae modules["text_encoder"] = text_encoder modules["tokenizer"] = tokenizer # tokenizer = modules["tokenizer"] unet.enable_xformers_memory_efficient_attention() unet.to(device, dtype=weight_dtype) if weight_dtype != torch.bfloat16: vae.to(device, dtype=torch.float32) else: vae.to(device, dtype=weight_dtype) text_encoder.to(device, dtype=weight_dtype) if lora_path is not None: network = get_lora_network(unet, lora_path, device=device, weight_dtype=weight_dtype) modules["network"] = network return modules @torch.no_grad() def inference(network: LoRANetwork, tokenizer: CLIPTokenizer, text_encoder: CLIPTextModel, vae: AutoencoderKL, unet: UNet2DConditionModel, noise_scheduler: LMSDiscreteScheduler, dataloader, height:int, width:int, scales:list = np.linspace(0,2,5),save_dir:str=None, seed:int = None, weight_dtype: torch.dtype = torch.float32, device: torch.device="cuda", batch_size:int=1, steps:int=50, guidance_scale:float=7.5, start_noise:int=800, uncond_prompt:str=None, uncond_embed=None, style_prompt = None, show:bool = False, no_load:bool=False, from_scratch=False): print(f"current time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print(f"save dir: {save_dir}") if start_noise < 0: assert from_scratch network = network.eval() unet = unet.eval() vae = vae.eval() do_convert = not from_scratch if not do_convert: try: dataloader.dataset.get_img = False except: pass scales = list(scales) else: scales = ["Real Image"] + list(scales) if not no_load and os.path.exists(os.path.join(save_dir, "infer_imgs.pickle")): with open(os.path.join(save_dir, "infer_imgs.pickle"), 'rb') as f: pred_images = pickle.load(f) take=True for key in scales: if key not in pred_images: take=False break if take: print(f"Found existing inference results in {save_dir}", flush=True) return pred_images max_length = tokenizer.model_max_length pred_images = {scale :[] for scale in scales} all_seeds = {scale:[] for scale in scales} prompts = [] ori_prompts = [] if save_dir is not None: img_output_dir = os.path.join(save_dir, "outputs") os.makedirs(img_output_dir, exist_ok=True) if uncond_embed is None: if uncond_prompt is None: uncond_input_text = [""] else: uncond_input_text = [uncond_prompt] uncond_embed = encode_prompts(tokenizer = tokenizer, text_encoder = text_encoder, prompts = uncond_input_text) for batch in dataloader: ori_prompt = batch["prompts"] image = batch["pixel_values"] if do_convert else None if do_convert: pred_images["Real Image"] += image if isinstance(ori_prompt, list): if isinstance(text_encoder, CLIPTextModel): # trunc prompts for clip encoder ori_prompt = [p.split(".")[0]+"." for p in ori_prompt] prompt = [f"{p.strip()[::-1].replace('.', '',1)[::-1]} in the style of {style_prompt}" for p in ori_prompt] if style_prompt is not None else ori_prompt else: if isinstance(text_encoder, CLIPTextModel): ori_prompt = ori_prompt.split(".")[0]+"." prompt = f"{prompt.strip()[::-1].replace('.', '',1)[::-1]} in the style of {style_prompt}" if style_prompt is not None else ori_prompt bcz = len(prompt) single_seed = seed if dataloader.batch_size == 1 and seed is None: if "seed" in batch: single_seed = batch["seed"][0] print(f"{prompt}, seed={single_seed}") # text_input = tokenizer(prompt, padding="max_length", max_length=tokenizer.model_max_length, truncation=True, return_tensors="pt").to(device) # original_embeddings = text_encoder(**text_input)[0] prompts += prompt ori_prompts += ori_prompt # style_input = tokenizer(prompt, padding="max_length", max_length=tokenizer.model_max_length, truncation=True, return_tensors="pt").to(device) # # style_embeddings = text_encoder(**style_input)[0] # style_embeddings = text_encoder(style_input.input_ids, return_dict=False)[0] style_embeddings = encode_prompts(tokenizer = tokenizer, text_encoder = text_encoder, prompts = prompt) original_embeddings = encode_prompts(tokenizer = tokenizer, text_encoder = text_encoder, prompts = ori_prompt) if uncond_embed.shape[0] == 1 and bcz > 1: uncond_embeddings = uncond_embed.repeat(bcz, 1, 1) else: uncond_embeddings = uncond_embed style_text_embeddings = torch.cat([uncond_embeddings, style_embeddings]).to(weight_dtype) # original_embeddings = torch.cat([uncond_embeddings, original_embeddings]).to(weight_dtype) generator = torch.manual_seed(single_seed) if single_seed is not None else None noise_scheduler.set_timesteps(steps) if do_convert: noised_latent, _, _ = get_noisy_image(image, vae, generator, unet, noise_scheduler, total_timesteps=int((1000-start_noise)/1000 *steps)) else: latent_shape = (bcz, 4, height//8, width//8) noised_latent = randn_tensor(latent_shape, generator=generator, device=vae.device) noised_latent = noised_latent.to(unet.dtype) noised_latent = noised_latent * noise_scheduler.init_noise_sigma for scale in scales: start_time = time.time() if not isinstance(scale, float) and not isinstance(scale, int): continue latents = noised_latent.clone().to(weight_dtype).to(device) noise_scheduler.set_timesteps(steps) for t in tqdm(noise_scheduler.timesteps): if do_convert and t>start_noise: continue else: if t > start_noise and start_noise >= 0: current_scale = 0 else: current_scale = scale network.set_lora_slider(scale=current_scale) text_embedding = style_text_embeddings # expand the latents if we are doing classifier-free guidance to avoid doing two forward passes. latent_model_input = torch.cat([latents] * 2).to(weight_dtype) latent_model_input = noise_scheduler.scale_model_input(latent_model_input, timestep=t).to(weight_dtype) # predict the noise residual with network: # print(f"dtype: {latent_model_input.dtype}, {text_embedding.dtype}, t={t}") noise_pred = unet(latent_model_input, t , encoder_hidden_states=text_embedding).sample # perform guidance noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) # compute the previous noisy sample x_t -> x_t-1 if isinstance(noise_scheduler, DDPMScheduler): latents = noise_scheduler.step(noise_pred, t, latents, generator=torch.manual_seed(single_seed+t) if single_seed is not None else None).prev_sample else: latents = noise_scheduler.step(noise_pred, t, latents).prev_sample # scale and decode the image latents with vae latents = 1 / 0.18215 * latents.to(vae.dtype) with torch.no_grad(): image = vae.decode(latents).sample image = (image / 2 + 0.5).clamp(0, 1) image = image.detach().cpu().permute(0, 2, 3, 1).to(torch.float32).numpy() images = (image * 255).round().astype("uint8") pil_images = [Image.fromarray(image) for image in images] pred_images[scale]+=pil_images all_seeds[scale] += [single_seed] * bcz end_time = time.time() print(f"Time taken for one batch, Art Adapter scale={scale}: {end_time-start_time}", flush=True) if save_dir is not None or show: end_idx = len(list(pred_images.values())[0]) for i in range(end_idx-bcz, end_idx): keys = list(pred_images.keys()) images_list = [pred_images[key][i] for key in keys] prompt = prompts[i] if len(scales)==1: plt.imshow(images_list[0]) plt.axis('off') plt.title(f"{prompt}_{single_seed}_start{start_noise}", fontsize=20) else: fig, ax = plt.subplots(1, len(images_list), figsize=(len(scales)*5,6), layout="constrained") for id, a in enumerate(ax): a.imshow(images_list[id]) if isinstance(scales[id], float) or isinstance(scales[id], int): a.set_title(f"Art Adapter scale={scales[id]}", fontsize=20) else: a.set_title(f"{keys[id]}", fontsize=20) a.axis('off') # plt.suptitle(f"{os.path.basename(lora_weight).replace('.pt','')}", fontsize=20) # plt.tight_layout() # if do_convert: # plt.suptitle(f"{prompt}\nseed{single_seed}_start{start_noise}_guidance{guidance_scale}", fontsize=20) # else: # plt.suptitle(f"{prompt}\nseed{single_seed}_from_scratch_guidance{guidance_scale}", fontsize=20) if save_dir is not None: plt.savefig(f"{img_output_dir}/{prompt.replace(' ', '_')[:100]}_seed{single_seed}_start{start_noise}.png") if show: plt.show() plt.close() flush() if save_dir is not None: with open(os.path.join(save_dir, "infer_imgs.pickle" ), 'wb') as f: pickle.dump(pred_images, f) with open(os.path.join(save_dir, "all_seeds.pickle"), 'wb') as f: to_save={"all_seeds":all_seeds, "batch_size":batch_size} pickle.dump(to_save, f) for scale, images in pred_images.items(): subfolder = os.path.join(save_dir,"images", f"{scale}") os.makedirs(subfolder, exist_ok=True) used_prompt = ori_prompts if (isinstance(scale, float) or isinstance(scale, int)): #and scale != 0: used_prompt = prompts for i, image in enumerate(images): if scale == "Real Image": suffix = "" else: suffix = f"_seed{all_seeds[scale][i]}" image.save(os.path.join(subfolder, f"{used_prompt[i].replace(' ', '_')[:100]}{suffix}.jpg")) with open(os.path.join(save_dir, "infer_prompts.txt"), 'w') as f: for prompt in prompts: f.write(f"{prompt}\n") with open(os.path.join(save_dir, "ori_prompts.txt"), 'w') as f: for prompt in ori_prompts: f.write(f"{prompt}\n") print(f"Saved inference results to {save_dir}", flush=True) return pred_images, prompts @torch.no_grad() def infer_metric(ref_image_folder,pred_images, prompts, save_dir, start_noise=""): prompts = [prompt.split(" in the style of ")[0] for prompt in prompts] scores = {} original_images = pred_images["Real Image"] if "Real Image" in pred_images else None metric = StyleContentMetric(ref_image_folder) for scale, images in pred_images.items(): score = metric(images, original_images, prompts) scores[scale] = score print(f"Style transfer score at scale {scale}: {score}") scores["ref_path"] = ref_image_folder save_name = f"scores_start{start_noise}.json" os.makedirs(save_dir, exist_ok=True) with open(os.path.join(save_dir, save_name), 'w') as f: json.dump(scores, f, indent=2) return scores def parse_args(): parser = argparse.ArgumentParser(description='Inference with LoRA') parser.add_argument('--lora_weights', type=str, default=["None"], nargs='+', help='path to your model file') parser.add_argument('--prompts', type=str, default=[], nargs='+', help='prompts to try') parser.add_argument("--prompt_file", type=str, default=None, help="path to the prompt file") parser.add_argument("--prompt_file_key", type=str, default="prompts", help="key to the prompt file") parser.add_argument('--resolution', type=int, default=512, help='resolution of the image') parser.add_argument('--seed', type=int, default=None, help='seed for the random number generator') parser.add_argument("--start_noise", type=int, default=800, help="start noise") parser.add_argument("--from_scratch", default=False, action="store_true", help="from scratch") parser.add_argument("--ref_image_folder", type=str, default=None, help="folder containing reference images") parser.add_argument("--show", action="store_true", help="show the image") parser.add_argument("--batch_size", type=int, default=1, help="batch size") parser.add_argument("--scales", type=float, default=[0.,1.], nargs='+', help="scales to test") parser.add_argument("--train_method", type=str, default=None, help="train method") # parser.add_argument("--vae_path", type=str, default="CompVis/stable-diffusion-v1-4", help="Path to the VAE model.") # parser.add_argument("--text_encoder_path", type=str, default="CompVis/stable-diffusion-v1-4", help="Path to the text encoder model.") parser.add_argument("--pretrained_model_name_or_path", type=str, default="rhfeiyang/art-free-diffusion-v1", help="Path to the pretrained model.") parser.add_argument("--unet_ckpt", default=None, type=str, help="Path to the unet checkpoint") parser.add_argument("--guidance_scale", type=float, default=5.0, help="guidance scale") parser.add_argument("--infer_mode", default="sks_art", help="inference mode") #, choices=["style", "ori", "artist", "sks_art","Peter"] parser.add_argument("--save_dir", type=str, default="inference_output", help="save directory") parser.add_argument("--num_workers", type=int, default=4, help="number of workers") parser.add_argument("--no_load", action="store_true", help="no load the pre-inferred results") parser.add_argument("--infer_prompts", type=str, default=None, nargs="+", help="prompts to infer") parser.add_argument("--infer_images", type=str, default=None, nargs="+", help="images to infer") parser.add_argument("--rank", type=int, default=1, help="rank of the lora") parser.add_argument("--val_set", type=str, default="laion_pop500", help="validation set") parser.add_argument("--folder_name", type=str, default=None, help="folder name") parser.add_argument("--scheduler_type",type=str, choices=["ddpm", "ddim", "pndm","lms"], default="ddpm", help="scheduler type") parser.add_argument("--infer_steps", type=int, default=50, help="inference steps") parser.add_argument("--weight_dtype", type=str, default="fp32", help="weight dtype") parser.add_argument("--custom_coco_cap", action="store_true", help="use custom coco caption") args = parser.parse_args() if args.infer_prompts is not None and len(args.infer_prompts) == 1 and os.path.isfile(args.infer_prompts[0]): if args.infer_prompts[0].endswith(".txt") and args.custom_coco_cap: args.infer_prompts = CustomCocoCaptions(custom_file=args.infer_prompts[0]) elif args.infer_prompts[0].endswith(".txt"): with open(args.infer_prompts[0], 'r') as f: args.infer_prompts = f.readlines() args.infer_prompts = [prompt.strip() for prompt in args.infer_prompts] elif args.infer_prompts[0].endswith(".csv"): from custom_datasets.custom_caption import Caption_set caption_set = Caption_set(args.infer_prompts[0]) args.infer_prompts = caption_set if args.infer_mode == "style": with open(os.path.join(args.ref_image_folder, "style_label.txt"), 'r') as f: args.style_label = f.readlines()[0].strip() elif args.infer_mode == "artist": with open(os.path.join(args.ref_image_folder, "style_label.txt"), 'r') as f: args.style_label = f.readlines()[0].strip() args.style_label = args.style_label.split(",")[0].strip() elif args.infer_mode == "ori": args.style_label = None else: args.style_label = args.infer_mode.replace("_", " ") if args.ref_image_folder is not None: args.ref_image_folder = os.path.join(args.ref_image_folder, "paintings") if args.start_noise < 0: args.from_scratch = True print(args.__dict__) return args def main(args): lora_weights = args.lora_weights if len(lora_weights) == 1 and isinstance(lora_weights[0], str) and os.path.isdir(lora_weights[0]): lora_weights = glob.glob(os.path.join(lora_weights[0], "*.pt")) lora_weights=sorted(lora_weights, reverse=True) width = args.resolution height = args.resolution steps = args.infer_steps revision = None device = 'cuda' rank = args.rank if args.weight_dtype == "fp32": weight_dtype = torch.float32 elif args.weight_dtype=="fp16": weight_dtype = torch.float16 elif args.weight_dtype=="bf16": weight_dtype = torch.bfloat16 modules = get_model(args.pretrained_model_name_or_path, unet_ckpt=args.unet_ckpt, revision=revision, variant=None, lora_path=None, weight_dtype=weight_dtype, device=device, ) if args.scheduler_type == "pndm": noise_scheduler = PNDMScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler") elif args.scheduler_type == "ddpm": noise_scheduler = DDPMScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler") elif args.scheduler_type == "ddim": noise_scheduler = DDIMScheduler( beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, clip_sample=False, prediction_type="epsilon", ) elif args.scheduler_type == "lms": noise_scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000) else: raise ValueError("Unknown scheduler type") cache=EasyDict() cache.modules = modules unet = modules["unet"] vae = modules["vae"] text_encoder = modules["text_encoder"] tokenizer = modules["tokenizer"] unet.requires_grad_(False) # Move unet, vae and text_encoder to device and cast to weight_dtype vae.requires_grad_(False) text_encoder.requires_grad_(False) ## dataloader dataloader = get_validation_dataloader(infer_prompts=args.infer_prompts, infer_images=args.infer_images, resolution=args.resolution, batch_size=args.batch_size, num_workers=args.num_workers, val_set=args.val_set) for lora_weight in lora_weights: print(f"Testing {lora_weight}") # for different seeds on same prompt seed = args.seed network_ret = get_lora_network(unet, lora_weight, train_method=args.train_method, rank=rank, alpha=1.0, device=device, weight_dtype=weight_dtype) network = network_ret["network"] train_method = network_ret["train_method"] if args.save_dir is not None: save_dir = args.save_dir if args.style_label is not None: save_dir = os.path.join(save_dir, f"{args.style_label.replace(' ', '_')}") else: save_dir = os.path.join(save_dir, f"ori/{args.start_noise}") else: if args.folder_name is not None: folder_name = args.folder_name else: folder_name = "validation" if args.infer_prompts is None else "validation_prompts" save_dir = os.path.join(os.path.dirname(lora_weight), f"{folder_name}/{train_method}", os.path.basename(lora_weight).replace('.pt','').split('_')[-1]) if args.infer_prompts is None: save_dir = os.path.join(save_dir, f"{args.val_set}") infer_config = f"{args.scheduler_type}{args.infer_steps}_{args.weight_dtype}_guidance{args.guidance_scale}" save_dir = os.path.join(save_dir, infer_config) os.makedirs(save_dir, exist_ok=True) if args.from_scratch: save_dir = os.path.join(save_dir, "from_scratch") else: save_dir = os.path.join(save_dir, "transfer") save_dir = os.path.join(save_dir, f"start{args.start_noise}") os.makedirs(save_dir, exist_ok=True) with open(os.path.join(save_dir, "infer_args.yaml"), 'w') as f: yaml.dump(vars(args), f) # save code code_dir = os.path.join(save_dir, "code") os.makedirs(code_dir, exist_ok=True) current_file = os.path.basename(__file__) shutil.copy(__file__, os.path.join(code_dir, current_file)) with torch.no_grad(): pred_images, prompts = inference(network, tokenizer, text_encoder, vae, unet, noise_scheduler, dataloader, height, width, args.scales, save_dir, seed, weight_dtype, device, args.batch_size, steps, guidance_scale=args.guidance_scale, start_noise=args.start_noise, show=args.show, style_prompt=args.style_label, no_load=args.no_load, from_scratch=args.from_scratch) if args.ref_image_folder is not None: flush() print("Calculating metrics") infer_metric(args.ref_image_folder, pred_images, save_dir, args.start_noise) if __name__ == "__main__": args = parse_args() main(args)