Update app.py

app.py

@@ -19,341 +19,346 @@ from lora_w2w import LoRAw2w
 from huggingface_hub import snapshot_download
 import spaces
 
+models_path = snapshot_download(repo_id="Snapchat/w2w")
 
-def main():
-    
-    models_path = snapshot_download(repo_id="Snapchat/w2w")
-    
-    device = "cuda"
-    mean = torch.load(f"{models_path}/files/mean.pt", map_location=torch.device('cpu')).bfloat16().to(device)
-    std = torch.load(f"{models_path}/files/std.pt", map_location=torch.device('cpu')).bfloat16().to(device)
-    v = torch.load(f"{models_path}/files/V.pt", map_location=torch.device('cpu')).bfloat16().to(device)
-    proj = torch.load(f"{models_path}/files/proj_1000pc.pt", map_location=torch.device('cpu')).bfloat16().to(device)
-    df = torch.load(f"{models_path}/files/identity_df.pt")
-    weight_dimensions = torch.load(f"{models_path}/files/weight_dimensions.pt")
-    pinverse = torch.load(f"{models_path}/files/pinverse_1000pc.pt", map_location=torch.device('cpu')).bfloat16().to(device)
-    
-    
-    unet, vae, text_encoder, tokenizer, noise_scheduler = load_models(device)
-    network = None
-    
-    young = get_direction(df, "Young", pinverse, 1000, device)
-    young = debias(young, "Male", df, pinverse, device)
-    young = debias(young, "Pointy_Nose", df, pinverse, device)
-    young = debias(young, "Wavy_Hair", df, pinverse, device)
-    young = debias(young, "Chubby", df, pinverse, device)
-    young = debias(young, "No_Beard", df, pinverse, device)
-    young = debias(young, "Mustache", df, pinverse, device)
-    
-    pointy = get_direction(df, "Pointy_Nose", pinverse, 1000, device)
-    pointy = debias(pointy, "Young", df, pinverse, device)
-    pointy = debias(pointy, "Male", df, pinverse, device)
-    pointy = debias(pointy, "Wavy_Hair", df, pinverse, device)
-    pointy = debias(pointy, "Chubby", df, pinverse, device)
-    pointy = debias(pointy, "Heavy_Makeup", df, pinverse, device)
-    
-    wavy = get_direction(df, "Wavy_Hair", pinverse, 1000, device)
-    wavy = debias(wavy, "Young", df, pinverse, device)
-    wavy = debias(wavy, "Male", df, pinverse, device)
-    wavy = debias(wavy, "Pointy_Nose", df, pinverse, device)
-    wavy = debias(wavy, "Chubby", df, pinverse, device)
-    wavy = debias(wavy, "Heavy_Makeup", df, pinverse, device)
-    
-    
-    thick = get_direction(df, "Bushy_Eyebrows", pinverse, 1000, device)
-    thick = debias(thick, "Male", df, pinverse, device)
-    thick = debias(thick, "Young", df, pinverse, device)
-    thick = debias(thick, "Pointy_Nose", df, pinverse, device)
-    thick = debias(thick, "Wavy_Hair", df, pinverse, device)
-    thick = debias(thick, "Mustache", df, pinverse, device)
-    thick = debias(thick, "No_Beard", df, pinverse, device)
-    thick = debias(thick, "Sideburns", df, pinverse, device)
-    thick = debias(thick, "Big_Nose", df, pinverse, device)
-    thick = debias(thick, "Big_Lips", df, pinverse, device)
-    thick = debias(thick, "Black_Hair", df, pinverse, device)
-    thick = debias(thick, "Brown_Hair", df, pinverse, device)
-    thick = debias(thick, "Pale_Skin", df, pinverse, device)
-    thick = debias(thick, "Heavy_Makeup", df, pinverse, device)
+class main():
+    def __init__(self):
+        super(main, self).__init__()
+        
+        device = "cuda"
+        mean = torch.load(f"{models_path}/files/mean.pt", map_location=torch.device('cpu')).bfloat16().to(device)
+        std = torch.load(f"{models_path}/files/std.pt", map_location=torch.device('cpu')).bfloat16().to(device)
+        v = torch.load(f"{models_path}/files/V.pt", map_location=torch.device('cpu')).bfloat16().to(device)
+        proj = torch.load(f"{models_path}/files/proj_1000pc.pt", map_location=torch.device('cpu')).bfloat16().to(device)
+        df = torch.load(f"{models_path}/files/identity_df.pt")
+        weight_dimensions = torch.load(f"{models_path}/files/weight_dimensions.pt")
+        pinverse = torch.load(f"{models_path}/files/pinverse_1000pc.pt", map_location=torch.device('cpu')).bfloat16().to(device)
 
-    def sample_model():
-        nonlocal network
-        mean.to(device)
-        std.to(device)
-        v.to(device)
-        proj.to(device)
-        unet, _, _, _, _ = load_models(device)
-        network = sample_weights(unet, proj, mean, std, v[:, :1000], device, factor = 1.00)
-        return network
+        self.device = device
+        self.mean = mean
+        self.std = std
+        self.v = v
+        self.proj = proj
+        self.df = df
+        self.weight_dimensions = weight_dimensions
+        self.pinverse = pinverse
+        
+        self.unet, self.vae, self.text_encoder, self.tokenizer, self.noise_scheduler = load_models(device)
+        self.network = None
+    
+        young = get_direction(df, "Young", pinverse, 1000, device)
+        young = debias(young, "Male", df, pinverse, device)
+        young = debias(young, "Pointy_Nose", df, pinverse, device)
+        young = debias(young, "Wavy_Hair", df, pinverse, device)
+        young = debias(young, "Chubby", df, pinverse, device)
+        young = debias(young, "No_Beard", df, pinverse, device)
+        young = debias(young, "Mustache", df, pinverse, device)
+        self.young = young
+    
+        pointy = get_direction(df, "Pointy_Nose", pinverse, 1000, device)
+        pointy = debias(pointy, "Young", df, pinverse, device)
+        pointy = debias(pointy, "Male", df, pinverse, device)
+        pointy = debias(pointy, "Wavy_Hair", df, pinverse, device)
+        pointy = debias(pointy, "Chubby", df, pinverse, device)
+        pointy = debias(pointy, "Heavy_Makeup", df, pinverse, device)
+        self.pointy = pointy
+    
+        wavy = get_direction(df, "Wavy_Hair", pinverse, 1000, device)
+        wavy = debias(wavy, "Young", df, pinverse, device)
+        wavy = debias(wavy, "Male", df, pinverse, device)
+        wavy = debias(wavy, "Pointy_Nose", df, pinverse, device)
+        wavy = debias(wavy, "Chubby", df, pinverse, device)
+        wavy = debias(wavy, "Heavy_Makeup", df, pinverse, device)
+        self.wavy = wavy
+    
+    
+        thick = get_direction(df, "Bushy_Eyebrows", pinverse, 1000, device)
+        thick = debias(thick, "Male", df, pinverse, device)
+        thick = debias(thick, "Young", df, pinverse, device)
+        thick = debias(thick, "Pointy_Nose", df, pinverse, device)
+        thick = debias(thick, "Wavy_Hair", df, pinverse, device)
+        thick = debias(thick, "Mustache", df, pinverse, device)
+        thick = debias(thick, "No_Beard", df, pinverse, device)
+        thick = debias(thick, "Sideburns", df, pinverse, device)
+        thick = debias(thick, "Big_Nose", df, pinverse, device)
+        thick = debias(thick, "Big_Lips", df, pinverse, device)
+        thick = debias(thick, "Black_Hair", df, pinverse, device)
+        thick = debias(thick, "Brown_Hair", df, pinverse, device)
+        thick = debias(thick, "Pale_Skin", df, pinverse, device)
+        thick = debias(thick, "Heavy_Makeup", df, pinverse, device)
+        self.thick = thick
+
+        def sample_model(self):
+            self.unet, _, _, _, _ = load_models(self.device)
+            self.network = sample_weights(self.unet, self.proj, self.mean, self.std, self.v[:, :1000], self.device, factor = 1.00)
+            
  
-    @torch.no_grad()
-    @spaces.GPU
-    def inference( prompt, negative_prompt, guidance_scale, ddim_steps, seed):
-        nonlocal network
-        generator = torch.Generator(device=device).manual_seed(seed)
-        latents = torch.randn(
-            (1, unet.in_channels, 512 // 8, 512 // 8),
-            generator = generator,
-            device = device
-        ).bfloat16()
-       
-    
-        text_input = tokenizer(prompt, padding="max_length", max_length=tokenizer.model_max_length, truncation=True, return_tensors="pt")
-    
-        text_embeddings = text_encoder(text_input.input_ids.to(device))[0]
-    
-        max_length = text_input.input_ids.shape[-1]
-        uncond_input = tokenizer(
-                                [negative_prompt], padding="max_length", max_length=max_length, return_tensors="pt"
-                            )
-        uncond_embeddings = text_encoder(uncond_input.input_ids.to(device))[0]
-        text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
-        noise_scheduler.set_timesteps(ddim_steps) 
-        latents = latents * noise_scheduler.init_noise_sigma
-        
-        for i,t in enumerate(tqdm.tqdm(noise_scheduler.timesteps)):
-            latent_model_input = torch.cat([latents] * 2)
-            latent_model_input = noise_scheduler.scale_model_input(latent_model_input, timestep=t)
-            with network:
-                noise_pred = unet(latent_model_input, t, encoder_hidden_states=text_embeddings, timestep_cond= None).sample
-            #guidance
-            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
-            noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
-            latents = noise_scheduler.step(noise_pred, t, latents).prev_sample
-        
-        latents = 1 / 0.18215 * latents
-        image = vae.decode(latents).sample
-        image = (image / 2 + 0.5).clamp(0, 1)
-        image = image.detach().cpu().float().permute(0, 2, 3, 1).numpy()[0]
-    
-        image = Image.fromarray((image * 255).round().astype("uint8"))
-    
-        return image
+        @torch.no_grad()
+        @spaces.GPU
+        def inference(self, prompt, negative_prompt, guidance_scale, ddim_steps, seed):
+            generator = torch.Generator(device=device).manual_seed(seed)
+            latents = torch.randn(
+                (1, self.unet.in_channels, 512 // 8, 512 // 8),
+                generator = generator,
+                device = self.device
+            ).bfloat16()
+           
+        
+            text_input = self.tokenizer(prompt, padding="max_length", max_length=self.tokenizer.model_max_length, truncation=True, return_tensors="pt")
+        
+            text_embeddings = self.text_encoder(text_input.input_ids.to(device))[0]
+        
+            max_length = text_input.input_ids.shape[-1]
+            uncond_input = self.tokenizer(
+                                    [negative_prompt], padding="max_length", max_length=max_length, return_tensors="pt"
+                                )
+            uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(device))[0]
+            text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+            self.noise_scheduler.set_timesteps(ddim_steps) 
+            latents = latents * self.noise_scheduler.init_noise_sigma
+            
+            for i,t in enumerate(tqdm.tqdm(self.noise_scheduler.timesteps)):
+                latent_model_input = torch.cat([latents] * 2)
+                latent_model_input = self.noise_scheduler.scale_model_input(latent_model_input, timestep=t)
+                with self.network:
+                    noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings, timestep_cond= None).sample
+                #guidance
+                noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+                latents = noise_scheduler.step(noise_pred, t, latents).prev_sample
+            
+            latents = 1 / 0.18215 * latents
+            image = self.vae.decode(latents).sample
+            image = (image / 2 + 0.5).clamp(0, 1)
+            image = image.detach().cpu().float().permute(0, 2, 3, 1).numpy()[0]
+        
+            image = Image.fromarray((image * 255).round().astype("uint8"))
+        
+            return image
 
 
-    @torch.no_grad()
-    @spaces.GPU
-    def edit_inference(prompt, negative_prompt, guidance_scale, ddim_steps, seed, start_noise, a1, a2, a3, a4): 
-        nonlocal network
-        original_weights = network.proj.clone()
-        
-        #pad to same number of PCs
-        pcs_original = original_weights.shape[1]
-        pcs_edits = young.shape[1]
-        padding =  torch.zeros((1,pcs_original-pcs_edits)).to(device)
-        young_pad = torch.cat((young, padding), 1)
-        pointy_pad = torch.cat((pointy, padding), 1)
-        wavy_pad = torch.cat((wavy, padding), 1)
-        thick_pad = torch.cat((thick, padding), 1)
+        @torch.no_grad()
+        @spaces.GPU
+        def edit_inference(self, prompt, negative_prompt, guidance_scale, ddim_steps, seed, start_noise, a1, a2, a3, a4): 
+            original_weights = self,network.proj.clone()
+            
+            #pad to same number of PCs
+            pcs_original = original_weights.shape[1]
+            pcs_edits = self.young.shape[1]
+            padding =  torch.zeros((1,pcs_original-pcs_edits)).to(device)
+            young_pad = torch.cat((self.young, padding), 1)
+            pointy_pad = torch.cat((self.pointy, padding), 1)
+            wavy_pad = torch.cat((self.wavy, padding), 1)
+            thick_pad = torch.cat((self.thick, padding), 1)
+            
         
-    
-        edited_weights = original_weights+a1*1e6*young_pad+a2*1e6*pointy_pad+a3*1e6*wavy_pad+a4*2e6*thick_pad
-       
-        generator = torch.Generator(device=device).manual_seed(seed)
-        latents = torch.randn(
-            (1, unet.in_channels, 512 // 8, 512 // 8),
-            generator = generator,
-            device = device
-        ).bfloat16()
-       
-    
-        text_input = tokenizer(prompt, padding="max_length", max_length=tokenizer.model_max_length, truncation=True, return_tensors="pt")
-    
-        text_embeddings = text_encoder(text_input.input_ids.to(device))[0]
-    
-        max_length = text_input.input_ids.shape[-1]
-        uncond_input = tokenizer(
-                                [negative_prompt], padding="max_length", max_length=max_length, return_tensors="pt"
-                            )
-        uncond_embeddings = text_encoder(uncond_input.input_ids.to(device))[0]
-        text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
-        noise_scheduler.set_timesteps(ddim_steps) 
-        latents = latents * noise_scheduler.init_noise_sigma
+            edited_weights = original_weights+a1*1e6*young_pad+a2*1e6*pointy_pad+a3*1e6*wavy_pad+a4*2e6*thick_pad
+           
+            generator = torch.Generator(device=device).manual_seed(seed)
+            latents = torch.randn(
+                (1, self.unet.in_channels, 512 // 8, 512 // 8),
+                generator = generator,
+                device = self.device
+            ).bfloat16()
+           
         
-    
-     
-        for i,t in enumerate(tqdm.tqdm(noise_scheduler.timesteps)):
-            latent_model_input = torch.cat([latents] * 2)
-            latent_model_input = noise_scheduler.scale_model_input(latent_model_input, timestep=t)
+            text_input = self.tokenizer(prompt, padding="max_length", max_length=self.tokenizer.model_max_length, truncation=True, return_tensors="pt")
+        
+            text_embeddings = text_encoder(text_input.input_ids.to(device))[0]
+        
+            max_length = text_input.input_ids.shape[-1]
+            uncond_input = tokenizer(
+                                    [negative_prompt], padding="max_length", max_length=max_length, return_tensors="pt"
+                                )
+            uncond_embeddings = text_encoder(uncond_input.input_ids.to(device))[0]
+            text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+            noise_scheduler.set_timesteps(ddim_steps) 
+            latents = latents * noise_scheduler.init_noise_sigma
             
-            if t>start_noise:
-                pass
-            elif t<=start_noise:
-                network.proj = torch.nn.Parameter(edited_weights)
-                network.reset()
-    
-    
-            with network:
-                noise_pred = unet(latent_model_input, t, encoder_hidden_states=text_embeddings, timestep_cond= None).sample
+        
+         
+            for i,t in enumerate(tqdm.tqdm(self.noise_scheduler.timesteps)):
+                latent_model_input = torch.cat([latents] * 2)
+                latent_model_input = self.noise_scheduler.scale_model_input(latent_model_input, timestep=t)
                 
+                if t>start_noise:
+                    pass
+                elif t<=start_noise:
+                    self.network.proj = torch.nn.Parameter(edited_weights)
+                    self.network.reset()
+        
+        
+                with self.network:
+                    noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings, timestep_cond= None).sample
+                    
+                
+                #guidance
+                noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+                latents = noise_scheduler.step(noise_pred, t, latents).prev_sample
             
-            #guidance
-            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
-            noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
-            latents = noise_scheduler.step(noise_pred, t, latents).prev_sample
-        
-        latents = 1 / 0.18215 * latents
-        image = vae.decode(latents).sample
-        image = (image / 2 + 0.5).clamp(0, 1)
-    
-        image = image.detach().cpu().float().permute(0, 2, 3, 1).numpy()[0]
-    
-        image = Image.fromarray((image * 255).round().astype("uint8"))
-    
-        #reset weights back to original 
-        network.proj = torch.nn.Parameter(original_weights)
-        network.reset()
-    
-        return image
+            latents = 1 / 0.18215 * latents
+            image = self.vae.decode(latents).sample
+            image = (image / 2 + 0.5).clamp(0, 1)
+        
+            image = image.detach().cpu().float().permute(0, 2, 3, 1).numpy()[0]
+        
+            image = Image.fromarray((image * 255).round().astype("uint8"))
+        
+            #reset weights back to original 
+            self.network.proj = torch.nn.Parameter(original_weights)
+            self.network.reset()
+        
+            return image
 
-    @spaces.GPU
-    def sample_then_run():
-        nonlocal network 
-        network = sample_model()    
-        prompt = "sks person"
-        negative_prompt = "low quality, blurry, unfinished, nudity, weapon"
-        seed = 5
-        cfg = 3.0
-        steps = 25
-        image = inference( prompt, negative_prompt, cfg, steps, seed)
-        torch.save(network.proj, "model.pt" )
-        return image, "model.pt"
+        @spaces.GPU
+        def sample_then_run(self):
+            sample_model()    
+            prompt = "sks person"
+            negative_prompt = "low quality, blurry, unfinished, nudity, weapon"
+            seed = 5
+            cfg = 3.0
+            steps = 25
+            image = inference( prompt, negative_prompt, cfg, steps, seed)
+            torch.save(self.network.proj, "model.pt" )
+            return image, "model.pt"
 
 
 
-    class CustomImageDataset(Dataset):
-        def __init__(self, images, transform=None):
-            self.images = images
-            self.transform = transform
-    
-        def __len__(self):
-            return len(self.images)
-    
-        def __getitem__(self, idx):
-            image = self.images[idx]
-            if self.transform:
-                image = self.transform(image)
-            return image
-    
-    @spaces.GPU
-    def invert(image, mask, pcs=10000, epochs=400, weight_decay = 1e-10, lr=1e-1):
-        
-        del unet
-        del network
-        unet, _, _, _, _ = load_models(device)
-        
-        proj = torch.zeros(1,pcs).bfloat16().to(device)
-        network = LoRAw2w( proj, mean, std, v[:, :pcs], 
-                            unet,
-                            rank=1,
-                            multiplier=1.0,
-                            alpha=27.0,
-                            train_method="xattn-strict"
-                        ).to(device, torch.bfloat16)
-    
-        ### load mask
-        mask = transforms.Resize((64,64), interpolation=transforms.InterpolationMode.BILINEAR)(mask)
-        mask = torchvision.transforms.functional.pil_to_tensor(mask).unsqueeze(0).to(device).bfloat16()[:,0,:,:].unsqueeze(1)
-        ### check if an actual mask was draw, otherwise mask is just all ones
-        if torch.sum(mask) == 0:
-            mask = torch.ones((1,1,64,64)).to(device).bfloat16()
+        class CustomImageDataset(Dataset):
+            def __init__(self, images, transform=None):
+                self.images = images
+                self.transform = transform
+        
+            def __len__(self):
+                return len(self.images)
+        
+            def __getitem__(self, idx):
+                image = self.images[idx]
+                if self.transform:
+                    image = self.transform(image)
+                return image
+        
+        @spaces.GPU
+        def invert(image, mask, pcs=10000, epochs=400, weight_decay = 1e-10, lr=1e-1):
             
-        ### single image dataset
-        image_transforms = transforms.Compose([transforms.Resize(512, interpolation=transforms.InterpolationMode.BILINEAR),
-                                                    transforms.RandomCrop(512),
-                                                    transforms.ToTensor(),
-                                                    transforms.Normalize([0.5], [0.5])])
-    
-    
-        train_dataset = CustomImageDataset(image, transform=image_transforms)
-        train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=1, shuffle=True) 
-    
-        ### optimizer 
-        optim = torch.optim.Adam(network.parameters(), lr=lr, weight_decay=weight_decay)    
-    
-        ### training loop
-        unet.train()
-        for epoch in tqdm.tqdm(range(epochs)):
-            for batch in train_dataloader:
-                ### prepare inputs
-                batch = batch.to(device).bfloat16()
-                latents = vae.encode(batch).latent_dist.sample()
-                latents = latents*0.18215
-                noise = torch.randn_like(latents)
-                bsz = latents.shape[0]
-             
-                timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device)
-                timesteps = timesteps.long()
-                noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps)
-                text_input = tokenizer("sks person", padding="max_length", max_length=tokenizer.model_max_length, truncation=True, return_tensors="pt")
-                text_embeddings = text_encoder(text_input.input_ids.to(device))[0]
-    
-                ### loss + sgd step
-                with network:
-                    model_pred = unet(noisy_latents, timesteps, text_embeddings).sample
-                    loss = torch.nn.functional.mse_loss(mask*model_pred.float(), mask*noise.float(), reduction="mean")
-                    optim.zero_grad()
-                    loss.backward()
-                    optim.step()
-    
-        ### return optimized network
-        return network
+            del unet
+            del network
+            unet, _, _, _, _ = load_models(device)
+            
+            proj = torch.zeros(1,pcs).bfloat16().to(device)
+            network = LoRAw2w( proj, mean, std, v[:, :pcs], 
+                                unet,
+                                rank=1,
+                                multiplier=1.0,
+                                alpha=27.0,
+                                train_method="xattn-strict"
+                            ).to(device, torch.bfloat16)
+    
+            ### load mask
+            mask = transforms.Resize((64,64), interpolation=transforms.InterpolationMode.BILINEAR)(mask)
+            mask = torchvision.transforms.functional.pil_to_tensor(mask).unsqueeze(0).to(device).bfloat16()[:,0,:,:].unsqueeze(1)
+            ### check if an actual mask was draw, otherwise mask is just all ones
+            if torch.sum(mask) == 0:
+                mask = torch.ones((1,1,64,64)).to(device).bfloat16()
+                
+            ### single image dataset
+            image_transforms = transforms.Compose([transforms.Resize(512, interpolation=transforms.InterpolationMode.BILINEAR),
+                                                        transforms.RandomCrop(512),
+                                                        transforms.ToTensor(),
+                                                        transforms.Normalize([0.5], [0.5])])
+        
+        
+            train_dataset = CustomImageDataset(image, transform=image_transforms)
+            train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=1, shuffle=True) 
+        
+            ### optimizer 
+            optim = torch.optim.Adam(network.parameters(), lr=lr, weight_decay=weight_decay)    
+        
+            ### training loop
+            unet.train()
+            for epoch in tqdm.tqdm(range(epochs)):
+                for batch in train_dataloader:
+                    ### prepare inputs
+                    batch = batch.to(device).bfloat16()
+                    latents = vae.encode(batch).latent_dist.sample()
+                    latents = latents*0.18215
+                    noise = torch.randn_like(latents)
+                    bsz = latents.shape[0]
+                 
+                    timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device)
+                    timesteps = timesteps.long()
+                    noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps)
+                    text_input = tokenizer("sks person", padding="max_length", max_length=tokenizer.model_max_length, truncation=True, return_tensors="pt")
+                    text_embeddings = text_encoder(text_input.input_ids.to(device))[0]
+        
+                    ### loss + sgd step
+                    with network:
+                        model_pred = unet(noisy_latents, timesteps, text_embeddings).sample
+                        loss = torch.nn.functional.mse_loss(mask*model_pred.float(), mask*noise.float(), reduction="mean")
+                        optim.zero_grad()
+                        loss.backward()
+                        optim.step()
+        
+            ### return optimized network
+            return network
 
 
-    @spaces.GPU
-    def run_inversion(dict, pcs, epochs, weight_decay,lr):
-        init_image = dict["image"].convert("RGB").resize((512, 512))
-        mask = dict["mask"].convert("RGB").resize((512, 512))
-        network = invert([init_image], mask, pcs, epochs, weight_decay,lr)
-    
-    
-        #sample an image
-        prompt = "sks person"
-        negative_prompt = "low quality, blurry, unfinished, nudity"
-        seed = 5
-        cfg = 3.0
-        steps = 25
-        image = inference( prompt, negative_prompt, cfg, steps, seed)
-        torch.save(network.proj, "model.pt" )
-        return image, "model.pt"
-    
-    
-    @spaces.GPU
-    def file_upload(file):
-        del unet
-        del network
+        @spaces.GPU
+        def run_inversion(dict, pcs, epochs, weight_decay,lr):
+            init_image = dict["image"].convert("RGB").resize((512, 512))
+            mask = dict["mask"].convert("RGB").resize((512, 512))
+            network = invert([init_image], mask, pcs, epochs, weight_decay,lr)
         
-        proj = torch.load(file.name).to(device)
-    
-        #pad to 10000 Principal components to keep everything consistent
-        pcs = proj.shape[1]
-        padding =  torch.zeros((1,10000-pcs)).to(device)
-        proj = torch.cat((proj, padding), 1)
-    
-        unet, _, _, _, _ = load_models(device)
+        
+            #sample an image
+            prompt = "sks person"
+            negative_prompt = "low quality, blurry, unfinished, nudity"
+            seed = 5
+            cfg = 3.0
+            steps = 25
+            image = inference( prompt, negative_prompt, cfg, steps, seed)
+            torch.save(network.proj, "model.pt" )
+            return image, "model.pt"
     
     
-        network = LoRAw2w( proj, mean, std, v[:, :10000], 
-                            unet,
-                            rank=1,
-                            multiplier=1.0,
-                            alpha=27.0,
-                            train_method="xattn-strict"
-                        ).to(device, torch.bfloat16)
+    @spaces.GPU
+        def file_upload(file):
+            del unet
+            del network
             
+            proj = torch.load(file.name).to(device)
+        
+            #pad to 10000 Principal components to keep everything consistent
+            pcs = proj.shape[1]
+            padding =  torch.zeros((1,10000-pcs)).to(device)
+            proj = torch.cat((proj, padding), 1)
+        
+            unet, _, _, _, _ = load_models(device)
         
-        prompt = "sks person"
-        negative_prompt = "low quality, blurry, unfinished, nudity"
-        seed = 5
-        cfg = 3.0
-        steps = 25
-        image = inference( prompt, negative_prompt, cfg, steps, seed)
-        return image
+        
+            network = LoRAw2w( proj, mean, std, v[:, :10000], 
+                                unet,
+                                rank=1,
+                                multiplier=1.0,
+                                alpha=27.0,
+                                train_method="xattn-strict"
+                            ).to(device, torch.bfloat16)
+                
+            
+            prompt = "sks person"
+            negative_prompt = "low quality, blurry, unfinished, nudity"
+            seed = 5
+            cfg = 3.0
+            steps = 25
+            image = inference( prompt, negative_prompt, cfg, steps, seed)
+            return image
     
     
 
     
-    intro = """
-    <div style="display: flex;align-items: center;justify-content: center">
+intro = """
+<div style="display: flex;align-items: center;justify-content: center">
         <h1 style="margin-left: 12px;text-align: center;margin-bottom: 7px;display: inline-block"><em>weights2weights</em> Demo</h1>
         <h3 style="display: inline-block;margin-left: 10px;margin-top: 6px;font-weight: 500">Interpreting the Weight Space of Customized Diffusion Models</h3>
     </div>
@@ -369,103 +374,101 @@ def main():
     
     
     
-    with gr.Blocks(css="style.css") as demo:
-    
-    
-        
-        gr.HTML(intro)
+with gr.Blocks(css="style.css") as demo:
+    model = main()    
+    gr.HTML(intro)
         
-        gr.Markdown("""<div style="text-align: justify;"> In this demo, you can get an identity-encoding model by sampling or inverting. To use a model previously downloaded from this demo see \"Uploading a model\" in the Advanced Options. Next, you can generate new images from it, or edit the identity encoded in the model and generate images from the edited model. We provide detailed instructions and tips at the bottom of the page.""")
-        with gr.Column():
-                with gr.Row():
-                    with gr.Column(): 
-                        gr.Markdown("""1) Either sample a new model, or upload an image (optionally draw a mask over the head) and click `invert`.""")
-                        sample = gr.Button("🎲 Sample New Model")
-                        input_image = gr.ImageEditor(elem_id="image_upload", type='pil', label="Reference Identity",
+    gr.Markdown("""<div style="text-align: justify;"> In this demo, you can get an identity-encoding model by sampling or inverting. To use a model previously downloaded from this demo see \"Uploading a model\" in the Advanced Options. Next, you can generate new images from it, or edit the identity encoded in the model and generate images from the edited model. We provide detailed instructions and tips at the bottom of the page.""")
+    with gr.Column():
+            with gr.Row():
+                with gr.Column(): 
+                    gr.Markdown("""1) Either sample a new model, or upload an image (optionally draw a mask over the head) and click `invert`.""")
+                    sample = gr.Button("🎲 Sample New Model")
+                    input_image = gr.ImageEditor(elem_id="image_upload", type='pil', label="Reference Identity",
                                                width=512, height=512)    
                       
-                        with gr.Row():
-                            invert_button = gr.Button("⬆️ Invert")
+                    with gr.Row():
+                        invert_button = gr.Button("⬆️ Invert")
     
     
     
-                    with gr.Column():
-                        gr.Markdown("""2) Generate images of the sampled/inverted identity or edit the identity with the sliders and generate new images with various prompts and seeds.""")
-                        gallery = gr.Image(label="Generated Image",height=512, width=512, interactive=False)
-                        submit = gr.Button("Generate")
+                with gr.Column():
+                    gr.Markdown("""2) Generate images of the sampled/inverted identity or edit the identity with the sliders and generate new images with various prompts and seeds.""")
+                    gallery = gr.Image(label="Generated Image",height=512, width=512, interactive=False)
+                    submit = gr.Button("Generate")
     
                         
-                prompt = gr.Textbox(label="Prompt",
+            prompt = gr.Textbox(label="Prompt",
                                                 info="Make sure to include 'sks person'" ,
                                                 placeholder="sks person", 
                                                 value="sks person")
                 
-                seed = gr.Number(value=5, label="Seed", precision=0, interactive=True)
+            seed = gr.Number(value=5, label="Seed", precision=0, interactive=True)
                     
-                # Editing 
-                with gr.Column():
-                    with gr.Row():
-                        a1 = gr.Slider(label="- Young +", value=0, step=0.001, minimum=-1, maximum=1, interactive=True)
-                        a2 = gr.Slider(label="- Pointy Nose +", value=0, step=0.001, minimum=-1, maximum=1, interactive=True)       
-                    with gr.Row():        
-                        a3 = gr.Slider(label="- Curly Hair +", value=0, step=0.001, minimum=-1, maximum=1, interactive=True)
-                        a4 = gr.Slider(label="- Thick Eyebrows +", value=0, step=0.001, minimum=-1, maximum=1, interactive=True)
+            # Editing 
+            with gr.Column():
+                with gr.Row():
+                    a1 = gr.Slider(label="- Young +", value=0, step=0.001, minimum=-1, maximum=1, interactive=True)
+                    a2 = gr.Slider(label="- Pointy Nose +", value=0, step=0.001, minimum=-1, maximum=1, interactive=True)       
+                with gr.Row():        
+                    a3 = gr.Slider(label="- Curly Hair +", value=0, step=0.001, minimum=-1, maximum=1, interactive=True)
+                    a4 = gr.Slider(label="- Thick Eyebrows +", value=0, step=0.001, minimum=-1, maximum=1, interactive=True)
     
                 
-                with gr.Accordion("Advanced Options", open=False):
-                    with gr.Tab("Inversion"):
-                        with gr.Row():     
-                            lr = gr.Number(value=1e-1, label="Learning Rate", interactive=True)
-                            pcs = gr.Slider(label="# Principal Components", value=10000, step=1, minimum=1, maximum=10000, interactive=True)
-                        with gr.Row():
-                            epochs = gr.Slider(label="Epochs", value=800, step=1, minimum=1, maximum=2000, interactive=True)
-                            weight_decay = gr.Number(value=1e-10, label="Weight Decay", interactive=True)
-                    with gr.Tab("Sampling"):
-                        with gr.Row():     
-                                cfg= gr.Slider(label="CFG", value=3.0, step=0.1, minimum=0, maximum=10, interactive=True)
-                                steps = gr.Slider(label="Inference Steps",  value=25, step=1, minimum=0, maximum=100, interactive=True)
-                        with gr.Row():
-                                negative_prompt = gr.Textbox(label="Negative Prompt", placeholder="low quality, blurry, unfinished, nudity, weapon", value="low quality, blurry, unfinished, nudity, weapon")
-                                injection_step = gr.Slider(label="Injection Step",  value=800, step=1, minimum=0, maximum=1000, interactive=True)
+            with gr.Accordion("Advanced Options", open=False):
+                with gr.Tab("Inversion"):
+                    with gr.Row():     
+                        lr = gr.Number(value=1e-1, label="Learning Rate", interactive=True)
+                        pcs = gr.Slider(label="# Principal Components", value=10000, step=1, minimum=1, maximum=10000, interactive=True)
+                    with gr.Row():
+                        epochs = gr.Slider(label="Epochs", value=800, step=1, minimum=1, maximum=2000, interactive=True)
+                        weight_decay = gr.Number(value=1e-10, label="Weight Decay", interactive=True)
+                with gr.Tab("Sampling"):
+                    with gr.Row():     
+                        cfg= gr.Slider(label="CFG", value=3.0, step=0.1, minimum=0, maximum=10, interactive=True)
+                        steps = gr.Slider(label="Inference Steps",  value=25, step=1, minimum=0, maximum=100, interactive=True)
+                    with gr.Row():
+                        negative_prompt = gr.Textbox(label="Negative Prompt", placeholder="low quality, blurry, unfinished, nudity, weapon", value="low quality, blurry, unfinished, nudity, weapon")
+                        injection_step = gr.Slider(label="Injection Step",  value=800, step=1, minimum=0, maximum=1000, interactive=True)
                                 
-                    with gr.Tab("Uploading a model"):
-                        gr.Markdown("""<div style="text-align: justify;">Upload a model below downloaded from this demo.""")
+                with gr.Tab("Uploading a model"):
+                    gr.Markdown("""<div style="text-align: justify;">Upload a model below downloaded from this demo.""")
     
-                        file_input = gr.File(label="Upload Model", container=True)
+                    file_input = gr.File(label="Upload Model", container=True)
     
          
     
     
-        gr.Markdown("""<div style="text-align: justify;"> After sampling a new model or inverting, you can download the model below.""")
+    gr.Markdown("""<div style="text-align: justify;"> After sampling a new model or inverting, you can download the model below.""")
     
-        with gr.Row():
-            file_output = gr.File(label="Download Sampled/Inverted Model", container=True, interactive=False)
+    with gr.Row():
+        file_output = gr.File(label="Download Sampled/Inverted Model", container=True, interactive=False)
             
     
     
     
-            invert_button.click(fn=run_inversion,
+        invert_button.click(fn=run_inversion,
                         inputs=[input_image, pcs, epochs, weight_decay,lr], 
                         outputs = [input_image, file_output])
             
                 
-            sample.click(fn=sample_then_run, outputs=[input_image, file_output])
+        sample.click(fn=model.sample_then_run, outputs=[input_image, file_output])
             
-            submit.click(
-                fn=edit_inference, inputs=[prompt, negative_prompt, cfg, steps, seed, injection_step, a1, a2, a3, a4], outputs=[gallery]
+        submit.click(
+                fn=model.edit_inference, inputs=[prompt, negative_prompt, cfg, steps, seed, injection_step, a1, a2, a3, a4], outputs=[gallery]
             )
-            file_input.change(fn=file_upload, inputs=file_input, outputs = gallery)
+        file_input.change(fn=file_upload, inputs=file_input, outputs = gallery)
     
     
     
-        help_text1 = """
+    help_text1 = """
             <b>Instructions</b>:
             1. To get results faster without waiting in queue, you can duplicate into a private space with an A100 GPU.
             2. To begin, you will have to get an identity-encoding model. You can either sample one from *weights2weights* space by clicking `Sample New Model` or by uploading an image and clicking `invert` to invert the identity into a model. You can optionally draw over the head to define a mask in the image for better results. Sampling a model takes around 10 seconds and inversion takes around 2 minutes. After this is done, you can optionally download this model for later use. A model can be uploaded in the \"Uploading a model\" tab in the `Advanced Options`.   
             3. After getting a model, an image of the identity will be displayed on the right. You can sample from the model by changing seeds as well as prompts and then clicking `Generate`. Make sure to include \"sks person\" in your prompt to keep the same identity.
             4. The identity in the model can be edited by changing the sliders for various attributes. After clicking `Generate`, you can see how the identity has changed and the effects are maintained across different seeds and prompts. 
             """
-        help_text2 = """<b>Tips</b>:
+    help_text2 = """<b>Tips</b>:
             1. Editing and Identity Generation
                 * If you are interested in preserving more of the image during identity-editing (i.e., where the same seed and prompt results in the same image with only the identity changed), you can play with the "Injection Step" parameter in the \"Sampling\" tab in the `Advanced Options`. During the first *n* timesteps, the original model's weights will be used, and then the edited weights will be set during the remaining steps. Values closer to 1000 will set the edited weights early, having a more pronounced effect, which may disrupt some semantics and structure of the generated image. Lower values will set the edited weights later, better preserving image context. We notice that around 600-800 tends to produce the best results. Larger values in the range (700-1000) are helpful for more global attribute changes, while smaller (400-700) can be used for more finegrained edits. Although it is not always needed.
                 * You can play around with negative prompts, number of inference steps, and CFG in the \"Sampling\" tab in the `Advanced Options` to affect the ultimate image quality.
@@ -477,11 +480,10 @@ def main():
                 * Note that if you change the number of PCs, you will probably need to change the learning rate. For less PCs, higher learning rates are typically required.""" 
         
             
-        gr.Markdown(help_text1)
-        gr.Markdown(help_text2)
+    gr.Markdown(help_text1)
+    gr.Markdown(help_text2)
         
-    demo.queue().launch()
+demo.queue().launch()
+
 
 
-if __name__ == "__main__":
-    main()