app.py

import torch
import gradio as gr


import argparse, os, sys, glob
import torch
import pickle
import numpy as np
from omegaconf import OmegaConf
from PIL import Image
from tqdm import tqdm, trange
from einops import rearrange
from torchvision.utils import make_grid

from ldm.util import instantiate_from_config
from ldm.models.diffusion.ddim import DDIMSampler
from ldm.models.diffusion.plms import PLMSSampler


def load_model_from_config(config, ckpt, verbose=False):
    print(f"Loading model from {ckpt}")
    pl_sd = torch.load(ckpt, map_location="cpu") # TODO: change for GPU resources
    # pl_sd = torch.load(ckpt)#, map_location="cpu")
    sd = pl_sd["state_dict"]
    model = instantiate_from_config(config.model)
    m, u = model.load_state_dict(sd, strict=False)
    if len(m) > 0 and verbose:
        print("missing keys:")
        print(m)
    if len(u) > 0 and verbose:
        print("unexpected keys:")
        print(u)

    # model.cuda() # TODO: change for GPU resources
    model.eval()
    return model


def masking_embed(embedding, levels=1):
    """
    size of embedding - nx1xd, n: number of samples, d - 512
    replacing the last 128*levels from the embedding
    """
    replace_size = 128*levels
    random_noise = torch.randn(embedding.shape[0], embedding.shape[1], replace_size)
    embedding[:, :, -replace_size:] = random_noise
    return embedding


# LOAD MODEL GLOBALLY
ckpt_path = './model_files/fishes/epoch=000119.ckpt'
config_path = './model_files/fishes/2024-03-01T23-15-36-project.yaml'
config = OmegaConf.load(config_path)
model = load_model_from_config(config, ckpt_path) 

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
model = model.to(device)

def generate_image(fish_name, masking_level_input,
                        swap_fish_name, swap_level_input):
    
    fish_name = fish_name.lower()


    label_to_class_mapping = {0: 'Alosa-chrysochloris', 1: 'Carassius-auratus', 2: 'Cyprinus-carpio', 3: 'Esox-americanus', 
    4: 'Gambusia-affinis', 5: 'Lepisosteus-osseus', 6: 'Lepisosteus-platostomus', 7: 'Lepomis-auritus', 8: 'Lepomis-cyanellus', 
    9: 'Lepomis-gibbosus', 10: 'Lepomis-gulosus', 11: 'Lepomis-humilis', 12: 'Lepomis-macrochirus', 13: 'Lepomis-megalotis', 
    14: 'Lepomis-microlophus', 15: 'Morone-chrysops', 16: 'Morone-mississippiensis', 17: 'Notropis-atherinoides', 
    18: 'Notropis-blennius', 19: 'Notropis-boops', 20: 'Notropis-buccatus', 21: 'Notropis-buchanani', 22: 'Notropis-dorsalis', 
    23: 'Notropis-hudsonius', 24: 'Notropis-leuciodus', 25: 'Notropis-nubilus', 26: 'Notropis-percobromus', 
    27: 'Notropis-stramineus', 28: 'Notropis-telescopus', 29: 'Notropis-texanus', 30: 'Notropis-volucellus', 
    31: 'Notropis-wickliffi', 32: 'Noturus-exilis', 33: 'Noturus-flavus', 34: 'Noturus-gyrinus', 35: 'Noturus-miurus', 
    36: 'Noturus-nocturnus', 37: 'Phenacobius-mirabilis'}

    def get_label_from_class(class_name):
        for key, value in label_to_class_mapping.items():
            if value == class_name:
                return key


    sampler = DDIMSampler(model)

    prompt = opt.prompt
    all_images = []
    labels = []

    class_to_node = './model_files/fishes/class_to_ancestral_label.pkl'
    with open(class_to_node, 'rb') as pickle_file:
        class_to_node_dict = pickle.load(pickle_file)

    class_to_node_dict =  {key.lower(): value for key, value in class_to_node_dict.items()}
    

    prompt = class_to_node_dict[fish_name]

    ### Trait Swapping
    if swap_fish_name:
        swap_fish_name = swap_fish_name.lower()
        swap_level = int(swap_level_input.split(" ")[-1]) - 1
        swap_fish = class_to_node_dict[swap_fish_name]

        swap_fish_split = swap_fish[0].split(',')
        fish_name_split = prompt[0].split(',')
        fish_name_split[swap_level] = swap_fish_split[swap_level]

        prompt = [','.join(fish_name_split)]

    all_samples=list()
    with torch.no_grad():
        with model.ema_scope():
            uc = None
            for n in trange(opt.n_iter, desc="Sampling"):

                all_prompts = opt.n_samples * (prompt)
                all_prompts = [tuple(all_prompts)]
                c = model.get_learned_conditioning({'class_to_node': all_prompts})
                if masking_level_input != "None":
                    masked_level = int(masking_level_input.split(" ")[-1])
                    masked_level = 4-masked_level
                    c = masking_embed(c, levels=masked_level)
                shape = [3, 64, 64]
                samples_ddim, _ = sampler.sample(S=opt.ddim_steps,
                                                conditioning=c,
                                                batch_size=opt.n_samples,
                                                shape=shape,
                                                verbose=False,
                                                unconditional_guidance_scale=opt.scale,
                                                unconditional_conditioning=uc,
                                                eta=opt.ddim_eta)

                x_samples_ddim = model.decode_first_stage(samples_ddim)
                x_samples_ddim = torch.clamp((x_samples_ddim+1.0)/2.0, min=0.0, max=1.0)

                all_samples.append(x_samples_ddim)

    ###### to make grid
    # additionally, save as grid
    grid = torch.stack(all_samples, 0)
    grid = rearrange(grid, 'n b c h w -> (n b) c h w')
    grid = make_grid(grid, nrow=opt.n_samples)

    # to image
    grid = 255. * rearrange(grid, 'c h w -> h w c').cpu().numpy()
    final_image = Image.fromarray(grid.astype(np.uint8))

    return final_image


if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--prompt",
        type=str,
        nargs="?",
        default="a painting of a virus monster playing guitar",
        help="the prompt to render"
    )

    parser.add_argument(
        "--ddim_steps",
        type=int,
        default=200,
        help="number of ddim sampling steps",
    )

    parser.add_argument(
        "--ddim_eta",
        type=float,
        default=1.0,
        help="ddim eta (eta=0.0 corresponds to deterministic sampling",
    )
    parser.add_argument(
        "--n_iter",
        type=int,
        default=1,
        help="sample this often",
    )

    parser.add_argument(
        "--n_samples",
        type=int,
        default=1,
        help="how many samples to produce for the given prompt",
    )

    parser.add_argument(
        "--scale",
        type=float,
        # default=5.0,
        default=1.0,
        help="unconditional guidance scale: eps = eps(x, empty) + scale * (eps(x, cond) - eps(x, empty))",
    )
    opt = parser.parse_args()

    title = "🎞️ Phylo Diffusion - Generating Fish Images Tool"

    def load_example(prompt, level, option, components):
        components['prompt_input'].value = prompt
        components['masking_level_input'].value = level
    
    def setup_interface():
        with gr.Blocks() as demo:

            gr.Markdown("# Phylo-Diffusion: Generating Fish Images Tool")
            gr.Markdown("### Write the Species name to generate a fish image")
            gr.Markdown("### 1. Trait Masking: Specify the Level information to mask")
            gr.Markdown("### 2. Trait Swapping: Specify the species name to swap trait with and at what level")
    
            with gr.Row():
                with gr.Column():
                    # gr.Markdown("## Generate Images Based on Prompts")
                    gr.Markdown("**NOTE:** The demo is currently running on free CPU resources provided by Hugging Face, so it may take up to 10 minutes to generate an image. We're working on securing additional resources to speed up the process. Thank you for your patience!")
                    prompt_input = gr.Textbox(label="Species Name")
    
                    # Radio button to select experiment type, with no default selection
                    experiment_choice = gr.Radio(label="Select Experiment", choices=["Trait Masking", "Trait Swapping"], value=None)
    
                    # Trait Masking Inputs (hidden initially)
                    masking_level_input = gr.Dropdown(label="Select Ancestral Level", choices=["None", "Level 3", "Level 2"], value="None", visible=False)
    
                    # Trait Swapping Inputs (hidden initially)
                    swap_fish_name = gr.Textbox(label="Species Name to swap trait with:", visible=False)
                    swap_level_input = gr.Dropdown(label="Level of swapping", choices=["Level 3", "Level 2"], value="Level 3", visible=False)
    
                    submit_button = gr.Button("Generate")
    
                    gr.Markdown("## Phylogeny Tree")
                    architecture_image = "phylogeny_tree.jpg"  # Update this with the actual path
                    gr.Image(value=architecture_image, label="Phylogeny Tree")
    
                with gr.Column():
                    gr.Markdown("## Generated Image")
                    output_image = gr.Image(label="Generated Image", width=256, height=256)
    
                    # Place to put example buttons
                    gr.Markdown("## Select an example:")
                    examples = [
                        ("Gambusia Affinis", "None", "", "Level 3"),
                        ("Lepomis Auritus", "Level 3", "", "Level 3"),
                        ("Noturus nocturnus", "None", "Notropis dorsalis", "Level 2")
                    ]
    
                    for text, level, swap_text, swap_level in examples:
                        if level == "None" and swap_text == "":
                            button = gr.Button(f"Species: {text}")
                            experiment_type = "None"
                        elif level != "None":
                            button = gr.Button(f"Species: {text} | Masking: {level}")
                            experiment_type = "Trait Masking"
                        elif swap_text != "":
                            button = gr.Button(f"Species: {text} | Swapping with {swap_text} at {swap_level} ")
                            experiment_type = "Trait Swapping"
    
                        # Update radio button, fields and auto-trigger the "Generate" action
                        button.click(
                            fn=lambda text=text, level=level, swap_text=swap_text, swap_level=swap_level, experiment_type=experiment_type: (
                                text, level, swap_text, swap_level, experiment_type
                            ),
                            inputs=[],
                            outputs=[prompt_input, masking_level_input, swap_fish_name, swap_level_input, experiment_choice]
                        ).then(
                            fn=generate_image, 
                            inputs=[prompt_input, masking_level_input, swap_fish_name, swap_level_input], 
                            outputs=output_image
                        )
    
            # Update visibility of inputs based on experiment selection
            def update_inputs(experiment_type):
                if experiment_type == "Trait Masking":
                    return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)
                elif experiment_type == "Trait Swapping":
                    return gr.update(visible=False), gr.update(visible=True), gr.update(visible=True)
                else:
                    # No experiment selected
                    return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
    
            experiment_choice.change(
                fn=update_inputs,
                inputs=[experiment_choice],
                outputs=[masking_level_input, swap_fish_name, swap_level_input]
            )
    
            # Submit button functionality
            submit_button.click(
                fn=generate_image,
                inputs=[prompt_input, masking_level_input, swap_fish_name, swap_level_input],
                outputs=output_image
            )
    
        return demo
        

    iface = setup_interface()
    iface.launch(share=True)