Update app.py

app.py

@@ -1,25 +1,140 @@
 import os
 import random
+import spaces
 import gradio as gr
+import numpy as np
+import PIL.Image
 import torch
-from PIL import Image
+import torchvision.transforms.functional as TF
+
 from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL
-from diffusers import EulerAncestralDiscreteScheduler
-from huggingface_hub import snapshot_download
-import numpy as np
-from gradio_imageslider import ImageSlider
-from controlnet_aux import HEDdetector
+from diffusers import DDIMScheduler, EulerAncestralDiscreteScheduler
+from controlnet_aux import PidiNetDetector, HEDdetector
+from diffusers.utils import load_image
+from huggingface_hub import HfApi, snapshot_download
+from pathlib import Path
+from PIL import Image, ImageOps
 import cv2
+from gradio_imageslider import ImageSlider
+
+js_func = """
+function refresh() {
+    const url = new URL(window.location);
+}
+"""
+def nms(x, t, s):
+    x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s)
+
+    f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8)
+    f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8)
+    f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8)
+    f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8)
+
+    y = np.zeros_like(x)
+
+    for f in [f1, f2, f3, f4]:
+        np.putmask(y, cv2.dilate(x, kernel=f) == x, x)
+
+    z = np.zeros_like(y, dtype=np.uint8)
+    z[y > t] = 255
+    return z
+
+def HWC3(x):
+    assert x.dtype == np.uint8
+    if x.ndim == 2:
+        x = x[:, :, None]
+    assert x.ndim == 3
+    H, W, C = x.shape
+    assert C == 1 or C == 3 or C == 4
+    if C == 3:
+        return x
+    if C == 1:
+        return np.concatenate([x, x, x], axis=2)
+    if C == 4:
+        color = x[:, :, 0:3].astype(np.float32)
+        alpha = x[:, :, 3:4].astype(np.float32) / 255.0
+        y = color * alpha + 255.0 * (1.0 - alpha)
+        y = y.clip(0, 255).astype(np.uint8)
+        return y
+
+DESCRIPTION = ''''''
+
+if not torch.cuda.is_available():
+    DESCRIPTION += ""
+
+style_list = [
+    {
+        "name": "(No style)",
+        "prompt": "{prompt}",
+        "negative_prompt": "longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality",
+    },
+    {
+        "name": "Cinematic",
+        "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy",
+        "negative_prompt": "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured",
+    },
+    {
+        "name": "3D Model",
+        "prompt": "professional 3d model {prompt} . octane render, highly detailed, volumetric, dramatic lighting",
+        "negative_prompt": "ugly, deformed, noisy, low poly, blurry, painting",
+    },
+    {
+        "name": "Anime",
+        "prompt": "anime artwork {prompt} . anime style, key visual, vibrant, studio anime,  highly detailed",
+        "negative_prompt": "photo, deformed, black and white, realism, disfigured, low contrast",
+    },
+    {
+        "name": "Digital Art",
+        "prompt": "concept art {prompt} . digital artwork, illustrative, painterly, matte painting, highly detailed",
+        "negative_prompt": "photo, photorealistic, realism, ugly",
+    },
+    {
+        "name": "Photographic",
+        "prompt": "cinematic photo {prompt} . 35mm photograph, film, bokeh, professional, 4k, highly detailed",
+        "negative_prompt": "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly",
+    },
+    {
+        "name": "Pixel art",
+        "prompt": "pixel-art {prompt} . low-res, blocky, pixel art style, 8-bit graphics",
+        "negative_prompt": "sloppy, messy, blurry, noisy, highly detailed, ultra textured, photo, realistic",
+    },
+    {
+        "name": "Fantasy art",
+        "prompt": "ethereal fantasy concept art of  {prompt} . magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy",
+        "negative_prompt": "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white",
+    },
+    {
+        "name": "Neonpunk",
+        "prompt": "neonpunk style {prompt} . cyberpunk, vaporwave, neon, vibes, vibrant, stunningly beautiful, crisp, detailed, sleek, ultramodern, magenta highlights, dark purple shadows, high contrast, cinematic, ultra detailed, intricate, professional",
+        "negative_prompt": "painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured",
+    },
+    {
+        "name": "Manga",
+        "prompt": "manga style {prompt} . vibrant, high-energy, detailed, iconic, Japanese comic style",
+        "negative_prompt": "ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, Western comic style",
+    },
+]
+
+styles = {k["name"]: (k["prompt"], k["negative_prompt"]) for k in style_list}
+STYLE_NAMES = list(styles.keys())
+DEFAULT_STYLE_NAME = "(No style)"
+
+
+def apply_style(style_name: str, positive: str, negative: str = "") -> tuple[str, str]:
+    p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
+    return p.replace("{prompt}", positive), n + negative
 
-MAX_SEED = np.iinfo(np.int32).max
 
-DESCRIPTION = '''This is a demo for generating images using Stable Diffusion XL models and ControlNet.'''
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-# Setup models
+eulera_scheduler = EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
+
+# Download the model files
 ckpt_dir_pony = snapshot_download(repo_id="John6666/pony-realism-v21main-sdxl")
 ckpt_dir_cyber = snapshot_download(repo_id="John6666/cyberrealistic-pony-v61-sdxl")
 ckpt_dir_stallion = snapshot_download(repo_id="John6666/stallion-dreams-pony-realistic-v1-sdxl")
 
+# Load the models
 vae_pony = AutoencoderKL.from_pretrained(os.path.join(ckpt_dir_pony, "vae"), torch_dtype=torch.float16)
 vae_cyber = AutoencoderKL.from_pretrained(os.path.join(ckpt_dir_cyber, "vae"), torch_dtype=torch.float16)
 vae_stallion = AutoencoderKL.from_pretrained(os.path.join(ckpt_dir_stallion, "vae"), torch_dtype=torch.float16)
@@ -29,84 +144,128 @@ controlnet_cyber = ControlNetModel.from_pretrained("xinsir/controlnet-union-sdxl
 controlnet_stallion = ControlNetModel.from_pretrained("xinsir/controlnet-union-sdxl-1.0", torch_dtype=torch.float16)
 
 pipe_pony = StableDiffusionXLControlNetPipeline.from_pretrained(
-    ckpt_dir_pony, controlnet=controlnet_pony, vae=vae_pony, torch_dtype=torch.float16, scheduler=EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
+    ckpt_dir_pony, controlnet=controlnet_pony, vae=vae_pony, torch_dtype=torch.float16, scheduler=eulera_scheduler
 )
 pipe_cyber = StableDiffusionXLControlNetPipeline.from_pretrained(
-    ckpt_dir_cyber, controlnet=controlnet_cyber, vae=vae_cyber, torch_dtype=torch.float16, scheduler=EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
+    ckpt_dir_cyber, controlnet=controlnet_cyber, vae=vae_cyber, torch_dtype=torch.float16, scheduler=eulera_scheduler
 )
 pipe_stallion = StableDiffusionXLControlNetPipeline.from_pretrained(
-    ckpt_dir_stallion, controlnet=controlnet_stallion, vae=vae_stallion, torch_dtype=torch.float16, scheduler=EulerAncestralDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
+    ckpt_dir_stallion, controlnet=controlnet_stallion, vae=vae_stallion, torch_dtype=torch.float16, scheduler=eulera_scheduler
 )
 
-styles = {
-    "(No style)": ("{prompt}", "longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality"),
-    "Cinematic": ("cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy", "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured"),
-    "3D Model": ("professional 3d model {prompt} . octane render, highly detailed, volumetric, dramatic lighting", "ugly, deformed, noisy, low poly, blurry, painting"),
-    "Anime": ("anime artwork {prompt} . anime style, key visual, vibrant, studio anime, highly detailed", "photo, deformed, black and white, realism, disfigured, low contrast"),
-    "Digital Art": ("concept art {prompt} . digital artwork, illustrative, painterly, matte painting, highly detailed", "photo, photorealistic, realism, ugly"),
-    "Photographic": ("cinematic photo {prompt} . 35mm photograph, film, bokeh, professional, 4k, highly detailed", "drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly"),
-    "Pixel art": ("pixel-art {prompt} . low-res, blocky, pixel art style, 8-bit graphics", "sloppy, messy, blurry, noisy, highly detailed, ultra textured, photo, realistic"),
-    "Fantasy art": ("ethereal fantasy concept art of  {prompt} . magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy", "photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white"),
-    "Neonpunk": ("neonpunk style {prompt} . cyberpunk, vaporwave, neon, vibes, vibrant, stunningly beautiful, crisp, detailed, sleek, ultramodern, magenta highlights, dark purple shadows, high contrast, cinematic, ultra detailed, intricate, professional", "painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured"),
-    "Manga": ("manga style {prompt} . vibrant, high-energy, detailed, iconic, Japanese comic style", "ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, Western comic style"),
-}
+MAX_SEED = np.iinfo(np.int32).max
+processor = HEDdetector.from_pretrained('lllyasviel/Annotators')
+def nms(x, t, s):
+    x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s)
 
-DEFAULT_STYLE_NAME = "(No style)"
+    f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8)
+    f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8)
+    f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8)
+    f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8)
 
-def apply_style(style_name: str, positive: str, negative: str = "") -> tuple[str, str]:
-    p, n = styles.get(style_name, styles[DEFAULT_STYLE_NAME])
-    return p.replace("{prompt}", positive), n + negative
+    y = np.zeros_like(x)
 
-def generate_weighted_prompt(prompt: str) -> str:
-    from compel import Compel
-    comp = Compel(prompt)
-    return comp.with_strength(0.8)
+    for f in [f1, f2, f3, f4]:
+        np.putmask(y, cv2.dilate(x, kernel=f) == x, x)
+
+    z = np.zeros_like(y, dtype=np.uint8)
+    z[y > t] = 255
+    return z
 
 def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
     if randomize_seed:
-        return random.randint(0, MAX_SEED)
+        seed = random.randint(0, MAX_SEED)
     return seed
 
-def run(image, prompt, negative_prompt, model_choice, style_name, num_steps, guidance_scale, controlnet_conditioning_scale, seed, use_hed, use_canny):
+@spaces.GPU(duration=120)
+def run(
+    image: dict,
+    prompt: str,
+    negative_prompt: str,
+    model_choice: str,  # Add this new input
+    style_name: str = DEFAULT_STYLE_NAME,
+    num_steps: int = 25,
+    guidance_scale: float = 5,
+    controlnet_conditioning_scale: float = 1.0,
+    seed: int = 0,
+    use_hed: bool = False,
+    use_canny: bool = False,
+    progress=gr.Progress(track_tqdm=True),
+) -> PIL.Image.Image:
+    # Get the composite image from the EditorValue dict
     composite_image = image['composite']
     width, height = composite_image.size
     
+    # Calculate new dimensions to fit within 1024x1024 while maintaining aspect ratio
     max_size = 1024
     ratio = min(max_size / width, max_size / height)
     new_width = int(width * ratio)
     new_height = int(height * ratio)
+    
+    # Resize the image
     resized_image = composite_image.resize((new_width, new_height), Image.LANCZOS)
     
+    if use_canny:
+        controlnet_img = np.array(resized_image)
+        controlnet_img = cv2.Canny(controlnet_img, 100, 200)
+        controlnet_img = HWC3(controlnet_img)
+        image = Image.fromarray(controlnet_img)
+    elif not use_hed:
+        controlnet_img = resized_image
+        image = resized_image
+    else:
+        controlnet_img = processor(resized_image, scribble=False)
+        controlnet_img = np.array(controlnet_img)
+        controlnet_img = nms(controlnet_img, 127, 3)
+        controlnet_img = cv2.GaussianBlur(controlnet_img, (0, 0), 3)
+        random_val = int(round(random.uniform(0.01, 0.10), 2) * 255)
+        controlnet_img[controlnet_img > random_val] = 255
+        controlnet_img[controlnet_img < 255] = 0
+        image = Image.fromarray(controlnet_img)
+    
     prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
-    prompt = generate_weighted_prompt(prompt)
 
-    generator = torch.Generator(device="cuda" if torch.cuda.is_available() else "cpu").manual_seed(seed)
+    generator = torch.Generator(device=device).manual_seed(seed)
     
+    # Select the appropriate pipe based on the model choice
     if model_choice == "Pony Realism v21":
         pipe = pipe_pony
     elif model_choice == "Cyber Realistic Pony v61":
         pipe = pipe_cyber
-    else:
+    else:  # "Stallion Dreams Pony Realistic v1"
         pipe = pipe_stallion
     
-    pipe.to("cuda" if torch.cuda.is_available() else "cpu")
-
-    out = pipe(
-        prompt=prompt,
-        negative_prompt=negative_prompt,
-        image=resized_image,
-        num_inference_steps=num_steps,
-        generator=generator,
-        controlnet_conditioning_scale=controlnet_conditioning_scale,
-        guidance_scale=guidance_scale,
-        width=new_width,
-        height=new_height,
-    ).images[0]
+    pipe.to(device)
+    
+    if use_canny:
+        out = pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image=image,
+            num_inference_steps=num_steps,
+            generator=generator,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            guidance_scale=guidance_scale,
+            width=new_width,
+            height=new_height,
+        ).images[0]
+    else:
+        out = pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image=image,
+            num_inference_steps=num_steps,
+            generator=generator,
+            controlnet_conditioning_scale=controlnet_conditioning_scale,
+            guidance_scale=guidance_scale,
+            width=new_width,
+            height=new_height,
+        ).images[0]
 
     pipe.to("cpu")
     torch.cuda.empty_cache()
 
-    return out
+    return (controlnet_img, out)
 
 with gr.Blocks(css="style.css", js=js_func) as demo:
     gr.Markdown(DESCRIPTION, elem_id="description")
@@ -121,7 +280,7 @@ with gr.Blocks(css="style.css", js=js_func) as demo:
             with gr.Group():
                 image = gr.ImageEditor(type="pil", label="Sketch your image or upload one", width=512, height=512)
                 prompt = gr.Textbox(label="Prompt")
-                style = gr.Dropdown(label="Style", choices=list(styles.keys()), value=DEFAULT_STYLE_NAME)
+                style = gr.Dropdown(label="Style", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME)
                 model_choice = gr.Dropdown(
                     ["Pony Realism v21", "Cyber Realistic Pony v61", "Stallion Dreams Pony Realistic v1"],
                     label="Model Choice",
@@ -169,6 +328,7 @@ with gr.Blocks(css="style.css", js=js_func) as demo:
             with gr.Group():
                 image_slider = ImageSlider(position=0.5)
 
+
     inputs = [
         image,
         prompt,
@@ -193,4 +353,6 @@ with gr.Blocks(css="style.css", js=js_func) as demo:
         fn=run, inputs=inputs, outputs=outputs
     )
     
-demo.queue().launch(show_error=True, ssl_verify=False)
+    
+
+demo.queue().launch(show_error=True, ssl_verify=False)