test gradio

app.py

@@ -3,64 +3,49 @@ from huggingface_hub import login
 import os
 import spaces
 import torch
-from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, StableDiffusionPipeline
-from diffusers.utils import load_image, make_image_grid
+from diffusers import StableDiffusionXLPipeline
+from PIL import Image
+
+from ip_adapter import IPAdapterXL
 
 token = os.getenv("HF_TOKEN")
 login(token=token)
 
 
+base_model_path = "stabilityai/stable-diffusion-xl-base-1.0"
+image_encoder_path = "sdxl_models/image_encoder"
+ip_ckpt = "sdxl_models/ip-adapter_sdxl.bin"
+device = "cuda"
 
-# # Load the T2I-Style Adapter and the SDXL pipeline
-# adapter = T2IAdapter.from_pretrained("TencentARC/t2i-adapter-style-sdxl")
-# pipe = StableDiffusionXLAdapterPipeline.from_pretrained(
-#     "stabilityai/stable-diffusion-xl-base-1.0",
-#     adapter=adapter,
-# )
-#
-# # Set up the scheduler and device
-# pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
-# pipe.to("cuda", torch.float16)
-
-# controlnet = SD3ControlNetModel.from_pretrained("alimama-creative/SD3-Controlnet-Softedge", torch_dtype=torch.float16)
-
-# pipe = StableDiffusion3ControlNetPipeline.from_pretrained("stabilityai/stable-diffusion-3-medium-diffusers", controlnet=controlnet)
-# adapter = T2IAdapter.from_pretrained(
-#   "TencentARC/t2iadapter_color_sd14v1", torch_dtype=torch.float16, varient="fp16"
-# )
-#
-# model_id = 'stabilityai/stable-diffusion-xl-base-1.0'
-# euler_a = EulerAncestralDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler")
-# vae=AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
-# pipe = StableDiffusionXLAdapterPipeline.from_pretrained(
-#     model_id, vae=vae, adapter=adapter, scheduler=euler_a, torch_dtype=torch.float16, variant="fp16",
-# )
-
-
-pipe = StableDiffusionPipeline.from_pretrained(
-    "Yntec/AbsoluteReality",
-    torch_dtype=torch.float16)
-pipe.load_ip_adapter("h94/IP-Adapter",
-                     subfolder="models",
-                     weight_name="ip-adapter_sd15.bin")
+# load SDXL pipeline
+pipe = StableDiffusionXLPipeline.from_pretrained(
+    base_model_path,
+    torch_dtype=torch.float16,
+    add_watermarker=False,
+)
 
-pipe.to("cuda", torch.float16)
+# reduce memory consumption
+pipe.enable_vae_tiling()
+ip_model = IPAdapterXL(pipe, image_encoder_path, ip_ckpt, device, target_blocks=["up_blocks.0.attentions.1"])
 
 
 @spaces.GPU
 def generate_image(prompt, reference_image, controlnet_conditioning_scale):
+    image = Image.open(reference_image)
+    image.resize((512, 512))
+    images = ip_model.generate(pil_image=image,
+                               prompt=prompt,
+                               negative_prompt="",
+                               scale=controlnet_conditioning_scale,
+                               guidance_scale=5,
+                               num_samples=1,
+                               num_inference_steps=30,
+                               seed=42,
+                               # neg_content_prompt="a rabbit",
+                               # neg_content_scale=0.5,
+                               )
 
-    # Generate the image with ControlNet conditioning
-    pipe.set_ip_adapter_scale(controlnet_conditioning_scale)
-    generated_image = pipe(prompt=prompt,
-                  negative_prompt="low quality",
-                  height=768,
-                  width=768,
-                  ip_adapter_image=load_image(reference_image),
-                  guidance_scale=6,
-                  num_inference_steps=20,
-                  num_images_per_prompt=3).images[0]
-    return generated_image
+    return images[0]
 
 # Set up Gradio interface
 interface = gr.Interface(

requirements.txt

@@ -5,4 +5,5 @@ accelerate
 gradio
 sentencepiece
 spaces
-pillow
+pillow
+ip-adapter