test gradio

app.py

@@ -5,46 +5,49 @@ import spaces
 import torch
 from diffusers import StableDiffusionXLPipeline
 from PIL import Image
-from ip_adapter import IPAdapterXL
+import torch
+from diffusers import AutoPipelineForText2Image, DDIMScheduler
+from transformers import CLIPVisionModelWithProjection
+from diffusers.utils import load_image
 
 token = os.getenv("HF_TOKEN")
 login(token=token)
 
-
-base_model_path = "stabilityai/stable-diffusion-xl-base-1.0"
-image_encoder_path = "sdxl_models/image_encoder"
-ip_ckpt = "sdxl_models/ip-adapter_sdxl.bin"
-device = "cuda"
-
-# load SDXL pipeline
-pipe = StableDiffusionXLPipeline.from_pretrained(
-    base_model_path,
+image_encoder = CLIPVisionModelWithProjection.from_pretrained(
+    "h94/IP-Adapter",
+    subfolder="models/image_encoder",
     torch_dtype=torch.float16,
-    add_watermarker=False,
 )
 
-# reduce memory consumption
-pipe.enable_vae_tiling()
-ip_model = IPAdapterXL(pipe, image_encoder_path, ip_ckpt, device,token=token)
+pipeline = AutoPipelineForText2Image.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    torch_dtype=torch.float16,
+    image_encoder=image_encoder,
+)
+pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config)
+pipeline.load_ip_adapter(
+  "h94/IP-Adapter",
+  subfolder="sdxl_models",
+  weight_name=["ip-adapter-plus_sdxl_vit-h.safetensors", "ip-adapter-plus-face_sdxl_vit-h.safetensors"]
+)
+pipeline.set_ip_adapter_scale([0.7, 0.3])
+pipeline.enable_model_cpu_offload()
 
 
 @spaces.GPU
 def generate_image(prompt, reference_image, controlnet_conditioning_scale):
-    image = Image.open(reference_image)
-    image.resize((512, 512))
-    images = ip_model.generate(pil_image=image,
-                               prompt=prompt,
-                               negative_prompt="",
-                               scale=controlnet_conditioning_scale,
-                               guidance_scale=5,
-                               num_samples=1,
-                               num_inference_steps=30,
-                               seed=42,
-                               # neg_content_prompt="a rabbit",
-                               # neg_content_scale=0.5,
-                               )
-
-    return images[0]
+    reference_image = Image.open(reference_image)
+    # reference_image.resize((512, 512))
+    pipeline.set_ip_adapter_scale([controlnet_conditioning_scale])
+
+    image = pipeline(
+        prompt=prompt,
+        ip_adapter_image=[reference_image],
+        negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality",
+        num_inference_steps=50, num_images_per_prompt=1,
+    ).images[0]
+
+    return image
 
 # Set up Gradio interface
 interface = gr.Interface(

requirements.txt

@@ -5,6 +5,4 @@ accelerate
 gradio
 sentencepiece
 spaces
-pillow
-ip-adapter
-einops
+pillow