test gradio

app.py

@@ -1,70 +1,59 @@
-import torch
-from diffusers.models import MotionAdapter
-from diffusers import AnimateDiffSDXLPipeline, DDIMScheduler
-from diffusers.utils import export_to_gif
-
 import gradio as gr
 from huggingface_hub import login
 import os
 import spaces,tempfile
 import torch
-from diffusers import StableDiffusionXLPipeline
-from PIL import Image
-import torch
-from diffusers import AutoPipelineForText2Image, DDIMScheduler
-from diffusers import AutoPipelineForText2Image
-from diffusers.utils import load_image
-import torch
-from diffusers.models import MotionAdapter
-from diffusers import AnimateDiffSDXLPipeline, DDIMScheduler
-from diffusers.utils import export_to_gif
-
+from diffusers import AnimateDiffSparseControlNetPipeline
+from diffusers.models import AutoencoderKL, MotionAdapter, SparseControlNetModel
+from diffusers.schedulers import DPMSolverMultistepScheduler
+from diffusers.utils import export_to_gif, load_image
 token = os.getenv("HF_TOKEN")
 login(token=token)
 
 
-adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-sdxl-beta", torch_dtype=torch.float16)
+model_id = "SG161222/Realistic_Vision_V5.1_noVAE"
+motion_adapter_id = "guoyww/animatediff-motion-adapter-v1-5-3"
+controlnet_id = "guoyww/animatediff-sparsectrl-rgb"
+lora_adapter_id = "guoyww/animatediff-motion-lora-v1-5-3"
+vae_id = "stabilityai/sd-vae-ft-mse"
+device = "cuda"
 
-model_id = "stabilityai/sdxl-turbo"
-scheduler = DDIMScheduler.from_pretrained(
+motion_adapter = MotionAdapter.from_pretrained(motion_adapter_id, torch_dtype=torch.float16).to(device)
+controlnet = SparseControlNetModel.from_pretrained(controlnet_id, torch_dtype=torch.float16).to(device)
+vae = AutoencoderKL.from_pretrained(vae_id, torch_dtype=torch.float16).to(device)
+scheduler = DPMSolverMultistepScheduler.from_pretrained(
     model_id,
     subfolder="scheduler",
-    clip_sample=False,
-    timestep_spacing="linspace",
     beta_schedule="linear",
-    steps_offset=1,
+    algorithm_type="dpmsolver++",
+    use_karras_sigmas=True,
 )
-pipe = AnimateDiffSDXLPipeline.from_pretrained(
+pipe = AnimateDiffSparseControlNetPipeline.from_pretrained(
     model_id,
-    motion_adapter=adapter,
+    motion_adapter=motion_adapter,
+    controlnet=controlnet,
+    vae=vae,
     scheduler=scheduler,
     torch_dtype=torch.float16,
-    variant="fp16",
-).to("cuda")
-pipe.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin")
-
-# enable memory savings
-pipe.enable_vae_slicing()
-pipe.enable_vae_tiling()
-pipeline = pipe
+).to(device)
+pipe.load_lora_weights(lora_adapter_id, adapter_name="motion_lora")
 
 
 @spaces.GPU
 def generate_image(prompt, reference_image, controlnet_conditioning_scale,num_frames):
-    style_images = [load_image(f) for f in reference_image]
-
-    pipeline.set_ip_adapter_scale(controlnet_conditioning_scale)
+    style_images = [load_image(f.name) for f in reference_image]
 
-    output = pipeline(
+    video = pipe(
         prompt=prompt,
-        ip_adapter_image=[style_images],
-        negative_prompt="",
-        guidance_scale=5,
-        num_inference_steps=30,
+        negative_prompt="low quality, worst quality",
+        num_inference_steps=25,
         num_frames=num_frames,
-    )
-    frames = output.frames[0]
-    export_to_gif(frames, "animation.gif")
+        conditioning_frames=style_images,
+        controlnet_frame_indices=[0],
+        controlnet_conditioning_scale=controlnet_conditioning_scale,
+        generator=torch.Generator().manual_seed(42),
+    ).frames[0]
+    export_to_gif(video, "output.gif")
 
     return "animation.gif"
 
@@ -74,7 +63,7 @@ interface = gr.Interface(
     inputs=[
         gr.Textbox(label="Prompt"),
         # gr.Image( type= "filepath",label="Reference Image (Style)"),
-        gr.File(type="filepath",file_count="multiple",label="Reference Image (Style)"),
+        gr.File(type="file",file_count="multiple",label="Reference Image (Style)"),
         gr.Slider(label="Control Net Conditioning Scale", minimum=0, maximum=1.0, step=0.1, value=1.0),
         gr.Slider(label="Number of frames", minimum=0, maximum=1.0, step=0.1, value=1.0),