Update app.py

app.py

@@ -16,7 +16,9 @@ from glob import glob
 from pathlib import Path
 from typing import Optional
 
-from diffsynth import save_video, ModelManager, SVDVideoPipeline, HunyuanDiTImagePipeline
+from diffsynth import save_video, ModelManager, SVDVideoPipeline
+from diffsynth import SDVideoPipeline, ControlNetConfigUnit, VideoData, save_frames
+from diffsynth.extensions.RIFE import RIFESmoother
 
 import uuid
 
@@ -45,23 +47,58 @@ if torch.cuda.is_available():
         model_id_list=["stable-video-diffusion-img2vid-xt", "ExVideo-SVD-128f-v1"],
         downloading_priority=["HuggingFace"])
     pipe = SVDVideoPipeline.from_model_manager(model_manager)
+        
+
+    model_manager2 = ModelManager(torch_dtype=torch.float16, device="cuda")
+    model_manager2.load_textual_inversions("models/textual_inversion")
+    model_manager2.load_models([
+            "models/stable_diffusion/flat2DAnimerge_v45Sharp.safetensors",
+            "models/AnimateDiff/mm_sd_v15_v2.ckpt",
+            "models/ControlNet/control_v11p_sd15_lineart.pth",
+            "models/ControlNet/control_v11f1e_sd15_tile.pth",
+            "models/RIFE/flownet.pkl"
+    ])
+    pipe2 = SDVideoPipeline.from_model_manager(
+        model_manager2,
+        [
+            ControlNetConfigUnit(
+                processor_id="lineart",
+                model_path="models/ControlNet/control_v11p_sd15_lineart.pth",
+                scale=0.5
+            ),
+            ControlNetConfigUnit(
+                processor_id="tile",
+                model_path="models/ControlNet/control_v11f1e_sd15_tile.pth",
+                scale=0.5
+            )
+        ]
+    )
+    smoother = RIFESmoother.from_model_manager(model_manager2)
+
+
+def video_to_image(selected):
+    if selected == "ExVideo":
+        return gr.Image(label='Upload Image', height=600, scale=2, image_mode="RGB", type="filepath")
+
 
 
 @spaces.GPU(duration=120)
 def generate(
-    image,
+    media,
     seed: Optional[int] = -1,
     motion_bucket_id: int = 127,
     fps_id: int = 25,
     num_inference_steps: int = 10,
     num_frames: int = 50,
     output_folder: str = "outputs",
+    selected,
+    prompt,
     progress=gr.Progress(track_tqdm=True)):
+
+    print(media)
     
     if seed == -1:
         seed = random.randint(0, MAX_SEED)
-        
-    image = Image.open(image)
 
     torch.manual_seed(seed)
     
@@ -69,19 +106,42 @@ def generate(
     base_count = len(glob(os.path.join(output_folder, "*.mp4")))
     video_path = os.path.join(output_folder, f"{base_count:06d}.mp4")
 
-    video = pipe(
-        input_image=image.resize((512, 512)), 
-        num_frames=num_frames, 
-        fps=fps_id, 
-        height=512, 
-        width=512,
-        motion_bucket_id=motion_bucket_id,
-        num_inference_steps=num_inference_steps,
-        min_cfg_scale=2, 
-        max_cfg_scale=2, 
-        contrast_enhance_scale=1.2
-    )
-    model_manager.to("cpu")
+    if selected == "ExVideo":
+        image = Image.open(media)
+        video = pipe(
+            input_image=image.resize((512, 512)), 
+            num_frames=num_frames, 
+            fps=fps_id, 
+            height=512, 
+            width=512,
+            motion_bucket_id=motion_bucket_id,
+            num_inference_steps=num_inference_steps,
+            min_cfg_scale=2, 
+            max_cfg_scale=2, 
+            contrast_enhance_scale=1.2
+        )
+        model_manager.to("cpu")
+    else:
+        up_video = VideoData(
+            video_file=media,
+            height=1024, width=1024)
+        input_video = [up_video[i] for i in range(40*60, 41*60)]
+
+        video = pipe(
+            prompt=prompt,
+            negative_prompt="verybadimagenegative_v1.3",
+            cfg_scale=3, 
+            clip_skip=2,
+            controlnet_frames=input_video, num_frames=len(input_video),
+            num_inference_steps=10, 
+            height=1024, 
+            width=1024,
+            animatediff_batch_size=32, 
+            animatediff_stride=16,
+            vram_limit_level=0,
+        )
+        video = smoother(video)
+
     
     save_video(video, video_path, fps=fps_id)
     
@@ -92,6 +152,7 @@ examples = [
         "./train.jpg",
         "./girl.webp",
         "./robo.jpg",
+        './working.mp4',
     ]
 
 
@@ -99,20 +160,48 @@ examples = [
 # Gradio Interface
 
 with gr.Blocks(css=CSS, js=JS, theme="soft") as demo:
-    gr.HTML("<h1><center>Exvideo📽️</center></h1>")
-    gr.HTML("<p><center><a href='https://huggingface.co/ECNU-CILab/ExVideo-SVD-128f-v1'>ExVideo</a> image-to-video generation<br><b>Update</b>: first version</center></p>")
+    gr.HTML("<h1><center>Exvideo📽️Diffutoon</center></h1>")
+    gr.HTML("<p><center>Exvideo and Diffutoon video generation<br><b>Update</b>: first version<br><b>Note</b>: ZeroGPU limited, Set the parameters appropriately.</center></p>")
     with gr.Row():
-        image = gr.Image(label='Upload Image', height=600, scale=2, image_mode="RGB", type="filepath")
+        media = gr.Video(label='Upload Video', height=600, scale=2)
         video = gr.Video(label="Generated Video", height=600, scale=2)
-        with gr.Accordion("Advanced Options", open=True):
-            with gr.Column(scale=1):
-                seed = gr.Slider(
-                    label="Seed (-1 Random)",
-                    minimum=-1,
-                    maximum=MAX_SEED,
+        with gr.Column(scale=1):
+            selected = gr.Radio(
+                label="Selected App",
+                    choices=["ExVideo", "Diffutoon"],
+                    value="Diffutoon"
+            )
+            seed = gr.Slider(
+                label="Seed (-1 Random)",
+                minimum=-1,
+                maximum=MAX_SEED,
+                step=1,
+                value=-1,
+                )
+            num_inference_steps = gr.Slider(
+                label="Inference steps", 
+                info="Inference steps",
+                step=1,
+                value=10, 
+                minimum=1, 
+                maximum=50
+                )
+            with gr.Accordion("Diffutoon Options", open=False):
+                animatediff_batch_size = gr.Slider(
+                    label="Animatediff batch size",
+                    minimum=1,
+                    maximum=50,
                     step=1,
-                    value=-1,
+                    value=32,
                 )
+                animatediff_stride = gr.Slider(
+                    label="Animatediff stride",
+                    minimum=1,
+                    maximum=50,
+                    step=1,
+                    value=16,
+                ) 
+            with gr.Accordion("ExVideo Options", open=False):
                 motion_bucket_id = gr.Slider(
                     label="Motion bucket id", 
                     info="Controls how much motion to add/remove from the image", 
@@ -129,14 +218,6 @@ with gr.Blocks(css=CSS, js=JS, theme="soft") as demo:
                     minimum=5, 
                     maximum=30
                 )
-                num_inference_steps = gr.Slider(
-                    label="Inference steps", 
-                    info="Inference steps",
-                    step=1,
-                    value=10, 
-                    minimum=1, 
-                    maximum=50
-                )
                 num_frames = gr.Slider(
                     label="Frames num", 
                     info="Frames num",
@@ -144,21 +225,19 @@ with gr.Blocks(css=CSS, js=JS, theme="soft") as demo:
                     value=50, 
                     minimum=1, 
                     maximum=128
-                )               
+                )
+    prompt = gr.Textbox(label="Prompt")
     with gr.Row():
         submit_btn = gr.Button(value="Generate")
         #stop_btn = gr.Button(value="Stop", variant="stop")
-        clear_btn = gr.ClearButton([image, seed, video])
+        clear_btn = gr.ClearButton([media, seed, video])
     gr.Examples(
         examples=examples,
-        inputs=image,
-        outputs=[video, seed],
-        fn=generate,
-        cache_examples="lazy",
         examples_per_page=4,
     )
 
-    submit_event = submit_btn.click(fn=generate, inputs=[image, seed, motion_bucket_id, fps_id,num_inference_steps, num_frames], outputs=[video, seed], api_name="video")
+    selected.change(fn=video_to_image, inputs=[selected], outputs=[media])
+    submit_event = submit_btn.click(fn=generate, inputs=[media, selected, seed, num_inference_steps, animatediff_batch_size, animatediff_stride, motion_bucket_id, fps_id, num_frames, prompt], outputs=[video, seed], api_name="video")
     #stop_btn.click(fn=None, inputs=None, outputs=None, cancels=[submit_event])
     
 demo.queue().launch()