Create app.py

app.py

@@ -0,0 +1,124 @@
+from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler
+import gradio as gr
+from PIL import Image
+import cv2
+import os, random, gc
+import numpy as np
+from transformers import pipeline
+import PIL.Image
+from diffusers.utils import load_image, export_to_video
+from accelerate import Accelerator
+from diffusers import StableDiffusionControlNetImg2ImgPipeline, ControlNetModel, EulerDiscreteScheduler
+import torch
+from moviepy.video.fx.all import crop
+from diffusers.utils import export_to_gif
+import mediapy
+from image_tools.sizes import resize_and_crop
+from moviepy.editor import *
+from pathlib import Path
+from typing import Optional, List
+from tqdm import tqdm
+import supervision as sv
+
+
+accelerator = Accelerator(cpu=True)
+pipe = accelerator.prepare(StableDiffusionControlNetImg2ImgPipeline.from_pretrained("stabilityai/sdxl-turbo", controlnet=controlnet, torch_dtype=torch.bfloat16, use_safetensors=False, variant="fp16", safety_checker=None))
+pipe.unet.to(memory_format=torch.channels_last)
+pipe.scheduler = accelerator.prepare(EulerDiscreteScheduler.from_config(pipe.scheduler.config))
+pipe = pipe.to("cpu")
+
+controlnet = accelerator.prepare(ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.bfloat16))
+def plex(fpath, text, neg_prompt, one, two, three, four, five):
+    gc.collect()
+    prompt = text
+    video = './video.mp4'
+    orvid = './orvid.mp4'
+    canvid = './canvid.mp4'
+    frames = []
+    canframes = []
+    orframes = []
+    fin_frames = []
+    max_frames=0
+    cap = cv2.VideoCapture(fpath)
+    clip = VideoFileClip(fpath)
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    aspect = width / height
+    if aspect == 1 and height >= 512:
+        nwidth = 512
+        nheight = 512
+        prep = clip.resize(height=nheight)
+        left = 0
+        top = 0
+        right = 512
+        bottom = 512
+    if aspect > 1 and height >= 512:
+        nheight = 512
+        nwidth = int(nheight * aspect)
+        prep = clip.resize(height=nheight)
+        left = (nwidth - width) / 2
+        top = 0
+        right = (nwidth + width) / 2
+        bottom = nheight
+    if aspect < 1 and width >= 512:
+        nwidth = 512
+        nheight = int(nwidth / aspect)
+        prep = clip.resize(height=nheight)
+        left = 0
+        top = (height - nheight) / 2
+        right = nwidth
+        bottom = (height + nheight) / 2
+    if aspect < 1 and width < 512:
+        return None
+    if aspect > 1 and height < 512:
+        return None
+    closer = crop(clip, x1=left, y1=top, x2=right, y2=bottom)
+    if fps > 33:
+        closer.write_videofile('./video.mp4', fps=33)
+        fps = 33
+    else:
+        closer.write_videofile('./video.mp4', fps=fps)
+        fps = fps
+    max_frames = int(fps * 4)
+    for frame in tqdm(sv.get_video_frames_generator(source_path=video,)):
+        frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+    cap.release()
+    cv2.destroyAllWindows()
+    ncap = cv2.VideoCapture(video)
+    total_frames = int(ncap.get(cv2.CAP_PROP_FRAME_COUNT))
+    if total_frames <= 0:
+        return None
+    b = 0
+    if total_frames > max_frames:
+        max_frames = int(max_frames)
+    if total_frames < max_frames:
+        max_frames = int(total_frames)
+    for b in range(int(max_frames)):
+        frame = frames[b]
+        original = load_image(Image.fromarray(frame))
+        original.save('./image.png', 'PNG')
+        original = original.resize((512, 512))
+        original = original.convert("RGB")
+        original.save('./image.png', 'PNG')
+        orframes.append(original)
+        cannyimage = np.array(original)
+        cannyimage = cv2.Canny(cannyimage, 100, 200)
+        cannyimage = cannyimage[:, :, None]
+        cannyimage = np.concatenate([cannyimage, cannyimage, cannyimage], axis=2)
+        cannyimage = Image.fromarray(cannyimage)
+        canframes.append(cannyimage)
+        generator = torch.Generator(device="cpu").manual_seed(five)
+        imoge = pipe(prompt=prompt,image=[original],control_image=[cannyimage],guidance_scale=four,num_inference_steps=one,generator=generator,strength=two,negative_prompt=neg_prompt,controlnet_conditioning_scale=three,width=512,height=512)
+        fin_frames.append(imoge.images[0])
+        b += 1
+    ncap.release()
+    cv2.destroyAllWindows()
+    export_to_video(fin_frames, video, fps=fps)
+    export_to_video(orframes, orvid, fps=fps)
+    export_to_video(canframes, canvid, fps=fps)
+    return video, canvid, orvid
+
+iface = gr.Interface(fn=plex, inputs=[gr.File(label="Your video",interactive=True),gr.Textbox(label="prompt"),gr.Textbox(label="neg prompt"), gr.Slider(label="num inference steps", minimum=1, step=1, maximum=5, value=2), gr.Slider(label="prompt strength", minimum=0.01, step=0.01, maximum=19.99, value=5.00), gr.Slider(label="controlnet scale", minimum=0.01, step=0.01, maximum=0.99, value=0.80), gr.Slider(label="Guidance scale", minimum=0.01, step=0.01, maximum=9.99, value=2.00), gr.Slider(label="Manual seed", minimum=0, step=32, maximum=4836928, value=0)], outputs=[gr.Video(label="final"), gr.Video(label="canny vid"), gr.Video(label="orig")],description="Running on cpu, very slow! by JoPmt.")
+iface.queue(max_size=1,api_open=False)
+iface.launch(max_threads=1)