app.py

import subprocess
subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)

from huggingface_hub import snapshot_download, hf_hub_download

snapshot_download(
    repo_id="Wan-AI/Wan2.1-T2V-1.3B",
    local_dir="wan_models/Wan2.1-T2V-1.3B",
    local_dir_use_symlinks=False,
    resume_download=True,
    repo_type="model" 
)

hf_hub_download(
    repo_id="gdhe17/Self-Forcing",
    filename="checkpoints/self_forcing_dmd.pt",
    local_dir=".",              
    local_dir_use_symlinks=False 
)

import os
import re
import random
import argparse
import hashlib
import urllib.request
import time
from PIL import Image
import spaces
import numpy as np
import torch
import gradio as gr
from omegaconf import OmegaConf
from tqdm import tqdm
import imageio

# Original project imports
from pipeline import CausalInferencePipeline
from demo_utils.constant import ZERO_VAE_CACHE
from demo_utils.vae_block3 import VAEDecoderWrapper
from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder

from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM

device = "cuda" if torch.cuda.is_available() else "cpu"
model_checkpoint = "gokaygokay/Flux-Prompt-Enhance"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)
enhancer = pipeline(
    'text2text-generation',
    model=model,
    tokenizer=tokenizer,
    repetition_penalty= 1.2,
    device=device
)
max_target_length = 256

@spaces.GPU
def enhance_prompt(prompt):
    prefix = "enhance prompt: "
    short_prompt = prompt
    answer = enhancer(prefix + short_prompt, max_length=max_target_length)
    final_answer = answer[0]['generated_text']
    return final_answer

# --- Argument Parsing ---
parser = argparse.ArgumentParser(description="Gradio Demo for Self-Forcing with Frame Streaming")
parser.add_argument('--port', type=int, default=7860, help="Port to run the Gradio app on.")
parser.add_argument('--host', type=str, default='0.0.0.0', help="Host to bind the Gradio app to.")
parser.add_argument("--checkpoint_path", type=str, default='./checkpoints/self_forcing_dmd.pt', help="Path to the model checkpoint.")
parser.add_argument("--config_path", type=str, default='./configs/self_forcing_dmd.yaml', help="Path to the model config.")
parser.add_argument('--share', action='store_true', help="Create a public Gradio link.")
parser.add_argument('--trt', action='store_true', help="Use TensorRT optimized VAE decoder.")
parser.add_argument('--fps', type=float, default=15.0, help="Playback FPS for frame streaming.")
args = parser.parse_args()

gpu = "cuda"

try:
    config = OmegaConf.load(args.config_path)
    default_config = OmegaConf.load("configs/default_config.yaml")
    config = OmegaConf.merge(default_config, config)
except FileNotFoundError as e:
    print(f"Error loading config file: {e}\n. Please ensure config files are in the correct path.")
    exit(1)

# Initialize Models
print("Initializing models...")
text_encoder = WanTextEncoder()
transformer = WanDiffusionWrapper(is_causal=True)

try:
    state_dict = torch.load(args.checkpoint_path, map_location="cpu")
    transformer.load_state_dict(state_dict.get('generator_ema', state_dict.get('generator')))
except FileNotFoundError as e:
    print(f"Error loading checkpoint: {e}\nPlease ensure the checkpoint '{args.checkpoint_path}' exists.")
    exit(1)

text_encoder.eval().to(dtype=torch.float16).requires_grad_(False)
transformer.eval().to(dtype=torch.float16).requires_grad_(False)

text_encoder.to(gpu)
transformer.to(gpu)

APP_STATE = {
    "torch_compile_applied": False,
    "fp8_applied": False,
    "current_use_taehv": False,
    "current_vae_decoder": None,
}

def initialize_vae_decoder(use_taehv=False, use_trt=False):
    if use_trt:
        from demo_utils.vae import VAETRTWrapper
        print("Initializing TensorRT VAE Decoder...")
        vae_decoder = VAETRTWrapper()
        APP_STATE["current_use_taehv"] = False
    elif use_taehv:
        print("Initializing TAEHV VAE Decoder...")
        from demo_utils.taehv import TAEHV
        taehv_checkpoint_path = "checkpoints/taew2_1.pth"
        if not os.path.exists(taehv_checkpoint_path):
            print(f"Downloading TAEHV checkpoint to {taehv_checkpoint_path}...")
            os.makedirs("checkpoints", exist_ok=True)
            download_url = "https://github.com/madebyollin/taehv/raw/main/taew2_1.pth"
            try:
                urllib.request.urlretrieve(download_url, taehv_checkpoint_path)
            except Exception as e:
                raise RuntimeError(f"Failed to download taew2_1.pth: {e}")
        
        class DotDict(dict): __getattr__ = dict.get
        
        class TAEHVDiffusersWrapper(torch.nn.Module):
            def __init__(self):
                super().__init__()
                self.dtype = torch.float16
                self.taehv = TAEHV(checkpoint_path=taehv_checkpoint_path).to(self.dtype)
                self.config = DotDict(scaling_factor=1.0)
            def decode(self, latents, return_dict=None):
                return self.taehv.decode_video(latents, parallel=not LOW_MEMORY).mul_(2).sub_(1)
        
        vae_decoder = TAEHVDiffusersWrapper()
        APP_STATE["current_use_taehv"] = True
    else:
        print("Initializing Default VAE Decoder...")
        vae_decoder = VAEDecoderWrapper()
        try:
            vae_state_dict = torch.load('wan_models/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth', map_location="cpu")
            decoder_state_dict = {k: v for k, v in vae_state_dict.items() if 'decoder.' in k or 'conv2' in k}
            vae_decoder.load_state_dict(decoder_state_dict)
        except FileNotFoundError:
            print("Warning: Default VAE weights not found.")
        APP_STATE["current_use_taehv"] = False

    vae_decoder.eval().to(dtype=torch.float16).requires_grad_(False).to(gpu)
    APP_STATE["current_vae_decoder"] = vae_decoder
    print(f"✅ VAE decoder initialized: {'TAEHV' if use_taehv else 'Default VAE'}")

# Initialize with default VAE
initialize_vae_decoder(use_taehv=False, use_trt=args.trt)

pipeline = CausalInferencePipeline(
    config, device=gpu, generator=transformer, text_encoder=text_encoder, 
    vae=APP_STATE["current_vae_decoder"]
)

pipeline.to(dtype=torch.float16).to(gpu)

# --- Frame Streaming Video Generation Handler ---
@torch.no_grad()
@spaces.GPU
def video_generation_handler(prompt, seed=42, fps=15):
    """
    Generator function that yields RGB frames for display in gr.Image.
    Includes timing delays for smooth playback.
    """
    if seed == -1: 
        seed = random.randint(0, 2**32 - 1)
    
    print(f"🎬 Starting video generation with prompt: '{prompt}' and seed: {seed}")
    
    # Calculate frame delay based on FPS
    frame_delay = 1.0 / fps if fps > 0 else 1.0 / 15.0
    
    print("🔤 Encoding text prompt...")
    conditional_dict = text_encoder(text_prompts=[prompt])
    for key, value in conditional_dict.items():
        conditional_dict[key] = value.to(dtype=torch.float16)
    
    # --- Generation Loop ---
    rnd = torch.Generator(gpu).manual_seed(int(seed))
    pipeline._initialize_kv_cache(1, torch.float16, device=gpu)
    pipeline._initialize_crossattn_cache(1, torch.float16, device=gpu)
    noise = torch.randn([1, 21, 16, 60, 104], device=gpu, dtype=torch.float16, generator=rnd)
    
    vae_cache, latents_cache = None, None
    if not APP_STATE["current_use_taehv"] and not args.trt:
        vae_cache = [c.to(device=gpu, dtype=torch.float16) for c in ZERO_VAE_CACHE]

    num_blocks = 7
    current_start_frame = 0
    all_num_frames = [pipeline.num_frame_per_block] * num_blocks
    
    total_frames_yielded = 0
    all_frames_for_video = []
    
    for idx, current_num_frames in enumerate(all_num_frames):
        print(f"📦 Processing block {idx+1}/{num_blocks} with {current_num_frames} frames")
        
        noisy_input = noise[:, current_start_frame : current_start_frame + current_num_frames]

        for step_idx, current_timestep in enumerate(pipeline.denoising_step_list):
            timestep = torch.ones([1, current_num_frames], device=noise.device, dtype=torch.int64) * current_timestep
            _, denoised_pred = pipeline.generator(
                noisy_image_or_video=noisy_input, conditional_dict=conditional_dict,
                timestep=timestep, kv_cache=pipeline.kv_cache1,
                crossattn_cache=pipeline.crossattn_cache,
                current_start=current_start_frame * pipeline.frame_seq_length
            )
            if step_idx < len(pipeline.denoising_step_list) - 1:
                next_timestep = pipeline.denoising_step_list[step_idx + 1]
                noisy_input = pipeline.scheduler.add_noise(
                    denoised_pred.flatten(0, 1), torch.randn_like(denoised_pred.flatten(0, 1)),
                    next_timestep * torch.ones([1 * current_num_frames], device=noise.device, dtype=torch.long)
                ).unflatten(0, denoised_pred.shape[:2])

        if idx < len(all_num_frames) - 1:
            pipeline.generator(
                noisy_image_or_video=denoised_pred, conditional_dict=conditional_dict,
                timestep=torch.zeros_like(timestep), kv_cache=pipeline.kv_cache1,
                crossattn_cache=pipeline.crossattn_cache,
                current_start=current_start_frame * pipeline.frame_seq_length,
            )

        # Decode to pixels
        if args.trt:
            pixels, vae_cache = pipeline.vae.forward(denoised_pred.half(), *vae_cache)
        elif APP_STATE["current_use_taehv"]:
            if latents_cache is None: 
                latents_cache = denoised_pred
            else:
                denoised_pred = torch.cat([latents_cache, denoised_pred], dim=1)
                latents_cache = denoised_pred[:, -3:]
            pixels = pipeline.vae.decode(denoised_pred)
        else:
            pixels, vae_cache = pipeline.vae(denoised_pred.half(), *vae_cache)
            
        # Handle frame skipping for first block
        if idx == 0 and not args.trt: 
            pixels = pixels[:, 3:]
        elif APP_STATE["current_use_taehv"] and idx > 0: 
            pixels = pixels[:, 12:]

        print(f"📹 Decoded pixels shape: {pixels.shape}")
        
        # Calculate actual frames that will be yielded for this block
        actual_frames_this_block = pixels.shape[1]
        
        # Yield individual frames with timing delays
        for frame_idx in range(actual_frames_this_block):
            frame_tensor = pixels[0, frame_idx]  # Get single frame [C, H, W]
            
            # Normalize from [-1, 1] to [0, 255]
            frame_np = torch.clamp(frame_tensor.float(), -1., 1.) * 127.5 + 127.5
            frame_np = frame_np.to(torch.uint8).cpu().numpy()
            
            # Convert from CHW to HWC format (RGB)
            frame_np = np.transpose(frame_np, (1, 2, 0))  # CHW -> HWC
            
            all_frames_for_video.append(frame_np)
            total_frames_yielded += 1
            
            # Calculate progress based on blocks completed + current block progress
            blocks_completed = idx
            current_block_progress = (frame_idx + 1) / actual_frames_this_block
            total_block_progress = (blocks_completed + current_block_progress) / num_blocks
            frame_progress_percent = total_block_progress * 100
            
            # Cap at 100% to avoid going over
            frame_progress_percent = min(frame_progress_percent, 100.0)
            
            print(f"📺 Yielding frame {total_frames_yielded}: shape {frame_np.shape}")
            
            # Create HTML status update
            if frame_idx == actual_frames_this_block - 1 and idx + 1 == num_blocks:  # Last frame
                status_html = (
                    f"<div style='padding: 16px; border: 1px solid #198754; background-color: #d1e7dd; border-radius: 8px; font-family: sans-serif; text-align: center;'>"
                    f"  <h4 style='margin: 0 0 8px 0; color: #0f5132; font-size: 18px;'>🎉 Generation Complete!</h4>"
                    f"  <p style='margin: 0; color: #0f5132;'>"
                    f"    Total frames: {total_frames_yielded}. The final video is now available."
                    f"  </p>"
                    f"</div>"
                )
            else:  # Regular frames
                status_html = (
                    f"<div style='padding: 10px; border: 1px solid #ddd; border-radius: 8px; font-family: sans-serif;'>"
                    f"  <p style='margin: 0 0 8px 0; font-size: 16px; font-weight: bold;'>Generating Video...</p>"
                    f"  <div style='background: #e9ecef; border-radius: 4px; width: 100%; overflow: hidden;'>"
                    f"    <div style='width: {frame_progress_percent:.1f}%; height: 20px; background-color: #0d6efd; transition: width 0.2s;'></div>"
                    f"  </div>"
                    f"  <p style='margin: 8px 0 0 0; color: #555; font-size: 14px; text-align: right;'>"
                    f"    Block {idx+1}/{num_blocks}   |   Frame {total_frames_yielded}   |   {frame_progress_percent:.1f}%"
                    f"  </p>"
                    f"</div>"
                )
            
            # Yield frame with a small delay to ensure UI updates
            yield gr.update(visible=True, value=frame_np), gr.update(visible=False), status_html
            
            # Sleep between frames for smooth playback (except for the last frame)
            # Add minimum delay to ensure UI can update
            if not (frame_idx == actual_frames_this_block - 1 and idx + 1 == num_blocks):
                time.sleep(max(frame_delay, 0.1))  # Minimum 100ms delay
            
        current_start_frame += current_num_frames
    
    print(f"✅ Video generation completed! Total frames yielded: {total_frames_yielded}")
    
    # Save final video
    try:
        video_path = f"gradio_tmp/{seed}_{hashlib.md5(prompt.encode()).hexdigest()}.mp4"
        imageio.mimwrite(video_path, all_frames_for_video, fps=fps, quality=8)
        print(f"✅ Video saved to {video_path}")
        final_status_html = (
            f"<div style='padding: 16px; border: 1px solid #198754; background-color: #d1e7dd; border-radius: 8px; font-family: sans-serif; text-align: center;'>"
            f"  <h4 style='margin: 0 0 8px 0; color: #0f5132; font-size: 18px;'>🎉 Generation Complete!</h4>"
            f"  <p style='margin: 0; color: #0f5132;'>"
            f"    Video saved successfully with {total_frames_yielded} frames at {fps} FPS."
            f"  </p>"
            f"</div>"
        )
        yield gr.update(visible=False), gr.update(value=video_path, visible=True), final_status_html
    except Exception as e:
        print(f"⚠️ Could not save final video: {e}")
        error_status_html = (
            f"<div style='padding: 16px; border: 1px solid #dc3545; background-color: #f8d7da; border-radius: 8px; font-family: sans-serif; text-align: center;'>"
            f"  <h4 style='margin: 0 0 8px 0; color: #721c24; font-size: 18px;'>⚠️ Video Save Error</h4>"
            f"  <p style='margin: 0; color: #721c24;'>"
            f"    Could not save final video: {str(e)}"
            f"  </p>"
            f"</div>"
        )
        yield None, None, error_status_html

@torch.no_grad()
@spaces.GPU
def video_generation_handler_example(prompt, seed=42, fps=15):
    """
    Simplified video generation function that returns the final video path.
    """
    if seed == -1: 
        seed = random.randint(0, 2**32 - 1)
    
    print(f"🎬 Starting video generation with prompt: '{prompt}' and seed: {seed}")
    
    # Encode text prompt
    print("🔤 Encoding text prompt...")
    conditional_dict = text_encoder(text_prompts=[prompt])
    for key, value in conditional_dict.items():
        conditional_dict[key] = value.to(dtype=torch.float16)
    
    # Initialize generation
    rnd = torch.Generator(gpu).manual_seed(int(seed))
    pipeline._initialize_kv_cache(1, torch.float16, device=gpu)
    pipeline._initialize_crossattn_cache(1, torch.float16, device=gpu)
    noise = torch.randn([1, 21, 16, 60, 104], device=gpu, dtype=torch.float16, generator=rnd)
    
    vae_cache, latents_cache = None, None
    if not APP_STATE["current_use_taehv"] and not args.trt:
        vae_cache = [c.to(device=gpu, dtype=torch.float16) for c in ZERO_VAE_CACHE]

    num_blocks = 7
    current_start_frame = 0
    all_num_frames = [pipeline.num_frame_per_block] * num_blocks
    all_frames_for_video = []
    
    # Generation loop
    for idx, current_num_frames in enumerate(all_num_frames):
        print(f"📦 Processing block {idx+1}/{num_blocks} with {current_num_frames} frames")
        
        noisy_input = noise[:, current_start_frame : current_start_frame + current_num_frames]

        # Denoising steps
        for step_idx, current_timestep in enumerate(pipeline.denoising_step_list):
            timestep = torch.ones([1, current_num_frames], device=noise.device, dtype=torch.int64) * current_timestep
            _, denoised_pred = pipeline.generator(
                noisy_image_or_video=noisy_input, conditional_dict=conditional_dict,
                timestep=timestep, kv_cache=pipeline.kv_cache1,
                crossattn_cache=pipeline.crossattn_cache,
                current_start=current_start_frame * pipeline.frame_seq_length
            )
            if step_idx < len(pipeline.denoising_step_list) - 1:
                next_timestep = pipeline.denoising_step_list[step_idx + 1]
                noisy_input = pipeline.scheduler.add_noise(
                    denoised_pred.flatten(0, 1), torch.randn_like(denoised_pred.flatten(0, 1)),
                    next_timestep * torch.ones([1 * current_num_frames], device=noise.device, dtype=torch.long)
                ).unflatten(0, denoised_pred.shape[:2])

        if idx < len(all_num_frames) - 1:
            pipeline.generator(
                noisy_image_or_video=denoised_pred, conditional_dict=conditional_dict,
                timestep=torch.zeros_like(timestep), kv_cache=pipeline.kv_cache1,
                crossattn_cache=pipeline.crossattn_cache,
                current_start=current_start_frame * pipeline.frame_seq_length,
            )

        # Decode to pixels
        if args.trt:
            pixels, vae_cache = pipeline.vae.forward(denoised_pred.half(), *vae_cache)
        elif APP_STATE["current_use_taehv"]:
            if latents_cache is None: 
                latents_cache = denoised_pred
            else:
                denoised_pred = torch.cat([latents_cache, denoised_pred], dim=1)
                latents_cache = denoised_pred[:, -3:]
            pixels = pipeline.vae.decode(denoised_pred)
        else:
            pixels, vae_cache = pipeline.vae(denoised_pred.half(), *vae_cache)
            
        # Handle frame skipping for first block
        if idx == 0 and not args.trt: 
            pixels = pixels[:, 3:]
        elif APP_STATE["current_use_taehv"] and idx > 0: 
            pixels = pixels[:, 12:]

        print(f"📹 Decoded pixels shape: {pixels.shape}")
        
        # Collect all frames from this block
        for frame_idx in range(pixels.shape[1]):
            frame_tensor = pixels[0, frame_idx]  # Get single frame [C, H, W]
            
            # Normalize from [-1, 1] to [0, 255]
            frame_np = torch.clamp(frame_tensor.float(), -1., 1.) * 127.5 + 127.5
            frame_np = frame_np.to(torch.uint8).cpu().numpy()
            
            # Convert from CHW to HWC format (RGB)
            frame_np = np.transpose(frame_np, (1, 2, 0))  # CHW -> HWC
            
            all_frames_for_video.append(frame_np)
            
        current_start_frame += current_num_frames
    
    print(f"✅ Video generation completed! Total frames: {len(all_frames_for_video)}")
    
    # Save final video
    video_path = f"gradio_tmp/{seed}_{hashlib.md5(prompt.encode()).hexdigest()}.mp4"
    imageio.mimwrite(video_path, all_frames_for_video, fps=fps, quality=8)
    print(f"✅ Video saved to {video_path}")
    
    return video_path
    
# --- Gradio UI Layout ---
frame_display = gr.Image(
    label="Generated Frames",
    height=480,
    width=832,
    show_label=True,
    container=True,
    visible=False
)
final_video = gr.Video(
    label="Final Rendered Video", 
    visible=True, 
    interactive=False,
    height=400,
    autoplay=True
)
status_html = gr.HTML(
    value="<div style='text-align: center; padding: 20px; color: #666;'>Ready to start generation...</div>",
    label="Generation Status"
)
with gr.Blocks(title="Self-Forcing Frame Streaming Demo") as demo:
    gr.Markdown("# 🚀 Self-Forcing Video Generation with Frame Streaming")
    gr.Markdown("Real-time video generation with frame-by-frame display. [[Model]](https://huggingface.co/gdhe17/Self-Forcing), [[Project page]](https://self-forcing.github.io), [[Paper]](https://huggingface.co/papers/2506.08009)")
    
    with gr.Row():
        with gr.Column(scale=2):
            gr.Markdown("### 📝 Configure Generation")
            prompt = gr.Textbox(
                label="Prompt", 
                placeholder="A stylish woman walks down a Tokyo street...", 
                lines=4,
            )
            enhance_button = gr.Button("Enhance prompt")
            gr.Examples(
                examples=[
                    "A close-up shot of a ceramic teacup slowly pouring water into a glass mug. The water flows smoothly from the spout of the teacup into the mug, creating gentle ripples as it fills up. Both cups have detailed textures, with the teacup having a matte finish and the glass mug showcasing clear transparency. The background is a blurred kitchen countertop, adding context without distracting from the central action. The pouring motion is fluid and natural, emphasizing the interaction between the two cups.",
                    "A playful cat is seen playing an electronic guitar, strumming the strings with its front paws. The cat has distinctive black facial markings and a bushy tail. It sits comfortably on a small stool, its body slightly tilted as it focuses intently on the instrument. The setting is a cozy, dimly lit room with vintage posters on the walls, adding a retro vibe. The cat's expressive eyes convey a sense of joy and concentration. Medium close-up shot, focusing on the cat's face and hands interacting with the guitar.",
                    "A dynamic over-the-shoulder perspective of a chef meticulously plating a dish in a bustling kitchen. The chef, a middle-aged woman, deftly arranges ingredients on a pristine white plate. Her hands move with precision, each gesture deliberate and practiced. The background shows a crowded kitchen with steaming pots, whirring blenders, and the clatter of utensils. Bright lights highlight the scene, casting shadows across the busy workspace. The camera angle captures the chef's detailed work from behind, emphasizing his skill and dedication.",
                ],
                inputs=[prompt],
                fn=video_generation_handler_example,
                outputs=[final_video],
                cache_examples="lazy"
            )
            
            with gr.Row():
                seed = gr.Number(label="Seed", value=-1, info="Use -1 for a random seed.")
                fps = gr.Slider(
                    label="Playback FPS", 
                    minimum=1, 
                    maximum=30, 
                    value=args.fps, 
                    step=1,
                    visible=False,
                    info="Frames per second for playback"
                )
                
            start_btn = gr.Button("🎬 Start Generation", variant="primary", size="lg")
            
        with gr.Column(scale=3):
            gr.Markdown("### 📺 Live Frame Stream")
            gr.Markdown("*Click 'Start Generation' to begin frame streaming*")

            final_video.render()
            
            frame_display.render()

            status_html.render()

    # Connect the generator to the image display
    start_btn.click(
        fn=video_generation_handler,
        inputs=[prompt, seed, fps],
        outputs=[frame_display, final_video, status_html]
    )
    enhance_button.click(
        fn=enhance_prompt,
        inputs=[prompt],
        outputs=[prompt]
    )

# --- Launch App ---
if __name__ == "__main__":
    if os.path.exists("gradio_tmp"):
        import shutil
        shutil.rmtree("gradio_tmp")
    os.makedirs("gradio_tmp", exist_ok=True)
    
    demo.queue().launch(
        server_name=args.host, 
        server_port=args.port, 
        share=args.share,
        show_error=True
    )