handler.py

from typing import Dict, Any, Union, Optional
import torch
from diffusers import LTXPipeline, LTXImageToVideoPipeline
from PIL import Image
import base64
import io

class EndpointHandler:
    def __init__(self, path: str = ""):
        """Initialize the LTX Video handler with both text-to-video and image-to-video pipelines.
        
        Args:
            path (str): Path to the model weights directory
        """
        # Load both pipelines with bfloat16 precision as recommended in docs
        self.text_to_video = LTXPipeline.from_pretrained(
            path,
            torch_dtype=torch.bfloat16
        ).to("cuda")
        
        self.image_to_video = LTXImageToVideoPipeline.from_pretrained(
            path,
            torch_dtype=torch.bfloat16
        ).to("cuda")

        # Enable memory optimizations
        self.text_to_video.enable_model_cpu_offload()
        self.image_to_video.enable_model_cpu_offload()

    def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """Process the input data and generate video using LTX.
        
        Args:
            data (Dict[str, Any]): Input data containing:
                - prompt (str): Text description for video generation
                - image (Optional[str]): Base64 encoded image for image-to-video generation
                - num_frames (Optional[int]): Number of frames to generate (default: 24)
                - guidance_scale (Optional[float]): Guidance scale (default: 7.5)
                - num_inference_steps (Optional[int]): Number of inference steps (default: 50)
        
        Returns:
            Dict[str, Any]: Dictionary containing:
                - frames: List of base64 encoded frames
        """
        # Extract parameters
        prompt = data.get("prompt")
        if not prompt:
            raise ValueError("'prompt' is required in the input data")

        # Get optional parameters with defaults
        num_frames = data.get("num_frames", 24)
        guidance_scale = data.get("guidance_scale", 7.5)
        num_inference_steps = data.get("num_inference_steps", 50)

        # Check if image is provided for image-to-video generation
        image_data = data.get("image")
        
        try:
            if image_data:
                # Decode base64 image
                image_bytes = base64.b64decode(image_data)
                image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
                
                # Generate video from image
                output = self.image_to_video(
                    prompt=prompt,
                    image=image,
                    num_frames=num_frames,
                    guidance_scale=guidance_scale,
                    num_inference_steps=num_inference_steps
                )
            else:
                # Generate video from text only
                output = self.text_to_video(
                    prompt=prompt,
                    num_frames=num_frames,
                    guidance_scale=guidance_scale,
                    num_inference_steps=num_inference_steps
                )

            # Convert frames to base64
            frames = []
            for frame in output.frames[0]:  # First element contains the frames
                buffer = io.BytesIO()
                frame.save(buffer, format="PNG")
                frame_base64 = base64.b64encode(buffer.getvalue()).decode()
                frames.append(frame_base64)

            return {"frames": frames}

        except Exception as e:
            raise RuntimeError(f"Error generating video: {str(e)}")