app.py

import gradio as gr
import torch
import numpy as np
import cv2
import time
import re
import spaces
from PIL import Image
from threading import Thread
from transformers import AutoProcessor, Gemma3ForConditionalGeneration, TextIteratorStreamer

#####################################
# 1. Load Model & Processor
#####################################
MODEL_ID = "google/gemma-3-12b-it"  # Adjust to your needs

processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)
model = Gemma3ForConditionalGeneration.from_pretrained(
    MODEL_ID,
    trust_remote_code=True,
    torch_dtype=torch.bfloat16
).to("cuda")
model.eval()

#####################################
# 2. Helper Function: Capture Live Frames
#####################################
def capture_live_frames(duration=5, num_frames=10):
    """
    Captures live frames from the default webcam for a specified duration.
    Returns a list of (PIL image, timestamp) tuples.
    """
    cap = cv2.VideoCapture(0)  # Use default webcam
    if not cap.isOpened():
        return []
    
    # Try to get FPS, default to 30 if not available.
    fps = cap.get(cv2.CAP_PROP_FPS)
    if fps <= 0:
        fps = 30
    total_frames_to_capture = int(duration * fps)
    frame_indices = np.linspace(0, total_frames_to_capture - 1, num_frames, dtype=int)
    
    captured_frames = []
    frame_count = 0
    start_time = time.time()
    
    while frame_count < total_frames_to_capture:
        ret, frame = cap.read()
        if not ret:
            break
        if frame_count in frame_indices:
            # Convert BGR (OpenCV) to RGB (PIL)
            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            pil_image = Image.fromarray(frame_rgb)
            timestamp = round(frame_count / fps, 2)
            captured_frames.append((pil_image, timestamp))
        frame_count += 1
        # Break if the elapsed time exceeds the duration.
        if time.time() - start_time > duration:
            break
    cap.release()
    return captured_frames

#####################################
# 3. Live Inference Function
#####################################
@spaces.GPU
def live_inference(duration=5):
    """
    Captures live frames from the webcam, builds a prompt, and returns the generated text.
    """
    frames = capture_live_frames(duration=duration, num_frames=10)
    if not frames:
        return "Could not capture live frames from the webcam."
    
    # Build prompt using the captured frames.
    messages = [{
        "role": "user",
        "content": [{"type": "text", "text": "Please describe what's happening in this live video."}]
    }]
    for (image, ts) in frames:
        messages[0]["content"].append({"type": "text", "text": f"Frame at {ts} seconds:"})
        messages[0]["content"].append({"type": "image", "image": image})
    
    prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
    frame_images = [img for (img, _) in frames]
    
    inputs = processor(
        text=[prompt],
        images=frame_images,
        return_tensors="pt",
        padding=True
    ).to("cuda")
    
    # Generate text using streaming.
    streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
    generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=512)
    
    thread = Thread(target=model.generate, kwargs=generation_kwargs)
    thread.start()
    
    generated_text = ""
    for new_text in streamer:
        generated_text += new_text
        time.sleep(0.01)
        
    return generated_text

#####################################
# 4. Build Gradio Live App
#####################################
def build_live_app():
    with gr.Blocks() as demo:
        gr.Markdown("# **Live Video Analysis**\n\nPress **Start** to capture a few seconds of live video from your webcam and analyze the content.")
        with gr.Column():
            duration_input = gr.Number(label="Capture Duration (seconds)", value=5, precision=0)
            start_btn = gr.Button("Start")
            output_text = gr.Textbox(label="Model Output")
            restart_btn = gr.Button("Start Again", visible=False)
        
        # This function triggers the live inference and also makes the restart button visible.
        def start_inference(duration):
            text = live_inference(duration)
            return text, gr.update(visible=True)
        
        start_btn.click(fn=start_inference, inputs=duration_input, outputs=[output_text, restart_btn])
        restart_btn.click(fn=start_inference, inputs=duration_input, outputs=[output_text, restart_btn])
    return demo

if __name__ == "__main__":
    app = build_live_app()
    app.launch(debug=True)