Update app.py

app.py

@@ -6,6 +6,7 @@ from threading import Thread
 import gradio as gr
 import spaces
 import torch
+import re
 from transformers import AutoProcessor, Gemma3ForConditionalGeneration, TextIteratorStreamer
 
 model_id = "google/gemma-3-12b-it"
@@ -14,9 +15,69 @@ model = Gemma3ForConditionalGeneration.from_pretrained(
     model_id, device_map="auto", torch_dtype=torch.bfloat16, attn_implementation="eager"
 )
 
+import cv2
+from PIL import Image
+import numpy as np
+import tempfile
+
+def downsample_video(video_path):
+    vidcap = cv2.VideoCapture(video_path)
+    fps = vidcap.get(cv2.CAP_PROP_FPS)
+    total_frames = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
+    
+    frame_interval = int(fps / 3) 
+    frames = []
+
+    for i in range(0, total_frames, frame_interval):
+        vidcap.set(cv2.CAP_PROP_POS_FRAMES, i)
+        success, image = vidcap.read()
+        if success:
+            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+            pil_image = Image.fromarray(image)
+            timestamp = round(i / fps, 2) 
+            frames.append((pil_image, timestamp))
+
+    vidcap.release()
+    return frames
+
 
 def process_new_user_message(message: dict) -> list[dict]:
-    return [{"type": "text", "text": message["text"]}, *[{"type": "image", "url": path} for path in message["files"]]]
+    if message["files"]:
+        if "<image>" in message["text"]:
+            content = []
+            print("message[files]", message["files"])
+            parts = re.split(r'(<image>)', message["text"]) 
+            image_index = 0  
+            print("parts", parts)
+            for part in parts:
+                print("part", part)
+                if part == "<image>":
+                    content.append({"type": "image", "url": message["files"][image_index]})
+                    print("file", message["files"][image_index])
+                    image_index += 1
+                elif part.strip():  
+                    content.append({"type": "text", "text": part.strip()})
+                elif isinstance(part, str) and not part == "<image>":
+                    content.append({"type": "text", "text": part})
+            print(content)
+            return content
+        elif message["files"][0].endswith(".mp4"): 
+            content = []
+            video = message["files"].pop(0)
+            frames = downsample_video(video)
+            for frame in frames:
+                pil_image, timestamp = frame
+                with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as temp_file:
+                    pil_image.save(temp_file.name)
+                    content.append({"type": "text", "text": f"Frame {timestamp}:"})
+                    content.append({"type": "image", "url": temp_file.name})
+            print(content)
+            return content
+        else:
+            # non interleaved images
+            return [{"type": "text", "text": message["text"]}, *[{"type": "image", "url": path} for path in message["files"]]]
+    else:
+        return [{"type": "text", "text": message["text"]}]
 
 
 def process_history(history: list[dict]) -> list[dict]:
@@ -71,32 +132,26 @@ def run(message: dict, history: list[dict], system_prompt: str = "", max_new_tok
 examples = [
     [
         {
-            "text": "caption this image",
-            "files": ["assets/sample-images/01.png"],
+            "text": "I need to be in Japan for 10 days, going to Tokyo, Kyoto and Osaka. Think about number of attractions in each of them and allocate number of days to each city. Make public transport recommendations.",
+            "files": [],
         }
     ],
     [
         {
-            "text": "What's the sign says?",
-            "files": ["assets/sample-images/02.png"],
+            "text": "Write the matplotlib code to generate the same bar chart.",
+            "files": ["assets/sample-images/barchart.png"],
         }
     ],
     [
         {
-            "text": "Compare and contrast the two images.",
-            "files": ["assets/sample-images/03.png"],
+            "text": "What is odd about this video?",
+            "files": ["assets/sample-images/tmp.mp4"],
         }
     ],
     [
         {
-            "text": "List all the objects in the image and their colors.",
-            "files": ["assets/sample-images/04.png"],
-        }
-    ],
-    [
-        {
-            "text": "Describe the atmosphere of the scene.",
-            "files": ["assets/sample-images/05.png"],
+            "text": "I already have this supplement <image> and I want to buy this one <image>. Do they have known interactions?",
+            "files": ["assets/sample-images/pill1.png", "assets/sample-images/pill2.png"],
         }
     ],
     [
@@ -164,20 +219,50 @@ examples = [
             "files": ["assets/additional-examples/4.png"],
         }
     ],
+    [
+        {
+            "text": "caption this image",
+            "files": ["assets/sample-images/01.png"],
+        }
+    ],
+    [
+        {
+            "text": "What's the sign says?",
+            "files": ["assets/sample-images/02.png"],
+        }
+    ],
+    [
+        {
+            "text": "Compare and contrast the two images.",
+            "files": ["assets/sample-images/03.png"],
+        }
+    ],
+    [
+        {
+            "text": "List all the objects in the image and their colors.",
+            "files": ["assets/sample-images/04.png"],
+        }
+    ],
+    [
+        {
+            "text": "Describe the atmosphere of the scene.",
+            "files": ["assets/sample-images/05.png"],
+        }
+    ],
 ]
 
 demo = gr.ChatInterface(
     fn=run,
     type="messages",
-    textbox=gr.MultimodalTextbox(file_types=["image"], file_count="multiple"),
+    textbox=gr.MultimodalTextbox(file_types=["image", ".mp4"], file_count="multiple"),
     multimodal=True,
     additional_inputs=[
         gr.Textbox(label="System Prompt", value="You are a helpful assistant."),
-        gr.Slider(label="Max New Tokens", minimum=100, maximum=2000, step=10, value=500),
+        gr.Slider(label="Max New Tokens", minimum=100, maximum=2000, step=10, value=700),
     ],
     stop_btn=False,
     title="Gemma 3 12B it",
-    description="<img src='https://huggingface.co/spaces/huggingface-projects/gemma-3-12b-it/resolve/main/assets/logo.png' id='logo' />",
+    description="<img src='https://huggingface.co/spaces/huggingface-projects/gemma-3-12b-it/resolve/main/assets/logo.png' id='logo' /><br>This is a demo of Gemma 3 12B it, a vision language model with outstanding performance on a wide range of tasks. You can upload images, interleaved images and videos. Note that video input only supports single-turn conversation and mp4 input.",
     examples=examples,
     run_examples_on_click=False,
     cache_examples=False,