Update app.py

update demo test

app.py

@@ -5,8 +5,13 @@ import re
 import sys
 import copy
 import warnings
+warnings.filterwarnings("ignore", category=UserWarning)
 from typing import Optional
 
+import threading
+from transformers import TextIteratorStreamer
+
+
 # Third-party imports
 import numpy as np
 import torch
@@ -22,6 +27,9 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 
 import gradio as gr
 import spaces
+import json
+from datetime import datetime
+import shutil
 
 # Local imports
 from egogpt.model.builder import load_pretrained_model
@@ -45,19 +53,21 @@ from huggingface_hub import snapshot_download
 #     ignore_patterns=["*.md", "*.txt"]  # 可以忽略一些不必要的文件（可选）
 # )
 
-from huggingface_hub import hf_hub_download
+# from huggingface_hub import hf_hub_download
 
-# Download the model checkpoint file (large-v3.pt)
-ego_gpt_path = hf_hub_download(
-    repo_id="EgoLife-v1/EgoGPT",
-    filename="large-v3.pt",
-    local_dir="./"
-)
+# # Download the model checkpoint file (large-v3.pt)
+# ego_gpt_path = hf_hub_download(
+#     repo_id="EgoLife-v1/EgoGPT",
+#     filename="large-v3.pt",
+#     local_dir="./"
+# )
 
 
 # pretrained = "/mnt/sfs-common/jkyang/EgoGPT/checkpoints/EgoGPT-llavaov-7b-EgoIT-109k-release"
 # pretrained = "/mnt/sfs-common/jkyang/EgoGPT/checkpoints/EgoGPT-llavaov-7b-EgoIT-EgoLife-Demo"
-pretrained = 'EgoLife-v1/EgoGPT'
+# pretrained = 'EgoLife-v1/EgoGPT'
+pretrained = 'EgoLife-v1/EgoGPT-0.5b-Demo'
+# pretrained = "/mnt/sfs-common/jkyang/EgoGPT_release/checkpoints/EgoGPT-7b-Demo"
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 device_map = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
@@ -100,15 +110,8 @@ title_markdown = """
 """
 notice_html = """
 <div style="background-color: #f9f9f9; border-left: 5px solid #48dbfb; padding: 20px; margin-top: 20px; border-radius: 10px; box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);">
-    <ul style="list-style-type: none; padding-left: 0; font-size: 1.1em; color: #555;">
-        <li>- Due to hardware limitations on this demo page, we recommend users only try 10-second videos.</li>
-        <li>- The demo model is used for the egocentric video captioning step for the EgoRAG framework. The recommended prompt includes:</li>
-        <ul style="padding-left: 20px; margin-top: 10px; color: #333;">
-            <li>Can you help me log everything I do and the key things I see, like a personal journal? Describe them in a natural style. 
-            <li>Please provide your response using the first person, with "I" as the subject. Make sure the descriptions are detailed and natural.</li>
-            <li>Can you write down important things I notice or interact with? Please respond in the first person, using "I" as the subject. Describe them in a natural style.</li>
-        </ul>
-    </ul>
+    <p style="font-size: 1.1em; color: #ff9933; margin-bottom: 10px; font-weight: bold;">💡 Pro Tip: Try accessing this demo from your phone's browser. You can use your phone's camera to capture and analyze egocentric videos, making the experience more interactive and personal.</p>
+    <p style="font-size: 1.1em; color: #555; margin-bottom: 10px;">EgoGPT-7B is built upon LLaVA-OV and has been finetuned on the EgoIT dataset and a partially de-identified EgoLife dataset. Its primary goal is to serve as an egocentric captioner, supporting EgoRAG for EgoLifeQA tasks. Please note that due to inherent biases in the EgoLife dataset, the model may occasionally hallucinate details about people in custom videos based on patterns from the training data (for example, describing someone as "wearing a blue t-shirt" or "with pink hair"). We are actively working on improving the model to make it more universally applicable and will continue to release updates regularly. If you're interested in contributing to the development of future iterations of EgoGPT or the EgoLife project, we welcome you to reach out and contact us. (Contact us at <a href="mailto:[email protected]">[email protected]</a>)</p>
 </div>
 """
 
@@ -174,15 +177,13 @@ def load_video(
     vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
     target_sr = 16000
     
-    # Add new time-based processing logic
+    # Process video frames first
     if time_based_processing:
         # Initialize video reader
         vr = decord.VideoReader(video_path, ctx=decord.cpu(0), num_threads=1)
         total_frame_num = len(vr)
-
-        # Get the actual FPS of the video
         video_fps = vr.get_avg_fps()
-
+        
         # Convert time to frame index based on the actual video FPS
         video_start_frame = int(time_to_frame_idx(video_start_time, video_fps))
         start_frame = int(time_to_frame_idx(start_time, video_fps))
@@ -208,20 +209,8 @@ def load_video(
 
         # Get the video frames for the sampled indices
         video = vr.get_batch(frame_idx).asnumpy()
-        target_sr = 16000  # Set target sample rate to 16kHz
-    
-        # Load audio from video with resampling
-        y, _ = librosa.load(video_path, sr=target_sr)
-        
-        # Convert time to audio samples (using 16kHz sample rate)
-        start_sample = int(start_time * target_sr)
-        end_sample = int(end_time * target_sr)
-        
-        # Extract audio segment
-        speech = y[start_sample:end_sample]
     else:
-        # Original processing logic
-        speech, _ = librosa.load(video_path, sr=target_sr)
+        # Original video processing logic
         total_frame_num = len(vr)
         avg_fps = round(vr.get_avg_fps() / fps)
         frame_idx = [i for i in range(0, total_frame_num, avg_fps)]
@@ -233,12 +222,32 @@ def load_video(
         
         video = vr.get_batch(frame_idx).asnumpy()
 
-    # Process audio
-    speech = whisper.pad_or_trim(speech.astype(np.float32))
-    speech = whisper.log_mel_spectrogram(speech, n_mels=128).permute(1, 0)
-    speech_lengths = torch.LongTensor([speech.shape[0]])
-    
-    return video, speech, speech_lengths
+    # Try to load audio, return None for speech if failed
+    try:
+        if time_based_processing:
+            y, _ = librosa.load(video_path, sr=target_sr)
+            start_sample = int(start_time * target_sr)
+            end_sample = int(end_time * target_sr)
+            speech = y[start_sample:end_sample]
+        else:
+            speech, _ = librosa.load(video_path, sr=target_sr)
+            
+        # Process audio if it exists
+        speech = whisper.pad_or_trim(speech.astype(np.float32))
+        speech = whisper.log_mel_spectrogram(speech, n_mels=128).permute(1, 0)
+        speech_lengths = torch.LongTensor([speech.shape[0]])
+        
+        return video, speech, speech_lengths, True  # True indicates real audio
+        
+    except Exception as e:
+        print(f"Warning: Could not load audio from video: {e}")
+        # Create dummy silent audio
+        duration = 10  # 10 seconds
+        speech = np.zeros(duration * target_sr, dtype=np.float32)
+        speech = whisper.pad_or_trim(speech)
+        speech = whisper.log_mel_spectrogram(speech, n_mels=128).permute(1, 0)
+        speech_lengths = torch.LongTensor([speech.shape[0]])
+        return video, speech, speech_lengths, False  # False indicates no real audio
 
 class PromptRequest(BaseModel):
     prompt: str
@@ -251,94 +260,188 @@ class PromptRequest(BaseModel):
     time_based_processing: bool = False
 
 # @spaces.GPU(duration=120)
+def save_interaction(video_path, prompt, output, audio_path=None):
+    """Save user interaction data and files"""
+    if not video_path:
+        return
+    
+    # Create timestamped directory for this interaction
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    interaction_dir = os.path.join(UPLOADS_DIR, timestamp)
+    os.makedirs(interaction_dir, exist_ok=True)
+    
+    # Copy video file
+    video_ext = os.path.splitext(video_path)[1]
+    new_video_path = os.path.join(interaction_dir, f"video{video_ext}")
+    shutil.copy2(video_path, new_video_path)
+    
+    # Save metadata
+    metadata = {
+        "timestamp": timestamp,
+        "prompt": prompt,
+        "output": output,
+        "video_path": new_video_path,
+    }
+    
+    # Only try to save audio if it's a file path (str), not audio data (tuple)
+    if audio_path and isinstance(audio_path, (str, bytes, os.PathLike)):
+        audio_ext = os.path.splitext(audio_path)[1]
+        new_audio_path = os.path.join(interaction_dir, f"audio{audio_ext}")
+        shutil.copy2(audio_path, new_audio_path)
+        metadata["audio_path"] = new_audio_path
+    
+    with open(os.path.join(interaction_dir, "metadata.json"), "w") as f:
+        json.dump(metadata, f, indent=4)
+
+def extract_audio_from_video(video_path, audio_path=None):
+    print('Processing audio from video...', video_path, audio_path)
+    if video_path is None:
+        return None
+        
+    if isinstance(video_path, dict) and 'name' in video_path:
+        video_path = video_path['name']
+    
+    try:
+        y, sr = librosa.load(video_path, sr=8000, mono=True, res_type='kaiser_fast')
+        # Check if the audio is silent
+        if np.abs(y).mean() < 0.001:
+            print("Video appears to be silent")
+            return None
+        return (sr, y)
+    except Exception as e:
+        print(f"Warning: Could not extract audio from video: {e}")
+        return None
+
+import time
+
 def generate_text(video_path, audio_track, prompt):
+    streamer = TextIteratorStreamer(tokenizer, skip_special_tokens=True)
+
     max_frames_num = 30
     fps = 1
-    # model.eval()
+    conv_template = "qwen_1_5"
+    if video_path is None and audio_track is None:
+        question = prompt
+        speech = None
+        speech_lengths = None
+        has_real_audio = False
+        image = None
+        image_sizes= None
+        modalities = ["image"]
+        image_tensor=None
+    # Load video and potentially audio
+    else:
+        video, speech, speech_lengths, has_real_audio = load_video(
+            video_path=video_path,
+            max_frames_num=max_frames_num,
+            fps=fps,
+        )
+
+        # Prepare the prompt based on whether we have real audio
+        if not has_real_audio:
+            question = f"<image>\n{prompt}"  # Video-only prompt
+        else:
+            question = f"<speech>\n<image>\n{prompt}"  # Video + speech prompt
+
+        speech = torch.stack([speech]).to("cuda").half()
+        processor = model.get_vision_tower().image_processor
+        processed_video = processor.preprocess(video, return_tensors="pt")["pixel_values"]
+        image = [(processed_video, video[0].size, "video")]
+        image_tensor = [image[0][0].half()]
+        image_sizes = [image[0][1]]
+        modalities = ["video"]
 
-    # Video + speech branch
-    conv_template = "qwen_1_5"  # Make sure you use correct chat template for different models
-    question = f"<image>\n{prompt}"
     conv = copy.deepcopy(conv_templates[conv_template])
     conv.append_message(conv.roles[0], question)
     conv.append_message(conv.roles[1], None)
     prompt_question = conv.get_prompt()
 
-    video, speech, speech_lengths = load_video(
-        video_path=video_path,
-        max_frames_num=max_frames_num,
-        fps=fps,
-    )
-    speech=torch.stack([speech]).to("cuda").half()
-    processor = model.get_vision_tower().image_processor
-    processed_video = processor.preprocess(video, return_tensors="pt")["pixel_values"]
-    image = [(processed_video, video[0].size, "video")]
-
-    print(prompt_question)
-    parts=split_text(prompt_question,["<image>","<speech>"])
-    input_ids=[]
+
+
+    parts = split_text(prompt_question, ["<image>", "<speech>"])
+    input_ids = []
     for part in parts:
-        if "<image>"==part:
-            input_ids+=[IMAGE_TOKEN_INDEX]
-        elif "<speech>"==part:
-            input_ids+=[SPEECH_TOKEN_INDEX]
+        if "<image>" == part:
+            input_ids += [IMAGE_TOKEN_INDEX]
+        elif "<speech>" == part and speech is not None:  # Only add speech token if we have audio
+            input_ids += [SPEECH_TOKEN_INDEX]
         else:
-            input_ids+=tokenizer(part).input_ids
-            
-    input_ids = torch.tensor(input_ids,dtype=torch.long).unsqueeze(0).to(device)
-    image_tensor = [image[0][0].half()]
-    image_sizes = [image[0][1]]
-
-    generate_kwargs={"eos_token_id":tokenizer.eos_token_id}
-    print(input_ids)
-    cont = model.generate(
-        input_ids,
-        images=image_tensor,
-        image_sizes=image_sizes,
-        speech=speech,
-        speech_lengths=speech_lengths,
-        do_sample=False,
-        temperature=0.5,
-        max_new_tokens=4096,
-        modalities=["video"],
-        **generate_kwargs
-    )
-    
-    text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
-    
-    return text_outputs[0]
+            input_ids += tokenizer(part).input_ids
+
+    input_ids = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0).to(device)
+
+
+    generate_kwargs = {"eos_token_id": tokenizer.eos_token_id}
+
+    def generate_response():
+        model.generate(
+            input_ids,
+            images=image_tensor,
+            image_sizes=image_sizes,
+            speech=speech,
+            speech_lengths=speech_lengths,
+            do_sample=False,
+            temperature=0.7,
+            max_new_tokens=512,
+            repetition_penalty=1.2,
+            modalities=modalities,
+            streamer=streamer,
+            **generate_kwargs
+        )
+
+    # Start generation in a separate thread
+    thread = threading.Thread(target=generate_response)
+    thread.start()
+
+    # Stream the output word by word
+    generated_text = ""
+    partial_word = ""
+    cursor = "|"  
+    cursor_visible = True
+    last_cursor_toggle = time.time()
+
+    for new_text in streamer:
+        partial_word += new_text
+        # Toggle the cursor visibility every 0.5 seconds
+        if time.time() - last_cursor_toggle > 0.5:
+            cursor_visible = not cursor_visible
+            last_cursor_toggle = time.time()
+        current_cursor = cursor if cursor_visible else " "
+        if partial_word.endswith(" ") or partial_word.endswith("\n"):
+            generated_text += partial_word
+            # Yield the current text with the cursor appended
+            yield generated_text + current_cursor
+            partial_word = ""
+        else:
+            # Yield the current text plus the partial word and the cursor
+            yield generated_text + partial_word + current_cursor
 
-def extract_audio_from_video(video_path, audio_path=None):
-    if audio_path:
-        try:
-            y, sr = librosa.load(audio_path, sr=8000, mono=True, res_type='kaiser_fast')
-            return (sr, y)
-        except Exception as e:
-            print(f"Error loading audio from {audio_path}: {e}")
-            return None
-    if video_path is None:
-        return None
-    try:
-        y, sr = librosa.load(video_path, sr=8000, mono=True, res_type='kaiser_fast')
-        return (sr, y)
-    except Exception as e:
-        print(f"Error extracting audio from video: {e}")
-        return None
+    # Handle any remaining partial word at the end
+    if partial_word:
+        generated_text += partial_word
+        yield generated_text
+
+    # Save the interaction after generation is complete
+    save_interaction(video_path, prompt, generated_text, audio_track)
 
 head = """
+<head>
+    <title>EgoGPT Demo - EgoLife</title>
+    <link rel="icon" type="image/x-icon" href="./egolife_circle.ico">
+</head>
 <style>
 /* Submit按钮默认和悬停效果 */
-button.lg.secondary.svelte-1gz44hr {
+button.lg.secondary.svelte-5st68j {
     background-color: #ff9933 !important;
     transition: background-color 0.3s ease !important;
 }
 
-button.lg.secondary.svelte-1gz44hr:hover {
+button.lg.secondary.svelte-5st68j:hover {
     background-color: #ff7777 !important;  /* 悬停时颜色加深 */
 }
 
 /* 确保按钮文字始终清晰可见 */
-button.lg.secondary.svelte-1gz44hr span {
+button.lg.secondary.svelte-5st68j span {
     color: white !important;
 }
 
@@ -360,137 +463,164 @@ button.lg.secondary.svelte-1gz44hr span {
 </style>
 
 <script>
-// 新版同步控制代码
-function syncMediaElements() {
-    // 获取视频和音频元素
-    const video = document.querySelector('[data-testid="Video-player"] video');
-    const waveform = document.querySelector('#waveform');
-    const audio = waveform?.querySelector('audio') || waveform?.shadowRoot?.querySelector('audio');
-
-    // 如果任一元素不存在，则退出
-    if (!video || !audio) return;
-
-    // 解除旧的事件监听（避免重复绑定）
-    video.removeEventListener('play', syncPlay);
-    audio.removeEventListener('play', syncPlay);
-    video.removeEventListener('timeupdate', syncVideoTime);
-    audio.removeEventListener('timeupdate', syncAudioTime);
-
-    // 定义同步函数
-    function syncPlay(e) {
-        if(e.target === video && audio.paused) audio.play();
-        if(e.target === audio && video.paused) video.play();
+function initializeControls() {
+    const video = document.querySelector('[data-testid="Video-player"]');
+    const waveform = document.getElementById('waveform');
+    
+    // 如果元素还没准备好，直接返回
+    if (!video || !waveform) {
+        return;
+    }
+    
+    // 尝试获取音频元素
+    const audio = waveform.querySelector('div')?.shadowRoot?.querySelector('audio');
+    if (!audio) {
+        return;
     }
 
-    function syncVideoTime() {
-        if(!audio.seeking && Math.abs(video.currentTime - audio.currentTime) > 0.1){
-            audio.currentTime = video.currentTime;
-        }
+    console.log('Elements found:', { video, audio });
+    
+   // 监听视频播放进度
+  video.addEventListener("play", () => {
+    if (audio.paused) {
+      audio.play();  // 如果音频暂停，开始播放
     }
+  });
 
-    function syncAudioTime() {
-        if(!video.seeking && Math.abs(audio.currentTime - video.currentTime) > 0.1){
-            video.currentTime = audio.currentTime;
-        }
+  // 监听音频播放进度
+  audio.addEventListener("play", () => {
+    if (video.paused) {
+      video.play();  // 如果视频暂停，开始播放
     }
+  });
 
-    // 绑定新的事件监听
-    video.addEventListener('play', syncPlay);
-    audio.addEventListener('play', syncPlay);
-    video.addEventListener('timeupdate', syncVideoTime);
-    audio.addEventListener('timeupdate', syncAudioTime);
+  // 同步视频和音频的播放进度
+  video.addEventListener("timeupdate", () => {
+    if (Math.abs(video.currentTime - audio.currentTime) > 0.1) {
+      audio.currentTime = video.currentTime; // 如果时间差超过0.1秒，同步
+    }
+  });
 
-    // 同步暂停事件
-    video.addEventListener('pause', () => audio.pause());
-    audio.addEventListener('pause', () => video.pause());
+  audio.addEventListener("timeupdate", () => {
+    if (Math.abs(audio.currentTime - video.currentTime) > 0.1) {
+      video.currentTime = audio.currentTime; // 如果时间差超过0.1秒，同步
+    }
+  });
+
+  // 监听暂停事件，确保视频和音频都暂停
+  video.addEventListener("pause", () => {
+    if (!audio.paused) {
+      audio.pause();  // 如果音频未暂停，暂停音频
+    }
+  });
 
-    console.log('Media elements synced successfully!');
+  audio.addEventListener("pause", () => {
+    if (!video.paused) {
+      video.pause();  // 如果视频未暂停，暂停视频
+    }
+  });
 }
 
-// 智能DOM观察器
+// 创建观察器监听DOM变化
 const observer = new MutationObserver((mutations) => {
-    mutations.forEach((mutation) => {
+    for (const mutation of mutations) {
         if (mutation.addedNodes.length) {
-            mutation.addedNodes.forEach((node) => {
-                // 深度检查新增节点
-                if (node.nodeType === 1) { // Element node
-                    // 检查是否包含视频组件
-                    if (node.querySelector?.('[data-testid="Video-player"]')) {
-                        // 当视频组件出现时，开始查找音频
-                        const audioObserver = new MutationObserver(() => {
-                            if(document.querySelector('#waveform audio')) {
-                                audioObserver.disconnect();
-                                setTimeout(syncMediaElements, 500); // 等待组件完全加载
-                            }
-                        });
-                        audioObserver.observe(document.body, {
-                            childList: true,
-                            subtree: true
-                        });
-                    }
-                }
-            });
+            // 当有新节点添加时，尝试初始化
+            const waveform = document.getElementById('waveform');
+            if (waveform?.querySelector('div')?.shadowRoot?.querySelector('audio')) {
+                console.log('Audio element detected');
+                initializeControls();
+                // 可选：如果不需要继续监听，可以断开观察器
+                // observer.disconnect();
+            }
         }
-    });
+    }
 });
 
-// 开始观察整个文档
+// 开始观察
 observer.observe(document.body, {
     childList: true,
     subtree: true
 });
 
-// 初始检查（应对组件已存在的情况）
-setTimeout(() => {
-    if(document.querySelector('[data-testid="Video-player"] video') && 
-       document.querySelector('#waveform audio')){
-        syncMediaElements();
+// 页面加载完成时也尝试初始化
+document.addEventListener('DOMContentLoaded', () => {
+    console.log('DOM Content Loaded');
+    initializeControls();
+
+    // Ensure title and favicon are set correctly
+    document.title = "EgoGPT Demo - EgoLife";
+    
+    // Create/update favicon link
+    let link = document.querySelector("link[rel~='icon']");
+    if (!link) {
+        link = document.createElement('link');
+        link.rel = 'icon';
+        document.head.appendChild(link);
     }
-}, 1000);
+    link.href = './egolife_circle.ico';
+
+});
+
 </script>
 """
 
-with gr.Blocks(head=head) as demo:
+with gr.Blocks(title="EgoGPT Demo - EgoLife", head=head) as demo:
     gr.HTML(title_markdown)
     gr.HTML(notice_html)
     
     with gr.Row():
         with gr.Column():
             video_input = gr.Video(label="Video", autoplay=True, loop=True, format="mp4", width=600, height=400, show_label=False, elem_id='video')
-            # Audio input synchronized with video playback
+            # Make audio display conditionally visible
             audio_display = gr.Audio(label="Video Audio Track", autoplay=False, show_label=True, visible=True, interactive=False, elem_id="audio")
-            text_input = gr.Textbox(label="Question", placeholder="Enter your message here...")
+            text_input = gr.Textbox(label="Question", placeholder="Enter your message here...", value="Describe everything I saw, did, and heard, using the first perspective. Transcribe all the speech.")
         
-        with gr.Column():  # Create a separate column for output and examples
+        with gr.Column():
             output_text = gr.Textbox(label="Response", lines=14, max_lines=14)
             gr.Examples(
                 examples=[
-                    [f"{cur_dir}/bike.mp4", f"{cur_dir}/bike.mp3", "Can you tell me what I'm doing in short words. Describe them in a natural style."],
-                    [f"{cur_dir}/bike.mp4", f"{cur_dir}/bike.mp3", "Can you tell me what I'm doing in short words. Describe them in a natural style."],
-                    [f"{cur_dir}/bike.mp4", f"{cur_dir}/bike.mp3", "Can you tell me what I'm doing in short words. Describe them in a natural style."],
-                    [f"{cur_dir}/bike.mp4", f"{cur_dir}/bike.mp3", "Can you tell me what I'm doing in short words. Describe them in a natural style."]
+                    [f"{cur_dir}/videos/cheers.mp4", f"{cur_dir}/videos/cheers.mp3", "Describe everything I saw, did, and heard from the first perspective."],
+                    [f"{cur_dir}/videos/DAY3_A6_SHURE_14550000.mp4", f"{cur_dir}/videos/DAY3_A6_SHURE_14550000.mp3", "请按照时间顺序描述我所见所为，并转录所有声音。"],
+                    [f"{cur_dir}/videos/shopping.mp4", f"{cur_dir}/videos/shopping.mp3", "Please only transcribe all the speech."],
+                    [f"{cur_dir}/videos/japan.mp4", f"{cur_dir}/videos/japan.mp3", "Describe everything I see, do, and hear from the first-person view."],
                 ],
                 inputs=[video_input, audio_display, text_input],
                 outputs=[output_text]
             )
 
-    # Add event handler for video changes
+    def handle_video_change(video):
+        if video is None:
+            return gr.update(visible=False), None
+        
+        audio = extract_audio_from_video(video)
+        # Update audio display visibility based on whether we have audio
+        return gr.update(visible=audio is not None), audio
+    
+    # Update the video input change event
     video_input.change(
-        fn=lambda video_path: extract_audio_from_video(video_path, audio_path=None),
+        fn=handle_video_change,
         inputs=[video_input],
-        outputs=[audio_display]
+        outputs=[audio_display, audio_display]  # First for visibility, second for audio data
     )
 
-    # Add event handler for video clear/delete
+    # Add clear handler
     def clear_outputs(video):
-        if video is None:  # Video is cleared/deleted
-            return ""
-        return gr.skip()  # Keep existing text if video exists
-    
-    video_input.change(
+        if video is None:
+            return gr.update(visible=False), "", None
+        return gr.skip()
+
+    video_input.clear(
         fn=clear_outputs,
         inputs=[video_input],
-        outputs=[output_text]
+        outputs=[audio_display, output_text, audio_display]
+    )
+
+    text_input.submit(
+        fn=generate_text,
+        inputs=[video_input, audio_display, text_input],
+        outputs=[output_text],
+        api_name="generate_streaming"
     )
 
     # Add submit button and its event handler
@@ -498,11 +628,11 @@ with gr.Blocks(head=head) as demo:
     submit_btn.click(
         fn=generate_text,
         inputs=[video_input, audio_display, text_input],
-        outputs=[output_text]
+        outputs=[output_text],
+        api_name="generate_streaming"
     )
 
     gr.Markdown(bibtext)
 # Launch the Gradio app
 if __name__ == "__main__":
-    demo.launch(share=True) 
-
+    demo.launch(share=True)