update some files

app.py

@@ -1,31 +1,42 @@
-# app.py（主入口简化版）
+# app.py
 import gradio as gr
 from src.model_loader import load_model
 from src.video_utils import process_video_for_internvl3
+from src.ar_prompts import generate_conversation_questions
 
-# === 初始化模型 ===
 tokenizer, model = load_model()
 
-# === 推理接口 ===
-def evaluate_ar(video):
-    pixel_values, num_patches_list, prompt = process_video_for_internvl3(video)
-    generation_config = dict(max_new_tokens=512)
-    output, _ = model.chat(
-        tokenizer,
-        pixel_values,
-        prompt,
-        generation_config=generation_config,
-        num_patches_list=num_patches_list,
-        history=None,
-        return_history=True
-    )
-    return output
-
-# === Gradio 接口 ===
+def evaluate_ar_multi_turn(video):
+    pixel_values, num_patches_list, image_prefix = process_video_for_internvl3(video)
+    conversation = generate_conversation_questions(include_descriptions=True)
+
+    history = None
+    visible_outputs = []
+
+    for i, question in enumerate(conversation):
+        prompt = image_prefix + question if i == 0 else question
+
+        output, history = model.chat(
+            tokenizer,
+            pixel_values,
+            prompt,
+            generation_config={"max_new_tokens": 1024},
+            num_patches_list=num_patches_list,
+            history=history,
+            return_history=True
+        )
+
+        # 仅保留评测和拓展部分的回答（即从第3轮开始）
+        if i >= 2:
+            visible_outputs.append(output)
+
+    # 多个输出拼接成文本显示
+    return "\n\n".join(visible_outputs)
+
 gr.Interface(
-    fn=evaluate_ar,
+    fn=evaluate_ar_multi_turn,
     inputs=gr.Video(label="Upload your AR video"),
     outputs="text",
-    title="InternVL3 AR Evaluation (Single-turn)",
-    description="Upload a short AR video clip. The model will sample frames and assess occlusion/rendering quality."
-).launch()
+    title="InternVL3 AR Evaluation (Multi-turn)",
+    description="Upload a short AR video clip. The model will sample frames and conduct a multi-turn dialogue to assess occlusion/rendering/placement/lighting."
+).launch()

src/prompt.py

@@ -1,7 +0,0 @@
-# src/prompt.py
-
-def build_video_prompt(num_frames: int) -> str:
-    """构建适用于 InternVL3 的单轮 AR 视频评估提示语。"""
-    frame_descriptors = ''.join([f"Frame{i+1}: <image>\n" for i in range(num_frames)])
-    final_prompt = frame_descriptors + "Evaluate the quality of AR occlusion and rendering in the uploaded video."
-    return final_prompt

src/video_utils.py

@@ -1,16 +1,15 @@
-# src/video_utils.py
+# ✅ src/video_utils.py（返回 <image> prefix 支持多轮对话）
 import numpy as np
 import torch
 from PIL import Image
 from decord import VideoReader, cpu
 import torchvision.transforms as T
 from torchvision.transforms.functional import InterpolationMode
-from src.prompt import build_video_prompt
 
 IMAGENET_MEAN = (0.485, 0.456, 0.406)
 IMAGENET_STD = (0.229, 0.224, 0.225)
 
-# === 构建标准图像预处理 transform ===
+# 图像预处理 transform
 def build_transform(input_size=448):
     return T.Compose([
         T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
@@ -19,12 +18,16 @@ def build_transform(input_size=448):
         T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
     ])
 
-# === InternVL3 视频帧采样策略 ===
+# 视频帧采样策略
 def get_frame_indices(num_frames, total_frames):
     indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
     return indices
 
-# === 从视频中提取图像帧并预处理成 patch tensor ===
+# 构建 <image> token 的前缀信息
+def build_image_prefix(num_frames: int) -> str:
+    return ''.join([f"Frame{i+1}: <image>\n" for i in range(num_frames)])
+
+# 视频处理为 patch tensor，并返回 <image> 前缀
 def process_video_for_internvl3(video_path, num_segments=8, max_patch_per_frame=1, input_size=448):
     vr = VideoReader(video_path, ctx=cpu(0))
     total_frames = len(vr)
@@ -41,16 +44,15 @@ def process_video_for_internvl3(video_path, num_segments=8, max_patch_per_frame=
         num_patches_list.append(patch_tensor.shape[0])
 
     pixel_values = torch.cat(pixel_values_list, dim=0).to(torch.bfloat16).cuda()
-    prompt = build_video_prompt(len(num_patches_list))
+    image_prefix = build_image_prefix(len(num_patches_list))
 
-    return pixel_values, num_patches_list, prompt
+    return pixel_values, num_patches_list, image_prefix
 
-# === 图像切片为 patch 区块 ===
+# 图像切块
 def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=True):
     orig_width, orig_height = image.size
     aspect_ratio = orig_width / orig_height
 
-    # 构造备选分块比率
     target_ratios = set(
         (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1)
         if i * j <= max_num and i * j >= min_num
@@ -80,7 +82,7 @@ def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbna
 
     return processed_images
 
-# === 找出最接近原图比例的块切方案 ===
+# 找最接近原图比例的切块方案
 def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
     best_ratio_diff = float('inf')
     best_ratio = (1, 1)