I2V

app.py

@@ -74,6 +74,15 @@ def get_normalized_df(df):
         normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
     return normalize_df
 
+def get_normalized_i2v_df(df):
+    normalize_df = df.copy().fillna(0.0)
+    for column in normalize_df.columns[1:]:
+        min_val = NORMALIZE_DIC_I2V[column]['Min']
+        max_val = NORMALIZE_DIC_I2V[column]['Max']
+        normalize_df[column] = (normalize_df[column] - min_val) / (max_val - min_val)
+    return normalize_df
+
+
 def calculate_selected_score(df, selected_columns):
     # selected_score = df[selected_columns].sum(axis=1)
     selected_QUALITY = [i for i in selected_columns if i in QUALITY_LIST]
@@ -91,6 +100,23 @@ def calculate_selected_score(df, selected_columns):
     selected_score =  (selected_quality_score * QUALITY_WEIGHT + selected_semantic_score * SEMANTIC_WEIGHT) / (QUALITY_WEIGHT + SEMANTIC_WEIGHT)
     return selected_score.fillna(0.0)
 
+def calculate_selected_score_i2v(df, selected_columns):
+    # selected_score = df[selected_columns].sum(axis=1)
+    selected_QUALITY = [i for i in selected_columns if i in I2V_QUALITY_LIST]
+    selected_I2V = [i for i in selected_columns if i in I2V_LIST]
+    selected_quality_score = df[selected_QUALITY].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_QUALITY])
+    selected_i2v_score = df[selected_I2V].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in selected_I2V ])
+    if selected_quality_score.isna().any().any() and selected_i2v_score.isna().any().any():
+        selected_score =  (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
+        return selected_score.fillna(0.0)
+    if selected_quality_score.isna().any().any():
+        return selected_i2v_score
+    if selected_i2v_score.isna().any().any():
+        return selected_quality_score
+    print(selected_i2v_score,selected_quality_score )
+    selected_score =  (selected_quality_score * I2V_QUALITY_WEIGHT + selected_i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
+    return selected_score.fillna(0.0)
+
 def get_final_score(df, selected_columns):
     normalize_df = get_normalized_df(df)
     #final_score = normalize_df.drop('name', axis=1).sum(axis=1)
@@ -118,6 +144,34 @@ def get_final_score(df, selected_columns):
         df.insert(1, 'Selected Score', selected_score)
     return df
 
+def get_final_score_i2v(df, selected_columns):
+    normalize_df = get_normalized_i2v_df(df)
+    #final_score = normalize_df.drop('name', axis=1).sum(axis=1)
+    for name in normalize_df.drop('Model Name (clickable)', axis=1).drop('Video-Text Camera Motion', axis=1):
+        normalize_df[name] = normalize_df[name]*DIM_WEIGHT_I2V[name]
+    quality_score = normalize_df[I2V_QUALITY_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_QUALITY_LIST])
+    i2v_score = normalize_df[I2V_LIST].sum(axis=1)/sum([DIM_WEIGHT_I2V[i] for i in I2V_LIST ])
+    final_score =  (quality_score * I2V_QUALITY_WEIGHT + i2v_score * I2V_WEIGHT) / (I2V_QUALITY_WEIGHT + I2V_WEIGHT)
+    if 'Total Score' in df:
+        df['Total Score'] = final_score
+    else:
+        df.insert(1, 'Total Score', final_score)
+    if 'I2V Score' in df:
+        df['I2V Score'] = i2v_score
+    else:
+        df.insert(2, 'I2V Score', i2v_score)
+    if 'Quality Score' in df:
+        df['Quality Score'] = quality_score
+    else:
+        df.insert(3, 'Quality Score', quality_score)
+    selected_score = calculate_selected_score(normalize_df, selected_columns)
+    if 'Selected Score' in df:
+        df['Selected Score'] = selected_score
+    else:
+        df.insert(1, 'Selected Score', selected_score)
+    return df
+
+
 
 def get_final_score_quality(df, selected_columns):
     normalize_df = get_normalized_df(df)
@@ -138,8 +192,8 @@ def get_final_score_quality(df, selected_columns):
     return df
 
 def get_baseline_df():
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
+    # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    # submission_repo.git_pull()
     df = pd.read_csv(CSV_DIR)
     df = get_final_score(df, checkbox_group.value)
     df = df.sort_values(by="Selected Score", ascending=False)
@@ -149,8 +203,8 @@ def get_baseline_df():
     return df
 
 def get_baseline_df_quality():
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
-    submission_repo.git_pull()
+    # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    # submission_repo.git_pull()
     df = pd.read_csv(QUALITY_DIR)
     df = get_final_score_quality(df, checkbox_group_quality.value)
     df = df.sort_values(by="Selected Score", ascending=False)
@@ -159,6 +213,17 @@ def get_baseline_df_quality():
     df = convert_scores_to_percentage(df)
     return df
 
+def get_baseline_df_i2v():
+    # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    # submission_repo.git_pull()
+    df = pd.read_csv(I2V_DIR)
+    df = get_final_score_i2v(df, checkbox_group_i2v.value)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    present_columns = MODEL_INFO_TAB_I2V + checkbox_group_i2v.value
+    df = df[present_columns]
+    df = convert_scores_to_percentage(df)
+    return df
+
 def get_all_df(selected_columns, dir=CSV_DIR):
     submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
@@ -175,6 +240,13 @@ def get_all_df_quality(selected_columns, dir=QUALITY_DIR):
     df = df.sort_values(by="Selected Score", ascending=False)
     return df
 
+def get_all_df_i2v(selected_columns, dir=I2V_DIR):
+    # submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    # submission_repo.git_pull()
+    df = pd.read_csv(dir)
+    df = get_final_score_i2v(df, selected_columns)
+    df = df.sort_values(by="Selected Score", ascending=False)
+    return df
 
 def convert_scores_to_percentage(df):
     # 对DataFrame中的每一列（除了'name'列）进行操作
@@ -239,6 +311,28 @@ def on_filter_model_size_method_change_quality(selected_columns):
         )
     return filter_component#.value
 
+def on_filter_model_size_method_change_i2v(selected_columns):
+
+    updated_data = get_all_df_i2v(selected_columns, I2V_DIR)
+    selected_columns = [item for item in I2V_TAB if item in selected_columns]
+    present_columns = MODEL_INFO_TAB_I2V + selected_columns
+    updated_data = updated_data[present_columns]
+    updated_data = updated_data.sort_values(by="Selected Score", ascending=False)
+    updated_data = convert_scores_to_percentage(updated_data)
+    updated_headers = present_columns
+    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+    import ipdb
+    ipdb.set_trace()
+    # print(updated_data,present_columns,update_datatype)
+    filter_component = gr.components.Dataframe(
+        value=updated_data, 
+        headers=updated_headers,
+        type="pandas", 
+        datatype=update_datatype,
+        interactive=False,
+        visible=True,
+        )
+    return filter_component#.value
 
 block = gr.Blocks()
 
@@ -322,8 +416,37 @@ with block:
     
             checkbox_group_quality.change(fn=on_filter_model_size_method_change_quality, inputs=[checkbox_group_quality], outputs=data_component_quality)
             
+        with gr.TabItem("VBench-I2V", elem_id="vbench-tab-table", id=3):
+            with gr.Accordion("NOTE", open=False):
+                    i2v_note_button = gr.Textbox(
+                        value=I2V_CLAIM_TEXT,
+                        label="",
+                        elem_id="quality-button",
+                        lines=3,
+                    )
+            with gr.Row():
+                with gr.Column(scale=1.0):
+                    # selection for column part:
+                    checkbox_group_i2v = gr.CheckboxGroup(
+                        choices=I2V_TAB,
+                        value=I2V_TAB,
+                        label="Evaluation Quality Dimension",
+                        interactive=True,
+                    )
+
+            data_component_i2v = gr.components.Dataframe(
+                value=get_baseline_df_i2v, 
+                headers=COLUMN_NAMES_I2V,
+                type="pandas", 
+                datatype=I2V_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            checkbox_group_i2v.change(fn=on_filter_model_size_method_change_i2v, inputs=[checkbox_group_i2v], outputs=data_component_i2v)
+            
         # table 2
-        with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=3):
+        with gr.TabItem("📝 About", elem_id="mvbench-tab-table", id=4):
             gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
         
         # table 3 

constants.py

@@ -14,6 +14,14 @@ MODEL_INFO_TAB_QUALITY = [
     "Selected Score"
 ]
 
+MODEL_INFO_TAB_I2V = [
+    "Model Name (clickable)",
+    "Total Score",
+    "I2V Score",
+    "Quality Score",
+    "Selected Score"
+]
+
 TASK_INFO = [
     "subject consistency",
     "background consistency",
@@ -71,6 +79,32 @@ QUALITY_TAB = [
     "imaging quality",
     "dynamic degree",]
 
+I2V_LIST = [
+    "Video-Image Subject Consistency",
+    "Video-Image Background Consistency",
+]
+
+I2V_QUALITY_LIST = [
+    "Subject Consistency",
+    "Background Consistency",
+    "Motion Smoothness",
+    "Dynamic Degree",
+    "Aesthetic Quality",
+    "Imaging Quality"
+]
+
+I2V_TAB = [
+    "Video-Text Camera Motion",
+    "Video-Image Subject Consistency",
+    "Video-Image Background Consistency",
+    "Subject Consistency",
+    "Background Consistency",
+    "Motion Smoothness",
+    "Dynamic Degree",
+    "Aesthetic Quality",
+    "Imaging Quality"
+]
+
 DIM_WEIGHT = {
 "subject consistency":1,
 "background consistency":1,
@@ -90,18 +124,35 @@ DIM_WEIGHT = {
 "overall consistency":1
 }
 
+DIM_WEIGHT_I2V = {
+"Video-Text Camera Motion": 0.1,
+"Video-Image Subject Consistency": 1,
+"Video-Image Background Consistency": 1,
+"Subject Consistency": 1,
+"Background Consistency": 1,
+"Motion Smoothness": 1,
+"Dynamic Degree": 0.5,
+"Aesthetic Quality": 1,
+"Imaging Quality": 1
+}
+
 SEMANTIC_WEIGHT = 1
 QUALITY_WEIGHT = 4
+I2V_WEIGHT = 1.0
+I2V_QUALITY_WEIGHT = 1.0
 
 DATA_TITILE_TYPE = ['markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
+I2V_TITILE_TYPE =  ['markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
 
 SUBMISSION_NAME = "vbench_leaderboard_submission"
 SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/Vchitect/", SUBMISSION_NAME)
 CSV_DIR = "./vbench_leaderboard_submission/results.csv"
 QUALITY_DIR = "./vbench_leaderboard_submission/quality.csv"
+I2V_DIR = "./vbench_leaderboard_submission/i2v_results.csv"
 
 COLUMN_NAMES = MODEL_INFO + TASK_INFO
 COLUMN_NAMES_QUALITY = MODEL_INFO_TAB_QUALITY + QUALITY_TAB
+COLUMN_NAMES_I2V = MODEL_INFO_TAB_I2V + I2V_TAB
 
 LEADERBORAD_INTRODUCTION = """# VBench Leaderboard
     
@@ -145,6 +196,8 @@ CITATION_BUTTON_TEXT = r"""@inproceedings{huang2023vbench,
 
 QUALITY_CLAIM_TEXT = "We use all the videos on Sora website (https://openai.com/sora) for a preliminary evaluation, including the failure case videos Sora provided."
 
+I2V_CLAIM_TEXT = "Since the open-sourced SVD models do not accept text input during the I2V stage, we are unable to evaluate its `camera motion` in terms of `video-text consistency`. The total score is calculated based on all dimensions except `camera motion`."
+
 NORMALIZE_DIC = {
   "subject consistency": {"Min": 0.1462, "Max": 1.0},
   "background consistency": {"Min": 0.2615, "Max": 1.0},
@@ -162,4 +215,16 @@ NORMALIZE_DIC = {
   "appearance style": {"Min": 0.0009, "Max": 0.2855},
   "temporal style": {"Min": 0.0, "Max": 0.364},
   "overall consistency": {"Min": 0.0, "Max": 0.364}
+}
+
+NORMALIZE_DIC_I2V = {
+    "Video-Text Camera Motion" :{"Min": 0.0, "Max":1.0 },
+    "Video-Image Subject Consistency":{"Min": 0.1462, "Max": 1.0},
+    "Video-Image Background Consistency":{"Min": 0.2615, "Max":1.0 },
+    "Subject Consistency":{"Min": 0.1462, "Max": 1.0},
+    "Background Consistency":{"Min": 0.2615, "Max": 1.0 },
+    "Motion Smoothness":{"Min": 0.7060, "Max": 0.9975},
+    "Dynamic Degree":{"Min": 0.0, "Max": 1.0},
+    "Aesthetic Quality":{"Min": 0.0, "Max": 1.0},
+    "Imaging Quality":{"Min": 0.0, "Max": 1.0}
 }