Upload 11 files

README.md

@@ -1,14 +1,13 @@
----
-title: AV Odyssey Bench Leaderboard
-emoji: 👁
-colorFrom: blue
-colorTo: indigo
-sdk: gradio
-sdk_version: 5.6.0
-app_file: app.py
-pinned: false
-license: apache-2.0
-short_description: This is the leaderboard for AV-Odyssey Bench
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Av-Odyssey Bench Leaderboard
+emoji: 🏆
+colorFrom: gray
+colorTo: blue
+sdk: gradio
+sdk_version: 3.40.1
+app_file: app.py
+pinned: false
+license: cc-by-4.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,303 @@
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+import gradio as gr
+import pandas as pd
+import json
+import pdb
+import tempfile
+import re
+from constants import *
+from src.auto_leaderboard.model_metadata_type import ModelType
+
+
+global data_component, filter_component
+
+def validate_model_size(s):
+    pattern = r'^\d+B$|^-$'
+    if re.match(pattern, s):
+        return s
+    else:
+        return '-'
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+def prediction_analyse(prediction_content):
+    # pdb.set_trace()
+    predictions = prediction_content.split("\n")
+
+    # 读取 ground_truth JSON 文件
+    with open("./file/SEED-Bench-1.json", "r") as file:
+        ground_truth_data = json.load(file)["questions"]
+
+    # 将 ground_truth 数据转换为以 question_id 为键的字典
+    ground_truth = {item["question_id"]: item for item in ground_truth_data}
+
+    # 初始化结果统计字典
+    results = {i: {"correct": 0, "total": 0} for i in range(1, 13)}
+
+    # 遍历 predictions，计算每个 question_type_id 的正确预测数和总预测数
+    for prediction in predictions:
+        # pdb.set_trace()
+        prediction = prediction.strip()
+        if not prediction:
+            continue
+        try:
+            prediction = json.loads(prediction)
+        except json.JSONDecodeError:
+            print(f"Warning: Skipping invalid JSON data in line: {prediction}")
+            continue
+        question_id = prediction["question_id"]
+        if question_id not in ground_truth:
+            continue
+        gt_item = ground_truth[question_id]
+        question_type_id = gt_item["question_type_id"]
+
+        if prediction["prediction"] == gt_item["answer"]:
+            results[question_type_id]["correct"] += 1
+
+        results[question_type_id]["total"] += 1
+    
+    return results
+
+
+def add_new_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_link: str,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    else:
+        model_size = validate_model_size(model_size)
+        # v1 evaluation
+        content = input_file.decode("utf-8")
+        prediction = prediction_analyse(content)
+        csv_data = pd.read_csv(CSV_DIR)
+
+        Start_dimension, End_dimension = 1, 13
+        if Evaluation_dimension == 'Image':
+            End_dimension = 10
+        elif Evaluation_dimension == 'Video':
+            Start_dimension = 10
+        each_task_accuracy = {i: round(prediction[i]["correct"] / prediction[i]["total"] * 100, 1) if i >= Start_dimension and i < End_dimension else 0 for i in range(1, 13)}
+
+        # count for average image\video\all
+        total_correct_image = sum(prediction[i]["correct"] for i in range(1, 10))
+        total_correct_video = sum(prediction[i]["correct"] for i in range(10, 13))
+
+        total_image = sum(prediction[i]["total"] for i in range(1, 10))
+        total_video = sum(prediction[i]["total"] for i in range(10, 13))
+
+        if Evaluation_dimension != 'Video':
+            average_accuracy_image = round(total_correct_image / total_image * 100, 1)
+        else:
+            average_accuracy_image = 0
+        
+        if Evaluation_dimension != 'Image':
+            average_accuracy_video = round(total_correct_video / total_video * 100, 1)
+        else:
+            average_accuracy_video = 0
+        
+        if Evaluation_dimension == 'All':
+            overall_accuracy = round((total_correct_image + total_correct_video) / (total_image + total_video) * 100, 1)
+        else:
+            overall_accuracy = 0
+
+        if LLM_type == 'Other':
+            LLM_name = LLM_name_textbox
+        else:
+            LLM_name = LLM_type
+        
+        if revision_name_textbox == '':
+            col = csv_data.shape[0]
+            model_name = model_name_textbox
+        else:
+            model_name = revision_name_textbox
+            model_name_list = csv_data['Model']
+            name_list = [name.split(']')[0][1:] for name in model_name_list]
+            if revision_name_textbox not in name_list:
+                col = csv_data.shape[0]
+            else:
+                col = name_list.index(revision_name_textbox)    
+        
+        if model_link == '':
+            model_name = model_name  # no url
+        else:
+            model_name = '[' + model_name + '](' + model_link + ')'
+
+        # add new data
+        new_data = [
+            model_name, 
+            LLM_name,
+            model_size,
+            overall_accuracy,
+            average_accuracy_image,
+            average_accuracy_video,
+            each_task_accuracy[1],
+            each_task_accuracy[2],
+            each_task_accuracy[3],
+            each_task_accuracy[4],
+            each_task_accuracy[5],
+            each_task_accuracy[6],
+            each_task_accuracy[7],
+            each_task_accuracy[8],
+            each_task_accuracy[9],
+            each_task_accuracy[10],
+            each_task_accuracy[11],
+            each_task_accuracy[12], 
+            ]
+        csv_data.loc[col] = new_data
+        csv_data = csv_data.to_csv(CSV_DIR, index=False)
+
+        csv_task_data.loc[col] = new_data
+        csv_task_data = csv_task_data.to_csv(CSV_TASK_DIR, index=False)
+    return 0
+
+def get_baseline_df():
+    df = pd.read_csv(CSV_DIR)
+    df = df.sort_values(by="Avg. All", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    df = df[present_columns]
+    return df
+
+def get_all_df():
+    df = pd.read_csv(CSV_DIR)
+    df = df.sort_values(by="Avg. All", ascending=False)
+    return df
+
+
+def switch_version(version):
+    return f"当前版本: {version}"
+
+block = gr.Blocks()
+
+
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # table seed-bench-v1
+        with gr.TabItem("🏅 AV-Odyssey Benchmark", elem_id="av-odyssey-tab-table", id=1):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                    ).style(show_copy_button=True)
+    
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+
+            # selection for column part:
+            checkbox_group = gr.CheckboxGroup(
+                choices=TASK_INFO,
+                value=AVG_INFO,
+                label="Evaluation Dimension",
+                interactive=True,
+            )
+
+
+            baseline_value = get_baseline_df()
+            baseline_header = MODEL_INFO + checkbox_group.value
+            baseline_datatype = ['markdown'] * len(MODEL_INFO) + ['number'] * len(checkbox_group.value)
+            # 创建数据帧组件
+            data_component = gr.components.Dataframe(
+                value=baseline_value, 
+                headers=baseline_header,
+                type="pandas", 
+                datatype=baseline_datatype,
+                interactive=False,
+                visible=True,
+                )            
+    
+            def on_filter_model_size_method_change(selected_columns):
+
+                updated_data = get_all_df()
+
+                # columns:
+                selected_columns = [item for item in TASK_INFO if item in selected_columns]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = updated_data[present_columns]
+                updated_data = updated_data.sort_values(by=selected_columns[0], ascending=False)
+                updated_headers = present_columns
+                update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data, 
+                    headers=updated_headers,
+                    type="pandas", 
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                    )
+                # pdb.set_trace()
+        
+                return filter_component.value
+            
+            def on_average_type_change(average_type):
+                return get_baseline_df()
+
+            checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component)
+
+        # table 2
+        with gr.TabItem("📝 About", elem_id="av-odyssey-tab-table", id=2):
+            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+        
+        # table 3 
+        with gr.TabItem("🚀 Submit here! ", elem_id="av-odyssey-tab-table", id=3):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="VideoLLaMA2"
+                        )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name", placeholder="VideoLLaMA2"
+                    )
+                    model_link = gr.Textbox(
+                        label="Model Link", placeholder="https://huggingface.co/DAMO-NLP-SG/VideoLLaMA2.1-7B-16F"
+                    )
+
+
+            with gr.Column():
+
+                input_file = gr.inputs.File(label = "Click to Upload a json File", file_count="single", type='binary')
+                submit_button = gr.Button("Submit Eval")
+    
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs = [
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_link
+                    ],
+                )
+
+
+    def refresh_data():
+        value1 = get_baseline_df()
+
+        return value1
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(
+            refresh_data, outputs=data_component
+        )
+
+
+block.launch()

constants.py

@@ -0,0 +1,67 @@
+# this is .py for store constants 
+MODEL_INFO = ["Model"]
+TASK_INFO = [
+    "Avg. All", "Avg. Timbre", "Avg. Tone", "Avg. Melody", "Avg. Space", "Avg. Time", "Avg. Hallucination", "Avg. Intricacy", 
+    "Instrument Recognition", "Singer Recognition", "Gunshot Recognition", "Bird Recognition", "Animal Recognition", 
+    "Transportation Recognition", "Material Recognition", "Scene Recognition", "Hazard Recognition", "Action Recognition", 
+    "Eating Sound Recognition", "Speech Sentiment Analysis", "Meme Understanding", "Music Sentiment Analysis", "Music Genre Classification", 
+    "Dance and Music Matching", "Film and Music Matching", "Music Score Matching", "Audio 3D Angle Estimation", "Audio Distance Estimation", 
+    "Audio Time Estimation", "Audio-Visual Synchronization", "Action Sequencing", "Hallucination Evaluation", 
+    "Action Prediction", "Action Tracing"]
+
+AVG_INFO = ["Avg. All", "Avg. Timbre", "Avg. Tone", "Avg. Melody", "Avg. Space", "Avg. Time", "Avg. Hallucination", "Avg. Intricacy"]
+DATA_TITILE_TYPE = ["markdown"] * len(MODEL_INFO) + ["number"] * len(TASK_INFO)
+CSV_DIR = "./file/AV-Odyssey_performance.csv"
+
+COLUMN_NAMES = MODEL_INFO + TASK_INFO
+
+DATA_NUM = [200, 200, 200, 200, 200, 200, 200, 200, 108, 196, 200, 200, 20, 97, 200, 200, 200, 200, 20, 20, 200, 200, 200, 200, 199, 195]
+
+LEADERBORAD_INTRODUCTION = """# AV-Odyssey Bench Leaderboard
+    Welcome to the leaderboard of the AV-Odyssey Bench! 🏆 
+
+    AV-Odyssey Bench, a comprehensive audio-visual benchmark designed to assess whether those MLLMs can truly understand the audio-visual information. This benchmark encompasses 4,555 carefully crafted problems, each incorporating text, visual, and audio components. To successfully infer answers, models must effectively leverage clues from both visual and audio inputs.
+    Please refer to [AV-Odyssey paper](https://arxiv.org/abs/2307.16125) for more details.
+
+    """
+
+
+SUBMIT_INTRODUCTION = """# Submit on AV-Odyssey Bench Introduction
+    
+
+    Note: The format of the submitted json file is a dict for each line. This dict contains two keys: question_id and prediction. Specific examples are as follows: 
+    ```shell
+    {"question_id": "5_0", "prediction": "B"}
+    {"question_id": "3_0", "prediction": "B"}
+    ```
+
+    ## Submit Example
+
+
+    ## If you have any questions, please contact [[email protected]]([email protected]).
+"""
+
+TABLE_INTRODUCTION = """In the table below, we summarize each task performance of all the models.
+        We use accurancy(%) as the primary evaluation metric for each tasks. 
+
+        Performance Average Type is All Average means that calculates the overall accuracy by dividing the total number of correct QA answers by the total number of QA questions.
+        
+        If you have any questions, please feel free to contact us.
+    """
+
+LEADERBORAD_INFO = """
+      Recently, multimodal large language models (MLLMs), such as GPT-4o, Gemini 1.5 Pro, and Reka Core, have expanded their capabilities to include vision and audio modalities. While these models demonstrate impressive performance across a wide range of audio-visual applications, our proposed DeafTest reveals that MLLMs often struggle with simple tasks humans find trivial: 
+      1) determining which of two sounds is louder, and 
+      2) determining which of two sounds has a higher pitch. 
+      Motivated by these observations, we introduce AV-Odyssey Bench. This benchmark encompasses 26 different tasks and 4,555 carefully crafted problems, each incorporating text, visual, and audio components. All data are newly collected and annotated by humans, not from any existing audio-visual dataset. AV-Odyssey Bench demonstrates three major features: 
+      1. Comprehensive Audio Attributes; 
+      2. Extensive Domains; 
+      3. Interleaved Text, Audio, and Visual components.
+"""
+
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""{
+
+}"""
+

file/AV-Odyssey_performance.csv

@@ -0,0 +1,19 @@
+Model,Avg. All,Avg. Timbre,Avg. Tone,Avg. Melody,Avg. Space,Avg. Time,Avg. Hallucination,Avg. Intricacy,Instrument Recognition,Singer Recognition,Gunshot Recognition,Bird Recognition,Animal Recognition,Transportation Recognition,Material Recognition,Scene Recognition,Hazard Recognition,Action Recognition,Eating Sound Recognition,Speech Sentiment Analysis,Meme Understanding,Music Sentiment Analysis,Music Genre Classification,Dance and Music Matching,Film and Music Matching,Music Score Matching,Audio 3D Angle Estimation,Audio Distance Estimation,Audio Time Estimation,Audio-Visual Synchronization,Action Sequencing,Hallucination Evaluation,Action Prediction,Action Tracing
+[Unified-IO-2 L](https://unified-io-2.allenai.org/),26.0,23.8,24.1,28.8,15.0,26.8,30.0,30.4,20.5,22.5,25.5,18.5,27.0,26.5,23.0,28.0,21.3,20.9,26.5,24.5,20.0,27.9,31.0,27.5,32.5,24.5,15.0,15.0,28.0,25.5,27.0,30.0,27.1,33.8
+[Unified-IO-2 XL](https://unified-io-2.allenai.org/),26.3,24.3,23.2,27.8,22.5,25.3,31.5,34.8,20.0,23.5,24.0,20.5,27.5,26.0,27.5,30.0,19.4,19.9,26.5,23.0,25.0,26.9,30.5,27.0,31.5,22.5,30.0,15.0,26.5,25.5,24.0,31.5,35.7,33.8
+[Unified-IO-2 XXL](https://unified-io-2.allenai.org/),27.2,26.3,22.7,26.4,32.5,26.8,24.5,33.8,29.5,24.0,23.5,29.0,23.5,25.5,30.5,26.5,23.1,27.0,25.5,23.0,20.0,23.9,31.5,27.5,24.5,23.5,50.0,15.0,28.0,25.0,27.5,24.5,33.2,34.4
+[OneLLM](https://github.com/csuhan/OneLLM),27.4,25.0,25.5,21.5,37.5,29.3,25.5,38.4,26.0,21.5,27.0,26.0,22.0,20.0,29.5,24.5,26.9,23.0,29.5,26.0,20.0,20.8,23.5,26.5,18.5,18.0,45.0,30.0,31.5,29.5,27.0,25.5,41.7,34.9
+[PandaGPT](https://panda-gpt.github.io/),26.7,23.5,23.2,27.6,45.0,23.8,28.0,23.9,20.0,21.5,23.0,17.5,26.0,26.5,28.0,27.0,23.1,21.4,24.5,23.5,20.0,21.6,28.0,27.0,32.5,26.0,45.0,45.0,18.5,26.0,27.0,28.0,19.6,28.2
+[Video-llama](https://github.com/DAMO-NLP-SG/Video-LLaMA),26.1,25.5,22.3,24.4,30.0,26.2,25.0,30.7,22.5,24.5,27.0,26.5,27.0,23.5,28.0,25.0,25.0,26.0,25.5,23.0,15.0,25.8,24.0,20.0,25.0,28.0,45.0,15.0,28.5,23.5,26.5,25.0,28.6,32.8
+[VideoLLaMA2](https://github.com/DAMO-NLP-SG/VideoLLaMA2),26.8,24.1,25.5,26.4,30.0,27.2,33.0,34.5,22.5,24.0,27.0,17.0,23.5,27.5,26.5,26.5,19.4,23.0,25.5,26.0,20.0,26.8,29.0,25.5,30.5,20.5,45.0,15.0,28.5,26.5,26.5,33.0,28.6,40.5
+[AnyGPT](https://junzhan2000.github.io/AnyGPT.github.io/),26.1,24.6,25.0,26.4,27.5,29.2,29.0,25.7,22.5,28.5,28.0,17.5,24.0,25.5,23.0,28.0,25.9,20.4,27.5,25.5,20.0,23.4,29.5,25.5,26.0,26.0,40.0,15.0,30.5,28.0,29.0,29.0,21.1,30.3
+[NExT-GPT](https://next-gpt.github.io/),25.5,23.2,20.9,27.8,30.0,28.8,28.5,23.6,21.0,23.5,25.5,21.5,25.5,25.5,21.0,24.0,19.4,23.0,24.0,21.5,15.0,23.7,26.0,28.0,31.0,28.0,45.0,15.0,31.5,24.0,31.0,28.5,20.6,26.7
+[VITA](https://vita-home.github.io/),26.4,24.1,26.4,27.8,22.5,26.3,31.0,36.8,22.0,20.5,24.5,21.5,27.5,25.0,23.5,28.5,21.3,19.4,29.5,24.5,45.0,26.9,26.0,27.5,33.5,24.5,25.0,20.0,26.5,25.5,27.0,31.0,34.2,39.5
+[Gemini 1.5 Flash](https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf),27.8,27.2,25.0,28.8,30.0,25.3,28.5,31.2,24.5,24.0,23.5,17.0,32.5,26.0,22.5,29.5,34.3,48.0,21.5,23.5,40.0,21.3,31.0,27.5,32.5,28.0,30.0,30.0,27.5,23.5,25.0,28.5,27.6,34.9
+[Gemini 1.5 Flash-8B](https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf),26.8,25.1,24.5,28.9,27.5,27.5,29.0,30.2,16.5,22.5,24.0,19.0,28.0,26.5,27.0,29.0,26.9,32.7,24.5,24.5,25.0,25.9,33.0,27.5,32.0,24.5,40.0,15.0,31.0,25.5,26.0,29.0,25.6,34.9
+[Gemini 1.5 Pro](https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf),30.8,30.8,31.4,31.3,37.5,27.7,20.5,33.0,33.0,26.0,29.0,25.0,25.5,26.0,29.5,30.0,38.0,57.7,22.5,29.5,50.0,25.4,42.5,28.0,28.5,29.0,35.0,40.0,30.0,24.5,28.5,20.5,32.2,33.8
+[Reka Core](https://arxiv.org/abs/2404.12387),26.9,26.7,27.7,26.4,22.5,26.5,24.0,34.3,32.5,20.0,26.5,25.0,24.0,27.0,30.0,27.0,25.0,34.2,21.5,28.5,20.0,22.8,24.5,27.5,30.0,25.5,25.0,20.0,30.0,25.5,24.0,24.0,33.7,34.9
+[Reka Flash](https://arxiv.org/abs/2404.12387),26.3,25.5,24.1,27.2,30.0,27.5,31.5,24.1,20.0,22.5,26.5,26.0,28.5,26.5,26.5,29.0,28.7,22.4,25.0,24.5,20.0,30.5,29.5,27.5,25.5,24.5,45.0,15.0,30.0,25.5,27.0,31.5,19.1,29.2
+[Reka Edge](https://arxiv.org/abs/2404.12387),25.0,23.8,20.5,26.3,22.5,25.5,22.5,36.8,21.5,24.0,30.5,20.0,19.5,22.5,20.5,25.5,25.9,23.5,29.0,20.5,20.0,24.9,24.5,27.5,30.0,24.0,30.0,15.0,30.0,25.5,21.0,22.5,38.2,35.4
+[GPT-4o visual caption](https://openai.com/index/hello-gpt-4o/),32.3,37.4,28.6,32.3,27.5,25.5,23.0,28.9,33.0,30.5,24.0,26.5,43.0,42.0,32.5,39.0,49.1,67.3,30.5,26.0,55.0,24.4,48.0,27.0,34.5,23.5,25.0,30.0,21.5,22.5,32.5,23.0,32.2,25.6
+[GPT-4o audio caption](https://openai.com/index/hello-gpt-4o/),34.5,38.6,31.8,33.6,32.5,27.5,25.0,26.1,40.0,38.0,27.5,26.5,45.0,42.0,27.0,41.0,42.6,62.2,35.5,28.0,70.0,24.4,56.5,27.5,32.5,22.5,30.0,35.0,23.5,25.5,33.5,25.0,30.2,22.0

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from enum import Enum
+import glob
+import json
+import os
+from typing import Dict, List
+
+from ..utils_display import AutoEvalColumn
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str # emoji
+
+model_type_symbols = {
+    "LLM": "🟢",
+    "ImageLLM": "🔶",
+    "VideoLLM": "⭕",
+    "Other": "🟦",
+}
+
+class ModelType(Enum):
+    PT = ModelInfo(name="LLM", symbol="🟢")
+    FT = ModelInfo(name="ImageLLM", symbol="🔶")
+    IFT = ModelInfo(name="VideoLLM", symbol="⭕")
+    RL = ModelInfo(name="Other", symbol="🟦")
+
+    def to_str(self, separator = " "):
+        return f"{self.value.symbol}{separator}{self.value.name}" 
+

src/utils_display.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+    displayed_by_default: bool 
+    hidden: bool = False
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model_type_symbol = ColumnContent("T", "str", True)
+    model = ColumnContent("Model", "markdown", True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC", "number", True)
+    hellaswag = ColumnContent("HellaSwag", "number", True)
+    mmlu = ColumnContent("MMLU", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    precision = ColumnContent("Precision", "str", False, True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+
+@dataclass(frozen=True)
+class EloEvalColumn: # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn: # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "bool", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+    #else:
+    #    link = MODEL_PAGE
+  
+    return model_hyperlink(link, model_name)
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"