update gradio

.pre-commit-config.yaml

@@ -0,0 +1,33 @@
+exclude: |
+  (?x)^(
+      meta_data.py
+  )
+repos:
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        args: ["--max-line-length=120", "--ignore=F401,F403,F405,E402"]
+        exclude: ^configs/
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+  - repo: https://github.com/pre-commit/mirrors-yapf
+    rev: v0.30.0
+    hooks:
+      - id: yapf
+        args: ["--style={column_limit=120}"]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.1.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: requirements-txt-fixer
+      - id: double-quote-string-fixer
+      - id: check-merge-conflict
+      - id: fix-encoding-pragma
+        args: ["--remove"]
+      - id: mixed-line-ending
+        args: ["--fix=lf"]

README.md

@@ -1,10 +1,10 @@
 ---
-title: OpenCompass MMBench Leaderboard
+title: MMBench Leaderboard
 emoji: 🚀
 colorFrom: blue
 colorTo: yellow
 sdk: gradio
-sdk_version: 3.44.3
+sdk_version: 4.36.1
 app_file: app.py
 pinned: true
 license: apache-2.0

app.py

@@ -1,12 +1,165 @@
-from flask import Flask, render_template
+import abc
 
-app = Flask(__name__)
+import gradio as gr
 
+from gen_table import *
+from meta_data import *
 
-@app.route("/")
-def index():
-    return render_template("index.html")
+with gr.Blocks() as demo:
+    struct = load_results()
+    timestamp = struct['time']
+    EVAL_TIME = format_timestamp(timestamp)
+    results = struct['results']
+    N_MODEL = len(results)
+    N_DATA = len(results['LLaVA-v1.5-7B']) - 1
+    DATASETS = list(results['LLaVA-v1.5-7B'])
+    DATASETS.remove('META')
+    print(DATASETS)
 
+    gr.Markdown(LEADERBORAD_INTRODUCTION.format(N_MODEL, N_DATA, EVAL_TIME))
+    structs = [abc.abstractproperty() for _ in range(N_DATA)]
 
-if __name__ == "__main__":
-    app.run(debug=False, port=7860, host="0.0.0.0")
+    with gr.Tabs(elem_classes='tab-buttons') as tabs:
+        with gr.TabItem('🏅 OpenVLM Main Leaderboard', elem_id='main', id=0):
+            gr.Markdown(LEADERBOARD_MD['MAIN'])
+            _, check_box = BUILD_L1_DF(results, MAIN_FIELDS)
+            table = generate_table(results, DEFAULT_BENCH)
+            table['Rank'] = list(range(1, len(table) + 1))
+
+            type_map = check_box['type_map']
+            type_map['Rank'] = 'number'
+
+            checkbox_group = gr.CheckboxGroup(
+                choices=check_box['all'],
+                value=check_box['required'],
+                label='Evaluation Dimension',
+                interactive=True,
+            )
+
+            headers = ['Rank'] + check_box['essential'] + checkbox_group.value
+            with gr.Row():
+                model_size = gr.CheckboxGroup(
+                    choices=MODEL_SIZE,
+                    value=MODEL_SIZE,
+                    label='Model Size',
+                    interactive=True
+                )
+                model_type = gr.CheckboxGroup(
+                    choices=MODEL_TYPE,
+                    value=MODEL_TYPE,
+                    label='Model Type',
+                    interactive=True
+                )
+            data_component = gr.components.DataFrame(
+                value=table[headers],
+                type='pandas',
+                datatype=[type_map[x] for x in headers],
+                interactive=False,
+                visible=True)
+
+            def filter_df(fields, model_size, model_type):
+                filter_list = ['Avg Score', 'Avg Rank', 'OpenSource', 'Verified']
+                headers = ['Rank'] + check_box['essential'] + fields
+
+                new_fields = [field for field in fields if field not in filter_list]
+                df = generate_table(results, new_fields)
+
+                df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
+                df = df[df['flag']]
+                df.pop('flag')
+                if len(df):
+                    df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
+                    df = df[df['flag']]
+                    df.pop('flag')
+                df['Rank'] = list(range(1, len(df) + 1))
+
+                comp = gr.components.DataFrame(
+                    value=df[headers],
+                    type='pandas',
+                    datatype=[type_map[x] for x in headers],
+                    interactive=False,
+                    visible=True)
+                return comp
+
+            for cbox in [checkbox_group, model_size, model_type]:
+                cbox.change(fn=filter_df, inputs=[checkbox_group, model_size, model_type], outputs=data_component)
+
+        with gr.TabItem('🔍 About', elem_id='about', id=1):
+            gr.Markdown(urlopen(VLMEVALKIT_README).read().decode())
+
+        for i, dataset in enumerate(DATASETS):
+            with gr.TabItem(f'📊 {dataset} Leaderboard', elem_id=dataset, id=i + 2):
+                if dataset in LEADERBOARD_MD:
+                    gr.Markdown(LEADERBOARD_MD[dataset])
+
+                s = structs[i]
+                s.table, s.check_box = BUILD_L2_DF(results, dataset)
+                s.type_map = s.check_box['type_map']
+                s.type_map['Rank'] = 'number'
+
+                s.checkbox_group = gr.CheckboxGroup(
+                    choices=s.check_box['all'],
+                    value=s.check_box['required'],
+                    label=f'{dataset} CheckBoxes',
+                    interactive=True,
+                )
+                s.headers = ['Rank'] + s.check_box['essential'] + s.checkbox_group.value
+                s.table['Rank'] = list(range(1, len(s.table) + 1))
+
+                with gr.Row():
+                    s.model_size = gr.CheckboxGroup(
+                        choices=MODEL_SIZE,
+                        value=MODEL_SIZE,
+                        label='Model Size',
+                        interactive=True
+                    )
+                    s.model_type = gr.CheckboxGroup(
+                        choices=MODEL_TYPE,
+                        value=MODEL_TYPE,
+                        label='Model Type',
+                        interactive=True
+                    )
+                s.data_component = gr.components.DataFrame(
+                    value=s.table[s.headers],
+                    type='pandas',
+                    datatype=[s.type_map[x] for x in s.headers],
+                    interactive=False,
+                    visible=True)
+                s.dataset = gr.Textbox(value=dataset, label=dataset, visible=False)
+
+                def filter_df_l2(dataset_name, fields, model_size, model_type):
+                    s = structs[DATASETS.index(dataset_name)]
+                    headers = ['Rank'] + s.check_box['essential'] + fields
+                    df = cp.deepcopy(s.table)
+                    df['flag'] = [model_size_flag(x, model_size) for x in df['Param (B)']]
+                    df = df[df['flag']]
+                    df.pop('flag')
+                    if len(df):
+                        df['flag'] = [model_type_flag(df.iloc[i], model_type) for i in range(len(df))]
+                        df = df[df['flag']]
+                        df.pop('flag')
+                    df['Rank'] = list(range(1, len(df) + 1))
+
+                    comp = gr.components.DataFrame(
+                        value=df[headers],
+                        type='pandas',
+                        datatype=[s.type_map[x] for x in headers],
+                        interactive=False,
+                        visible=True)
+                    return comp
+
+                for cbox in [s.checkbox_group, s.model_size, s.model_type]:
+                    cbox.change(
+                        fn=filter_df_l2,
+                        inputs=[s.dataset, s.checkbox_group, s.model_size, s.model_type],
+                        outputs=s.data_component)
+
+    with gr.Row():
+        with gr.Accordion('Citation', open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id='citation-button')
+
+if __name__ == '__main__':
+    demo.launch(server_name='0.0.0.0')

gen_table.py

@@ -0,0 +1,196 @@
+import copy as cp
+import json
+from collections import defaultdict
+from urllib.request import urlopen
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+
+from meta_data import DEFAULT_BENCH, META_FIELDS, URL
+
+
+def listinstr(lst, s):
+    assert isinstance(lst, list)
+    for item in lst:
+        if item in s:
+            return True
+    return False
+
+
+def load_results():
+    data = json.loads(urlopen(URL).read())
+    return data
+
+
+def nth_large(val, vals):
+    return sum([1 for v in vals if v > val]) + 1
+
+
+def format_timestamp(timestamp):
+    date = timestamp[:2] + '.' + timestamp[2:4] + '.' + timestamp[4:6]
+    time = timestamp[6:8] + ':' + timestamp[8:10] + ':' + timestamp[10:12]
+    return date + ' ' + time
+
+
+def model_size_flag(sz, FIELDS):
+    if pd.isna(sz) and 'Unknown' in FIELDS:
+        return True
+    if pd.isna(sz):
+        return False
+    if '<4B' in FIELDS and sz < 4:
+        return True
+    if '4B-10B' in FIELDS and sz >= 4 and sz < 10:
+        return True
+    if '10B-20B' in FIELDS and sz >= 10 and sz < 20:
+        return True
+    if '20B-40B' in FIELDS and sz >= 20 and sz < 40:
+        return True
+    if '>40B' in FIELDS and sz >= 40:
+        return True
+    return False
+
+
+def model_type_flag(line, FIELDS):
+    if 'OpenSource' in FIELDS and line['OpenSource'] == 'Yes':
+        return True
+    if 'API' in FIELDS and line['OpenSource'] == 'No' and line['Verified'] == 'Yes':
+        return True
+    if 'Proprietary' in FIELDS and line['OpenSource'] == 'No' and line['Verified'] == 'No':
+        return True
+    return False
+
+
+def BUILD_L1_DF(results, fields):
+    check_box = {}
+    check_box['essential'] = ['Method', 'Param (B)', 'Language Model', 'Vision Model']
+    # revise there to set default dataset
+    check_box['required'] = ['Avg Score', 'Avg Rank'] + DEFAULT_BENCH
+    check_box['avg'] = ['Avg Score', 'Avg Rank']
+    check_box['all'] = check_box['avg'] + fields
+    type_map = defaultdict(lambda: 'number')
+    type_map['Method'] = 'html'
+    type_map['Language Model'] = type_map['Vision Model'] = type_map['OpenSource'] = type_map['Verified'] = 'str'
+    check_box['type_map'] = type_map
+
+    df = generate_table(results, fields)
+    return df, check_box
+
+
+def BUILD_L2_DF(results, dataset):
+    res = defaultdict(list)
+    sub = [v for v in results.values() if dataset in v]
+    assert len(sub)
+    fields = list(sub[0][dataset].keys())
+
+    non_overall_fields = [x for x in fields if 'Overall' not in x]
+    overall_fields = [x for x in fields if 'Overall' in x]
+    if dataset == 'MME':
+        non_overall_fields = [x for x in non_overall_fields if not listinstr(['Perception', 'Cognition'], x)]
+        overall_fields = overall_fields + ['Perception', 'Cognition']
+    if dataset == 'OCRBench':
+        non_overall_fields = [x for x in non_overall_fields if not listinstr(['Final Score'], x)]
+        overall_fields = ['Final Score']
+
+    for m in results:
+        item = results[m]
+        if dataset not in item:
+            continue
+        meta = item['META']
+        for k in META_FIELDS:
+            if k == 'Param (B)':
+                param = meta['Parameters']
+                res[k].append(float(param.replace('B', '')) if param != '' else None)
+            elif k == 'Method':
+                name, url = meta['Method']
+                res[k].append(f'<a href="{url}">{name}</a>')
+            else:
+                res[k].append(meta[k])
+        fields = [x for x in fields]
+
+        for d in non_overall_fields:
+            res[d].append(item[dataset][d])
+        for d in overall_fields:
+            res[d].append(item[dataset][d])
+
+    df = pd.DataFrame(res)
+    all_fields = overall_fields + non_overall_fields
+    # Use the first 5 non-overall fields as required fields
+    required_fields = overall_fields if len(overall_fields) else non_overall_fields[:5]
+
+    if dataset == 'OCRBench':
+        df = df.sort_values('Final Score')
+    elif dataset == 'COCO_VAL':
+        df = df.sort_values('CIDEr')
+    else:
+        df = df.sort_values('Overall')
+    df = df.iloc[::-1]
+
+    check_box = {}
+    check_box['essential'] = ['Method', 'Param (B)', 'Language Model', 'Vision Model']
+    check_box['required'] = required_fields
+    check_box['all'] = all_fields
+    type_map = defaultdict(lambda: 'number')
+    type_map['Method'] = 'html'
+    type_map['Language Model'] = type_map['Vision Model'] = type_map['OpenSource'] = type_map['Verified'] = 'str'
+    check_box['type_map'] = type_map
+    return df, check_box
+
+
+def generate_table(results, fields):
+
+    def get_mmbench_v11(item):
+        assert 'MMBench_TEST_CN_V11' in item and 'MMBench_TEST_EN_V11' in item
+        val = (item['MMBench_TEST_CN_V11']['Overall'] + item['MMBench_TEST_EN_V11']['Overall']) / 2
+        val = float(f'{val:.1f}')
+        return val
+
+    res = defaultdict(list)
+    for i, m in enumerate(results):
+        item = results[m]
+        meta = item['META']
+        for k in META_FIELDS:
+            if k == 'Param (B)':
+                param = meta['Parameters']
+                res[k].append(float(param.replace('B', '')) if param != '' else None)
+            elif k == 'Method':
+                name, url = meta['Method']
+                res[k].append(f'<a href="{url}">{name}</a>')
+                res['name'].append(name)
+            else:
+                res[k].append(meta[k])
+        scores, ranks = [], []
+        for d in fields:
+            key_name = 'Overall' if d != 'OCRBench' else 'Final Score'
+            # Every Model should have MMBench_V11 results
+            if d == 'MMBench_V11':
+                val = get_mmbench_v11(item)
+                res[d].append(val)
+                scores.append(val)
+                ranks.append(nth_large(val, [get_mmbench_v11(x) for x in results.values()]))
+            elif d in item:
+                res[d].append(item[d][key_name])
+                if d == 'MME':
+                    scores.append(item[d][key_name] / 28)
+                elif d == 'OCRBench':
+                    scores.append(item[d][key_name] / 10)
+                else:
+                    scores.append(item[d][key_name])
+                ranks.append(nth_large(item[d][key_name], [x[d][key_name] for x in results.values() if d in x]))
+            else:
+                res[d].append(None)
+                scores.append(None)
+                ranks.append(None)
+
+        res['Avg Score'].append(round(np.mean(scores), 1) if None not in scores else None)
+        res['Avg Rank'].append(round(np.mean(ranks), 2) if None not in ranks else None)
+
+    df = pd.DataFrame(res)
+    valid, missing = df[~pd.isna(df['Avg Score'])], df[pd.isna(df['Avg Score'])]
+    valid = valid.sort_values('Avg Score')
+    valid = valid.iloc[::-1]
+    if len(fields):
+        missing = missing.sort_values('MMBench_V11' if 'MMBench_V11' in fields else fields[0])
+        missing = missing.iloc[::-1]
+    df = pd.concat([valid, missing])
+    return df

meta_data.py

@@ -0,0 +1,227 @@
+# CONSTANTS-URL
+URL = "http://opencompass.openxlab.space/assets/OpenVLM.json"
+VLMEVALKIT_README = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/README.md'
+# CONSTANTS-CITATION
+CITATION_BUTTON_TEXT = r"""@misc{2023opencompass,
+    title={OpenCompass: A Universal Evaluation Platform for Foundation Models},
+    author={OpenCompass Contributors},
+    howpublished = {\url{https://github.com/open-compass/opencompass}},
+    year={2023}
+}"""
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+# CONSTANTS-TEXT
+LEADERBORAD_INTRODUCTION = """# OpenVLM Leaderboard
+### Welcome to the OpenVLM Leaderboard! On this leaderboard we share the evaluation results of VLMs obtained by the OpenSource Framework:
+### [*VLMEvalKit*: A Toolkit for Evaluating Large Vision-Language Models](https://github.com/open-compass/VLMEvalKit) 🏆
+### Currently, OpenVLM Leaderboard covers {} different VLMs (including GPT-4v, Gemini, QwenVLPlus, LLaVA, etc.) and {} different multi-modal benchmarks. 
+
+This leaderboard was last updated: {}. 
+
+OpenVLM Leaderboard only includes open-source VLMs or API models that are publicly available. To add your own model to the leaderboard, please create a PR in [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) to support your VLM and then we will help with the evaluation and updating the leaderboard. For any questions or concerns, please feel free to contact us at [opencompass, duanhaodong]@pjlab.org.cn.
+"""
+# CONSTANTS-FIELDS
+META_FIELDS = ['Method', 'Param (B)', 'Language Model', 'Vision Model', 'OpenSource', 'Verified']
+MAIN_FIELDS = [
+    'MMBench_V11', 'MMStar', 'MME',
+    'MMMU_VAL', 'MathVista', 'OCRBench', 'AI2D', 
+    'HallusionBench', 'SEEDBench_IMG', 'MMVet', 
+    'LLaVABench', 'CCBench', 'RealWorldQA', 'POPE', 'ScienceQA_TEST',
+    'SEEDBench2_Plus', 'MMT-Bench_VAL', 'BLINK'
+]
+DEFAULT_BENCH = [
+    'MMBench_V11', 'MMStar', 'MMMU_VAL', 'MathVista', 'OCRBench', 'AI2D', 
+    'HallusionBench', 'MMVet'
+]
+MMBENCH_FIELDS = ['MMBench_TEST_EN_V11', 'MMBench_TEST_CN_V11', 'MMBench_TEST_EN', 'MMBench_TEST_CN', 'CCBench']
+MODEL_SIZE = ['<4B', '4B-10B', '10B-20B', '20B-40B', '>40B', 'Unknown']
+MODEL_TYPE = ['API', 'OpenSource', 'Proprietary']
+
+# The README file for each benchmark
+LEADERBOARD_MD = {}
+
+LEADERBOARD_MD['MAIN'] = f"""
+## Main Evaluation Results
+
+- Metrics:
+  - Avg Score: The average score on all VLM Benchmarks (normalized to 0 - 100, the higher the better). 
+  - Avg Rank: The average rank on all VLM Benchmarks (the lower the better). 
+  - Avg Score & Rank are calculated based on selected benchmark. **When results for some selected benchmarks are missing, Avg Score / Rank will be None!!!** 
+- By default, we present the overall evaluation results based on {len(DEFAULT_BENCH)} VLM benchmarks, sorted by the descending order of Avg Score. 
+  - The following datasets are included in the main results: {', '.join(DEFAULT_BENCH)}. 
+  - Detailed evaluation results for each dataset (included or not included in main) are provided in the consequent tabs. 
+"""
+
+for dataset in ['MMBench_DEV_CN', 'MMBench_TEST_CN', 'MMBench_DEV_EN', 'MMBench_TEST_EN', 'CCBench']:
+    LEADERBOARD_MD[dataset] = f"""
+## {dataset.replace('_', ' ')} Evaluation Results  
+
+- We adopt Circular Eval for benchmarks in MMBench series, you can check https://arxiv.org/pdf/2307.06281.pdf for the detailed definition of Circular Eval. 
+"""
+
+LEADERBOARD_MD['SEEDBench_IMG'] = """
+## SEEDBench_IMG Scores (ChatGPT Answer Extraction / Official Leaderboard)
+
+- **Overall**: The overall accuracy across all questions with **ChatGPT answer matching**.
+- **Overall (official)**: SEEDBench_IMG acc on the official leaderboard (if applicable). 
+"""
+
+LEADERBOARD_MD['MMVet'] = """
+## MMVet Evaluation Results
+
+- In MMVet Evaluation, we use GPT-4-Turbo (gpt-4-1106-preview) as the judge LLM to assign scores to the VLM outputs. We only perform the evaluation once due to the limited variance among results of multiple evaluation pass originally reported. 
+- No specific prompt template adopted for **ALL VLMs**.
+- We also provide performance on the [**Official Leaderboard**](https://paperswithcode.com/sota/visual-question-answering-on-mm-vet) for models that are applicable. Those results are obtained with GPT-4-0314 evaluator (which has been deperacted for new users).  
+"""
+
+LEADERBOARD_MD['MMMU_VAL'] = """
+## MMMU Validation Evaluation Results
+
+- For MMMU, we support the evaluation of the `dev` (150 samples) and `validation` (900 samples) set. Here we only report the results on the `validation` set. 
+- **Answer Inference:**
+  - For models with `interleave_generate` interface (accept interleaved images & texts as inputs), all testing samples can be inferred. **`interleave_generate` is adopted for inference.**
+  - For models without `interleave_generate` interface, samples with more than one images are skipped (42 out of 1050, directly count as wrong). **`generate` is adopted for inference.**
+- **Evaluation**:
+  - MMMU include two types of questions: **multi-choice questions** & **open-ended QA**. 
+  - For **open-ended QA (62/1050)**, we re-formulate it as multi-choice questions: `{'question': 'QQQ', 'answer': 'AAA'} -> {'question': 'QQQ', 'A': 'AAA', 'B': 'Other Answers', 'answer': 'A'}`, and then adopt the same evaluation paradigm for **multi-choice questions**. 
+  - For **multi-choice questions (988/1050)**, we use **GPT-3.5-Turbo-0613** for matching prediction with options if heuristic matching does not work. 
+"""
+
+LEADERBOARD_MD['MathVista'] = """
+## MMMU TestMini Evaluation Results
+
+- We report the evaluation results on MathVista **TestMini**, which include 1000 test samples. 
+- We adopt `GPT-4-Turbo (1106)` as the answer extractor when we failed to extract the answer with heuristic matching. 
+- The performance of **Human  (High school)** and **Random Choice** are copied from the official leaderboard. 
+**Category Definitions:** **FQA:** figure QA, **GPS:** geometry problem solving, **MWP:** math word problem, **TQA:** textbook QA, **VQA:** visual QA, **ALG:** algebraic, **ARI:** arithmetic, **GEO:** geometry, **LOG:** logical , **NUM:** numeric, **SCI:** scientific, **STA:** statistical.
+"""
+
+LEADERBOARD_MD['HallusionBench'] = """
+[**HallusionBench**](https://github.com/tianyi-lab/HallusionBench) is a benchmark to evaluate hallucination of VLMs. It asks a set of visual questions with one original image and one modified image (the answers for a question can be different, considering the image content). 
+
+**Examples in HallusionBench:**
+
+| Original Figure                                              | Modified Figure                                              |
+| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| ![](http://opencompass.openxlab.space/utils/Hallu0.png) | ![](http://opencompass.openxlab.space/utils/Hallu1.png) |
+| **Q1.** Is the right orange circle the same size as the left orange circle? **A1. Yes** | **Q1.** Is the right orange circle the same size as the left orange circle? **A1. No** |
+| **Q2.** Is the right orange circle larger than the left orange circle? **A2. No** | **Q2.** Is the right orange circle larger than the left orange circle? **A2. Yes** |
+| **Q3.** Is the right orange circle smaller than the left orange circle? **A3. No** | **Q3.** Is the right orange circle smaller than the left orange circle? **A3. No** |
+
+**Metrics**:
+
+>-  aAcc: The overall accuracy of **all** atomic questions. 
+>
+>- qAcc: The mean accuracy of unique **questions**. One question can be asked multiple times with different figures, we consider VLM correctly solved a unique question only if it succeeds in all <question, figure> pairs for this unique question.
+>- fAcc: The mean accuracy of all **figures**. One figure is associated with multiple questions, we consider VLM correct on a figure only if it succeeds to solve all questions of this figure. 
+
+**Evaluation Setting**:
+
+> 1. **No-visual** Questions (questions asked without the associated figure) in HallusionBench are **skipped** during evaluation.
+> 2. When we failed to extract Yes / No from the VLM prediction, we adopt **GPT-3.5-Turbo-0613** as the answer extractor.
+> 3. We report aAcc, qAcc, and fAcc for all evaluated VLMs. 
+
+## HallusionBench Evaluation Results
+"""
+
+LEADERBOARD_MD['LLaVABench'] = """
+## LLaVABench Evaluation Results
+
+- In LLaVABench Evaluation, we use GPT-4-Turbo (gpt-4-1106-preview) as the judge LLM to assign scores to the VLM outputs. We only perform the evaluation once due to the limited variance among results of multiple evaluation pass originally reported. 
+- No specific prompt template adopted for **ALL VLMs**.
+- We also include the official results (obtained by gpt-4-0314) for applicable models. 
+"""
+
+LEADERBOARD_MD['COCO_VAL'] = """
+## COCO Caption Results
+
+-  By default, we evaluate COCO Caption Validation set (5000 samples), and report the following metrics: BLEU-1, BLEU-4, CIDEr, ROUGE-L (default sorted by CIDEr).
+-  We use the following prompt to evaluate all VLMs: `Please describe this image in general. Directly provide the description, do not include prefix like "This image depicts". `
+- **No specific prompt is adopted for all VLMs.**
+"""
+
+LEADERBOARD_MD['ScienceQA_VAL'] = """
+## ScienceQA Evaluation Results
+
+- We benchmark the **image** subset of ScienceQA validation and test set, and report the Top-1 accuracy. 
+- During evaluation, we use `GPT-3.5-Turbo-0613` as the choice extractor for all VLMs if the choice can not be extracted via heuristic matching. **Zero-shot** inference is adopted. 
+"""
+
+LEADERBOARD_MD['ScienceQA_TEST'] = LEADERBOARD_MD['ScienceQA_VAL']
+
+LEADERBOARD_MD['OCRBench'] = """
+## OCRBench Evaluation Results
+
+- The evaluation of OCRBench is implemented by the official team: https://github.com/Yuliang-Liu/MultimodalOCR. 
+- The performance of GPT4V might be underestimated: GPT4V rejects to answer 12 percent of the questions due to the policy of OpenAI. For those questions, the returned answer is "Your input image may contain content that is not allowed by our safety system."
+"""
+
+LEADERBOARD_MD['MMStar'] = """
+## MMStar Evaluation Results
+
+- MMStar is an elite vision-indispensable multi-modal benchmark, including 1,500 challenging samples meticulously selected by humans.
+- During the evaluation of MMStar, we find that some API models may reject to answer some of the questions. Currently, we treat such cases as wrong answers when reporting the results. 
+"""
+
+LEADERBOARD_MD['RealWorldQA'] = """
+## RealWorldQA Evaluation Results
+
+- RealWorldQA is a benchmark designed to evaluate the real-world spatial understanding capabilities of multimodal AI models, contributed by XAI. It assesses how well these models comprehend physical environments. The benchmark consists of 700+ images, each accompanied by a question and a verifiable answer. These images are drawn from real-world scenarios, including those captured from vehicles. The goal is to advance AI models' understanding of our physical world.
+"""
+
+LEADERBOARD_MD['TextVQA_VAL'] = """
+## TextVQA Evaluation Results
+
+- TextVQA is a dataset to benchmark visual reasoning based on text in images. TextVQA requires models to read and reason about text in images to answer questions about them. Specifically, models need to incorporate a new modality of text present in the images and reason over it to answer TextVQA questions.
+- Note that some models may not be able to generate standardized responses based on the prompt. We currently do not have reports for these models.
+"""
+
+LEADERBOARD_MD['ChartQA_TEST'] = """
+## ChartQA Evaluation Results
+
+- ChartQA is a benchmark for question answering about charts with visual and logical reasoning. 
+- Note that some models may not be able to generate standardized responses based on the prompt. We currently do not have reports for these models.
+"""
+
+LEADERBOARD_MD['OCRVQA_TESTCORE'] = """
+## OCRVQA Evaluation Results
+
+- OCRVQA is a benchmark for visual question answering by reading text in images. It presents a large-scale dataset, OCR-VQA-200K, comprising over 200,000 images of book covers. The study combines techniques from the Optical Character Recognition (OCR) and Visual Question Answering (VQA) domains to address the challenges associated with this new task and dataset. 
+- Note that some models may not be able to generate standardized responses based on the prompt. We currently do not have reports for these models.
+"""
+
+LEADERBOARD_MD['POPE'] = """
+## POPE Evaluation Results
+
+- POPE is a benchmark for object hallucination evaluation. It includes three tracks of object hallucination: random, popular, and adversarial.
+- Note that the official POPE dataset contains approximately 8910 cases. POPE includes three tracks, and there are some overlapping samples among the three tracks. To reduce the data file size, we have kept only a single copy of the overlapping samples (about 5127 examples). However, the final accuracy will be calculated on the ~9k samples.
+- Some API models, due to safety policies, refuse to answer certain questions, so their actual capabilities may be higher than the reported scores.
+- We report the average F1 score across the three types of data as the overall score. Accuracy, precision, and recall are also shown in the table. F1 score = 2 * (precision * recall) / (precision + recall). 
+"""
+
+LEADERBOARD_MD['SEEDBench2_Plus'] = """
+## SEEDBench2 Plus Evaluation Results
+
+- SEEDBench2 Plus comprises 2.3K multiple-choice questions with precise human annotations, spanning three broad categories: Charts, Maps, and Webs, each of which covers a wide spectrum of textrich scenarios in the real world.
+"""
+
+LEADERBOARD_MD['MMT-Bench_VAL'] = """
+## MMT-Bench Validation Evaluation Results
+
+- MMT-Bench comprises 31,325 meticulously curated multi-choice visual questions from various multimodal scenarios such as vehicle driving and embodied navigation, covering 32 core meta-tasks and 162 subtasks in multimodal understanding.
+- MMT-Bench_VAL is the validation set of MMT-Bench. MMT-Bench_ALL includes both validation and test sets. The suffix `MI`, such as `MMT-Bench_VAL_MI`, represents the multi-image version of the dataset with several images input. 
+The defualt version is the single-image version, which concats the multiple images into a single image as input.
+"""
+
+LEADERBOARD_MD['SEEDBench2'] = """
+## SEEDBench2 Evaluation Results
+
+- SEEDBench2 comprises 24K multiple-choice questions with accurate human annotations, which spans 27 dimensions, including the evaluation of both text and image generation.
+- Note that we only evaluate and report the part of model's results on the SEEDBench2.
+"""
+
+LEADERBOARD_MD['BLINK'] = """
+## BLINK Test Evaluation Results
+
+- BLINK is a benchmark containing 14 visual perception tasks that can be solved by humans “within a blink”, but pose significant challenges for current multimodal large language models (LLMs).
+- We evaluate BLINK on the test set of the benchmark, which contains 1901 visual questions in multi-choice format. 
+"""

requirements.txt

@@ -1 +1,3 @@
-flask
+gradio==4.15.0
+numpy>=1.23.4
+pandas>=1.5.3

templates/index.html

@@ -1,35 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width" />
-    <title>My static Space</title>
-    <style>
-        body {
-            margin: 0;
-        }
-
-        .space {
-            max-width: 100%;
-            max-height: 100%;
-            width: 100vw;
-            height: 100vh;
-            overflow: hidden;
-        }
-
-        .iframe {
-            min-width: 100%;
-            min-height: 100%;
-        }
-    </style>
-</head>
-
-<body>
-    <div class="space">
-        <iframe class="iframe" allowfullscreen="true" frameborder="0" src="https://mmbench.opencompass.org.cn/leaderboard">
-        </iframe>
-    </div>
-</body>
-
-</html>