| import pandas as pd | |
| import gradio as gr | |
| import csv | |
| import json | |
| import os | |
| import shutil | |
| from huggingface_hub import Repository | |
| HF_TOKEN = os.environ.get("HUGGINGFACE_TOKEN") | |
| MODEL_INFO = [ | |
| "Model", | |
| "Avg", | |
| "Visual Quality", | |
| "Temporal Consistency", | |
| "Dynamic Degree", | |
| "Text-to-Video Alignment", | |
| ] | |
| DATA_TITILE_TYPE = ['markdown', 'number', 'number', 'number', 'number', 'number',] | |
| SUBMISSION_NAME = "VideoScore-Leaderboard" | |
| SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/hexuan21/", SUBMISSION_NAME) | |
| CSV_DIR = "./VideoScore-Leaderboard/results.csv" | |
| COLUMN_NAMES = MODEL_INFO | |
| LEADERBORAD_INTRODUCTION = """# VideoScore Leaderboard | |
| """ | |
| TABLE_INTRODUCTION = """ | |
| """ | |
| LEADERBORAD_INFO = """ | |
| We list the information of the used datasets as follows:<br> | |
| """ | |
| CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" | |
| CITATION_BUTTON_TEXT = r"""@inproceedings{hendrycks2021measuring, | |
| title={Measuring Mathematical Problem Solving With the MATH Dataset}, | |
| author={Hendrycks, Dan and Burns, Collin and Kadavath, Saurav and Arora, Akul and Basart, Steven and Tang, Eric and Song, Dawn and Steinhardt, Jacob}, | |
| booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)}, | |
| year={2021} | |
| } | |
| }""" | |
| SUBMIT_INTRODUCTION = """# Submit on Science Leaderboard Introduction | |
| ## ⚠ Please note that you need to submit the json file with following format: | |
| ```json | |
| { | |
| "Model": "[NAME]", | |
| "Repo": "https://huggingface.co/[MODEL_NAME]" | |
| "TheoremQA": 50, | |
| "MATH": 50, | |
| "GSM": 50, | |
| "GPQA": 50, | |
| "MMLU-STEM": 50 | |
| } | |
| ``` | |
| After submitting, you can click the "Refresh" button to see the updated leaderboard(it may takes few seconds). | |
| """ | |
| def get_df(): | |
| repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN) | |
| repo.git_pull() | |
| df = pd.read_csv(CSV_DIR) | |
| df['Model'] = df['Model'].apply(lambda x: f"[{x.split(']')[0][1:]}]({x.split('(')[1][:-1]})") | |
| df['Avg'] = df[["Visual Quality", | |
| "Temporal Consistency", | |
| "Dynamic Degree", | |
| "Text-to-Video Alignment",]].mean(axis=1).round(2) | |
| df = df.sort_values(by=['Avg'], ascending=False) | |
| return df[COLUMN_NAMES] | |
| def refresh_data(): | |
| return get_df() |