Update utils.py

utils.py

@@ -8,103 +8,42 @@ from huggingface_hub import Repository
 
 HF_TOKEN = os.environ.get("HF_TOKEN")
 
+SUBJECTS = ["Biology", "Business", "Chemistry", "Computer Science", "Economics", "Engineering",
+            "Health", "History", "Law", "Math", "Philosophy", "Physics", "Psychology", "Other"]
+
 MODEL_INFO = [
-    "Model (CoT)",
-    "Avg",
-    "TheoremQA",
-    "MATH",
-    "GSM",
-    "GPQA",
-    "MMLU-STEM"
-    ]
-
-DATA_TITILE_TYPE = ['markdown', 'number', 'number', 'number', 'number', 'number', 'number']
-
-SUBMISSION_NAME = "science_leaderboard_submission"
+    "Models",
+    "Overall",
+    "Biology", "Business", "Chemistry", "Computer Science", "Economics", "Engineering",
+    "Health", "History", "Law", "Math", "Philosophy", "Physics", "Psychology", "Other"]
+
+DATA_TITLE_TYPE = ['markdown', 'number', 'number', 'number', 'number', 'number', 'number',
+                   'number', 'number', 'number', 'number', 'number', 'number', 'number',
+                   'number', 'number']
+
+SUBMISSION_NAME = "mmlu_pro_leaderboard_submission"
 SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/TIGER-Lab/", SUBMISSION_NAME)
-CSV_DIR = "./science_leaderboard_submission/results.csv"
+CSV_DIR = "./mmlu_pro_leaderboard_submission/results.csv"
 
 COLUMN_NAMES = MODEL_INFO
 
-LEADERBORAD_INTRODUCTION = """# Science Leaderboard
-    
-    **"Which large language model is the BEST on scinece and engineering?"**<br>
-    🏆 Welcome to the **Science** leaderboard! The leaderboard covers the most popular evaluation for different science subjects including math, phyiscs, biology, chemistry, computer science, finance.
-    <div style="display: flex; flex-wrap: wrap; align-items: center; gap: 10px;">
-    </div>
-    The evaluation set from the following datasets are being included in the leaderboard.
-    <ul>
-        <li> MATH (4-shot): this contains the test set of 5000 questions from American Math contest covering different fields like algebra, calculus, statistics, geometry, linear algebra, number theory.
-        <li> GSM8K (4-shot): this contains the test set of 1320 questions from grade school math word problems. This dataset is mainly covering algebra problems.
-        <li> TheoremQA (5-shot): this contains the test set of 800 questions collected from college-level exams. This covers math, physics, engineering and finance.
-        <li> GPQA (5-shot): this contains the test of 198 questions from college-level dataset GPQA-diamond. This covers many fields like chemistry, genetics, biology, etc.
-        <li> MMLU-STEM (5-shot): this contains the test of 3.3K questions from MMLU dataset. This covers many fields like math, chemistry, genetics, biology, computer science, anatomy, astronomy, etc.
-    </ul>
-
-    **"How to evaluate your model and submit your results?"**<br>
-    Please refer to the guideline in <a href="https://github.com/TIGER-AI-Lab/MAmmoTH/blob/main/math_eval/README.md">Github</a> to evaluate your own model.
-
-    <a href='https://hits.seeyoufarm.com'><img src='https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fhuggingface.co%2Fspaces%2FTIGER-Lab%2FTheoremQA-Leaderboard&count_bg=%23C7C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false'></a>
+LEADERBOARD_INTRODUCTION = """# MMLU-Pro Leaderboard
+
+MMLU-Pro dataset, a more robust and challenging massive multi-task understanding dataset tailored to more rigorously benchmark large language models' capabilities. This dataset contains 12K complex questions across various disciplines. The following are the accuracies of various models evaluated on MMLU-Pro.
+
+Our dataset is available at [https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro](https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro). If you want to reproduce our results or evaluate your own models on MMLU-Pro, please check out our evaluation scripts at [https://github.com/TIGER-AI-Lab/MMLU-Pro](https://github.com/TIGER-AI-Lab/MMLU-Pro).
     """
 
 TABLE_INTRODUCTION = """
     """
 
-LEADERBORAD_INFO = """
+LEADERBOARD_INFO = """
 We list the information of the used datasets as follows:<br>
 
-MATH: Measuring Mathematical Problem Solving With the MATH Dataset<br>
-<a href='https://arxiv.org/pdf/2103.03874.pdf'>Paper</a><br>
-<a href='https://github.com/hendrycks/math'>Code</a><br>
-
-GSM8K: Training Verifiers to Solve Math Word Problems<br>
-<a href='https://arxiv.org/pdf/2110.14168.pdf'>Paper</a><br>
-<a href='https://github.com/openai/grade-school-math'>Code</a><br>
-
-TheoremQA: A Theorem-driven Question Answering dataset<br>
-<a href='https://arxiv.org/pdf/2305.12524.pdf'>Paper</a><br>
-<a href='https://github.com/TIGER-AI-Lab/TheoremQA'>Code</a><br>
-
-GPQA: A Graduate-Level Google-Proof Q&A Benchmark<br>
-<a href='https://arxiv.org/pdf/2311.12022.pdf'>Paper</a><br>
-<a href='https://github.com/idavidrein/gpqa'>Code</a>
-
-MMLU: Measuring Massive Multitask Language Understanding<br>
-<a href='https://arxiv.org/pdf/2009.03300.pdf'>Paper</a><br>
-<a href='https://github.com/hendrycks/test'>Code</a>
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""@inproceedings{hendrycks2021measuring,
-  title={Measuring Mathematical Problem Solving With the MATH Dataset},
-  author={Hendrycks, Dan and Burns, Collin and Kadavath, Saurav and Arora, Akul and Basart, Steven and Tang, Eric and Song, Dawn and Steinhardt, Jacob},
-  booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)},
-  year={2021}
-}
-@article{cobbe2021training,
-  title={Training verifiers to solve math word problems},
-  author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and others},
-  journal={arXiv preprint arXiv:2110.14168},
-  year={2021}
-}
-@inproceedings{chen2023theoremqa,
-  title={Theoremqa: A theorem-driven question answering dataset},
-  author={Chen, Wenhu and Yin, Ming and Ku, Max and Lu, Pan and Wan, Yixin and Ma, Xueguang and Xu, Jianyu and Wang, Xinyi and Xia, Tony},
-  booktitle={The 2023 Conference on Empirical Methods in Natural Language Processing},
-  year={2023}
-}
-@article{rein2023gpqa,
-  title={Gpqa: A graduate-level google-proof q\&a benchmark},
-  author={Rein, David and Hou, Betty Li and Stickland, Asa Cooper and Petty, Jackson and Pang, Richard Yuanzhe and Dirani, Julien and Michael, Julian and Bowman, Samuel R},
-  journal={arXiv preprint arXiv:2311.12022},
-  year={2023}
-}
-@inproceedings{hendrycks2020measuring,
-  title={Measuring Massive Multitask Language Understanding},
-  author={Hendrycks, Dan and Burns, Collin and Basart, Steven and Zou, Andy and Mazeika, Mantas and Song, Dawn and Steinhardt, Jacob},
-  booktitle={International Conference on Learning Representations},
-  year={2020}
-}"""
+CITATION_BUTTON_TEXT = r""""""
 
 SUBMIT_INTRODUCTION = """# Submit on Science Leaderboard Introduction
 
@@ -112,36 +51,40 @@ SUBMIT_INTRODUCTION = """# Submit on Science Leaderboard Introduction
 
 ```json
 {
-    "Model": "[NAME]",
-    "Repo": "https://huggingface.co/[MODEL_NAME]"
-    "TheoremQA": 50,
-    "MATH": 50,
-    "GSM": 50,
-    "GPQA": 50,
-    "MMLU-STEM": 50
+    "Model": "[MODEL_NAME]",
+    "Overall": 0.5678,
+    "Biology": 0.1234, 
+    "Business": 0.4567,
+    ...,
+    "Other: 0.3456"
 }
 ```
-After submitting, you can click the "Refresh" button to see the updated leaderboard(it may takes few seconds).
+After submitting, you can click the "Refresh" button to see the updated leaderboard (it may takes few seconds).
 
 """
+
+
 def get_df():
     repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN)
     repo.git_pull()
     df = pd.read_csv(CSV_DIR)
-    df['Avg'] = df[['TheoremQA', 'MATH', 'GSM', 'GPQA', 'MMLU-STEM']].mean(axis=1).round(1)
-    df = df.sort_values(by=['Avg'], ascending=False)
+    df = df.sort_values(by=['Overall'], ascending=False)
     return df[COLUMN_NAMES]
 
+
 def add_new_eval(
     input_file,
 ):
     if input_file is None:
         return "Error! Empty file!"
 
-    upload_data=json.loads(input_file)
-    data_row = [f'[{upload_data["Model"]}]({upload_data["Repo"]})', upload_data['TheoremQA'], upload_data['MATH'], upload_data['GSM'], upload_data['GPQA'], upload_data['MMLU-STEM']]
+    upload_data = json.loads(input_file)
+    data_row = [f'{upload_data["Model"]}', upload_data['Overall']]
+    for subject in SUBJECTS:
+        data_row += [upload_data[subject]]
 
-    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL,
+                                 use_auth_token=HF_TOKEN, repo_type="dataset")
     submission_repo.git_pull()
 
     already_submitted = []
@@ -162,4 +105,7 @@ def add_new_eval(
 
 
 def refresh_data():
-    return get_df()
+    return get_df()
+
+
+