Cleanup

Signed-off-by: Jonathan Bnayahu <[email protected]>

app.py

@@ -1,13 +1,10 @@
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
+from gradio_leaderboard import Leaderboard
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
 
 from src.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
-    # EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
@@ -17,73 +14,25 @@ from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
     COLS,
-    EVAL_COLS,
-    # EVAL_TYPES,
     AutoEvalColumn,
-    # ModelType,
     fields,
-    # WeightType,
-    # Precision
 )
-from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, TOKEN
+from src.envs import API, EVAL_RESULTS_PATH, REPO_ID
 from src.populate import get_leaderboard_df
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
-# try:
-#     print(EVAL_REQUESTS_PATH)
-#     snapshot_download(
-#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-#     )
-# except Exception:
-#     restart_space()
-# try:
-#     print(EVAL_RESULTS_PATH)
-#     snapshot_download(
-#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-#     )
-# except Exception:
-#     restart_space()
-
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
 
-# (
-#     finished_eval_queue_df,
-#     running_eval_queue_df,
-#     pending_eval_queue_df,
-# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
     return Leaderboard(
         value=dataframe,
         datatype=[c.type for c in fields(AutoEvalColumn)],
-        # select_columns=SelectColumns(
-        #     default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-        #     cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-        #     label="Select Columns to Display:",
-        # ),
         search_columns=[AutoEvalColumn.model.name],
-        # hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        # filter_columns=[
-        #     ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-        #     ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-        #     ColumnFilter(
-        #         AutoEvalColumn.params.name,
-        #         type="slider",
-        #         min=0.01,
-        #         max=150,
-        #         label="Select the number of parameters (B)",
-        #     ),
-        #     ColumnFilter(
-        #         AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-        #     ),
-        # ],
-        # bool_checkboxgroup_label="Hide models",
         interactive=False,
     )
 
@@ -101,93 +50,6 @@ with demo:
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        # with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-        #     with gr.Column():
-        #         with gr.Row():
-        #             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-        #         with gr.Column():
-        #             with gr.Accordion(
-        #                 f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-        #                 open=False,
-        #             ):
-        #                 with gr.Row():
-        #                     finished_eval_table = gr.components.Dataframe(
-        #                         value=finished_eval_queue_df,
-        #                         headers=EVAL_COLS,
-        #                         datatype=EVAL_TYPES,
-        #                         row_count=5,
-        #                     )
-        #             with gr.Accordion(
-        #                 f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-        #                 open=False,
-        #             ):
-        #                 with gr.Row():
-        #                     running_eval_table = gr.components.Dataframe(
-        #                         value=running_eval_queue_df,
-        #                         headers=EVAL_COLS,
-        #                         datatype=EVAL_TYPES,
-        #                         row_count=5,
-        #                     )
-
-        #             with gr.Accordion(
-        #                 f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-        #                 open=False,
-        #             ):
-        #                 with gr.Row():
-        #                     pending_eval_table = gr.components.Dataframe(
-        #                         value=pending_eval_queue_df,
-        #                         headers=EVAL_COLS,
-        #                         datatype=EVAL_TYPES,
-        #                         row_count=5,
-        #                     )
-        #     with gr.Row():
-        #         gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-        #     with gr.Row():
-        #         with gr.Column():
-        #             model_name_textbox = gr.Textbox(label="Model name")
-        #             revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-        #             model_type = gr.Dropdown(
-        #                 choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-        #                 label="Model type",
-        #                 multiselect=False,
-        #                 value=None,
-        #                 interactive=True,
-        #             )
-
-        #         with gr.Column():
-        #             precision = gr.Dropdown(
-        #                 choices=[i.value.name for i in Precision if i != Precision.Unknown],
-        #                 label="Precision",
-        #                 multiselect=False,
-        #                 value="float16",
-        #                 interactive=True,
-        #             )
-        #             weight_type = gr.Dropdown(
-        #                 choices=[i.value.name for i in WeightType],
-        #                 label="Weights type",
-        #                 multiselect=False,
-        #                 value="Original",
-        #                 interactive=True,
-        #             )
-        #             base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-        #     submit_button = gr.Button("Submit Eval")
-        #     submission_result = gr.Markdown()
-        #     submit_button.click(
-        #         add_new_eval,
-        #         [
-        #             model_name_textbox,
-        #             base_model_name_textbox,
-        #             revision_name_textbox,
-        #             precision,
-        #             weight_type,
-        #             model_type,
-        #         ],
-        #         submission_result,
-        #     )
-
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(

src/about.py

@@ -27,7 +27,6 @@ class Tasks(Enum):
     task11 = Task("summarization", "score", "Summarization")
     task12 = Task("translation", "score", "Translation")
 
-NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
 
 
@@ -39,7 +38,7 @@ TITLE = """<h1 align="center" id="space-title">BlueBench Leaderboard</h1>"""
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
 BlueBench is an open-source benchmark developed by domain experts to represent required needs of Enterprise users.
-It is constructed using state-of-the-art benchmarking methodologies to ensure validity, robustness, and efficiency by utilizing unitxt’s abilities for dynamic and flexible text processing.
+It is constructed using state-of-the-art benchmarking methodologies to ensure validity, robustness, and efficiency by utilizing <a href="https://www.unitxt.ai">unitxt</a>’s abilities for dynamic and flexible text processing.
 As a dynamic and evolving benchmark, BlueBench currently encompasses diverse domains such as legal, finance, customer support, and news. It also evaluates a range of capabilities, including RAG, pro-social behavior, summarization, and chatbot performance, with additional tasks and domains to be integrated over time.
 """
 
@@ -66,7 +65,7 @@ table th:nth-of-type(3) {
 | Reasoning               | <pre><p><b>Hellaswag</b></p>[Dataset](https://huggingface.co/datasets/Rowan/hellaswag), [Paper](https://arxiv.org/abs/1905.07830), [Unitxt Card](https://www.unitxt.ai/en/latest/catalog/catalog.cards.hellaswag.html)</pre>                                                      | <p>Commonsense natural language inference</p>Given an event description such as "A woman sits at a piano," a machine must select the most likely followup: "She sets her fingers on the keys." Gatherd via Adversarial Filtering (AF), a data collection paradigm wherein a series of discriminators iteratively select an adversarial set of machine-generated wrong answers. |
 | Reasoning               | <pre><p><b>Openbook QA</b></p>[Dataset](https://huggingface.co/datasets/allenai/openbookqa), [Paper](https://aclanthology.org/D18-1260/), [Unitxt Card](https://www.unitxt.ai/en/latest/catalog/catalog.cards.openbook_qa.html)</pre>                                             | <p>Question answering dataset using open book exams</p>Comes with our questions is a set of 1326 elementary level science facts. Roughly 6000 questions probe an understanding of these facts and their application to novel situations. This requires combining an open book fact (e.g., metals conduct electricity) with broad common knowledge (e.g., a suit of armor is made of metal) obtained from other sources. |
 | Machine Translation     | <pre><p><b>Flores 101</b></p>[Dataset](https://huggingface.co/datasets/gsarti/flores_101), [Paper](https://arxiv.org/abs/2106.03193), [Unitxt Card](https://www.unitxt.ai/en/latest/catalog/catalog.cards.mt.flores_101.__dir__.html)</pre>                                       | <p>Benchmark dataset for machine translation</p>There are 101 lanugages in this dataset, each sentence appears in all languages, and all a total of `2k` sentences. We use the following language pairs:  `["ara_eng", "deu_eng", "eng_ara", "eng_deu", "eng_fra", "eng_kor", "eng_por", "eng_ron", "eng_spa", "fra_eng", "jpn_eng", "kor_eng", "por_eng", "ron_eng", "spa_eng"]`. |
-| Chatbot Abilities       | <pre><p><b>Arena Hard</b></p>[Dataset](https://huggingface.co/datasets/lmsys/arena-hard-auto-v0.1), [Paper](https://arxiv.org/abs/2406.11939), [Unitxt Card](https://www.unitxt.ai/en/latest/catalog/catalog.cards.arena_hard.generation.english_gpt_4_0314_reference.html)</pre> | <p>An automatic evaluation tool for instruction-tuned LLMs</p>Contains 500 challenging user queries sourced from Chatbot Arena. We prompt GPT-4-Turbo as judge to compare the models" responses against a baseline model (default: GPT-4-0314 for here we are using `llama-3.1-70b`). |
+| Chatbot Abilities       | <pre><p><b>Arena Hard</b></p>[Dataset](https://huggingface.co/datasets/lmsys/arena-hard-auto-v0.1), [Paper](https://arxiv.org/abs/2406.11939), [Unitxt Card](https://www.unitxt.ai/en/latest/catalog/catalog.cards.arena_hard.generation.english_gpt_4_0314_reference.html)</pre> | <p>An automatic evaluation tool for instruction-tuned LLMs</p>Contains 500 challenging user queries sourced from Chatbot Arena. We prompt GPT-4-Turbo as judge to compare the models" responses against a baseline model (default: GPT-4-0314 for here we are using `llama-3.3-70b`). |
 | Classification          | <pre><p><b>20_newsgroups</b></p>[Dataset](https://huggingface.co/datasets/SetFit/20_newsgroups), [Unitxt Card](https://www.unitxt.ai/en/latest/catalog/catalog.cards.20_newsgroups.html)</pre>                                                                                    | <p>News article classification</p>The 20 newsgroups dataset comprises around 18000 newsgroups posts on 20 topics split in two subsets: one for training (or development) and the other one for testing (or for performance evaluation). The split between the train and test set is based upon a messages posted before and after a specific date. |
 | Bias                    | <pre><p><b>BBQ</b></p>[Dataset](https://huggingface.co/datasets/heegyu/bbq), [Paper](https://arxiv.org/abs/2110.08193), [Unitxt Card](https://www.unitxt.ai/en/latest/catalog/catalog.cards.safety.bbq.__dir__.html)</pre>                                                        | <p>Question sets constructed to highlight attested social biases against people belonging to protected classes along nine social dimensions relevant for U.S. English-speaking contexts.</p>It is well documented that NLP models learn social biases, but little work has been done on how these biases manifest in model outputs for applied tasks like question answering (QA). We introduce the Bias Benchmark for QA (BBQ), a dataset of question sets constructed by the authors that highlight attested social biases against people belonging to protected classes along nine social dimensions relevant for U.S. English-speaking contexts. Our task evaluates model responses at two levels: (i) given an under-informative context, we test how strongly responses refect social biases, and (ii) given an adequately informative context, we test whether the model's biases override a correct answer choice. We fnd that models often rely on stereotypes when the context is under-informative, meaning the model's outputs consistently reproduce harmful biases in this setting. Though models are more accurate when the context provides an informative answer, they still rely on stereotypes and average up to 3.4 percentage points higher accuracy when the correct answer aligns with a social bias than when it conficts, with this difference widening to over 5 points on examples targeting gender for most models tested. |
 | Legal Reasoning         | <pre><p><b>Legalbench</b></p>[Dataset](https://huggingface.co/datasets/nguha/legalbench), [Paper](https://arxiv.org/abs/2308.11462), [Unitxt Card](https://www.unitxt.ai/en/latest/catalog/catalog.cards.legalbench.__dir__.html)</pre>                                           | <p>Evaluating legal reasoning in English large language models (LLMs).</p>LegalBench tasks span multiple types (binary classification, multi-class classification, extraction, generation, entailment), multiple types of text (statutes, judicial opinions, contracts, etc.), and multiple areas of law (evidence, contracts, civil procedure, etc.). For more information on tasks, we recommend visiting the website, where you can search through task descriptions, or the Github repository, which contains more granular task descriptions. We also recommend reading the paper, which provides more background on task significance and construction process. |
@@ -88,36 +87,6 @@ unitxt-evaluate --tasks "benchmarks.bluebench" --model cross_provider --model_ar
 ```
 """
 
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
 """

src/display/formatting.py

@@ -1,12 +1,3 @@
-# def model_hyperlink(link, model_name):
-#     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-# def make_clickable_model(model_name):
-#     link = f"https://huggingface.co/{model_name}"
-#     return model_hyperlink(link, model_name)
-
-
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
 

src/display/utils.py

@@ -23,22 +23,11 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-# auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -60,46 +49,6 @@ class ModelDetails:
     display_name: str = ""
     symbol: str = "" # emoji
 
-
-# class ModelType(Enum):
-#     PT = ModelDetails(name="pretrained", symbol="🟢")
-#     FT = ModelDetails(name="fine-tuned", symbol="🔶")
-#     IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-#     RL = ModelDetails(name="RL-tuned", symbol="🟦")
-#     Unknown = ModelDetails(name="", symbol="?")
-
-#     def to_str(self, separator=" "):
-#         return f"{self.value.symbol}{separator}{self.value.name}"
-
-#     @staticmethod
-#     def from_str(type):
-#         if "fine-tuned" in type or "🔶" in type:
-#             return ModelType.FT
-#         if "pretrained" in type or "🟢" in type:
-#             return ModelType.PT
-#         if "RL-tuned" in type or "🟦" in type:
-#             return ModelType.RL
-#         if "instruction-tuned" in type or "⭕" in type:
-#             return ModelType.IFT
-#         return ModelType.Unknown
-
-# class WeightType(Enum):
-#     Adapter = ModelDetails("Adapter")
-#     Original = ModelDetails("Original")
-#     Delta = ModelDetails("Delta")
-
-# class Precision(Enum):
-#     float16 = ModelDetails("float16")
-#     bfloat16 = ModelDetails("bfloat16")
-#     Unknown = ModelDetails("?")
-
-#     def from_str(precision):
-#         if precision in ["torch.float16", "float16"]:
-#             return Precision.float16
-#         if precision in ["torch.bfloat16", "bfloat16"]:
-#             return Precision.bfloat16
-#         return Precision.Unknown
-
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 

src/envs.py

@@ -10,18 +10,9 @@ OWNER = "jbnayahu" # Change to your org - don't forget to create a results and r
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/bluebench"
-# QUEUE_REPO = f"{OWNER}/bluebench-requests"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")
-
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "results/bluebench")
 
-
-# Local caches
-# EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-# EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-# EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-# EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,6 +1,4 @@
-import glob
 import json
-import math
 import os
 from dataclasses import dataclass
 
@@ -8,8 +6,6 @@ import dateutil
 import numpy as np
 
 from src.display.utils import AutoEvalColumn, Tasks
-# from src.submission.check_validity import is_model_on_hub
-
 
 @dataclass
 class EvalResult:
@@ -17,19 +13,8 @@ class EvalResult:
     """
     eval_name: str # org_model_precision (uid)
     full_model: str # org/model (path on hub)
-    # org: str 
-    # model: str
-    # revision: str # commit hash, "" if main
     results: dict
-    # precision: Precision = Precision.Unknown
-    # model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    # weight_type: WeightType = WeightType.Original # Original or Adapter
-    # architecture: str = "Unknown" 
-    # license: str = "?"
-    # likes: int = 0
-    # num_params: int = 0
     date: str = "" # submission date of request file
-    # still_on_hub: bool = False
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -41,32 +26,6 @@ class EvalResult:
 
         full_model = env_info.get("model")
                
-        # Precision
-        # precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        # org_and_model = config.get("model_name", config.get("model_args", None))
-        # org_and_model = org_and_model.split("/", 1)
-
-        # if len(org_and_model) == 1:
-        #     org = None
-        #     model = org_and_model[0]
-        #     result_key = f"{model}_{precision.value.name}"
-        # else:
-        #     org = org_and_model[0]
-        #     model = org_and_model[1]
-        #     result_key = f"{org}_{model}_{precision.value.name}"
-        # full_model = "/".join(org_and_model)
-
-        # still_on_hub, _, model_config = is_model_on_hub(
-        #     full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        # )
-        # architecture = "?"
-        # if model_config is not None:
-        #     architectures = getattr(model_config, "architectures", None)
-        #     if architectures:
-        #         architecture = ";".join(architectures)
-
         # Extract results available in this file (some results are split in several files)
         results = {}
         for task in Tasks:
@@ -83,48 +42,16 @@ class EvalResult:
         return self(
             eval_name=full_model,
             full_model=full_model,
-            # org=org,
-            # model=model,
             results=results,
-            # precision=precision,  
-            # revision= config.get("model_sha", ""),
-            # still_on_hub=still_on_hub,
-            # architecture=architecture
         )
 
-    # def update_with_request_file(self, requests_path):
-    #     """Finds the relevant request file for the current model and updates info with it"""
-    #     request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-    #     try:
-    #         with open(request_file, "r") as f:
-    #             request = json.load(f)
-    #         self.model_type = ModelType.from_str(request.get("model_type", ""))
-    #         self.weight_type = WeightType[request.get("weight_type", "Original")]
-    #         self.license = request.get("license", "?")
-    #         self.likes = request.get("likes", 0)
-    #         self.num_params = request.get("params", 0)
-    #         self.date = request.get("submitted_time", "")
-    #     except Exception:
-    #         print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            # AutoEvalColumn.precision.name: self.precision.value.name,
-            # AutoEvalColumn.model_type.name: self.model_type.value.name,
-            # AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            # AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: self.full_model,
-            # AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
-            # AutoEvalColumn.license.name: self.license,
-            # AutoEvalColumn.likes.name: self.likes,
-            # AutoEvalColumn.params.name: self.num_params,
-            # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
         for task in Tasks:
@@ -133,28 +60,6 @@ class EvalResult:
         return data_dict
 
 
-# def get_request_file_for_model(requests_path, model_name, precision):
-#     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-#     request_files = os.path.join(
-#         requests_path,
-#         f"{model_name}_eval_request_*.json",
-#     )
-#     request_files = glob.glob(request_files)
-
-#     # Select correct request file (precision)
-#     request_file = ""
-#     request_files = sorted(request_files, reverse=True)
-#     for tmp_request_file in request_files:
-#         with open(tmp_request_file, "r") as f:
-#             req_content = json.load(f)
-#             if (
-#                 req_content["status"] in ["FINISHED"]
-#                 and req_content["precision"] == precision.split(".")[-1]
-#             ):
-#                 request_file = tmp_request_file
-#     return request_file
-
-
 def get_raw_eval_results(results_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []

src/populate.py

@@ -1,10 +1,7 @@
-import json
-import os
-
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.display.utils import AutoEvalColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
@@ -19,40 +16,4 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-# def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-#     """Creates the different dataframes for the evaluation queues requestes"""
-#     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-#     all_evals = []
-
-#     for entry in entries:
-#         if ".json" in entry:
-#             file_path = os.path.join(save_path, entry)
-#             with open(file_path) as fp:
-#                 data = json.load(fp)
-
-#             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-#             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-#             all_evals.append(data)
-#         elif ".md" not in entry:
-#             # this is a folder
-#             sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-#             for sub_entry in sub_entries:
-#                 file_path = os.path.join(save_path, entry, sub_entry)
-#                 with open(file_path) as fp:
-#                     data = json.load(fp)
-
-#                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-#                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-#                 all_evals.append(data)
-
-#     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-#     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-#     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-#     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-#     df_running = pd.DataFrame.from_records(running_list, columns=cols)
-#     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-#     return df_finished[cols], df_running[cols], df_pending[cols]
+    return df