Check in updated leaderboard

src/about.py

@@ -57,9 +57,9 @@ EVALUATION_QUEUE_TEXT = """
 ### 1) Make sure you can load your model and tokenizer using AutoClasses:
 ```python
 from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+config = AutoConfig.from_pretrained("your model name", revision="main")
+model = AutoModel.from_pretrained("your model name", revision="main")
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision="main")
 ```
 If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
 

src/display/formatting.py

@@ -2,8 +2,9 @@ def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
 
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
+def make_clickable_model(full_model_name):
+    model_name = full_model_name.split("/")[1]
+    link = f"https://huggingface.co/{full_model_name}"
     return model_hyperlink(link, model_name)
 
 

src/display/utils.py

@@ -23,22 +23,18 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["org", ColumnContent, ColumnContent("Organization", "str", True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Aiera Score ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("License", "str", False)])
+
+
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -47,10 +43,7 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "?")
     status = ColumnContent("status", "str", True)
 
 ## All the model information that we might need
@@ -60,72 +53,6 @@ class ModelDetails:
     display_name: str = ""
     symbol: str = "" # emoji
 
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-    Unknown = ModelDetails("?")
-
-    def from_str(weight):
-        if weight == "Adapter":
-            return WeightType.Adapter
-        
-        elif weight == "Original":
-            return WeightType.Original
-        
-        elif weight == "Delta":
-            return WeightType.Delta
-        
-        else:
-            return WeightType.Unknown
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    float32 = ModelDetails("float32")
-    #qt_8bit = ModelDetails("8bit")
-    #qt_4bit = ModelDetails("4bit")
-    #qt_GPTQ = ModelDetails("GPTQ")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        if precision in ["float32"]:
-            return Precision.float32
-        #if precision in ["8bit"]:
-        #    return Precision.qt_8bit
-        #if precision in ["4bit"]:
-        #    return Precision.qt_4bit
-        #if precision in ["GPTQ", "None"]:
-        #    return Precision.qt_GPTQ
-        return Precision.Unknown
-
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 

src/leaderboard/read_evals.py

@@ -8,11 +8,10 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, Tasks
 from src.submission.check_validity import is_model_on_hub
 from src.envs import EXTERNAL_PROVIDERS
 
-
 @dataclass
 class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run.
@@ -21,14 +20,8 @@ class EvalResult:
     full_model: str # org/model (path on hub)
     org: str 
     model: str
-    revision: str # commit hash, "" if main
     results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
     license: str = "?"
-    likes: int = 0
     num_params: int = 0
     date: str = "" # submission date of request file
     still_on_hub: bool = False
@@ -41,9 +34,6 @@ class EvalResult:
 
         config = data.get("config")
 
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
         org_and_model = org_and_model.split("/", 1)
@@ -51,26 +41,20 @@ class EvalResult:
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
+            result_key = model
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
+            result_key = f"{org}_{model}"
         full_model = "/".join(org_and_model)
 
-        architecture = None
-        model_config = None
         still_on_hub = False
         if not any([org.lower() in provider for provider in EXTERNAL_PROVIDERS]):
             still_on_hub, _, model_config = is_model_on_hub(
-                full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+                full_model, trust_remote_code=True, test_tokenizer=False
             )
-            architecture = "?"
 
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+        num_params = data.get("params")
 
         # Extract results available in this file (some results are split in several files)
         results = {}
@@ -91,10 +75,8 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
+            num_params=num_params, 
             still_on_hub=still_on_hub,
-            architecture=architecture
         )
 
     def update_with_request_file(self, requests_path):
@@ -104,10 +86,7 @@ class EvalResult:
         #try:
         with open(request_file, "r") as f:
             request = json.load(f)
-        self.model_type = ModelType.from_str(request.get("model_type", ""))
-        self.weight_type = WeightType[request.get("weight_type", "Unknown")]
         self.license = request.get("license", "?")
-        self.likes = request.get("likes", 0)
         self.num_params = request.get("params", "?")
         self.date = request.get("submitted_time", "")
         #except Exception:
@@ -118,16 +97,10 @@ class EvalResult:
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.org.name: self.org,
             AutoEvalColumn.params.name: self.num_params,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }

src/populate.py

@@ -34,7 +34,6 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 data = json.load(fp)
 
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
         elif ".md" not in entry:
@@ -46,7 +45,6 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                     data = json.load(fp)
 
                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
                 all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]

src/submission/check_validity.py

@@ -33,7 +33,7 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
     return True, ""
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+def is_model_on_hub(model_name: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
     """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
     model_info = API.model_info(model_name, revision="main")
     try:
@@ -41,7 +41,7 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
         config = model_info.config
         if test_tokenizer:
             try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+                tk = AutoTokenizer.from_pretrained(model_name, revision="main", trust_remote_code=trust_remote_code, token=token)
             except ValueError as e:
                 return (
                     False,
@@ -92,7 +92,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}")
+                    file_names.append(info['model'])
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -15,12 +15,7 @@ REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
 
 def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
+    model: str
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
@@ -36,27 +31,10 @@ def add_new_eval(
     precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=False)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=False)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
 
     # Is the model info correctly filled?
     try:
-        model_info = API.model_info(repo_id=model, revision=revision)
+        model_info = API.model_info(repo_id=model, revision="main")
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
@@ -78,12 +56,9 @@ def add_new_eval(
     eval_entry = {
         "model": model,
         "base_model": base_model,
-        "revision": revision,
         "precision": precision,
-        "weight_type": weight_type,
         "status": "PENDING",
         "submitted_time": current_time,
-        "model_type": model_type,
         "likes": model_info.likes,
         "params": model_size,
         "license": license,
@@ -91,13 +66,13 @@ def add_new_eval(
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+    if model in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_{revision}.json"
+    out_path = f"{OUT_DIR}/{model_path}.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))