update model metadata

scripts/create_request_file.py

@@ -12,7 +12,7 @@ EVAL_REQUESTS_PATH = "eval-queue"
 QUEUE_REPO = "open-llm-leaderboard/requests"
 
 precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
-model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")
+model_types = ("open-source", "close-source")
 weight_types = ("Original", "Delta", "Adapter")
 
 

src/display/utils.py

@@ -65,10 +65,8 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    OSM = ModelDetails(name="open-source", symbol="🟢")
+    CSM = ModelDetails(name="close-source", symbol="🔶")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -76,14 +74,10 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
+        if "close-source" in type or "🔶" in type:
+            return ModelType.CSM
+        if "open-source" in type or "🟢" in type:
+            return ModelType.OSM
         return ModelType.Unknown
 
 class WeightType(Enum):

src/leaderboard/read_evals.py

@@ -64,8 +64,6 @@ class EvalResult:
             architectures = getattr(model_config, "architectures", None)
             if architectures:
                 architecture = ";".join(architectures)
-            if model_type == ModelType.Unknown:
-                model_type = ModelType.from_str(getattr(model_config, "model_type", "Unknown"))
 
         # Extract results available in this file (some results are split in several files)
         results = {}