Fix submission checks

src/envs.py

@@ -15,6 +15,10 @@ RESULTS_REPO = f"{OWNER}/aiera-leaderboard-results"
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")
 
+# NO
+EXTERNAL_PROVIDERS = ["openai", "anthropic", "google"]
+
+
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")

src/leaderboard/read_evals.py

@@ -10,6 +10,7 @@ import numpy as np
 from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
 from src.submission.check_validity import is_model_on_hub
+from src.envs import EXTERNAL_PROVIDERS
 
 
 @dataclass
@@ -57,10 +58,15 @@ class EvalResult:
             result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
+        architecture = None
+        model_config = None
+        still_on_hub = False
+        if not any([org.lower() in provider for provider in EXTERNAL_PROVIDERS]):
+            still_on_hub, _, model_config = is_model_on_hub(
+                full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+            )
+            architecture = "?"
+
         if model_config is not None:
             architectures = getattr(model_config, "architectures", None)
             if architectures:

src/submission/check_validity.py

@@ -9,6 +9,8 @@ from huggingface_hub import ModelCard
 from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
 from transformers.models.auto.tokenization_auto import AutoTokenizer
+from src.envs import API
+
 
 def check_model_card(repo_id: str) -> tuple[bool, str]:
     """Checks if the model card and license exist and have been filled"""
@@ -33,8 +35,10 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
 def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
     """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    model_info = API.model_info(model_name, revision="main")
     try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        model_info = API.model_info(model_name)
+        config = model_info.config
         if test_tokenizer:
             try:
                 tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)

src/submission/submit.py

@@ -45,12 +45,12 @@ def add_new_eval(
 
     # Is the model on the hub?
     if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=False)
         if not base_model_on_hub:
             return styled_error(f'Base model "{base_model}" {error}')
 
     if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=False)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')