update

src/envs.py

@@ -31,4 +31,10 @@ DYNAMIC_INFO_PATH = os.path.join(CACHE_PATH, "dynamic-info")
 DYNAMIC_INFO_FILE_PATH = os.path.join(DYNAMIC_INFO_PATH, "model_infos.json")
 
 PATH_TO_COLLECTION = "open-cn-llm-leaderboard/flageval-vlm-leaderboard-best-models-677e51cdc44f8123e02cbda1"
+
+# Rate limit variables
+RATE_LIMIT_PERIOD = 7
+RATE_LIMIT_QUOTA = 5
+HAS_HIGHER_RATE_LIMIT = ["TheBloke"]
+
 API = HfApi(token=TOKEN)

src/submission/check_validity.py

@@ -10,6 +10,8 @@ from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
 from transformers.models.auto.tokenization_auto import AutoTokenizer
 
+from src.envs import HAS_HIGHER_RATE_LIMIT
+
 def check_model_card(repo_id: str) -> tuple[bool, str]:
     """Checks if the model card and license exist and have been filled"""
     try:
@@ -97,3 +99,54 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     users_to_submission_dates[organisation].append(info["submitted_time"])
 
     return set(file_names), users_to_submission_dates
+
+def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
+    if org_or_user not in users_to_submission_dates:
+        return True, ""
+    submission_dates = sorted(users_to_submission_dates[org_or_user])
+
+    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
+
+    num_models_submitted_in_period = len(submissions_after_timelimit)
+    if org_or_user in HAS_HIGHER_RATE_LIMIT:
+        rate_limit_quota = 2 * rate_limit_quota
+
+    if num_models_submitted_in_period > rate_limit_quota:
+        error_msg = f"Organisation or user `{org_or_user}`"
+        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
+        error_msg += f"in the last {rate_limit_period} days.\n"
+        error_msg += (
+            "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
+        )
+        return False, error_msg
+    return True, ""
+
+def get_model_tags(model_card, model: str):
+    is_merge_from_metadata = False
+    is_moe_from_metadata = False
+
+    tags = []
+    if model_card is None:
+        return tags
+    if model_card.data.tags:
+        is_merge_from_metadata = "merge" in model_card.data.tags
+        is_moe_from_metadata = "moe" in model_card.data.tags
+    merge_keywords = ["merged model", "merge model"]
+    # If the model is a merge but not saying it in the metadata, we flag it
+    is_merge_from_model_card = any(keyword in model_card.text.lower() for keyword in merge_keywords)
+    if is_merge_from_model_card or is_merge_from_metadata:
+        tags.append("merge")
+        if not is_merge_from_metadata:
+            tags.append("flagged:undisclosed_merge")
+    moe_keywords = ["moe", "mixtral"]
+    is_moe_from_model_card = any(keyword in model_card.text.lower() for keyword in moe_keywords)
+    is_moe_from_name = "moe" in model.lower().replace("/", "-").replace("_", "-").split("-")
+    if is_moe_from_model_card or is_moe_from_name or is_moe_from_metadata:
+        tags.append("moe")
+        # We no longer tag undisclosed MoEs
+        #if not is_moe_from_metadata:
+        #    tags.append("flagged:undisclosed_moe")
+
+
+    return tags