update

app.py

@@ -26,7 +26,7 @@ from src.display.utils import (
     Precision,
     NUMERIC_INTERVALS
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, DYNAMIC_INFO_REPO, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_PATH, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 from src.scripts.update_all_request_files import update_dynamic_files
@@ -67,7 +67,7 @@ def init_space():
         cols=COLS,
         benchmark_cols=BENCHMARK_COLS
     )
-    #update_collections(original_df.copy())
+    update_collections(original_df.copy())
     leaderboard_df = original_df.copy()
 
     plot_df = create_plot_df(create_scores_df(raw_data))
@@ -421,6 +421,7 @@ with demo:
                     model_name_textbox = gr.Textbox(label="Model name")
                     model_url_textbox = gr.Textbox(label="Model online api url")
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
                     model_type = gr.Dropdown(
                         choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
                         label="Model type",
@@ -458,6 +459,7 @@ with demo:
                     base_model_name_textbox,
                     revision_name_textbox,
                     precision,
+                    private,
                     weight_type,
                     model_type,
                 ],

src/envs.py

@@ -20,6 +20,7 @@ QUEUE_REPO = "open-cn-llm-leaderboard/vlm_requests"
 DYNAMIC_INFO_REPO = "open-cn-llm-leaderboard/vlm_dynamic_model_information"
 RESULTS_REPO = "open-cn-llm-leaderboard/vlm_results"
 
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")
 

src/submission/submit.py

@@ -3,7 +3,7 @@ import os
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, DYNAMIC_INFO_PATH, DYNAMIC_INFO_FILE_PATH, DYNAMIC_INFO_REPO, TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
 from src.submission.check_validity import (
     already_submitted_models,
     check_model_card,
@@ -21,6 +21,7 @@ def add_new_eval(
     base_model: str,
     revision: str,
     precision: str,
+    private: str,
     weight_type: str,
     model_type: str,
 ):
@@ -84,6 +85,7 @@ def add_new_eval(
         "base_model": base_model,
         "revision": revision,
         "precision": precision,
+        "private": private,
         "weight_type": weight_type,
         "status": "PENDING",
         "submitted_time": current_time,
@@ -113,7 +115,26 @@ def add_new_eval(
         repo_id=QUEUE_REPO,
         repo_type="dataset",
         commit_message=f"Add {model} to eval queue",
-        create_pr=1,
+    )
+  
+    # We want to grab the latest version of the submission file to not accidentally overwrite it
+    snapshot_download(
+        repo_id=DYNAMIC_INFO_REPO, local_dir=DYNAMIC_INFO_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    )
+
+    with open(DYNAMIC_INFO_FILE_PATH) as f:
+        all_supplementary_info = json.load(f)
+
+    all_supplementary_info[model] = supplementary_info
+    with open(DYNAMIC_INFO_FILE_PATH, "w") as f:
+        json.dump(all_supplementary_info, f, indent=2)
+
+    API.upload_file(
+        path_or_fileobj=DYNAMIC_INFO_FILE_PATH,
+        path_in_repo=DYNAMIC_INFO_FILE_PATH.split("/")[-1],
+        repo_id=DYNAMIC_INFO_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to dynamic info queue",
     )
 
     # Remove the local file