Added second leaderboard

app.py

@@ -23,7 +23,7 @@ from src.display.utils import (
     WeightType,
     Precision,
 )
-from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, EVAL_RESULTS_PATH_CDM, EVAL_RESULTS_PATH_CDM_FI, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_leaderboard_df
 
 
@@ -33,10 +33,10 @@ def restart_space():
 
 ### Space initialisation
 try:
-    print(EVAL_RESULTS_PATH)
+    print(EVAL_RESULTS_PATH_CDM)
     snapshot_download(
         repo_id=RESULTS_REPO,
-        local_dir=EVAL_RESULTS_PATH,
+        local_dir=EVAL_RESULTS_PATH_CDM,
         repo_type="dataset",
         tqdm_class=None,
         etag_timeout=30,
@@ -45,15 +45,22 @@ try:
 except Exception:
     restart_space()
 
+try:
+    print(EVAL_RESULTS_PATH_CDM_FI)
+    snapshot_download(
+        repo_id=RESULTS_REPO,
+        local_dir=EVAL_RESULTS_PATH_CDM_FI,
+        repo_type="dataset",
+        tqdm_class=None,
+        etag_timeout=30,
+        token=TOKEN,
+    )
+except Exception:
+    restart_space()
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
-
-# (
-#     finished_eval_queue_df,
-#     running_eval_queue_df,
-#     pending_eval_queue_df,
-# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
+LEADERBOARD_DF_CDM = get_leaderboard_df(EVAL_RESULTS_PATH_CDM, COLS, BENCHMARK_COLS)
+LEADERBOARD_DF_CDM_FI = get_leaderboard_df(EVAL_RESULTS_PATH_CDM_FI, COLS, BENCHMARK_COLS)
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -91,99 +98,15 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+        with gr.TabItem("MIMIC CDM", elem_id="llm-benchmark-tab-table", id=0):
+            leaderboard = init_leaderboard(LEADERBOARD_DF_CDM)
+
+        with gr.TabItem("MIMIC CDM FI", elem_id="llm-benchmark-tab-table", id=0):
+            leaderboard = init_leaderboard(LEADERBOARD_DF_CDM_FI)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        # with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-        #     with gr.Column():
-        #         with gr.Row():
-        #             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-        #         with gr.Column():
-        #             with gr.Accordion(
-        #                 f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-        #                 open=False,
-        #             ):
-        #                 with gr.Row():
-        #                     finished_eval_table = gr.components.Dataframe(
-        #                         value=finished_eval_queue_df,
-        #                         headers=EVAL_COLS,
-        #                         datatype=EVAL_TYPES,
-        #                         row_count=5,
-        #                     )
-        #             with gr.Accordion(
-        #                 f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-        #                 open=False,
-        #             ):
-        #                 with gr.Row():
-        #                     running_eval_table = gr.components.Dataframe(
-        #                         value=running_eval_queue_df,
-        #                         headers=EVAL_COLS,
-        #                         datatype=EVAL_TYPES,
-        #                         row_count=5,
-        #                     )
-
-        #             with gr.Accordion(
-        #                 f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-        #                 open=False,
-        #             ):
-        #                 with gr.Row():
-        #                     pending_eval_table = gr.components.Dataframe(
-        #                         value=pending_eval_queue_df,
-        #                         headers=EVAL_COLS,
-        #                         datatype=EVAL_TYPES,
-        #                         row_count=5,
-        #                     )
-        #     with gr.Row():
-        #         gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-        #     with gr.Row():
-        #         with gr.Column():
-        #             model_name_textbox = gr.Textbox(label="Model name")
-        #             revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-        #             model_type = gr.Dropdown(
-        #                 choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-        #                 label="Model type",
-        #                 multiselect=False,
-        #                 value=None,
-        #                 interactive=True,
-        #             )
-
-        #         with gr.Column():
-        #             precision = gr.Dropdown(
-        #                 choices=[i.value.name for i in Precision if i != Precision.Unknown],
-        #                 label="Precision",
-        #                 multiselect=False,
-        #                 value="float16",
-        #                 interactive=True,
-        #             )
-        #             weight_type = gr.Dropdown(
-        #                 choices=[i.value.name for i in WeightType],
-        #                 label="Weights type",
-        #                 multiselect=False,
-        #                 value="Original",
-        #                 interactive=True,
-        #             )
-        #             base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-        #     submit_button = gr.Button("Submit Eval")
-        #     submission_result = gr.Markdown()
-        #     submit_button.click(
-        #         add_new_eval,
-        #         [
-        #             model_name_textbox,
-        #             base_model_name_textbox,
-        #             revision_name_textbox,
-        #             precision,
-        #             weight_type,
-        #             model_type,
-        #         ],
-        #         submission_result,
-        #     )
-
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(

src/about.py

@@ -13,17 +13,10 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    task0 = Task("MIMIC_CDM_Appendicitis", "acc", "CDM App")
-    task1 = Task("MIMIC_CDM_Cholecystitis", "acc", "CDM Cholec")
-    task2 = Task("MIMIC_CDM_Diverticulitis", "acc", "CDM Divert")
-    task3 = Task("MIMIC_CDM_Pancreatitis", "acc", "CDM Pancr")
-    task4 = Task("MIMIC_CDM_Mean", "acc", "CDM Mean")
-
-    task5 = Task("MIMIC_CDM_FI_Appendicitis", "acc", "CDM FI App")
-    task6 = Task("MIMIC_CDM_FI_Cholecystitis", "acc", "CDM FI Cholec")
-    task7 = Task("MIMIC_CDM_FI_Diverticulitis", "acc", "CDM FI Divert")
-    task8 = Task("MIMIC_CDM_FI_Pancreatitis", "acc", "CDM FI Pancr")
-    task9 = Task("MIMIC_CDM_FI_Mean", "acc", "CDM FI Mean")
+    task0 = Task("Appendicitis", "acc", "Appendicits")
+    task1 = Task("Cholecystitis", "acc", "Cholecystitis")
+    task2 = Task("Diverticulitis", "acc", "Diverticulitis")
+    task3 = Task("Pancreatitis", "acc", "Pancreatitis")
 
 
 NUM_FEWSHOT = 0  # Change with your few shot

src/envs.py

@@ -13,14 +13,16 @@ OWNER = (
 
 REPO_ID = f"{OWNER}/leaderboard"
 # QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+RESULTS_REPO_CDM = f"{OWNER}/results-CDM"
+RESULTS_REPO_CDM_FI = f"{OWNER}/results-CDM-FI"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
 # EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_RESULTS_PATH_CDM = os.path.join(CACHE_PATH, "eval-results-CDM")
+EVAL_RESULTS_PATH_CDM_FI = os.path.join(CACHE_PATH, "eval-results-CDM-FI")
 # EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")