add term, domain, boomlet tables, and code to parse/display them correctly

app.py

@@ -61,10 +61,19 @@ def restart_space():
 
 
 LEADERBOARD_DF = get_leaderboard_df(
-    EVAL_RESULTS_PATH + "/" + "BOOM_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
+    EVAL_RESULTS_PATH + "/leaderboards/BOOM_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
 )
 LEADERBOARD_DF_DOMAIN = get_leaderboard_df(
-    EVAL_RESULTS_PATH + "/" + "BOOM_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
+    EVAL_RESULTS_PATH + "/leaderboards/BOOM_domain_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
+)
+# LEADERBOARD_DF_METRIC_TYPE = get_leaderboard_df(
+#     EVAL_RESULTS_PATH + "/leaderboards/BOOM_metric_type_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
+# )
+LEADERBOARD_DF_TERM = get_leaderboard_df(
+    EVAL_RESULTS_PATH + "/leaderboards/BOOM_term_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
+)
+LEADERBOARD_DF_BOOMLET = get_leaderboard_df(
+    EVAL_RESULTS_PATH + "/leaderboards/BOOMLET_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
 )
 model_info_df = get_model_info_df(EVAL_RESULTS_PATH)
 
@@ -76,23 +85,37 @@ model_info_df = get_model_info_df(EVAL_RESULTS_PATH)
 
 
 def init_leaderboard(dataframe, model_info_df):
-    # TODO: merge results df with model info df
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
 
     merged_df = get_merged_df(dataframe, model_info_df)
-    merged_df = merged_df.sort_values(by=[AutoEvalColumn.Rank_6750_scaled.name], ascending=True)
+
+    if "Rank" in merged_df.columns:
+        merged_df = merged_df.sort_values(by=["Rank"], ascending=True)
+    else:
+        # Sort by the first CRPS column if the Rank column is not present
+        crps_cols = [col for col in merged_df.columns if "CRPS" in col]
+        if crps_cols:
+            merged_df = merged_df.sort_values(by=crps_cols[0], ascending=True)
 
     # Move the model_type_symbol column to the beginning
-    cols = [AutoEvalColumn.model_type_symbol.name] + [
-        col for col in merged_df.columns if col != AutoEvalColumn.model_type_symbol.name
-    ]
+    cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + sorted(
+        [
+            col
+            for col in merged_df.columns
+            if col not in [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
+        ]
+    )
     merged_df = merged_df[cols]
+    col2type_dict = {c.name: c.type for c in fields(AutoEvalColumn)}
+    datatype_list = [col2type_dict[col] if col in col2type_dict else "number" for col in merged_df.columns]
+    model_info_col_list = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default]
+    default_selection_list = list(dataframe.columns) + model_info_col_list
     return Leaderboard(
         value=merged_df,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
+        datatype=datatype_list,
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            default_selection=default_selection_list,
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
@@ -102,7 +125,8 @@ def init_leaderboard(dataframe, model_info_df):
             ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
         ],
         bool_checkboxgroup_label="Hide models",
-        column_widths=[40, 150] + [180 for _ in range(len(merged_df.columns) - 2)],
+        column_widths=[40, 180] + [160 for _ in range(len(merged_df.columns) - 2)],
+        wrap=True,
         interactive=False,
     )
 
@@ -116,11 +140,19 @@ with demo:
         with gr.TabItem("🏅 Overall", elem_id="boom-benchmark-tab-table", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF, model_info_df)
 
-        # TODO - add other tabs if needed
-        # with gr.TabItem("🏅 By Domain - TODO", elem_id="boom-benchmark-tab-table", id=1):
-        #     leaderboard = init_leaderboard(LEADERBOARD_DF_DOMAIN)  # TODO - update table data
+        with gr.TabItem("🏅 By Domain", elem_id="boom-benchmark-tab-table", id=1):
+            leaderboard = init_leaderboard(LEADERBOARD_DF_DOMAIN, model_info_df)
+
+        # with gr.TabItem("🏅 By Metric Type", elem_id="boom-benchmark-tab-table", id=2):
+        #     leaderboard = init_leaderboard(LEADERBOARD_DF_METRIC_TYPE, model_info_df)
+
+        with gr.TabItem("🏅 By Forecast Horizon", elem_id="boom-benchmark-tab-table", id=3):
+            leaderboard = init_leaderboard(LEADERBOARD_DF_TERM, model_info_df)
+
+        with gr.TabItem("🏅 BOOMLET", elem_id="boom-benchmark-tab-table", id=4):
+            leaderboard = init_leaderboard(LEADERBOARD_DF_BOOMLET, model_info_df)
 
-        with gr.TabItem("📝 About", elem_id="boom-benchmark-tab-table", id=2):
+        with gr.TabItem("📝 About", elem_id="boom-benchmark-tab-table", id=5):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
     with gr.Row():

results/leaderboards/BOOMLET_leaderboard.csv

@@ -0,0 +1,15 @@
+model,MASE,CRPS,Rank
+Toto-Open-Base-1.0,0.617,0.519,1.244
+timesfm_2_0_500m,0.685,0.603,4.156
+moirai_1.1_large,0.767,0.621,4.267
+moirai_1.1_base,0.779,0.630,4.567
+moirai_1.1_small,0.786,0.631,4.944
+chronos_bolt_base,0.711,0.637,5.467
+chronos_bolt_small,0.717,0.642,5.667
+time-moe,0.810,0.788,8.989
+timer,0.807,0.793,9.244
+autoarima,0.922,0.880,9.667
+visionts,0.912,0.885,10.922
+seasonalnaive,1.000,1.000,11.400
+autoets,0.969,15.664,12.033
+autotheta,1.030,1.182,12.433

results/leaderboards/BOOM_domain_leaderboard.csv

@@ -0,0 +1,15 @@
+model,Application Usage (MASE),Database (MASE),Infrastructure (MASE),Networking (MASE),Security (MASE),Application Usage (CRPS),Database (CRPS),Infrastructure (CRPS),Networking (CRPS),Security (CRPS)
+Toto-Open-Base-1.0,0.639,0.635,0.568,0.635,0.682,0.378,0.362,0.391,0.4,0.476
+autoets,0.87,0.859,0.727,0.98,0.868,1.423,1.604,4.563,2.067,0.878
+autoarima,0.865,0.839,0.708,0.937,0.9,0.757,0.734,0.679,0.795,0.757
+autotheta,1.151,1.188,0.976,1.213,1.039,1.019,1.032,0.963,1.105,0.991
+chronos_bolt_base,0.748,0.757,0.663,0.757,0.729,0.451,0.441,0.466,0.489,0.535
+chronos_bolt_small,0.748,0.761,0.678,0.779,0.734,0.452,0.444,0.474,0.506,0.539
+moirai_1.1_base,0.721,0.738,0.65,0.786,0.739,0.422,0.414,0.446,0.484,0.504
+moirai_1.1_large,0.73,0.743,0.67,0.773,0.736,0.43,0.418,0.462,0.484,0.504
+moirai_1.1_small,0.747,0.751,0.692,0.795,0.741,0.44,0.429,0.476,0.493,0.505
+seasonalnaive,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0
+time-moe,0.863,0.714,0.791,0.856,0.77,0.633,0.618,0.713,0.721,0.625
+timer,0.871,0.716,0.728,0.871,0.828,0.636,0.619,0.655,0.725,0.664
+timesfm_2_0_500m,0.736,0.765,0.679,0.765,0.717,0.441,0.44,0.471,0.493,0.525
+visionts,1.042,1.017,0.863,1.035,0.924,0.691,0.647,0.666,0.734,0.735

results/{BOOM_leaderboard.csv → leaderboards/BOOM_leaderboard.csv}

@@ -1,4 +1,4 @@
-model,MASE_6750_scaled,CRPS_6750_scaled,Rank_6750_scaled
+model,MASE,CRPS,Rank
 Toto-Open-Base-1.0,0.617,0.375,2.336
 moirai_1.1_base,0.710,0.428,4.253
 moirai_1.1_large,0.720,0.436,4.481

results/leaderboards/BOOM_term_leaderboard.csv

@@ -0,0 +1,15 @@
+model,Long (MASE),Medium (MASE),Short (MASE),Long (CRPS),Medium (CRPS),Short (CRPS)
+Toto-Open-Base-1.0,0.688,0.657,0.535,0.424,0.406,0.318
+autoets,0.938,0.885,0.739,2.83,2.399,1.253
+autoarima,0.896,0.853,0.749,0.807,0.804,0.635
+autotheta,1.368,1.163,0.928,1.296,1.183,0.738
+chronos_bolt_base,0.798,0.782,0.632,0.519,0.507,0.365
+chronos_bolt_small,0.813,0.782,0.638,0.528,0.508,0.368
+moirai_1.1_base,0.78,0.753,0.627,0.473,0.46,0.37
+moirai_1.1_large,0.799,0.77,0.626,0.491,0.475,0.369
+moirai_1.1_small,0.795,0.771,0.67,0.482,0.476,0.399
+seasonalnaive,1.0,1.0,1.0,1.0,1.0,1.0
+time-moe,0.886,0.866,0.704,0.724,0.725,0.541
+timer,0.809,0.804,0.779,0.661,0.671,0.597
+timesfm_2_0_500m,0.817,0.78,0.619,0.522,0.499,0.359
+visionts,1.026,1.011,0.947,0.698,0.698,0.64

results/models_info/naive/config.json

@@ -1,5 +1,6 @@
 {
     "model": "Naive",
+    "tmp_name": "naive",
     "model_type": "statistical",
     "model_dtype": "float32"
 }

src/display/utils.py

@@ -29,10 +29,6 @@ auto_eval_column_dict.append(
     ["model_type_symbol", ColumnContent, ColumnContent("Type", "str", True, never_hidden=True)]
 )
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-# Scores
-auto_eval_column_dict.append(["MASE_6750_scaled", ColumnContent, ColumnContent("MASE", "number", True)])
-auto_eval_column_dict.append(["CRPS_6750_scaled", ColumnContent, ColumnContent("CRPS", "number", True)])
-auto_eval_column_dict.append(["Rank_6750_scaled", ColumnContent, ColumnContent("Rank", "number", True)])
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Model Type", "str", False, hidden=True)])
 # auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])

src/populate.py

@@ -21,7 +21,9 @@ def get_merged_df(result_df: pd.DataFrame, model_info_df: pd.DataFrame) -> pd.Da
     """Merges the model info dataframe with the results dataframe"""
     result_df = result_df.rename(columns={"Model": "tmp_name"})
     merged_df = pd.merge(model_info_df, result_df, on="tmp_name", how="inner")
-    assert len(merged_df) == len(result_df)
+    assert len(merged_df) == len(
+        result_df
+    ), f"missing model info for: {set(result_df['tmp_name'].unique()) - set(model_info_df['tmp_name'].unique())}"
     merged_df = merged_df.drop(columns=["Model", "tmp_name"])
     merged_df = merged_df.rename(columns={"model_w_link": "Model"})
     return merged_df
@@ -53,7 +55,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
         - Internal column names are mapped to display names using `AutoEvalColumn`.
         - A new column for model type symbols is created by parsing the `model_type` column.
         - The `model_type` column is updated to prepend the model type symbol.
-        - The DataFrame is sorted by the `Rank_6750_scaled` column in ascending order.
+        - The DataFrame is sorted by the `Rank_scaled` column in ascending order.
     """
 
     df = pd.read_csv(results_path)
@@ -62,8 +64,6 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     column_mapping = {field.name: getattr(AutoEvalColumn, field.name).name for field in fields(AutoEvalColumn)}
     # Assuming `df` is your DataFrame:
     df.rename(columns=column_mapping, inplace=True)
-
-    df = df.sort_values(by=[AutoEvalColumn.Rank_6750_scaled.name], ascending=True)
     return df