Check in failed eval col viewer

app.py

@@ -18,6 +18,8 @@ from src.display.utils import (
     COLS,
     EVAL_COLS,
     EVAL_TYPES,
+    FAILED_EVAL_COLS,
+    FAILED_EVAL_TYPES,
     AutoEvalColumn,
     fields,
 )
@@ -52,7 +54,8 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
     finished_eval_queue_df,
     running_eval_queue_df,
     pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+    failed_eval_queue_df
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS, FAILED_EVAL_COLS)
 
 
 def init_leaderboard(dataframe):
@@ -132,6 +135,20 @@ with demo:
                                 datatype=EVAL_TYPES,
                                 row_count=5,
                             )
+
+                    
+                    with gr.Accordion(
+                        f"Failed ({len(failed_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            failed_eval_table = gr.components.Dataframe(
+                                value=failed_eval_queue_df,
+                                headers=FAILED_EVAL_COLS,
+                                datatype=FAILED_EVAL_TYPES,
+                                row_count=5,
+                            )
+
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 

src/display/utils.py

@@ -35,7 +35,6 @@ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Avai
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("License", "str", False)])
 
 
-
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
@@ -46,6 +45,13 @@ class EvalQueueColumn:  # Queue column
     private = ColumnContent("private", "bool", True)
     status = ColumnContent("status", "str", True)
 
+@dataclass(frozen=True)
+class FailedEvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    private = ColumnContent("private", "bool", True)
+    status = ColumnContent("status", "str", True)
+    reason = ColumnContent("reason", "str", True)
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
@@ -59,4 +65,7 @@ COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
+FAILED_EVAL_COLS = [c.name for c in fields(FailedEvalQueueColumn)]
+FAILED_EVAL_TYPES = [c.type for c in fields(FailedEvalQueueColumn)]
+
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]

src/populate.py

@@ -22,7 +22,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     return df
 
 
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+def get_evaluation_queue_df(save_path: str, cols: list, failed_cols: list) -> list[pd.DataFrame]:
     """Creates the different dataframes for the evaluation queues requestes"""
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
@@ -50,7 +50,9 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    failed_list = [e for e in all_evals if e["status"] == "FAILED"]
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
     df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+    df_failed = pd.DataFrame.from_records(failed_list, columns=failed_cols)
+    return df_finished[cols], df_running[cols], df_pending[cols], df_failed[failed_cols]