Update

app.py

@@ -41,7 +41,7 @@ except Exception:
     restart_space()
 
 total_issues = load_dataset("dtcxzyw/llvm-apr-benchmark").num_rows["test"]
-LEADERBOARD_DF = get_leaderboard_df(EVAL_REQUESTS_PATH, COLS)
+LEADERBOARD_DF = get_leaderboard_df(EVAL_REQUESTS_PATH, COLS, total_issues)
 
 
 def init_leaderboard(dataframe):

src/display/utils.py

@@ -28,7 +28,8 @@ auto_eval_column_dict.append(
 )
 auto_eval_column_dict.append(["model_name", ColumnContent, ColumnContent("Base Model", "markdown", True)])
 # Scores
-auto_eval_column_dict.append(["full_pass_count", ColumnContent, ColumnContent("Repaired ⬆️", "number", True)])
+auto_eval_column_dict.append(["score", ColumnContent, ColumnContent("Score", "number", True)])
+auto_eval_column_dict.append(["full_pass_count", ColumnContent, ColumnContent("Repaired", "number", True)])
 auto_eval_column_dict.append(["fast_pass_count", ColumnContent, ColumnContent("Repaired (Fast)", "number", True)])
 auto_eval_column_dict.append(["with_hint", ColumnContent, ColumnContent("Hint", "str", True)])
 auto_eval_column_dict.append(["attempts", ColumnContent, ColumnContent("Number of attempts", "number", True)])

src/leaderboard/read_evals.py

@@ -61,12 +61,13 @@ class EvalResult:
             full_pass_count_miscompilation=full_pass_count_cat.get("miscompilation", 0),
         )
 
-    def to_dict(self):
+    def to_dict(self, total_issues):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         data_dict = {
             AutoEvalColumn.method_name.name: make_hyperlink(self.method_url, self.method_name),
             AutoEvalColumn.model_name.name: make_hyperlink(self.model_url, self.model_name),
             AutoEvalColumn.with_hint.name: "w/ hint" if self.with_hint else "w/o hint",
+            AutoEvalColumn.score.name: round(self.full_pass_count * 100.0 / total_issues, 1),
             AutoEvalColumn.attempts.name: self.attempts,
             AutoEvalColumn.fast_pass_count.name: self.fast_pass_count,
             AutoEvalColumn.full_pass_count.name: self.full_pass_count,

src/populate.py

@@ -7,10 +7,10 @@ from src.display.utils import AutoEvalColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(requests_path: str, cols: list) -> pd.DataFrame:
+def get_leaderboard_df(requests_path: str, cols: list, total_issues: int) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
+    all_data_json = [v.to_dict(total_issues) for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.full_pass_count.name], ascending=False)