add BSR

src/display/utils.py

@@ -40,6 +40,9 @@ auto_eval_column_dict.append(
     ["full_pass_count_miscompilation", ColumnContent, ColumnContent("Repaired (Miscompilation)", "number", True)]
 )
 auto_eval_column_dict.append(["full_pass_count_hang", ColumnContent, ColumnContent("Repaired (Hang)", "number", True)])
+auto_eval_column_dict.append(
+    ["build_success_rate", ColumnContent, ColumnContent("Build Success Rate", "number", True)]
+)
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)

src/leaderboard/read_evals.py

@@ -22,6 +22,8 @@ class EvalResult:
     full_pass_count_crash: int
     full_pass_count_hang: int
     full_pass_count_miscompilation: int
+    build_count: int
+    build_failure_count: int
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -39,6 +41,8 @@ class EvalResult:
         fast_pass_count = 0
         full_pass_count = 0
         full_pass_count_cat = {}
+        build_count = 0
+        build_failure_count = 0
         for fix in fixes:
             bug_type = fix.get("bug_type", "")
             if fix.get("fast_check_pass", False):
@@ -46,6 +50,8 @@ class EvalResult:
             if fix.get("full_check_pass", False):
                 full_pass_count += 1
                 full_pass_count_cat[bug_type] = full_pass_count_cat.get(bug_type, 0) + 1
+            build_count += fix.get("build_count", 0)
+            build_failure_count += fix.get("build_failure_count", 0)
 
         return self(
             method_name=method_name,
@@ -59,6 +65,8 @@ class EvalResult:
             full_pass_count_crash=full_pass_count_cat.get("crash", 0),
             full_pass_count_hang=full_pass_count_cat.get("hang", 0),
             full_pass_count_miscompilation=full_pass_count_cat.get("miscompilation", 0),
+            build_count=build_count,
+            build_failure_count=build_failure_count,
         )
 
     def to_dict(self, total_issues):
@@ -74,6 +82,9 @@ class EvalResult:
             AutoEvalColumn.full_pass_count_crash.name: self.full_pass_count_crash,
             AutoEvalColumn.full_pass_count_hang.name: self.full_pass_count_hang,
             AutoEvalColumn.full_pass_count_miscompilation.name: self.full_pass_count_miscompilation,
+            AutoEvalColumn.build_success_rate.name: round(
+                (self.build_count - self.build_failure_count) * 100.0 / self.build_count, 1
+            ),
         }
 
         return data_dict