Update

app.py

@@ -77,6 +77,7 @@ for cat, bug_ids in bug_id_by_cat.items():
         timeline_bugids.append(bug_id)
 LEADERBOARD_DF = get_leaderboard_df(EVAL_REQUESTS_PATH, total_issues)
 fixed_bug_ids = set()
+fixed_bug_ids_fast = set()
 for row in LEADERBOARD_DF.itertuples():
     print(row)
     model_cnt += 1
@@ -85,6 +86,8 @@ for row in LEADERBOARD_DF.itertuples():
         timeline_cols.append(row.method_id)
         timeline_bugids.append(fix)
         fixed_bug_ids.add(fix)
+    for fix in row.fixed_bug_ids_fast:
+        fixed_bug_ids_fast.add(fix)
 timeline_bugtypes = []
 for bug_id in timeline_bugids:
     timeline_xs.append(bug_id_to_time[bug_id])
@@ -99,10 +102,14 @@ timeline_df = pd.DataFrame(
     }
 )
 fixed_by_cat = dict()
+fixed_by_cat_fast = dict()
 for bug_id in fixed_bug_ids:
     fixed_by_cat[bug_id_to_type[bug_id]] = fixed_by_cat.get(bug_id_to_type[bug_id], 0) + 1
+for bug_id in fixed_bug_ids_fast:
+    fixed_by_cat_fast[bug_id_to_type[bug_id]] = fixed_by_cat_fast.get(bug_id_to_type[bug_id], 0) + 1
 fixed_by_cat["All"] = len(fixed_bug_ids)
 bug_id_by_cat["All"] = [0] * total_issues
+fixed_by_cat_fast["All"] = len(fixed_bug_ids_fast)
 fixed_by_cat_df = pd.DataFrame(
     {
         "Category": [str(cat).capitalize() for cat in fixed_by_cat.keys()],
@@ -111,6 +118,10 @@ fixed_by_cat_df = pd.DataFrame(
         "Repair Rate (%)": [
             round(fixed_by_cat[cat] / len(bug_id_by_cat[cat]) * 100, 1) for cat in fixed_by_cat.keys()
         ],
+        "Repaired (Fast)": [fixed_by_cat_fast.get(cat, 0) for cat in fixed_by_cat.keys()],
+        "Repair Rate (Fast) (%)": [
+            round(fixed_by_cat_fast.get(cat, 0) / len(bug_id_by_cat[cat]) * 100, 1) for cat in fixed_by_cat.keys()
+        ],
     }
 )
 fixed_by_cat_df.sort_values("Total", inplace=True, ascending=False)
@@ -118,6 +129,10 @@ fixed_by_comp = dict()
 for bug_id in fixed_bug_ids:
     for comp in bug_id_to_comp[bug_id]:
         fixed_by_comp[comp] = fixed_by_comp.get(comp, 0) + 1
+fixed_by_comp_fast = dict()
+for bug_id in fixed_bug_ids_fast:
+    for comp in bug_id_to_comp[bug_id]:
+        fixed_by_comp_fast[comp] = fixed_by_comp_fast.get(comp, 0) + 1
 fixed_by_comp_df = pd.DataFrame(
     {
         "Component": list(comp_bug_count.keys()),
@@ -126,6 +141,10 @@ fixed_by_comp_df = pd.DataFrame(
         "Repair Rate (%)": [
             round(fixed_by_comp.get(comp, 0) / comp_bug_count[comp] * 100, 1) for comp in comp_bug_count.keys()
         ],
+        "Repaired (Fast)": [fixed_by_comp_fast.get(comp, 0) for comp in comp_bug_count.keys()],
+        "Repair Rate (Fast) (%)": [
+            round(fixed_by_comp_fast.get(comp, 0) / comp_bug_count[comp] * 100, 1) for comp in comp_bug_count.keys()
+        ],
     }
 )
 fixed_by_comp_df.sort_values("Total", inplace=True, ascending=False)

src/leaderboard/read_evals.py

@@ -27,6 +27,7 @@ class EvalResult:
     mttr: float
     sample_count: float
     fixed_bug_ids: list[str]
+    fixed_bug_ids_fast: list[str]
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -48,11 +49,13 @@ class EvalResult:
         build_failure_count = 0
         ttr_sum = 0
         fixed_bug_ids = []
+        fixed_bug_ids_fast = []
         sample_count = 0
         for fix in fixes:
             bug_type = fix.get("bug_type", "")
             if fix.get("fast_check_pass", False):
                 fast_pass_count += 1
+                fixed_bug_ids_fast.append(fix.get("bug_id", ""))
             if fix.get("full_check_pass", False):
                 full_pass_count += 1
                 full_pass_count_cat[bug_type] = full_pass_count_cat.get(bug_type, 0) + 1
@@ -78,6 +81,7 @@ class EvalResult:
             build_failure_count=build_failure_count,
             mttr=round(ttr_sum / full_pass_count / 60, 1) if full_pass_count > 0 else 0,
             fixed_bug_ids=fixed_bug_ids,
+            fixed_bug_ids_fast=fixed_bug_ids_fast,
             sample_count=round(sample_count / full_pass_count, 1) if full_pass_count > 0 else 0,
         )
 
@@ -99,6 +103,7 @@ class EvalResult:
             ),
             AutoEvalColumn.mttr.name: self.mttr,
             "fixed_bug_ids": self.fixed_bug_ids,
+            "fixed_bug_ids_fast": self.fixed_bug_ids_fast,
             "method_id": self.method_name + "(" + self.model_name + ")",
             AutoEvalColumn.sample_count.name: self.sample_count,
         }