File size: 2,315 Bytes
246d201 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
import argparse
import numpy as np
import pandas as pd
def extract_test_results(df: pd.DataFrame) -> tuple[list[str], list[str]]:
passed = []
failed = []
for _, row in df.iterrows():
instance_id = row['instance_id']
resolved = False
if 'test_result' in row and 'exit_code' in row['test_result']:
resolved = row['test_result']['exit_code'] == 0
if resolved:
passed.append(instance_id)
else:
failed.append(instance_id)
return passed, failed
def visualize_results(df: pd.DataFrame):
df1 = pd.DataFrame()
df1['cost'] = df['metrics'].apply(pd.Series)['accumulated_cost']
df1['result'] = (
df['test_result'].apply(pd.Series)['exit_code'].map({0: 'Pass', 1: 'Fail'})
)
df1['actions'] = pd.Series([len(a) - 1 for a in df['history']])
passed = np.sum(df1['result'] == 'Pass')
total = df.shape[0]
resolve_rate = round((passed / total) * 100, 2)
print('Number of passed tests:', f'{passed}/{total} {resolve_rate:.2f}%')
print('\nDescriptive statistics for number of actions:')
print(df1['actions'].describe())
print('\nDescriptive statistics for costs:')
print(df1['cost'].describe())
# Bin counts for actions
action_bins = pd.cut(df1['actions'], bins=range(0, 32, 2))
print('\nAction bin counts:')
print(action_bins.value_counts().sort_index())
# Bin counts for costs
cost_bins = pd.cut(df1['cost'], bins=10)
print('\nCost bin counts:')
print(cost_bins.value_counts().sort_index())
return resolve_rate
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Summarize AiderBench results')
parser.add_argument('input_filepath', type=str, help='Path to the JSONL file')
args = parser.parse_args()
# Create DataFrame from JSONL file
df = pd.read_json(args.input_filepath, lines=True)
passed_tests, failed_tests = extract_test_results(df)
resolve_rate = visualize_results(df)
print(
f'\nPassed {len(passed_tests)} tests, failed {len(failed_tests)} tests, resolve rate = {resolve_rate:.2f}%'
)
print('PASSED TESTS:')
print(passed_tests)
print('FAILED TESTS:')
print(failed_tests)
|