File size: 3,660 Bytes
246d201 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
import argparse
import pandas as pd
from openhands.core.logger import openhands_logger as logger
def verify_instance_costs(row: pd.Series) -> float:
"""
Verifies that the accumulated_cost matches the sum of individual costs in metrics.
Also checks for duplicate consecutive costs which might indicate buggy counting.
If the consecutive costs are identical, the file is affected by this bug:
https://github.com/All-Hands-AI/OpenHands/issues/5383
Args:
row: DataFrame row containing instance data with metrics
Returns:
float: The verified total cost for this instance (corrected if needed)
"""
try:
metrics = row.get('metrics')
if not metrics:
logger.warning(f"Instance {row['instance_id']}: No metrics found")
return 0.0
accumulated = metrics.get('accumulated_cost')
costs = metrics.get('costs', [])
if accumulated is None:
logger.warning(
f"Instance {row['instance_id']}: No accumulated_cost in metrics"
)
return 0.0
# Check for duplicate consecutive costs and systematic even-odd pairs
has_duplicate = False
all_pairs_match = True
# Check each even-odd pair (0-1, 2-3, etc.)
for i in range(0, len(costs) - 1, 2):
if abs(costs[i]['cost'] - costs[i + 1]['cost']) < 1e-6:
has_duplicate = True
logger.debug(
f"Instance {row['instance_id']}: Possible buggy double-counting detected! "
f"Steps {i} and {i+1} have identical costs: {costs[i]['cost']:.2f}"
)
else:
all_pairs_match = False
break
# Calculate total cost, accounting for buggy double counting if detected
if len(costs) >= 2 and has_duplicate and all_pairs_match:
paired_steps_cost = sum(
cost_entry['cost']
for cost_entry in costs[: -1 if len(costs) % 2 else None]
)
real_paired_cost = paired_steps_cost / 2
unpaired_cost = costs[-1]['cost'] if len(costs) % 2 else 0
total_cost = real_paired_cost + unpaired_cost
else:
total_cost = sum(cost_entry['cost'] for cost_entry in costs)
if not abs(total_cost - accumulated) < 1e-6:
logger.warning(
f"Instance {row['instance_id']}: Cost mismatch: "
f"accumulated: {accumulated:.2f}, sum of costs: {total_cost:.2f}, "
)
return total_cost
except Exception as e:
logger.error(
f"Error verifying costs for instance {row.get('instance_id', 'UNKNOWN')}: {e}"
)
return 0.0
def main():
parser = argparse.ArgumentParser(
description='Verify costs in SWE-bench output file'
)
parser.add_argument(
'input_filepath', type=str, help='Path to the output.jsonl file'
)
args = parser.parse_args()
try:
# Load and verify the JSONL file
df = pd.read_json(args.input_filepath, lines=True)
logger.info(f'Loaded {len(df)} instances from {args.input_filepath}')
# Verify costs for each instance and sum up total
total_cost = df.apply(verify_instance_costs, axis=1).sum()
logger.info(f'Total verified cost across all instances: ${total_cost:.2f}')
except Exception as e:
logger.error(f'Failed to process file: {e}')
raise
if __name__ == '__main__':
main()
|