mirror of
https://github.com/OpenHands/OpenHands.git
synced 2025-12-25 21:36:52 +08:00
b5e00f577c
Co-authored-by: openhands <openhands@all-hands.dev> Co-authored-by: Engel Nyst <engel.nyst@gmail.com> Co-authored-by: Engel Nyst <enyst@users.noreply.github.com>
105 lines
3.5 KiB
Python
105 lines
3.5 KiB
Python
import argparse
|
|
|
|
import pandas as pd
|
|
|
|
from openhands.core.logger import openhands_logger as logger
|
|
|
|
|
|
def verify_instance_costs(row: pd.Series) -> float:
|
|
"""
|
|
Verifies that the accumulated_cost matches the sum of individual costs in metrics.
|
|
Also checks for duplicate consecutive costs which might indicate buggy counting.
|
|
If the consecutive costs are identical, the file is affected by this bug:
|
|
https://github.com/OpenHands/OpenHands/issues/5383
|
|
|
|
Args:
|
|
row: DataFrame row containing instance data with metrics
|
|
Returns:
|
|
float: The verified total cost for this instance (corrected if needed)
|
|
"""
|
|
try:
|
|
metrics = row.get('metrics')
|
|
if not metrics:
|
|
logger.warning(f'Instance {row["instance_id"]}: No metrics found')
|
|
return 0.0
|
|
|
|
accumulated = metrics.get('accumulated_cost')
|
|
costs = metrics.get('costs', [])
|
|
|
|
if accumulated is None:
|
|
logger.warning(
|
|
f'Instance {row["instance_id"]}: No accumulated_cost in metrics'
|
|
)
|
|
return 0.0
|
|
|
|
# Check for duplicate consecutive costs and systematic even-odd pairs
|
|
has_duplicate = False
|
|
all_pairs_match = True
|
|
|
|
# Check each even-odd pair (0-1, 2-3, etc.)
|
|
for i in range(0, len(costs) - 1, 2):
|
|
if abs(costs[i]['cost'] - costs[i + 1]['cost']) < 1e-6:
|
|
has_duplicate = True
|
|
logger.debug(
|
|
f'Instance {row["instance_id"]}: Possible buggy double-counting detected! '
|
|
f'Steps {i} and {i + 1} have identical costs: {costs[i]["cost"]:.2f}'
|
|
)
|
|
else:
|
|
all_pairs_match = False
|
|
break
|
|
|
|
# Calculate total cost, accounting for buggy double counting if detected
|
|
if len(costs) >= 2 and has_duplicate and all_pairs_match:
|
|
paired_steps_cost = sum(
|
|
cost_entry['cost']
|
|
for cost_entry in costs[: -1 if len(costs) % 2 else None]
|
|
)
|
|
real_paired_cost = paired_steps_cost / 2
|
|
|
|
unpaired_cost = costs[-1]['cost'] if len(costs) % 2 else 0
|
|
total_cost = real_paired_cost + unpaired_cost
|
|
|
|
else:
|
|
total_cost = sum(cost_entry['cost'] for cost_entry in costs)
|
|
|
|
if not abs(total_cost - accumulated) < 1e-6:
|
|
logger.warning(
|
|
f'Instance {row["instance_id"]}: Cost mismatch: '
|
|
f'accumulated: {accumulated:.2f}, sum of costs: {total_cost:.2f}, '
|
|
)
|
|
|
|
return total_cost
|
|
|
|
except Exception as e:
|
|
logger.error(
|
|
f'Error verifying costs for instance {row.get("instance_id", "UNKNOWN")}: {e}'
|
|
)
|
|
return 0.0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description='Verify costs in SWE-bench output file'
|
|
)
|
|
parser.add_argument(
|
|
'input_filepath', type=str, help='Path to the output.jsonl file'
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
# Load and verify the JSONL file
|
|
df = pd.read_json(args.input_filepath, lines=True)
|
|
logger.info(f'Loaded {len(df)} instances from {args.input_filepath}')
|
|
|
|
# Verify costs for each instance and sum up total
|
|
total_cost = df.apply(verify_instance_costs, axis=1).sum()
|
|
logger.info(f'Total verified cost across all instances: ${total_cost:.2f}')
|
|
|
|
except Exception as e:
|
|
logger.error(f'Failed to process file: {e}')
|
|
raise
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|