mirror of
https://github.com/OpenHands/OpenHands.git
synced 2026-03-22 05:37:20 +08:00
bd8b1bfa25
Co-authored-by: linhaowei <linhaowei@wizardquant.com> Co-authored-by: Graham Neubig <neubig@gmail.com>
53 lines
2.2 KiB
Bash
53 lines
2.2 KiB
Bash
#!/bin/bash
|
|
|
|
# This script is designed to first test a candidate "best" solver for performance,
|
|
# record its speedup, and then run a final validation test using the original solver.
|
|
|
|
# --- Step 1: Create the test file ---
|
|
# This file contains the tests that will be run against the solvers.
|
|
# It's created first to be available for both test runs.
|
|
cat > /workspace/test_outputs.py << 'EOF'
|
|
{test_output_content}
|
|
EOF
|
|
|
|
# --- Step 2: Backup the original solver ---
|
|
# Copy the existing solver.py to a backup location before overwriting it.
|
|
# This preserves the original code for the final test run.
|
|
echo "Backing up original solver..."
|
|
cp /workspace/solver.py /workspace/solver.py.bak
|
|
|
|
# --- Step 3: Write the new "best" solver ---
|
|
# The new solver code is written to /workspace/solver.py. This file will be
|
|
# used for the performance evaluation in the next step.
|
|
{solution_code_command}
|
|
|
|
# --- Step 4: Run performance test on the new solver ---
|
|
# Execute pytest and parse the output to find the "Calculated Speedup".
|
|
# The result is saved to a file. If the test fails or the speedup
|
|
# cannot be determined, it defaults to "1.0".
|
|
echo "Running performance test..."
|
|
(set -o pipefail; /usr/local/bin/pytest -s /workspace/test_outputs.py 2>&1 | tee /workspace/test_outputs.log | tee /dev/stderr | grep "Calculated Speedup" | awk '{print $3}' | sed 's/x$//' > /workspace/.best_solver_speedup.txt) \
|
|
|| echo "1.0" > /workspace/.best_solver_speedup.txt
|
|
|
|
# --- Output Speedup and Solver Code for Review ---
|
|
echo "--- TARGET SPEEDUP ---"
|
|
cat /workspace/.best_solver_speedup.txt
|
|
echo "--- TARGET SPEED END ---"
|
|
|
|
# --- Step 5: Restore the original solver ---
|
|
# Move the backup file back to /workspace/solver.py, overwriting the
|
|
# temporary "best" solver.
|
|
echo "Restoring original solver..."
|
|
rm -rf /workspace/solver.py
|
|
mv /workspace/solver.py.bak /workspace/solver.py
|
|
|
|
# --- Step 6: Run final tests on the original solver ---
|
|
# Now that the original solver is restored, run the final pytest
|
|
# validation to ensure correctness.
|
|
echo "Running final validation on original solver..."
|
|
/usr/local/bin/pytest /workspace/test_outputs.py -rA -s
|
|
|
|
# --- Step 7: Exit ---
|
|
# The script has completed its execution.
|
|
echo "Script finished."
|