github/OpenHands
1
0
Fork 0
mirror of https://github.com/OpenHands/OpenHands.git synced 2025-12-26 05:48:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Evaluation time travel: build sandbox on the fly (#2491)

Browse Source
This commit is contained in:
Boxuan Li 2024-06-20 19:22:02 -07:00 committed by GitHub
parent 6dd2491944
commit feabc97aba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 20 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
evaluation/EDA/scripts/run_infer.sh
View File

@ -57,5 +57,3 @@ fi
# Run the command
echo $COMMAND
eval $COMMAND
checkout_original_branch

2
evaluation/agent_bench/scripts/run_infer.sh
View File

@ -36,5 +36,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/biocoder/scripts/run_infer.sh
View File

@ -39,5 +39,3 @@ fi
# Run the command
echo $COMMAND
eval $COMMAND
checkout_original_branch

2
evaluation/bird/scripts/run_infer.sh
View File

@ -36,5 +36,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/gaia/scripts/run_infer.sh
View File

@ -47,5 +47,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/gorilla/scripts/run_infer.sh
View File

@ -45,5 +45,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/gpqa/scripts/run_infer.sh
View File

@ -44,5 +44,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/humanevalfix/scripts/run_infer.sh
View File

@ -74,5 +74,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/logic_reasoning/scripts/run_infer.sh
View File

@ -40,5 +40,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/miniwob/scripts/run_infer.sh
View File

@ -46,5 +46,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/mint/scripts/run_infer.sh
View File

@ -42,5 +42,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/ml_bench/scripts/run_infer.sh
View File

@ -46,5 +46,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

2
evaluation/toolqa/scripts/run_infer.sh
View File

@ -61,5 +61,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch

20
evaluation/utils/version_control.sh
View File

@ -1,8 +1,16 @@
checkout_eval_branch() {
if [ -z "$COMMIT_HASH" ]; then
echo "Commit hash not specified, use current git commit"
build_sandbox
return 0
fi
if git diff --quiet $COMMIT_HASH HEAD; then
echo "The given hash is equivalent to the current HEAD"
build_sandbox
return 0
fi
echo "Start to checkout opendevin version to $COMMIT_HASH, but keep current evaluation harness"
if ! git diff-index --quiet HEAD --; then
echo "There are uncommitted changes, please stash or commit them first"
@ -15,8 +23,20 @@ checkout_eval_branch() {
echo "Failed to check out to $COMMIT_HASH"
exit 1
fi
echo "Revert changes in evaluation folder"
git checkout $current_branch -- evaluation
# Trap the EXIT signal to checkout original branch
trap checkout_original_branch EXIT
build_sandbox
}
build_sandbox() {
echo "Build sandbox locally"
docker build -t eval-sandbox -f containers/sandbox/Dockerfile /tmp
export SANDBOX_CONTAINER_IMAGE="eval-sandbox"
}
checkout_original_branch() {

2
evaluation/webarena/scripts/run_infer.sh
View File

@ -44,5 +44,3 @@ fi
# Run the command
eval $COMMAND
checkout_original_branch