mirror of
https://github.com/OpenHands/OpenHands.git
synced 2025-12-26 05:48:36 +08:00
57 lines
2.3 KiB
YAML
57 lines
2.3 KiB
YAML
# Run evaluation on a PR
|
|
name: Run Eval
|
|
|
|
# Runs when a PR is labeled with one of the "run-eval-" labels
|
|
on:
|
|
pull_request:
|
|
types: [labeled]
|
|
|
|
jobs:
|
|
trigger-job:
|
|
name: Trigger remote eval job
|
|
if: ${{ github.event.label.name == 'run-eval-1' || github.event.label.name == 'run-eval-2' || github.event.label.name == 'run-eval-50' || github.event.label.name == 'run-eval-100' }}
|
|
runs-on: blacksmith-4vcpu-ubuntu-2204
|
|
|
|
steps:
|
|
- name: Checkout PR branch
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ github.head_ref }}
|
|
|
|
- name: Trigger remote job
|
|
env:
|
|
PR_BRANCH: ${{ github.head_ref }}
|
|
run: |
|
|
REPO_URL="https://github.com/${{ github.repository }}"
|
|
echo "Repository URL: $REPO_URL"
|
|
echo "PR Branch: $PR_BRANCH"
|
|
|
|
if [[ "${{ github.event.label.name }}" == "run-eval-1" ]]; then
|
|
EVAL_INSTANCES="1"
|
|
elif [[ "${{ github.event.label.name }}" == "run-eval-2" ]]; then
|
|
EVAL_INSTANCES="2"
|
|
elif [[ "${{ github.event.label.name }}" == "run-eval-50" ]]; then
|
|
EVAL_INSTANCES="50"
|
|
elif [[ "${{ github.event.label.name }}" == "run-eval-100" ]]; then
|
|
EVAL_INSTANCES="100"
|
|
fi
|
|
|
|
curl -X POST \
|
|
-H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
|
|
-H "Accept: application/vnd.github+json" \
|
|
-d "{\"ref\": \"main\", \"inputs\": {\"github-repo\": \"${REPO_URL}\", \"github-branch\": \"${PR_BRANCH}\", \"pr-number\": \"${{ github.event.pull_request.number }}\", \"eval-instances\": \"${EVAL_INSTANCES}\"}}" \
|
|
https://api.github.com/repos/All-Hands-AI/evaluation/actions/workflows/create-branch.yml/dispatches
|
|
|
|
# Send Slack message
|
|
PR_URL="https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}"
|
|
slack_text="PR $PR_URL has triggered evaluation on $EVAL_INSTANCES instances..."
|
|
curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$slack_text"'"}' \
|
|
https://hooks.slack.com/services/${{ secrets.SLACK_TOKEN }}
|
|
|
|
- name: Comment on PR
|
|
uses: KeisukeYamashita/create-comment@v1
|
|
with:
|
|
unique: false
|
|
comment: |
|
|
Running evaluation on the PR. Once eval is done, the results will be posted.
|