mirror of
https://github.com/OpenHands/OpenHands.git
synced 2026-10-07 17:08:34 +08:00
Add live Agent Server E2E framework (#195)
* Add demo flow E2E coverage * Add live Agent Server E2E * Stabilize live E2E CI * Stabilize live Agent Server E2E * Use default pull request workflow triggers * Organize Playwright E2E tests * Comment live E2E results on PR * Fix live E2E comment permissions * Stabilize live E2E PR reporting * Add collapsible live E2E evidence * Embed live E2E media in PR report * Use release assets for live E2E media * Stabilize live E2E media and auth * Use raw URLs for live E2E media * Update tests for SDK workspace API * Use PR artifacts for live E2E media * Move live E2E scripts under tests * chore: Update PR QA artifacts * Clarify live E2E test layout * Document and simplify live E2E local runs * Remove unrelated non-test diffs * Preserve HEAD git ref in workspace client * Strengthen live Agent Server E2E * chore: Update PR QA artifacts * Remove workspace session URL normalization * Remove obsolete mock E2E regressions * Disable live E2E trace capture * Harden live E2E workflow * Harden live E2E review fixes * Fix live E2E manual checkout * chore: Update PR QA artifacts * Address live E2E re-review feedback * Address live E2E security review feedback * Address live E2E approval suggestions * chore: address PR review feedback (#195) * chore: address live e2e review followups (#195) * chore: Remove PR-only artifacts * chore: address latest live e2e review * fix(ci): drop --ignore-scripts so typescript-client git dep builds After merging main (PR #278), source files import directly from @openhands/typescript-client subpath exports (e.g. /clients, /workspace/remote-workspace). These resolve to dist/ files that are generated by the package's prepare script. The --ignore-scripts flag on npm ci prevented that script from running, so CI's typecheck failed with TS2307 'Cannot find module' for every subpath import. Main's CI uses plain 'npm ci' (no --ignore-scripts) and passes. Align this branch to match. Co-authored-by: openhands <openhands@all-hands.dev> * fix: restore avatar-menu and css-isolation regression tests These were moved from tests/ to tests/e2e/regressions/ in 08b8e12 but then mistakenly deleted in 5c5a39b. The live E2E framework is additive — it should not remove existing browser regression coverage. The placeholder.spec.ts is not restored since it was a no-op stub. Co-authored-by: openhands <openhands@all-hands.dev> --------- Co-authored-by: allhands-bot <allhands-bot@users.noreply.github.com> Co-authored-by: openhands <openhands@all-hands.dev>
This commit is contained in:
Xingyao Wang
committed by
GitHub
GitHub
co-authored by
openhands
allhands-bot
openhands
allhands-bot
parent
951ca0af24
commit
bad60ed8de
@@ -5,6 +5,11 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: PR number to run live QA against when running manually.
|
||||
required: true
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -51,3 +56,390 @@ jobs:
|
||||
|
||||
- name: Verify package contents
|
||||
run: npm pack --dry-run
|
||||
|
||||
live-e2e:
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository &&
|
||||
contains(github.event.pull_request.labels.*.name, 'live-e2e'))
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
concurrency:
|
||||
group: pr-artifacts-live-e2e-${{ github.event.pull_request.number || inputs.pr_number || github.ref }}
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
env:
|
||||
LIVE_E2E_LLM_BASE_URL: ${{ vars.LIVE_E2E_LLM_BASE_URL || 'https://llm-proxy.app.all-hands.dev' }}
|
||||
LIVE_E2E_LLM_MODEL: ${{ vars.LIVE_E2E_LLM_MODEL || 'openhands/claude-haiku-4-5-20251001' }}
|
||||
LIVE_E2E_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number || '' }}
|
||||
LIVE_E2E_REPORT_PATH: live-e2e-report.md
|
||||
LIVE_E2E_RECORD_VIDEO: "on"
|
||||
LIVE_E2E_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
LIVE_E2E_PR_ARTIFACT_KEEP_RUNS: 3
|
||||
|
||||
steps:
|
||||
- name: Resolve live E2E PR context
|
||||
id: pr_context
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [ -z "$LIVE_E2E_PR_NUMBER" ]; then
|
||||
echo "::error::Live E2E QA must run against a PR. Provide workflow_dispatch.pr_number or add the live-e2e label to a PR."
|
||||
exit 1
|
||||
fi
|
||||
case "$LIVE_E2E_PR_NUMBER" in
|
||||
*[!0-9]*)
|
||||
echo "::error::Invalid PR number: $LIVE_E2E_PR_NUMBER"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
pr_api="repos/$GITHUB_REPOSITORY/pulls/$LIVE_E2E_PR_NUMBER"
|
||||
head_repo="$(gh api "$pr_api" --jq '.head.repo.full_name')"
|
||||
head_ref="$(gh api "$pr_api" --jq '.head.ref')"
|
||||
head_sha="$(gh api "$pr_api" --jq '.head.sha')"
|
||||
base_repo="$(gh api "$pr_api" --jq '.base.repo.full_name')"
|
||||
|
||||
if [ -z "$head_repo" ] || [ -z "$head_ref" ] || [ -z "$head_sha" ] || [ -z "$base_repo" ]; then
|
||||
echo "::error::Failed to fetch complete PR data from GitHub API."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "has_pr=true" >> "$GITHUB_OUTPUT"
|
||||
echo "head_repo=$head_repo" >> "$GITHUB_OUTPUT"
|
||||
echo "head_ref=$head_ref" >> "$GITHUB_OUTPUT"
|
||||
echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
|
||||
if [ "$head_repo" = "$base_repo" ]; then
|
||||
echo "is_fork=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "is_fork=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Skip live E2E for fork PRs
|
||||
if: steps.pr_context.outputs.is_fork == 'true'
|
||||
run: echo "::notice::Skipping live E2E for fork PRs so secrets are never exposed to untrusted code."
|
||||
|
||||
- name: Check live E2E credential
|
||||
id: live_credential
|
||||
if: steps.pr_context.outputs.is_fork != 'true'
|
||||
env:
|
||||
LIVE_E2E_LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
|
||||
run: |
|
||||
if [ -n "$LIVE_E2E_LLM_API_KEY" ]; then
|
||||
echo "has_key=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_key=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Check out PR head
|
||||
if: steps.pr_context.outputs.is_fork != 'true'
|
||||
env:
|
||||
PR_HEAD_REPO: ${{ steps.pr_context.outputs.head_repo }}
|
||||
PR_HEAD_SHA: ${{ steps.pr_context.outputs.head_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git init .
|
||||
git remote remove pr-head >/dev/null 2>&1 || true
|
||||
git remote add pr-head "https://github.com/$PR_HEAD_REPO.git"
|
||||
git fetch pr-head "$PR_HEAD_SHA"
|
||||
git checkout --detach FETCH_HEAD
|
||||
actual_sha="$(git rev-parse HEAD)"
|
||||
if [ "$actual_sha" != "$PR_HEAD_SHA" ]; then
|
||||
echo "::error::SHA mismatch: expected $PR_HEAD_SHA, got $actual_sha"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Detect PR artifact-only commit
|
||||
id: latest_commit
|
||||
if: steps.pr_context.outputs.is_fork != 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
changed_files="$(git diff-tree --no-commit-id --name-only -r --root HEAD)"
|
||||
|
||||
artifact_only=false
|
||||
if [ -n "$changed_files" ]; then
|
||||
artifact_only=true
|
||||
while IFS= read -r changed_file; do
|
||||
[ -n "$changed_file" ] || continue
|
||||
case "$changed_file" in
|
||||
.pr/*) ;;
|
||||
*) artifact_only=false ;;
|
||||
esac
|
||||
done < <(printf '%s\n' "$changed_files")
|
||||
fi
|
||||
|
||||
echo "pr_artifact_only=$artifact_only" >> "$GITHUB_OUTPUT"
|
||||
if [ "$artifact_only" = "true" ]; then
|
||||
echo "::notice::Skipping live E2E because the latest commit only changes .pr artifacts."
|
||||
fi
|
||||
|
||||
- name: Set up Node.js
|
||||
if: steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22.12.0
|
||||
cache: npm
|
||||
|
||||
- name: Create live E2E PR comment
|
||||
if: env.LIVE_E2E_PR_NUMBER != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
node tests/e2e/live/scripts/render-live-e2e-report.mjs \
|
||||
--status running \
|
||||
--model "$LIVE_E2E_LLM_MODEL" \
|
||||
--workflow-url "$LIVE_E2E_WORKFLOW_URL" \
|
||||
--commit "${{ steps.pr_context.outputs.head_sha }}" \
|
||||
--timestamp "$(date -u '+%Y-%m-%d %H:%M:%S UTC')" \
|
||||
--output "$LIVE_E2E_REPORT_PATH"
|
||||
node tests/e2e/live/scripts/upsert-pr-comment.mjs \
|
||||
--issue-number "$LIVE_E2E_PR_NUMBER" \
|
||||
--body-file "$LIVE_E2E_REPORT_PATH"
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Install uv
|
||||
if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install Playwright Chromium
|
||||
if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run live Agent Server E2E
|
||||
id: live_test
|
||||
if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
env:
|
||||
LIVE_E2E_LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
|
||||
run: |
|
||||
set +e
|
||||
npm run test:e2e:live
|
||||
exit_code=$?
|
||||
echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
|
||||
- name: Skip live Agent Server E2E
|
||||
if: steps.live_credential.outputs.has_key != 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
run: echo "Skipping live Agent Server E2E because LLM_API_KEY is not configured."
|
||||
|
||||
- name: Extract live E2E media
|
||||
id: live_media
|
||||
if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
run: |
|
||||
node tests/e2e/live/scripts/extract-live-e2e-media.mjs \
|
||||
--results "test-results-live/results.json" \
|
||||
--output-dir "test-results-live/media"
|
||||
|
||||
- name: Create live E2E video preview
|
||||
id: live_video_preview
|
||||
if: always() && steps.live_media.outputs.video_path != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
video_path="${{ steps.live_media.outputs.video_path }}"
|
||||
if [ ! -f "$video_path" ]; then
|
||||
echo "::warning::Live E2E video file was not found, skipping GIF preview generation."
|
||||
exit 0
|
||||
fi
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ffmpeg
|
||||
ffmpeg -y \
|
||||
-i "$video_path" \
|
||||
-vf "fps=4,scale=960:-1:flags=lanczos" \
|
||||
-loop 0 \
|
||||
test-results-live/media/live-agent-recording.gif
|
||||
echo "video_preview_path=test-results-live/media/live-agent-recording.gif" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Publish live E2E media
|
||||
id: publish_live_media
|
||||
if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
env:
|
||||
MEDIA_KEEP_RUNS: ${{ env.LIVE_E2E_PR_ARTIFACT_KEEP_RUNS }}
|
||||
MEDIA_RUN_DIR: ${{ github.run_id }}
|
||||
PR_ARTIFACT_PUSH_TOKEN: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}
|
||||
PR_HEAD_IS_FORK: ${{ steps.pr_context.outputs.is_fork }}
|
||||
PR_HEAD_REF: ${{ steps.pr_context.outputs.head_ref }}
|
||||
PR_HEAD_SHA: ${{ steps.pr_context.outputs.head_sha }}
|
||||
SCREENSHOT_PATH: ${{ steps.live_media.outputs.screenshot_path || '' }}
|
||||
VIDEO_PATH: ${{ steps.live_media.outputs.video_path || '' }}
|
||||
VIDEO_PREVIEW_PATH: ${{ steps.live_video_preview.outputs.video_preview_path || '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [ -z "$SCREENSHOT_PATH" ] && [ -z "$VIDEO_PREVIEW_PATH" ] && [ -z "$VIDEO_PATH" ]; then
|
||||
echo "No live E2E media was produced."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$PR_HEAD_IS_FORK" = "true" ]; then
|
||||
echo "::notice::Skipping .pr media publishing for fork PRs."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
media_dir=".pr/live-e2e/$MEDIA_RUN_DIR"
|
||||
mkdir -p "$media_dir"
|
||||
{
|
||||
echo "# PR Artifacts"
|
||||
echo
|
||||
echo "This directory contains generated PR-only QA artifacts. The PR Artifacts workflow removes it after approval so these files do not enter the final squash merge."
|
||||
} > .pr/README.md
|
||||
|
||||
if [ -n "$SCREENSHOT_PATH" ]; then
|
||||
cp "$SCREENSHOT_PATH" "$media_dir/live-agent-response.png"
|
||||
fi
|
||||
if [ -n "$VIDEO_PREVIEW_PATH" ]; then
|
||||
cp "$VIDEO_PREVIEW_PATH" "$media_dir/live-agent-recording.gif"
|
||||
fi
|
||||
if [ -n "$VIDEO_PATH" ]; then
|
||||
cp "$VIDEO_PATH" "$media_dir/live-agent-recording.webm"
|
||||
fi
|
||||
|
||||
if [ -d ".pr/live-e2e" ]; then
|
||||
find ".pr/live-e2e" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' |
|
||||
sort -nr |
|
||||
tail -n "+$((MEDIA_KEEP_RUNS + 1))" |
|
||||
while IFS= read -r stale_run; do
|
||||
[ -n "$stale_run" ] || continue
|
||||
if [[ ! "$stale_run" =~ ^[0-9]+$ ]]; then
|
||||
echo "::warning::Skipping invalid live E2E media directory name: $stale_run"
|
||||
continue
|
||||
fi
|
||||
rm -rf ".pr/live-e2e/$stale_run"
|
||||
done
|
||||
fi
|
||||
|
||||
git config user.name "allhands-bot"
|
||||
git config user.email "allhands-bot@users.noreply.github.com"
|
||||
git config credential.helper '!f() { echo "username=x-access-token"; echo "password=$PR_ARTIFACT_PUSH_TOKEN"; }; f'
|
||||
git remote remove pr-artifacts >/dev/null 2>&1 || true
|
||||
git remote add pr-artifacts "https://github.com/${GITHUB_REPOSITORY}.git"
|
||||
git add .pr
|
||||
if git diff --cached --quiet; then
|
||||
echo "No .pr media changes to publish."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -m "chore: Update PR QA artifacts"
|
||||
push_succeeded=false
|
||||
for attempt in 1 2 3; do
|
||||
if git push pr-artifacts "HEAD:refs/heads/$PR_HEAD_REF"; then
|
||||
push_succeeded=true
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$attempt" -lt 3 ]; then
|
||||
echo "::notice::Failed to push .pr media, rebasing and retrying."
|
||||
git fetch pr-artifacts "$PR_HEAD_REF"
|
||||
if ! git rebase FETCH_HEAD; then
|
||||
echo "::error::Failed to rebase .pr media commit. Manual resolution required."
|
||||
exit 1
|
||||
fi
|
||||
if ! git merge-base --is-ancestor "$PR_HEAD_SHA" HEAD; then
|
||||
echo "::error::Rebased .pr media commit no longer descends from the tested PR head $PR_HEAD_SHA."
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
if [ "$push_succeeded" != "true" ]; then
|
||||
echo "::error::Failed to push .pr media after retries."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
media_commit="$(git rev-parse HEAD)"
|
||||
media_url() {
|
||||
printf 'https://raw.githubusercontent.com/%s/%s/%s/%s\n' "$GITHUB_REPOSITORY" "$media_commit" "$media_dir" "$1"
|
||||
}
|
||||
|
||||
if [ -f "$media_dir/live-agent-response.png" ]; then
|
||||
echo "screenshot_url=$(media_url "live-agent-response.png")" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if [ -f "$media_dir/live-agent-recording.gif" ]; then
|
||||
echo "video_preview_url=$(media_url "live-agent-recording.gif")" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if [ -f "$media_dir/live-agent-recording.webm" ]; then
|
||||
echo "video_url=$(media_url "live-agent-recording.webm")" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Upload live E2E artifacts
|
||||
id: upload_live_artifacts
|
||||
if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: live-agent-server-e2e
|
||||
if-no-files-found: ignore
|
||||
path: |
|
||||
playwright-report-live/
|
||||
test-results-live/
|
||||
|
||||
- name: Determine live E2E result
|
||||
id: live_status
|
||||
if: always() && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
run: |
|
||||
if [ "${{ steps.live_credential.outputs.has_key }}" != "true" ]; then
|
||||
echo "status=skipped" >> "$GITHUB_OUTPUT"
|
||||
echo "reason=LLM_API_KEY is not configured." >> "$GITHUB_OUTPUT"
|
||||
echo "exit_code=0" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exit_code="${{ steps.live_test.outputs.exit_code }}"
|
||||
if [ -z "$exit_code" ]; then
|
||||
exit_code=1
|
||||
fi
|
||||
|
||||
if [ "$exit_code" = "0" ]; then
|
||||
echo "status=passed" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "status=failed" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "reason=" >> "$GITHUB_OUTPUT"
|
||||
echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Render live E2E report
|
||||
if: always() && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
env:
|
||||
LIVE_E2E_ARTIFACT_URL: ${{ steps.upload_live_artifacts.outputs.artifact-url || '' }}
|
||||
LIVE_E2E_SCREENSHOT_URL: ${{ steps.publish_live_media.outputs.screenshot_url || '' }}
|
||||
LIVE_E2E_STATUS: ${{ steps.live_status.outputs.status }}
|
||||
LIVE_E2E_REASON: ${{ steps.live_status.outputs.reason }}
|
||||
LIVE_E2E_VIDEO_PREVIEW_URL: ${{ steps.publish_live_media.outputs.video_preview_url || '' }}
|
||||
LIVE_E2E_VIDEO_URL: ${{ steps.publish_live_media.outputs.video_url || '' }}
|
||||
run: |
|
||||
node tests/e2e/live/scripts/render-live-e2e-report.mjs \
|
||||
--status "$LIVE_E2E_STATUS" \
|
||||
--reason "$LIVE_E2E_REASON" \
|
||||
--results "test-results-live/results.json" \
|
||||
--model "$LIVE_E2E_LLM_MODEL" \
|
||||
--workflow-url "$LIVE_E2E_WORKFLOW_URL" \
|
||||
--artifact-url "$LIVE_E2E_ARTIFACT_URL" \
|
||||
--screenshot-url "$LIVE_E2E_SCREENSHOT_URL" \
|
||||
--video-preview-url "$LIVE_E2E_VIDEO_PREVIEW_URL" \
|
||||
--video-url "$LIVE_E2E_VIDEO_URL" \
|
||||
--commit "${{ steps.pr_context.outputs.head_sha }}" \
|
||||
--timestamp "$(date -u '+%Y-%m-%d %H:%M:%S UTC')" \
|
||||
--output "$LIVE_E2E_REPORT_PATH"
|
||||
cat "$LIVE_E2E_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Update live E2E PR comment
|
||||
if: always() && env.LIVE_E2E_PR_NUMBER != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
node tests/e2e/live/scripts/upsert-pr-comment.mjs \
|
||||
--issue-number "$LIVE_E2E_PR_NUMBER" \
|
||||
--body-file "$LIVE_E2E_REPORT_PATH"
|
||||
|
||||
- name: Fail live E2E job when tests fail
|
||||
if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
|
||||
run: exit "${{ steps.live_status.outputs.exit_code }}"
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
name: PR Artifacts
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
branches:
|
||||
- main
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
cleanup-on-approval:
|
||||
if: github.event_name == 'pull_request_review' && github.event.review.state == 'approved'
|
||||
runs-on: ubuntu-latest
|
||||
concurrency:
|
||||
group: pr-artifacts-live-e2e-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Check if fork PR
|
||||
id: check_fork
|
||||
run: |
|
||||
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.event.pull_request.base.repo.full_name }}" ]; then
|
||||
echo "is_fork=true" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::Fork PR detected - skipping automatic .pr cleanup."
|
||||
else
|
||||
echo "is_fork=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Resolve PR context
|
||||
id: pr_context
|
||||
if: steps.check_fork.outputs.is_fork == 'false'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pr_api="repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}"
|
||||
head_ref="$(gh api "$pr_api" --jq '.head.ref')"
|
||||
head_sha="$(gh api "$pr_api" --jq '.head.sha')"
|
||||
echo "head_ref=$head_ref" >> "$GITHUB_OUTPUT"
|
||||
echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
|
||||
if gh api "repos/${{ github.repository }}/contents/.pr?ref=$head_sha" --silent >/dev/null 2>&1; then
|
||||
echo "has_pr_dir=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_pr_dir=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::No .pr directory exists at the approved PR head; skipping artifact cleanup."
|
||||
fi
|
||||
|
||||
- name: Check out PR branch
|
||||
if: steps.check_fork.outputs.is_fork == 'false' && steps.pr_context.outputs.has_pr_dir == 'true'
|
||||
env:
|
||||
PR_ARTIFACT_PUSH_TOKEN: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}
|
||||
PR_HEAD_SHA: ${{ steps.pr_context.outputs.head_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git init .
|
||||
git remote add origin "https://github.com/${GITHUB_REPOSITORY}.git"
|
||||
git fetch origin "$PR_HEAD_SHA"
|
||||
git checkout --detach FETCH_HEAD
|
||||
actual_sha="$(git rev-parse HEAD)"
|
||||
if [ "$actual_sha" != "$PR_HEAD_SHA" ]; then
|
||||
echo "::error::SHA mismatch: expected $PR_HEAD_SHA, got $actual_sha"
|
||||
exit 1
|
||||
fi
|
||||
git config credential.helper '!f() { echo "username=x-access-token"; echo "password=$PR_ARTIFACT_PUSH_TOKEN"; }; f'
|
||||
|
||||
- name: Remove .pr directory
|
||||
id: remove
|
||||
if: steps.check_fork.outputs.is_fork == 'false' && steps.pr_context.outputs.has_pr_dir == 'true'
|
||||
env:
|
||||
PR_ARTIFACT_PUSH_TOKEN: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}
|
||||
PR_HEAD_REF: ${{ steps.pr_context.outputs.head_ref }}
|
||||
PR_HEAD_SHA: ${{ steps.pr_context.outputs.head_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ ! -d ".pr" ]; then
|
||||
echo "removed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::No .pr directory to remove."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git config user.name "allhands-bot"
|
||||
git config user.email "allhands-bot@users.noreply.github.com"
|
||||
git rm -rf --ignore-unmatch .pr/
|
||||
if git diff --cached --quiet; then
|
||||
echo "removed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::No tracked .pr files to remove."
|
||||
exit 0
|
||||
fi
|
||||
git commit -m "chore: Remove PR-only artifacts"
|
||||
push_succeeded=false
|
||||
for attempt in 1 2 3; do
|
||||
if git push origin "HEAD:refs/heads/$PR_HEAD_REF"; then
|
||||
push_succeeded=true
|
||||
break
|
||||
fi
|
||||
|
||||
if [ "$attempt" -lt 3 ]; then
|
||||
echo "::notice::Failed to push cleanup commit, rebasing and retrying."
|
||||
git fetch origin "$PR_HEAD_REF"
|
||||
if ! git rebase FETCH_HEAD; then
|
||||
echo "::error::Failed to rebase .pr cleanup commit. Manual resolution required."
|
||||
exit 1
|
||||
fi
|
||||
if ! git merge-base --is-ancestor "$PR_HEAD_SHA" HEAD; then
|
||||
echo "::error::Rebased .pr cleanup commit no longer descends from the approved PR head $PR_HEAD_SHA."
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
if [ "$push_succeeded" != "true" ]; then
|
||||
echo "::error::Failed to push .pr cleanup commit after retries."
|
||||
exit 1
|
||||
fi
|
||||
echo "removed=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Update PR artifacts comment
|
||||
if: steps.check_fork.outputs.is_fork == 'false' && steps.pr_context.outputs.has_pr_dir == 'true' && steps.remove.outputs.removed == 'true'
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- pr-artifacts-notice -->';
|
||||
const body = [
|
||||
marker,
|
||||
'**PR Artifacts Cleaned Up**',
|
||||
'',
|
||||
'The `.pr/` directory has been removed after approval.',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find((comment) => comment.body.includes(marker));
|
||||
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
check-pr-artifacts:
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Check for .pr directory
|
||||
id: check
|
||||
run: |
|
||||
if [ -d ".pr" ]; then
|
||||
echo "exists=true" >> "$GITHUB_OUTPUT"
|
||||
echo "::warning::.pr directory exists and will be automatically removed when the PR is approved. Fork PRs require manual cleanup before merging."
|
||||
else
|
||||
echo "exists=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Post or update PR artifacts comment
|
||||
if: steps.check.outputs.exists == 'true'
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- pr-artifacts-notice -->';
|
||||
const body = [
|
||||
marker,
|
||||
'**PR Artifacts Notice**',
|
||||
'',
|
||||
'This PR contains a `.pr/` directory with PR-specific artifacts. This directory will be **automatically removed** when the PR is approved.',
|
||||
'',
|
||||
'> Fork PRs require manual cleanup before merging.',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find((comment) => comment.body.includes(marker));
|
||||
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user