OpenHands/.github/workflows/snapshot-tests.yml

name: Snapshot Tests

on:
  pull_request:
    types: [opened, synchronize, reopened, labeled, unlabeled]
  push:
    branches:
      - main
  workflow_dispatch:
    inputs:
      force_update:
        description: "Force-regenerate baseline snapshots from main"
        type: boolean
        default: false

concurrency:
  # One run at a time per PR — a new commit cancels the in-flight run.
  # Falls back to github.ref for main pushes and manual dispatch so those
  # don't interfere with each other or with PR runs.
  group: snapshot-tests-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  contents: write
  pull-requests: write
  actions: read

jobs:
  snapshot-tests:
    name: Visual Snapshot Tests
    runs-on: ubuntu-24.04
    timeout-minutes: 30

    steps:
      - name: Check out repository
        uses: actions/checkout@v6
        with:
          # On `pull_request` events `github.head_ref` is the head branch
          # name — but it lives on the PR head repo, which is a fork for
          # cross-repo PRs. Without `repository` set, actions/checkout
          # defaults to `github.repository` (the upstream repo) and fails
          # to find the branch. Resolve both the head repo and head SHA
          # from the PR event payload so fork PRs check out cleanly.
          # Falls back to the workflow's own ref for `push` /
          # `workflow_dispatch` events where there's no PR payload.
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
          ref: ${{ github.event.pull_request.head.sha || github.ref_name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          fetch-depth: 0

      - name: Set up Node.js with npm cache
        uses: actions/setup-node@v6
        with:
          # Pin to 24.15.x — Node 24.16.0 has a zip-extraction regression
          # (nodejs/node#63487) that hangs `playwright install` for Playwright
          # < 1.60.0. Remove this pin after upgrading to Playwright >= 1.60.0.
          node-version: "24.15"
          cache: npm

      - name: Install dependencies
        run: npm ci

      - name: Get Playwright version
        id: pw_version
        run: echo "version=$(npx playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"

      - name: Cache Playwright browsers
        id: pw_cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/ms-playwright
          key: playwright-${{ runner.os }}-${{ steps.pw_version.outputs.version }}

      - name: Install Playwright Chromium
        if: steps.pw_cache.outputs.cache-hit != 'true'
        run: npx playwright install chromium

      - name: Install Playwright system deps
        run: npx playwright install-deps chromium

      # ── MAIN BRANCH: generate fresh baselines and store as artifact ──────

      - name: Generate baseline snapshots
        if: github.ref == 'refs/heads/main' || github.event.inputs.force_update == 'true'
        run: npm run test:e2e:snapshots:update

      - name: Upload baseline snapshots artifact
        if: github.ref == 'refs/heads/main' || github.event.inputs.force_update == 'true'
        uses: actions/upload-artifact@v7
        with:
          name: snapshot-baselines
          path: tests/e2e/__snapshots__/
          retention-days: 90

      # ── PR BRANCH: download main baselines, compare, report diffs ────────

      # Wait for the snapshot-tests.yml run triggered by the *current* HEAD
      # commit of main to complete before we download baselines.  Without this,
      # a PR job that starts while main's baseline upload is still in flight
      # downloads the previous (stale) artifact and produces false failures.
      #
      # We resolve main's HEAD SHA first so we wait only for that specific run —
      # not for an older in-progress run from a different commit.
      - name: Wait for main baseline workflow if in progress
        if: github.event_name == 'pull_request'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          MAIN_SHA=$(gh api "/repos/${{ github.repository }}/git/ref/heads/main" \
            --jq '.object.sha' 2>/dev/null || echo "")
          if [ -z "$MAIN_SHA" ]; then
            echo "Could not resolve main HEAD SHA — skipping wait."
            exit 0
          fi
          echo "main HEAD SHA: $MAIN_SHA"

          # Find the snapshot-tests.yml run triggered by exactly this commit.
          if ! RUN_ID=$(gh api \
            "/repos/${{ github.repository }}/actions/workflows/snapshot-tests.yml/runs?branch=main&per_page=20" \
            --jq "[.workflow_runs[] | select(.head_sha == \"$MAIN_SHA\")] | .[0].id // empty"); then
            echo "::warning::Could not query main snapshot workflow runs; proceeding with latest available artifact."
            exit 0
          fi

          if [ -z "$RUN_ID" ]; then
            echo "No snapshot-tests run found for main SHA $MAIN_SHA — proceeding."
            exit 0
          fi

          if ! STATUS=$(gh api "/repos/${{ github.repository }}/actions/runs/$RUN_ID" --jq '.status'); then
            echo "::warning::Could not query main snapshot workflow run status; proceeding with latest available artifact."
            exit 0
          fi
          echo "Run $RUN_ID status: $STATUS"
          if [ "$STATUS" = "completed" ]; then
            echo "Already completed — proceeding."
            exit 0
          fi

          echo "Waiting for run $RUN_ID to complete (max 10 min)..."
          for i in $(seq 1 60); do
            sleep 10
            if ! STATUS=$(gh api "/repos/${{ github.repository }}/actions/runs/$RUN_ID" --jq '.status'); then
              echo "::warning::Could not query main snapshot workflow run status; proceeding with latest available artifact."
              exit 0
            fi
            echo "  [$((i * 10))s] status: $STATUS"
            if [ "$STATUS" = "completed" ]; then
              echo "Main baseline workflow completed."
              exit 0
            fi
          done
          echo "::warning::Timed out waiting for main baseline workflow — proceeding with latest available artifact."

      - name: Find latest snapshot-baselines artifact from main
        if: github.event_name == 'pull_request'
        id: find-run
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Query the artifacts API by name so we only match runs that actually
          # uploaded the artifact (not just any successful run of the workflow).
          ARTIFACT=$(gh api \
            "/repos/${{ github.repository }}/actions/artifacts?name=snapshot-baselines&per_page=10" \
            --jq '[.artifacts[] | select(.expired == false and .workflow_run.head_branch == "main")] | sort_by(.created_at) | reverse | .[0]')
          if [ -z "$ARTIFACT" ] || [ "$ARTIFACT" = "null" ]; then
            echo "has_baselines=false" >> "$GITHUB_OUTPUT"
            echo "::warning::No snapshot-baselines artifact found on main — all snapshots will be treated as new."
          else
            RUN_ID=$(echo "$ARTIFACT" | jq -r '.workflow_run.id')
            echo "has_baselines=true" >> "$GITHUB_OUTPUT"
            echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
            echo "Found snapshot-baselines artifact from run $RUN_ID"
          fi

      - name: Clear snapshot directory before downloading baselines
        if: github.event_name == 'pull_request'
        run: rm -rf tests/e2e/__snapshots__/

      - name: Download main-branch baselines
        if: github.event_name == 'pull_request' && steps.find-run.outputs.has_baselines == 'true'
        uses: actions/download-artifact@v8
        continue-on-error: true
        id: download-baselines
        with:
          name: snapshot-baselines
          path: tests/e2e/__snapshots__/
          run-id: ${{ steps.find-run.outputs.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Save main baselines for diff comparison
        if: github.event_name == 'pull_request'
        run: |
          if [ -d "tests/e2e/__snapshots__" ]; then
            cp -r tests/e2e/__snapshots__ /tmp/main-baselines
          else
            mkdir -p /tmp/main-baselines
          fi

      - name: Run snapshot comparison
        if: github.event_name == 'pull_request'
        id: compare
        run: npm run test:e2e:snapshots
        continue-on-error: true

      # Playwright clears test-results/ at the start of every new run, so the
      # diff files produced by the comparison step above would be wiped by the
      # update step below before post-snapshot-comment.mjs can read them.
      # Save the comparison output now so the comment script can still find the
      # diff/actual/expected images.
      - name: Save comparison test-results
        if: github.event_name == 'pull_request'
        run: |
          if [ -d "test-results" ]; then
            cp -r test-results /tmp/comparison-results
          else
            mkdir -p /tmp/comparison-results
          fi

      - name: Generate current PR snapshots
        if: github.event_name == 'pull_request'
        id: generate
        run: npm run test:e2e:snapshots:update
        continue-on-error: true

      - name: Post snapshot report to PR
        if: always() && github.event_name == 'pull_request'
        id: post-comment
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          HEAD_REF: ${{ github.head_ref }}
          MAIN_BASELINES_DIR: /tmp/main-baselines
          COMPARISON_RESULTS_DIR: /tmp/comparison-results
          SNAPSHOTS_APPROVED: ${{ contains(github.event.pull_request.labels.*.name, 'update-snapshots') }}
          GENERATE_OUTCOME: ${{ steps.generate.outcome }}
          COMPARE_OUTCOME: ${{ steps.compare.outcome }}
        run: node tests/e2e/snapshots/scripts/post-snapshot-comment.mjs

      - name: Upload test results artifact
        if: github.event_name == 'pull_request' && steps.compare.outcome == 'failure'
        uses: actions/upload-artifact@v7
        with:
          name: snapshot-test-results
          path: |
            test-results/
            playwright-report/
          retention-days: 7

      - name: Fail if snapshot comparison found differences
        # Only fail on real pixel-diff failures (changed snapshots), not on
        # missing-baseline failures which are expected for new tests added in
        # this PR. `has_changes` is written by post-snapshot-comment.mjs to
        # $GITHUB_OUTPUT; it is false when all failures are "new" snapshots.
        if: >-
          github.event_name == 'pull_request' &&
          steps.post-comment.outputs.has_changes == 'true' &&
          !contains(github.event.pull_request.labels.*.name, 'update-snapshots')
        run: |
          echo "::error::Snapshot differences detected. Add the 'update-snapshots' label to acknowledge intentional changes."
          exit 1

      - name: Fail if snapshot generation had test crashes
        if: >-
          github.event_name == 'pull_request' &&
          steps.generate.outcome == 'failure'
        run: |
          echo "::error::One or more snapshot tests crashed during generation. See the PR comment and CI logs for details."
          exit 1