OpenHands/.github/workflows/e2e-tests.yml

name: End-to-End Tests

on:
  pull_request:
    types: [opened, synchronize, reopened, labeled]
    branches:
      - main
      - develop
  workflow_dispatch:

jobs:
  e2e-tests:
    if: contains(github.event.pull_request.labels.*.name, 'end-to-end') || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    timeout-minutes: 60

    env:
      GITHUB_REPO_NAME: ${{ github.repository }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install poetry via pipx
        uses: abatilo/actions-poetry@v3
        with:
          poetry-version: 2.1.3

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
          cache: 'poetry'

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libgtk-3-0 libnotify4 libnss3 libxss1 libxtst6 xauth xvfb libgbm1 libasound2t64 netcat-openbsd

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '22'
          cache: 'npm'
          cache-dependency-path: 'frontend/package-lock.json'

      - name: Setup environment for end-to-end tests
        run: |
          # Create test results directory
          mkdir -p test-results

          # Create downloads directory for OpenHands (use a directory in the home folder)
          mkdir -p $HOME/downloads
          sudo chown -R $USER:$USER $HOME/downloads
          sudo chmod -R 755 $HOME/downloads

      - name: Build OpenHands
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4o' }}
          LLM_API_KEY: ${{ secrets.LLM_API_KEY || 'test-key' }}
          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
          INSTALL_DOCKER: 1
          RUNTIME: docker
          FRONTEND_PORT: 12000
          FRONTEND_HOST: 0.0.0.0
          BACKEND_HOST: 0.0.0.0
          BACKEND_PORT: 3000
          ENABLE_BROWSER: true
          INSTALL_PLAYWRIGHT: 1
        run: |
          # Fix poetry.lock file if needed
          echo "Fixing poetry.lock file if needed..."
          poetry lock

          # Build OpenHands using make build
          echo "Running make build..."
          make build

          # Install Chromium Headless Shell for Playwright (needed for pytest-playwright)
          echo "Installing Chromium Headless Shell for Playwright..."
          poetry run playwright install chromium-headless-shell

          # Verify Playwright browsers are installed (for e2e tests only)
          echo "Verifying Playwright browsers installation for e2e tests..."
          BROWSER_CHECK=$(poetry run python tests/e2e/check_playwright.py 2>/dev/null)

          if [ "$BROWSER_CHECK" != "chromium_found" ]; then
            echo "ERROR: Chromium browser not found or not working for e2e tests"
            echo "$BROWSER_CHECK"
            exit 1
          else
            echo "Playwright browsers are properly installed for e2e tests."
          fi

          # Docker runtime will handle workspace directory creation

          # Start the application using make run with custom parameters and reduced logging
          echo "Starting OpenHands using make run..."
          # Set environment variables to reduce logging verbosity
          export PYTHONUNBUFFERED=1
          export LOG_LEVEL=WARNING
          export UVICORN_LOG_LEVEL=warning
          export OPENHANDS_LOG_LEVEL=WARNING
          FRONTEND_PORT=12000 FRONTEND_HOST=0.0.0.0 BACKEND_HOST=0.0.0.0 make run > /tmp/openhands-e2e-test.log 2>&1 &

          # Store the PID of the make run process
          MAKE_PID=$!
          echo "OpenHands started with PID: $MAKE_PID"

          # Wait for the application to start
          echo "Waiting for OpenHands to start..."
          max_attempts=15
          attempt=1

          while [ $attempt -le $max_attempts ]; do
            echo "Checking if OpenHands is running (attempt $attempt of $max_attempts)..."

            # Check if the process is still running
            if ! ps -p $MAKE_PID > /dev/null; then
              echo "ERROR: OpenHands process has terminated unexpectedly"
              echo "Last 50 lines of the log:"
              tail -n 50 /tmp/openhands-e2e-test.log
              exit 1
            fi

            # Check if frontend port is open
            if nc -z localhost 12000; then
              # Verify we can get HTML content
              if curl -s http://localhost:12000 | grep -q "<html"; then
                echo "SUCCESS: OpenHands is running and serving HTML content on port 12000"
                break
              else
                echo "Port 12000 is open but not serving HTML content yet"
              fi
            else
              echo "Frontend port 12000 is not open yet"
            fi

            # Show log output on each attempt
            echo "Recent log output:"
            tail -n 20 /tmp/openhands-e2e-test.log

            # Wait before next attempt
            echo "Waiting 10 seconds before next check..."
            sleep 10
            attempt=$((attempt + 1))

            # Exit if we've reached the maximum number of attempts
            if [ $attempt -gt $max_attempts ]; then
              echo "ERROR: OpenHands failed to start after $max_attempts attempts"
              echo "Last 50 lines of the log:"
              tail -n 50 /tmp/openhands-e2e-test.log
              exit 1
            fi
          done

          # Final verification that the app is running
          if ! nc -z localhost 12000 || ! curl -s http://localhost:12000 | grep -q "<html"; then
            echo "ERROR: OpenHands is not running properly on port 12000"
            echo "Last 50 lines of the log:"
            tail -n 50 /tmp/openhands-e2e-test.log
            exit 1
          fi

          # Print success message
          echo "OpenHands is running successfully on port 12000"

      - name: Run end-to-end tests
        env:
          GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN }}
          LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4o' }}
          LLM_API_KEY: ${{ secrets.LLM_API_KEY || 'test-key' }}
          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
        run: |
          # Check if the application is running
          if ! nc -z localhost 12000; then
            echo "ERROR: OpenHands is not running on port 12000"
            echo "Last 50 lines of the log:"
            tail -n 50 /tmp/openhands-e2e-test.log
            exit 1
          fi

          # Run the tests with detailed output
          cd tests/e2e
          poetry run python -m pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --no-header --capture=no --timeout=600

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: playwright-report
          path: tests/e2e/test-results/
          retention-days: 30

      - name: Upload OpenHands logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: openhands-logs
          path: |
            /tmp/openhands-e2e-test.log
            /tmp/openhands-e2e-build.log
            /tmp/openhands-backend.log
            /tmp/openhands-frontend.log
            /tmp/backend-health-check.log
            /tmp/frontend-check.log
            /tmp/vite-config.log
            /tmp/makefile-contents.log
          retention-days: 30

      - name: Cleanup
        if: always()
        run: |
          # Stop OpenHands processes
          echo "Stopping OpenHands processes..."
          pkill -f "python -m openhands.server" || true
          pkill -f "npm run dev" || true
          pkill -f "make run" || true

          # Print process status for debugging
          echo "Checking if any OpenHands processes are still running:"
          ps aux | grep -E "openhands|npm run dev" || true