OpenHands/.github/workflows/py-tests.yml

# Workflow that runs python tests
name: Run Python Tests

# The jobs in this workflow are required, so they must run at all times
# * Always run on "main"
# * Always run on PRs
on:
  push:
    branches:
      - main
  pull_request:

# If triggered by a PR, it will be in the same group. However, each commit on main will be in its own unique group
concurrency:
  group: ${{ github.workflow }}-${{ (github.head_ref && github.ref) || github.run_id }}
  cancel-in-progress: true

jobs:
  # Run python tests on Linux
  test-on-linux:
    name: Python Tests on Linux
    runs-on: blacksmith-4vcpu-ubuntu-2404
    env:
      INSTALL_DOCKER: "0" # Set to '0' to skip Docker installation
    strategy:
      matrix:
        python-version: ["3.12"]
    permissions:
      # For coverage comment and python-coverage-comment-action branch
      pull-requests: write
      contents: write
    steps:
      - uses: actions/checkout@v4
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
      - name: Install tmux
        run: sudo apt-get update && sudo apt-get install -y tmux
      - name: Setup Node.js
        uses: useblacksmith/setup-node@v5
        with:
          node-version: "22.x"
      - name: Install poetry via pipx
        run: pipx install poetry
      - name: Set up Python
        uses: useblacksmith/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
          cache: "poetry"
      - name: Install Python dependencies using Poetry
        run: |
          poetry install --with dev,test,runtime
          poetry run pip install pytest-xdist
          poetry run pip install pytest-rerunfailures
      - name: Build Environment
        run: make build
      - name: Run Unit Tests
        run: PYTHONPATH=".:$PYTHONPATH" poetry run pytest --forked -n auto -s ./tests/unit --cov=openhands --cov-branch
        env:
          COVERAGE_FILE: ".coverage.${{ matrix.python_version }}"
      - name: Run Runtime Tests with CLIRuntime
        run: PYTHONPATH=".:$PYTHONPATH" TEST_RUNTIME=cli poetry run pytest -n 5 --reruns 2 --reruns-delay 3 -s tests/runtime/test_bash.py --cov=openhands --cov-branch
        env:
          COVERAGE_FILE: ".coverage.runtime.${{ matrix.python_version }}"
      - name: Store coverage file
        uses: actions/upload-artifact@v4
        with:
          name: coverage-openhands
          path: |
            .coverage.${{ matrix.python_version }}
            .coverage.runtime.${{ matrix.python_version }}
          include-hidden-files: true

  test-enterprise:
    name: Enterprise Python Unit Tests
    runs-on: blacksmith-4vcpu-ubuntu-2404
    strategy:
      matrix:
        python-version: ["3.12"]
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry via pipx
        run: pipx install poetry
      - name: Set up Python
        uses: useblacksmith/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
          cache: "poetry"
      - name: Install Python dependencies using Poetry
        working-directory: ./enterprise
        run: poetry install --with dev,test
      - name: Run Unit Tests
        # Use base working directory for coverage paths to line up.
        run: PYTHONPATH=".:$PYTHONPATH" poetry run --project=enterprise pytest --forked -n auto -s -p no:ddtrace -p no:ddtrace.pytest_bdd -p no:ddtrace.pytest_benchmark ./enterprise/tests/unit --cov=enterprise --cov-branch
        env:
          COVERAGE_FILE: ".coverage.enterprise.${{ matrix.python_version }}"
      - name: Store coverage file
        uses: actions/upload-artifact@v4
        with:
          name: coverage-enterprise
          path: ".coverage.enterprise.${{ matrix.python_version }}"
          include-hidden-files: true

  coverage-comment:
    name: Coverage Comment
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    needs: [test-on-linux, test-enterprise]

    permissions:
      pull-requests: write
      contents: write
    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v5
        id: download
        with:
          pattern: coverage-*
          merge-multiple: true

      - name: Coverage comment
        id: coverage_comment
        uses: py-cov-action/python-coverage-comment-action@v3
        with:
          GITHUB_TOKEN: ${{ github.token }}
          MERGE_COVERAGE_FILES: true