Remove runtime unit tests (#12331)

.github/workflows/ghcr-build.yml

@@ -252,151 +252,15 @@ jobs:
             -d "{\"ref\": \"main\", \"inputs\": {\"openhandsPrNumber\": \"${{ github.event.pull_request.number }}\", \"deployEnvironment\": \"feature\", \"enterpriseImageTag\": \"pr-${{ github.event.pull_request.number }}\" }}" \
             https://api.github.com/repos/OpenHands/deploy/actions/workflows/deploy.yaml/dispatches
 
-  # Run unit tests with the Docker runtime Docker images as root
-  test_runtime_root:
-    name: RT Unit Tests (Root)
-    needs: [ghcr_build_runtime, define-matrix]
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    strategy:
-      fail-fast: false
-      matrix:
-        base_image: ${{ fromJson(needs.define-matrix.outputs.base_image) }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Download runtime source for fork
-        if: github.event.pull_request.head.repo.fork
-        uses: actions/download-artifact@v6
-        with:
-          name: runtime-src-${{ matrix.base_image.tag }}
-          path: containers/runtime
-      - name: Lowercase Repository Owner
-        run: |
-          echo REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
-      # Forked repos can't push to GHCR, so we need to rebuild using cache
-      - name: Build runtime image ${{ matrix.base_image.image }} for fork
-        if: github.event.pull_request.head.repo.fork
-        uses: useblacksmith/build-push-action@v1
-        with:
-          load: true
-          tags: ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}
-          context: containers/runtime
-      - name: Install poetry via pipx
-        run: pipx install poetry
-      - name: Set up Python
-        uses: useblacksmith/setup-python@v6
-        with:
-          python-version: "3.12"
-          cache: poetry
-      - name: Install Python dependencies using Poetry
-        run: make install-python-dependencies INSTALL_PLAYWRIGHT=0
-      - name: Run docker runtime tests
-        shell: bash
-        run: |
-          # We install pytest-xdist in order to run tests across CPUs
-          poetry run pip install pytest-xdist
-
-          # Install to be able to retry on failures for flakey tests
-          poetry run pip install pytest-rerunfailures
-
-          image_name=ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}
-
-          # Setting RUN_AS_OPENHANDS to false means use root.
-          # That should mean SANDBOX_USER_ID is ignored but some tests do not check for RUN_AS_OPENHANDS.
-
-          TEST_RUNTIME=docker \
-          SANDBOX_USER_ID=$(id -u) \
-          SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
-          TEST_IN_CI=true \
-          RUN_AS_OPENHANDS=false \
-          poetry run pytest -n 5 -raRs --reruns 2 --reruns-delay 3 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
-        env:
-          DEBUG: "1"
-
-  # Run unit tests with the Docker runtime Docker images as openhands user
-  test_runtime_oh:
-    name: RT Unit Tests (openhands)
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [ghcr_build_runtime, define-matrix]
-    strategy:
-      matrix:
-        base_image: ${{ fromJson(needs.define-matrix.outputs.base_image) }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Download runtime source for fork
-        if: github.event.pull_request.head.repo.fork
-        uses: actions/download-artifact@v6
-        with:
-          name: runtime-src-${{ matrix.base_image.tag }}
-          path: containers/runtime
-      - name: Lowercase Repository Owner
-        run: |
-          echo REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
-      # Forked repos can't push to GHCR, so we need to rebuild using cache
-      - name: Build runtime image ${{ matrix.base_image.image }} for fork
-        if: github.event.pull_request.head.repo.fork
-        uses: useblacksmith/build-push-action@v1
-        with:
-          load: true
-          tags: ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}
-          context: containers/runtime
-      - name: Install poetry via pipx
-        run: pipx install poetry
-      - name: Set up Python
-        uses: useblacksmith/setup-python@v6
-        with:
-          python-version: "3.12"
-          cache: poetry
-      - name: Install Python dependencies using Poetry
-        run: make install-python-dependencies POETRY_GROUP=main,test,runtime INSTALL_PLAYWRIGHT=0
-      - name: Run runtime tests
-        shell: bash
-        run: |
-          # We install pytest-xdist in order to run tests across CPUs
-          poetry run pip install pytest-xdist
-
-          # Install to be able to retry on failures for flaky tests
-          poetry run pip install pytest-rerunfailures
-
-          image_name=ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}
-
-          TEST_RUNTIME=docker \
-          SANDBOX_USER_ID=$(id -u) \
-          SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
-          TEST_IN_CI=true \
-          RUN_AS_OPENHANDS=true \
-          poetry run pytest -n 5 -raRs --reruns 2 --reruns-delay 3 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
-        env:
-          DEBUG: "1"
-
-  # The two following jobs (named identically) are to check whether all the runtime tests have passed as the
   # "All Runtime Tests Passed" is a required job for PRs to merge
-  # Due to this bug: https://github.com/actions/runner/issues/2566, we want to create a job that runs when the
-  # prerequisites have been cancelled or failed so merging is disallowed, otherwise Github considers "skipped" as "success"
+  # We can remove this once the config changes
   runtime_tests_check_success:
     name: All Runtime Tests Passed
-    if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
     runs-on: blacksmith-4vcpu-ubuntu-2204
-    needs: [test_runtime_root, test_runtime_oh]
     steps:
       - name: All tests passed
         run: echo "All runtime tests have passed successfully!"
 
-  runtime_tests_check_fail:
-    name: All Runtime Tests Passed
-    if: ${{ cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
-    runs-on: blacksmith-4vcpu-ubuntu-2204
-    needs: [test_runtime_root, test_runtime_oh]
-    steps:
-      - name: Some tests failed
-        run: |
-          echo "Some runtime tests failed or were cancelled"
-          exit 1
   update_pr_description:
     name: Update PR Description
     if: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'