From 99e493b3a46f7d2425c3b616c10fb8f00e2e4b55 Mon Sep 17 00:00:00 2001 From: Ray Myers Date: Mon, 7 Apr 2025 19:04:17 -0500 Subject: [PATCH] feature - Build alternate ubuntu images (#7691) --- .github/workflows/ghcr-build.yml | 16 ++++++++-- .../runtime/impl/docker/docker_runtime.py | 5 ++- .../utils/runtime_templates/Dockerfile.j2 | 31 ++++++++++++++----- tests/runtime/test_bash.py | 4 +-- tests/unit/test_runtime_build.py | 2 +- 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ghcr-build.yml b/.github/workflows/ghcr-build.yml index 8979750a8a..94c73296f1 100644 --- a/.github/workflows/ghcr-build.yml +++ b/.github/workflows/ghcr-build.yml @@ -25,7 +25,6 @@ concurrency: cancel-in-progress: true env: - BASE_IMAGE_FOR_HASH_EQUIVALENCE_TEST: nikolaik/python-nodejs:python3.12-nodejs22 RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} jobs: @@ -37,6 +36,7 @@ jobs: contents: read packages: write outputs: + # Since this job uses outputs it cannot use matrix hash_from_app_image: ${{ steps.get_hash_in_app_image.outputs.hash_from_app_image }} steps: - name: Checkout @@ -89,6 +89,8 @@ jobs: base_image: - image: 'nikolaik/python-nodejs:python3.12-nodejs22' tag: nikolaik + - image: 'ubuntu:24.04' + tag: ubuntu steps: - name: Checkout uses: actions/checkout@v4 @@ -116,6 +118,7 @@ jobs: with: path: | ~/.cache/pypoetry + ~/.cache/ms-playwright ~/.virtualenvs key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | @@ -156,6 +159,8 @@ jobs: fail-fast: false matrix: base_image: ['nikolaik'] + env: + BASE_IMAGE_FOR_HASH_EQUIVALENCE_TEST: nikolaik/python-nodejs:python3.12-nodejs22 steps: - uses: actions/checkout@v4 with: @@ -165,6 +170,7 @@ jobs: with: path: | ~/.cache/pypoetry + ~/.cache/ms-playwright ~/.virtualenvs key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | @@ -208,7 +214,7 @@ jobs: strategy: fail-fast: false matrix: - base_image: ['nikolaik'] + base_image: ['nikolaik', 'ubuntu'] steps: - uses: actions/checkout@v4 - name: Set up Docker Buildx @@ -230,6 +236,7 @@ jobs: with: path: | ~/.cache/pypoetry + ~/.cache/ms-playwright ~/.virtualenvs key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} restore-keys: | @@ -255,6 +262,9 @@ jobs: image_name=ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image }} + # Setting RUN_AS_OPENHANDS to false means use root. + # That should mean SANDBOX_USER_ID is ignored but some tests do not check for RUN_AS_OPENHANDS. + TEST_RUNTIME=docker \ SANDBOX_USER_ID=$(id -u) \ SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \ @@ -273,7 +283,7 @@ jobs: needs: [ghcr_build_runtime] strategy: matrix: - base_image: ['nikolaik'] + base_image: [nikolaik, ubuntu] steps: - uses: actions/checkout@v4 - name: Set up Docker Buildx diff --git a/openhands/runtime/impl/docker/docker_runtime.py b/openhands/runtime/impl/docker/docker_runtime.py index 9cbe319891..e7e0834618 100644 --- a/openhands/runtime/impl/docker/docker_runtime.py +++ b/openhands/runtime/impl/docker/docker_runtime.py @@ -252,8 +252,9 @@ class DockerRuntime(ActionExecutionClient): # Combine environment variables environment = { 'port': str(self._container_port), - 'PYTHONUNBUFFERED': 1, + 'PYTHONUNBUFFERED': '1', 'VSCODE_PORT': str(self._vscode_port), + 'PIP_BREAK_SYSTEM_PACKAGES': '1', } if self.config.debug or DEBUG: environment['DEBUG'] = 'true' @@ -293,6 +294,8 @@ class DockerRuntime(ActionExecutionClient): self.container = self.docker_client.containers.run( self.runtime_container_image, command=command, + # Override the default 'bash' entrypoint because the command is a binary. + entrypoint=[], network_mode=network_mode, ports=port_mapping, working_dir='/openhands/code/', # do not change this! diff --git a/openhands/runtime/utils/runtime_templates/Dockerfile.j2 b/openhands/runtime/utils/runtime_templates/Dockerfile.j2 index 74e75dc39a..c645777ed1 100644 --- a/openhands/runtime/utils/runtime_templates/Dockerfile.j2 +++ b/openhands/runtime/utils/runtime_templates/Dockerfile.j2 @@ -14,24 +14,34 @@ ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry \ # Install base system dependencies RUN apt-get update && \ - apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ - wget curl sudo apt-utils git jq tmux \ + wget curl ca-certificates sudo apt-utils git jq tmux build-essential \ {%- if 'ubuntu' in base_image and (base_image.endswith(':latest') or base_image.endswith(':24.04')) -%} libgl1 \ {%- else %} libgl1-mesa-glx \ {% endif -%} libasound2-plugins libatomic1 && \ - # Remove packages with CVEs and no updates yet, if present - (apt-get remove -y libaom3 || true) && \ - (apt-get remove -y libjxl0.7 || true) && \ - (apt-get remove -y libopenexr-3-1-30 || true) && \ + {%- if 'ubuntu' in base_image -%} + curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \ + TZ=Etc/UTC DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends nodejs python3.12 python-is-python3 python3-pip python3.12-venv && \ + corepack enable yarn && \ + {% endif -%} apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Remove UID 1000 if it's called pn--this fixes the nikolaik image for ubuntu users -RUN if getent passwd 1000 | grep -q pn; then userdel pn; fi +{% if 'ubuntu' in base_image %} +RUN ln -s "$(dirname $(which node))/corepack" /usr/local/bin/corepack && \ + npm install -g corepack && corepack enable yarn && \ + curl -fsSL --compressed https://install.python-poetry.org | python - && \ + curl -LsSf https://astral.sh/uv/install.sh | sh +{% endif %} + +# Remove UID 1000 named pn or ubuntu, so the 'openhands' user can be created from ubuntu hosts +RUN (if getent passwd 1000 | grep -q pn; then userdel pn; fi) && \ + (if getent passwd 1000 | grep -q ubuntu; then userdel ubuntu; fi) + # Create necessary directories RUN mkdir -p /openhands && \ @@ -71,6 +81,8 @@ RUN if [ -z "${RELEASE_TAG}" ]; then \ cp ${OPENVSCODE_SERVER_ROOT}/bin/remote-cli/openvscode-server ${OPENVSCODE_SERVER_ROOT}/bin/remote-cli/code && \ rm -f ${RELEASE_TAG}-linux-${arch}.tar.gz + + {% endmacro %} {% macro install_vscode_extensions() %} @@ -80,6 +92,9 @@ RUN mkdir -p ${OPENVSCODE_SERVER_ROOT}/extensions/openhands-hello-world && \ RUN mkdir -p ${OPENVSCODE_SERVER_ROOT}/extensions/openhands-memory-monitor && \ cp -r /openhands/code/openhands/runtime/utils/vscode-extensions/memory-monitor/* ${OPENVSCODE_SERVER_ROOT}/extensions/openhands-memory-monitor/ + +# Some extension dirs are removed because they trigger false positives in vulnerability scans. +RUN rm -rf ${OPENVSCODE_SERVER_ROOT}/extensions/{handlebars,pug,json,diff,grunt,ini,npm} {% endmacro %} {% macro install_dependencies() %} diff --git a/tests/runtime/test_bash.py b/tests/runtime/test_bash.py index d5911c89fc..76fc768ad4 100644 --- a/tests/runtime/test_bash.py +++ b/tests/runtime/test_bash.py @@ -153,10 +153,10 @@ def test_multiple_multiline_commands(temp_dir, runtime_cls, run_as_openhands): _close_test_runtime(runtime) -def test_complex_commands(temp_dir, runtime_cls): +def test_complex_commands(temp_dir, runtime_cls, run_as_openhands): cmd = """count=0; tries=0; while [ $count -lt 3 ]; do result=$(echo "Heads"); tries=$((tries+1)); echo "Flip $tries: $result"; if [ "$result" = "Heads" ]; then count=$((count+1)); else count=0; fi; done; echo "Got 3 heads in a row after $tries flips!";""" - runtime, config = _load_runtime(temp_dir, runtime_cls) + runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands) try: obs = _run_cmd_action(runtime, cmd) logger.info(obs, extra={'msg_type': 'OBSERVATION'}) diff --git a/tests/unit/test_runtime_build.py b/tests/unit/test_runtime_build.py index 7c1cedfa57..ae04f499e3 100644 --- a/tests/unit/test_runtime_build.py +++ b/tests/unit/test_runtime_build.py @@ -136,7 +136,7 @@ def test_generate_dockerfile_build_from_scratch(): ) assert base_image in dockerfile_content assert 'apt-get update' in dockerfile_content - assert 'wget curl sudo apt-utils git' in dockerfile_content + assert 'wget curl' in dockerfile_content assert 'poetry' in dockerfile_content and '-c conda-forge' in dockerfile_content assert 'python=3.12' in dockerfile_content