From c2fa99b4a10aa12d6cf323de69966d31e5d35573 Mon Sep 17 00:00:00 2001 From: Boxuan Li Date: Tue, 18 Jun 2024 10:44:51 -0700 Subject: [PATCH] Split container image build & push (#2456) * Split container image build & push * Code cleanup * Cleanup * Add back useless docker_build_success step to make CI happy * Revert "Cleanup" This reverts commit 2a260791a95a110a54335141d017a418397eecf9. * Use fresh built sandbox image in integration test * fix dependency * DEBUG: only build * Attempt to fix dependency * Change dependency * Combine both jobs * Fix env * Remove Mac integration tests as they are too unstable * Move sandbox tests to ghcr * Use loaded image --- .github/workflows/ghcr.yml | 216 ++++++++++++++++++-- .github/workflows/run-integration-tests.yml | 104 ---------- .github/workflows/run-unit-tests.yml | 30 --- containers/build.sh | 55 ++--- 4 files changed, 230 insertions(+), 175 deletions(-) delete mode 100644 .github/workflows/run-integration-tests.yml diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 426a6e47fc..9e8d6228b3 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -1,4 +1,4 @@ -name: Publish Docker Image +name: Build Publish and Test Docker Image concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -7,7 +7,7 @@ concurrency: on: push: branches: - - main + - main tags: - '*' pull_request: @@ -19,19 +19,23 @@ on: default: '' jobs: - ghcr_build_and_push: + ghcr_build: runs-on: ubuntu-latest + outputs: + tags: ${{ steps.capture-tags.outputs.tags }} + permissions: contents: read packages: write strategy: matrix: - image: ["app", "sandbox"] + image: ["sandbox", "opendevin"] + platform: ["amd64", "arm64"] steps: - - name: checkout + - name: Checkout uses: actions/checkout@v4 - name: Free Disk Space (Ubuntu) @@ -40,7 +44,6 @@ jobs: # this might remove tools that are actually needed, # if set to "true" but frees about 6 GB tool-cache: true - # all of these default to true, but feel free to set to # "false" if necessary for your workflow android: true @@ -57,26 +60,207 @@ jobs: id: buildx uses: docker/setup-buildx-action@v3 - - name: Login to ghcr - uses: docker/login-action@v1 + - name: Build and export image + id: build + run: ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} ${{ matrix.platform }} + + - name: Capture tags + id: capture-tags + run: | + tags=$(cat tags.txt) + echo "tags=$tags" + echo "tags=$tags" >> $GITHUB_OUTPUT + + - name: Upload Docker image as artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.image }}-docker-image-${{ matrix.platform }} + path: /tmp/${{ matrix.image }}_image_${{ matrix.platform }}.tar + + test-for-sandbox: + name: Test for Sandbox + runs-on: ubuntu-latest + needs: ghcr_build + env: + PERSIST_SANDBOX: "false" + steps: + - uses: actions/checkout@v4 + + - name: Install poetry via pipx + run: pipx install poetry + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "poetry" + + - name: Install Python dependencies using Poetry + run: make install-python-dependencies + + - name: Download sandbox Docker image + uses: actions/download-artifact@v4 + with: + name: sandbox-docker-image-amd64 + path: /tmp/ + + - name: Load sandbox image and run sandbox tests + run: | + # Load the Docker image and capture the output + output=$(docker load -i /tmp/sandbox_image_amd64.tar) + + # Extract the image name from the output + image_name=$(echo "$output" | grep -oP 'Loaded image: \K.*') + + # Print the full name of the image + echo "Loaded Docker image: $image_name" + + SANDBOX_CONTAINER_IMAGE=$image_name poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml -s ./tests/unit/test_sandbox.py + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + integration-tests-on-linux: + name: Integration Tests on Linux + runs-on: ubuntu-latest + needs: ghcr_build + env: + PERSIST_SANDBOX: "false" + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + sandbox: ["ssh", "exec", "local"] + steps: + - uses: actions/checkout@v4 + + - name: Install poetry via pipx + run: pipx install poetry + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + + - name: Install Python dependencies using Poetry + run: make install-python-dependencies + + - name: Download sandbox Docker image + uses: actions/download-artifact@v4 + with: + name: sandbox-docker-image-amd64 + path: /tmp/ + + - name: Load sandbox image and run integration tests + env: + SANDBOX_TYPE: ${{ matrix.sandbox }} + run: | + # Load the Docker image and capture the output + output=$(docker load -i /tmp/sandbox_image_amd64.tar) + + # Extract the image name from the output + image_name=$(echo "$output" | grep -oP 'Loaded image: \K.*') + + # Print the full name of the image + echo "Loaded Docker image: $image_name" + + SANDBOX_CONTAINER_IMAGE=$image_name TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + ghcr_push: + runs-on: ubuntu-latest + # don't push if integration tests or sandbox tests fail + needs: [ghcr_build, integration-tests-on-linux, test-for-sandbox] + if: github.ref == 'refs/heads/main' + + env: + tags: ${{ needs.ghcr_build.outputs.tags }} + + permissions: + contents: read + packages: write + + strategy: + matrix: + image: ["sandbox", "opendevin"] + platform: ["amd64", "arm64"] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Login to GHCR + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push ${{ matrix.image }} - if: "!github.event.pull_request.head.repo.fork" - run: | - ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} --push + - name: Download Docker images + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.image }}-docker-image-${{ matrix.platform }} + path: /tmp/${{ matrix.platform }} - - name: Build ${{ matrix.image }} - if: "github.event.pull_request.head.repo.fork" + - name: Load images and push to registry run: | - ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} + mv /tmp/${{ matrix.platform }}/${{ matrix.image }}_image_${{ matrix.platform }}.tar . + loaded_image=$(docker load -i ${{ matrix.image }}_image_${{ matrix.platform }}.tar | grep "Loaded image:" | awk '{print $3}') + tags=$(echo ${tags} | tr ' ' '\n') + for tag in $tags; do + echo "tag = $tag" + docker tag $loaded_image ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_${{ matrix.platform }} + docker push ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_${{ matrix.platform }} + done + create_manifest: + runs-on: ubuntu-latest + needs: [ghcr_build, ghcr_push] + if: github.ref == 'refs/heads/main' + + env: + tags: ${{ needs.ghcr_build.outputs.tags }} + + strategy: + matrix: + image: ["sandbox", "opendevin"] + + permissions: + contents: read + packages: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create and push multi-platform manifest + run: | + tags=$(echo ${tags} | tr ' ' '\n') + for tag in $tags; do + echo 'tag = $tag' + docker buildx imagetools create --tag ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:$tag \ + ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_amd64 \ + ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_arm64 + done + + # FIXME: an admin needs to mark this as non-mandatory, and then we can remove it docker_build_success: name: Docker Build Success runs-on: ubuntu-latest - needs: ghcr_build_and_push + needs: ghcr_build steps: - run: echo Done! diff --git a/.github/workflows/run-integration-tests.yml b/.github/workflows/run-integration-tests.yml deleted file mode 100644 index 3f0f80e3e5..0000000000 --- a/.github/workflows/run-integration-tests.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Run Integration Tests - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -on: - push: - branches: - - main - paths-ignore: - - '**/*.md' - - 'frontend/**' - - 'docs/**' - - 'evaluation/**' - pull_request: - -env: - PERSIST_SANDBOX : "false" - -jobs: - integration-tests-on-linux: - name: Integration Tests on Linux - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.11"] - sandbox: ["ssh", "exec", "local"] - steps: - - uses: actions/checkout@v4 - - - name: Install poetry via pipx - run: pipx install poetry - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'poetry' - - - name: Install Python dependencies using Poetry - run: poetry install - - - name: Build Environment - run: make build - - - name: Run Integration Tests - env: - SANDBOX_TYPE: ${{ matrix.sandbox }} - run: | - TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - integration-tests-on-mac: - name: Integration Tests on MacOS - runs-on: macos-13 - if: contains(github.event.pull_request.title, 'mac') || contains(github.event.pull_request.title, 'Mac') - strategy: - fail-fast: false - matrix: - python-version: ["3.11"] - sandbox: ["ssh"] - steps: - - uses: actions/checkout@v4 - - - name: Install poetry via pipx - run: pipx install poetry - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'poetry' - - - name: Install Python dependencies using Poetry - run: poetry install - - - name: Install & Start Docker - run: | - brew install colima docker - colima start - - # For testcontainers to find the Colima socket - # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running - sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - - - name: Build Environment - run: make build - - - name: Run Integration Tests - env: - SANDBOX_TYPE: ${{ matrix.sandbox }} - run: | - TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index bedf4d380d..86f1b96dba 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -97,33 +97,3 @@ jobs: uses: codecov/codecov-action@v4 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - test-for-sandbox: - name: Test for Sandbox - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install poetry via pipx - run: pipx install poetry - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: "poetry" - - - name: Install Python dependencies using Poetry - run: poetry install - - - name: Build Environment - run: make build - - - name: Run Integration Test for Sandbox - run: | - poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml -s ./tests/unit/test_sandbox.py - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/containers/build.sh b/containers/build.sh index 66a3a31c41..dd4eaf7075 100755 --- a/containers/build.sh +++ b/containers/build.sh @@ -3,12 +3,9 @@ set -eo pipefail image_name=$1 org_name=$2 -push=0 -if [[ $3 == "--push" ]]; then - push=1 -fi +platform=$3 -echo -e "Building: $image_name" +echo "Building: $image_name for platform: $platform" tags=() OPEN_DEVIN_BUILD_VERSION="dev" @@ -19,49 +16,57 @@ cache_tag="$cache_tag_base" if [[ -n $GITHUB_REF_NAME ]]; then # check if ref name is a version number if [[ $GITHUB_REF_NAME =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - major_version=$(echo $GITHUB_REF_NAME | cut -d. -f1) - minor_version=$(echo $GITHUB_REF_NAME | cut -d. -f1,2) - tags+=($major_version $minor_version) + major_version=$(echo "$GITHUB_REF_NAME" | cut -d. -f1) + minor_version=$(echo "$GITHUB_REF_NAME" | cut -d. -f1,2) + tags+=("$major_version" "$minor_version") fi - sanitized=$(echo $GITHUB_REF_NAME | sed 's/[^a-zA-Z0-9.-]\+/-/g') + sanitized=$(echo "$GITHUB_REF_NAME" | sed 's/[^a-zA-Z0-9.-]\+/-/g') OPEN_DEVIN_BUILD_VERSION=$sanitized cache_tag+="-${sanitized}" - tags+=($sanitized) + tags+=("$sanitized") fi echo "Tags: ${tags[@]}" -dir=./containers/$image_name -if [ ! -f $dir/Dockerfile ]; then +if [[ "$image_name" == "opendevin" ]]; then + dir="./containers/app" +else + dir="./containers/$image_name" +fi + +if [[ ! -f "$dir/Dockerfile" ]]; then echo "No Dockerfile found" exit 1 fi -if [ ! -f $dir/config.sh ]; then +if [[ ! -f "$dir/config.sh" ]]; then echo "No config.sh found for Dockerfile" exit 1 fi -source $dir/config.sh + +source "$dir/config.sh" + if [[ -n "$org_name" ]]; then DOCKER_ORG="$org_name" fi -DOCKER_REPOSITORY=$DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE + +DOCKER_REPOSITORY="$DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE" DOCKER_REPOSITORY=${DOCKER_REPOSITORY,,} # lowercase echo "Repo: $DOCKER_REPOSITORY" echo "Base dir: $DOCKER_BASE_DIR" args="" -for tag in ${tags[@]}; do +for tag in "${tags[@]}"; do args+=" -t $DOCKER_REPOSITORY:$tag" done -if [[ $push -eq 1 ]]; then - args+=" --push" - args+=" --cache-to=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag,mode=max" -fi + +output_image="/tmp/${image_name}_image_${platform}.tar" docker buildx build \ $args \ - --build-arg OPEN_DEVIN_BUILD_VERSION=$OPEN_DEVIN_BUILD_VERSION \ - --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag \ - --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag_base-main \ - --platform linux/amd64,linux/arm64 \ + --build-arg OPEN_DEVIN_BUILD_VERSION="$OPEN_DEVIN_BUILD_VERSION" \ + --platform linux/$platform \ --provenance=false \ - -f $dir/Dockerfile $DOCKER_BASE_DIR + -f "$dir/Dockerfile" \ + --output type=docker,dest="$output_image" \ + "$DOCKER_BASE_DIR" + +echo "${tags[*]}" > tags.txt