From b63dec4b2e022ce29ac948e4e4a72ac4f38aa791 Mon Sep 17 00:00:00 2001 From: Robert Brennan Date: Fri, 23 Aug 2024 13:01:18 -0400 Subject: [PATCH] Add back docker caching, simplify docker builds (#3546) * fix multiarch * remove extra push * add back tag file * fix cache tag * add login step * fix login * try to fix save * fix output maybe * rm outputs * remove tars * fix refs * fix runtime dep * force rebuild * lowercase image * add suffix to build tags for runtime * update matrix * fix cut * fix cut again * add back matrix * Update containers/build.sh Co-authored-by: Xingyao Wang --------- Co-authored-by: Xingyao Wang --- .github/workflows/ghcr_app.yml | 111 +--------- .github/workflows/ghcr_runtime.yml | 255 +++-------------------- containers/build.sh | 48 +++-- openhands/runtime/utils/runtime_build.py | 2 +- 4 files changed, 75 insertions(+), 341 deletions(-) diff --git a/.github/workflows/ghcr_app.yml b/.github/workflows/ghcr_app.yml index c9c6ef2d88..28ee7d6324 100644 --- a/.github/workflows/ghcr_app.yml +++ b/.github/workflows/ghcr_app.yml @@ -25,16 +25,9 @@ jobs: ghcr_build: name: Build App Image runs-on: ubuntu-latest - outputs: - tags: ${{ steps.capture-tags.outputs.tags }} - last_tag: ${{ steps.capture-last-tag.outputs.last_tag }} permissions: contents: read packages: write - strategy: - matrix: - image: ['openhands'] - platform: ['amd64', 'arm64'] steps: - name: Checkout uses: actions/checkout@v4 @@ -54,105 +47,15 @@ jobs: swap-storage: true - name: Set up QEMU uses: docker/setup-qemu-action@v3 + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - name: Build and export image id: build - run: ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} ${{ matrix.platform }} - - name: Capture tags - id: capture-tags - run: | - tags=$(cat tags.txt) - echo "tags=$tags" - echo "tags=$tags" >> $GITHUB_OUTPUT - - name: Capture last tag - id: capture-last-tag - run: | - last_tag=$(cat tags.txt | awk '{print $NF}') - echo "last_tag=$last_tag" - echo "last_tag=$last_tag" >> $GITHUB_OUTPUT - - name: Upload Docker image as artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.image }}_${{ steps.capture-last-tag.outputs.last_tag }}_${{ matrix.platform }} - path: /tmp/${{ matrix.image }}_${{ steps.capture-last-tag.outputs.last_tag }}_${{ matrix.platform }}.tar - retention-days: 14 - - # Push the OpenHands Docker images to the ghcr.io repository - ghcr_push: - name: Push App Image - runs-on: ubuntu-latest - needs: [ghcr_build] - if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') || (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') - env: - tags: ${{ needs.ghcr_build.outputs.tags }} - permissions: - contents: read - packages: write - strategy: - matrix: - image: ['openhands'] - last_tag: ['${{ needs.ghcr_build.outputs.last_tag }}'] - platform: ['amd64', 'arm64'] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Download Docker images - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }} - path: /tmp - - name: Load images and push to registry - run: | - mv /tmp/${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar . - loaded_image=$(docker load -i ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar | grep "Loaded image:" | head -n 1 | awk '{print $3}') - echo "loaded image = $loaded_image" - tags=$(echo ${tags} | tr ' ' '\n') - image_name=$(echo "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}" | tr '[:upper:]' '[:lower:]') - echo "image name = $image_name" - for tag in $tags; do - echo "tag = $tag" - docker tag $loaded_image $image_name:${tag}_${{ matrix.platform }} - docker push $image_name:${tag}_${{ matrix.platform }} - done - # Creates and pushes the OpenHands Docker image manifests - create_manifest: - name: Create Manifest - runs-on: ubuntu-latest - needs: [ghcr_build, ghcr_push] - if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') || (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') - env: - tags: ${{ needs.ghcr_build.outputs.tags }} - strategy: - matrix: - image: ['openhands'] - permissions: - contents: read - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Create and push multi-platform manifest - run: | - image_name=$(echo "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}" | tr '[:upper:]' '[:lower:]') - echo "image name = $image_name" - tags=$(echo ${tags} | tr ' ' '\n') - for tag in $tags; do - echo 'tag = $tag' - docker buildx imagetools create --tag $image_name:$tag \ - $image_name:${tag}_amd64 \ - $image_name:${tag}_arm64 - done + run: ./containers/build.sh openhands ${{ github.repository_owner }} --push diff --git a/.github/workflows/ghcr_runtime.yml b/.github/workflows/ghcr_runtime.yml index 7fbae04ffa..660e7a4788 100644 --- a/.github/workflows/ghcr_runtime.yml +++ b/.github/workflows/ghcr_runtime.yml @@ -31,11 +31,7 @@ jobs: packages: write strategy: matrix: - image: ['runtime'] base_image: ['nikolaik/python-nodejs:python3.11-nodejs22', 'python:3.11-bookworm', 'node:22-bookworm'] - platform: ['amd64', 'arm64'] - outputs: - tags: ${{ steps.capture-tags.outputs.tags }} steps: - name: Checkout uses: actions/checkout@v4 @@ -55,6 +51,12 @@ jobs: swap-storage: true - name: Set up QEMU uses: docker/setup-qemu-action@v3 + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -72,77 +74,17 @@ jobs: - name: Build and export image id: build run: | - if [ -f 'containers/runtime/Dockerfile' ]; then - echo 'Dockerfile detected, building runtime image...' - ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} ${{ matrix.platform }} - # Capture the last tag to use in the artifact name - last_tag=$(cat tags.txt | awk '{print $NF}') - else - echo 'No Dockerfile detected which means an exact image is already built. Pulling the image and saving it to a tar file...' - source containers/runtime/config.sh - echo "$DOCKER_IMAGE_HASH_TAG $DOCKER_IMAGE_TAG" >> tags.txt - export last_tag=$DOCKER_IMAGE_TAG - echo "Pulling image $DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE:$DOCKER_IMAGE_HASH_TAG to /tmp/${{ matrix.image }}_${last_tag}_${{ matrix.platform }}.tar" - docker pull $DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE:$DOCKER_IMAGE_HASH_TAG - docker save $DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE:$DOCKER_IMAGE_HASH_TAG -o /tmp/${{ matrix.image }}_${last_tag}_${{ matrix.platform }}.tar - fi - echo "last_tag=${last_tag}" >> $GITHUB_OUTPUT - - name: Capture tags - id: capture-tags - run: | - tags=$(cat tags.txt) - echo "tags=$tags" - echo "tags=$tags" >> $GITHUB_OUTPUT - - name: Upload Docker image as artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.image }}_${{ steps.build.outputs.last_tag }}_${{ matrix.platform }} - path: /tmp/${{ matrix.image }}_${{ steps.build.outputs.last_tag }}_${{ matrix.platform }}.tar - retention-days: 14 - - name: Capture last tag - id: capture-last-tag - run: | - last_tag=$(cat tags.txt | awk '{print $NF}') - echo "$last_tag" > /tmp/last-tag-${{ matrix.image }}-${{ matrix.platform }}-${{ steps.build.outputs.last_tag }}.txt - echo "Saved last tag to /tmp/last-tag-${{ matrix.image }}-${{ matrix.platform }}-${{ steps.build.outputs.last_tag }}.txt" - - name: Upload last tag as artifact - uses: actions/upload-artifact@v4 - with: - name: last-tag-${{ matrix.image }}-${{ matrix.platform }}-${{ steps.build.outputs.last_tag }} - path: /tmp/last-tag-${{ matrix.image }}-${{ matrix.platform }}-${{ steps.build.outputs.last_tag }}.txt - retention-days: 1 - - prepare_test_image_tags: - name: Prepare Test Images Tags - needs: ghcr_build_runtime - runs-on: ubuntu-latest - outputs: - test_image_tags: ${{ steps.set-matrix.outputs.test_image_tags }} - steps: - - name: Download last tags - uses: actions/download-artifact@v4 - with: - pattern: last-tag-* - path: /tmp/ - merge-multiple: true - - name: Set up test matrix - id: set-matrix - run: | - matrix=$(cat /tmp/last-tag-*.txt | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "test_image_tags=$matrix" >> $GITHUB_OUTPUT - echo "Generated test_image_tags: $matrix" + suffix=$(echo "${{ matrix.base_image }}" | cut -d ':' -f 1 | cut -d '/' -f 1) + ./containers/build.sh runtime ${{ github.repository_owner }} --push $suffix # Run unit tests with the EventStream runtime Docker images test_runtime: name: Test Runtime runs-on: ubuntu-latest - needs: prepare_test_image_tags + needs: [ghcr_build_runtime] strategy: matrix: - image: ['runtime'] - runtime_type: ['eventstream'] - platform: ['amd64'] - last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }} + base_image: ['nikolaik', 'python', 'node'] steps: - uses: actions/checkout@v4 - name: Free Disk Space (Ubuntu) @@ -163,35 +105,17 @@ jobs: cache: 'poetry' - name: Install Python dependencies using Poetry run: make install-python-dependencies - - name: Download Runtime Docker image - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }} - path: /tmp/ - - name: Load Runtime image and run runtime tests + - name: Run runtime tests run: | - image_file=$(find /tmp -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1) + git_hash=$(git rev-parse --short "$GITHUB_SHA") + image_name=ghcr.io/${{ github.repository_owner }}/runtime:$git_hash-${{ matrix.base_image }} + image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]') - if [ -z "$image_file" ]; then - echo "No matching image file found for tag: ${{ matrix.last_tag }}" - exit 1 - fi - - echo "Loading image from file: $image_file" - output=$(docker load -i "$image_file") - - # Extract the image name from the output - # Print all tags - echo "All tags:" - all_tags=$(echo "$output" | grep -oP 'Loaded image: \K.*') - echo "$all_tags" - # Choose the last tag - image_name=$(echo "$all_tags" | tail -n 1) - - # Print the full name of the image - echo "Loaded Docker image: $image_name" - - TEST_RUNTIME=${{ matrix.runtime_type }} SANDBOX_USER_ID=$(id -u) SANDBOX_CONTAINER_IMAGE=$image_name TEST_IN_CI=true poetry run pytest --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime + TEST_RUNTIME=eventstream \ + SANDBOX_USER_ID=$(id -u) \ + SANDBOX_CONTAINER_IMAGE=$image_name \ + TEST_IN_CI=true \ + poetry run pytest --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 env: @@ -201,14 +125,11 @@ jobs: runtime_integration_tests_on_linux: name: Runtime Integration Tests on Linux runs-on: ubuntu-latest - needs: prepare_test_image_tags + needs: [ghcr_build_runtime] strategy: fail-fast: false matrix: - image: ['runtime'] - runtime_type: ['eventstream'] - platform: ['amd64'] - last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }} + base_image: ['nikolaik', 'python', 'node'] steps: - uses: actions/checkout@v4 - name: Install poetry via pipx @@ -220,30 +141,18 @@ jobs: cache: 'poetry' - name: Install Python dependencies using Poetry run: make install-python-dependencies - - name: Download Runtime Docker image - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }} - path: /tmp/ - - name: Load runtime image and run integration tests + - name: Run integration tests run: | - image_file=$(find /tmp -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1) + git_hash=$(git rev-parse --short "$GITHUB_SHA") + image_name=ghcr.io/${{ github.repository_owner }}/runtime:$git_hash-${{ matrix.base_image }} + image_name=$(echo $image_name | tr '[:upper:]' '[:lower:]') - if [ -z "$image_file" ]; then - echo "No matching image file found for tag: ${{ matrix.last_tag }}" - exit 1 - fi - - echo "Loading image from file: $image_file" - output=$(docker load -i "$image_file") - - # Extract the image name from the output - image_name=$(echo "$output" | grep -oP 'Loaded image: \K.*' | head -n 1) - - # Print the full name of the image - echo "Loaded Docker image: $image_name" - - TEST_RUNTIME=${{ matrix.runtime_type }} SANDBOX_USER_ID=$(id -u) SANDBOX_CONTAINER_IMAGE=$image_name TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh + TEST_RUNTIME=eventstream \ + SANDBOX_USER_ID=$(id -u) \ + SANDBOX_CONTAINER_IMAGE=$image_name \ + TEST_IN_CI=true \ + TEST_ONLY=true \ + ./tests/integration/regenerate.sh - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 env: @@ -257,105 +166,3 @@ jobs: steps: - name: All tests passed run: echo "All runtime tests have passed successfully!" - - # Push the runtime Docker images to the ghcr.io repository - ghcr_push_runtime: - name: Push Image - runs-on: ubuntu-latest - needs: [ghcr_build_runtime, prepare_test_image_tags, all_runtime_tests_passed] - if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') || (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') - env: - RUNTIME_TAGS: ${{ needs.ghcr_build_runtime.outputs.tags }} - permissions: - contents: read - packages: write - strategy: - matrix: - image: ['runtime'] - runtime_type: ['eventstream'] - platform: ['amd64', 'arm64'] - last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - tool-cache: true - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: false - swap-storage: true - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Download Docker images - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }} - path: /tmp/ - - name: Load images and push to registry - run: | - image_file=$(find /tmp -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1) - if [ -z "$image_file" ]; then - echo "No matching image file found for tag: ${{ matrix.last_tag }}" - exit 1 - fi - - echo "Loading image from file: $image_file" - if ! loaded_image=$(docker load -i "$image_file" | grep "Loaded image:" | head -n 1 | awk '{print $3}'); then - echo "Failed to load Docker image" - exit 1 - fi - echo "loaded image = $loaded_image" - image_name=$(echo "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}" | tr '[:upper:]' '[:lower:]') - echo "image name = $image_name" - echo "$RUNTIME_TAGS" | tr ' ' '\n' | while read -r tag; do - echo "tag = $tag" - if [ -n "$image_name" ] && [ -n "$tag" ]; then - docker tag $loaded_image $image_name:${tag}_${{ matrix.platform }} - docker push $image_name:${tag}_${{ matrix.platform }} - else - echo "Skipping tag and push due to empty image_name or tag" - fi - done - - # Creates and pushes the runtime Docker image manifest - create_manifest_runtime: - name: Create Manifest - runs-on: ubuntu-latest - needs: [ghcr_build_runtime, prepare_test_image_tags, ghcr_push_runtime] - if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') || (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') - env: - tags: ${{ needs.ghcr_build_runtime.outputs.tags }} - strategy: - matrix: - image: ['runtime'] - permissions: - contents: read - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Create and push multi-platform manifest - run: | - image_name=$(echo "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}" | tr '[:upper:]' '[:lower:]') - echo "image name = $image_name" - tags=$(echo ${tags} | tr ' ' '\n') - for tag in $tags; do - echo 'tag = $tag' - docker buildx imagetools create --tag $image_name:$tag \ - $image_name:${tag}_amd64 \ - $image_name:${tag}_arm64 - done diff --git a/containers/build.sh b/containers/build.sh index 420a050fb9..3624eadc2a 100755 --- a/containers/build.sh +++ b/containers/build.sh @@ -3,13 +3,25 @@ set -eo pipefail image_name=$1 org_name=$2 -platform=$3 +push=0 +if [[ $3 == "--push" ]]; then + push=1 +fi +tag_suffix=$4 -echo "Building: $image_name for platform: $platform" +echo "Building: $image_name" tags=() OPENHANDS_BUILD_VERSION="dev" +cache_tag_base="buildcache" +cache_tag="$cache_tag_base" + +if [[ -n $GITHUB_SHA ]]; then + git_hash=$(git rev-parse --short "$GITHUB_SHA") + tags+=("$git_hash") +fi + if [[ -n $GITHUB_REF_NAME ]]; then # check if ref name is a version number if [[ $GITHUB_REF_NAME =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -18,11 +30,20 @@ if [[ -n $GITHUB_REF_NAME ]]; then tags+=("$major_version" "$minor_version") tags+=("latest") fi - sanitized=$(echo "$GITHUB_REF_NAME" | sed 's/[^a-zA-Z0-9.-]\+/-/g') - OPENHANDS_BUILD_VERSION=$sanitized - tag=$(echo "$sanitized" | tr '[:upper:]' '[:lower:]') # lower case is required in tagging - tags+=("$tag") + sanitized_ref_name=$(echo "$GITHUB_REF_NAME" | sed 's/[^a-zA-Z0-9.-]\+/-/g') + OPENHANDS_BUILD_VERSION=$sanitized_ref_name + sanitized_ref_name=$(echo "$sanitized_ref_name" | tr '[:upper:]' '[:lower:]') # lower case is required in tagging + tags+=("$sanitized_ref_name") + cache_tag+="-${sanitized_ref_name}" fi + +if [[ -n $tag_suffix ]]; then + cache_tag+="-${tag_suffix}" + for i in "${!tags[@]}"; do + tags[$i]="${tags[$i]}-$tag_suffix" + done +fi + echo "Tags: ${tags[@]}" if [[ "$image_name" == "openhands" ]]; then @@ -68,16 +89,19 @@ for tag in "${tags[@]}"; do args+=" -t $DOCKER_REPOSITORY:$tag" done -output_image="/tmp/${image_name}_${tags[-1]}_${platform}.tar" -echo "Output image will be saved to: $output_image" +if [[ $push -eq 1 ]]; then + args+=" --push" + args+=" --cache-to=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag,mode=max" +fi + +echo "Args: $args" docker buildx build \ $args \ --build-arg OPENHANDS_BUILD_VERSION="$OPENHANDS_BUILD_VERSION" \ - --platform linux/$platform \ + --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag \ + --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag_base-main \ + --platform linux/amd64,linux/arm64 \ --provenance=false \ -f "$dir/Dockerfile" \ - --output type=docker,dest="$output_image" \ "$DOCKER_BASE_DIR" - -echo "${tags[*]}" > tags.txt diff --git a/openhands/runtime/utils/runtime_build.py b/openhands/runtime/utils/runtime_build.py index 5a4270903a..70bfc9b080 100644 --- a/openhands/runtime/utils/runtime_build.py +++ b/openhands/runtime/utils/runtime_build.py @@ -243,7 +243,7 @@ def build_runtime_image( # Scenario 1: If we already have an image with the exact same hash, then it means the image is already built # with the exact same source code and Dockerfile, so we will reuse it. Building it is not required. - if runtime_builder.image_exists(hash_runtime_image_name): + if not force_rebuild and runtime_builder.image_exists(hash_runtime_image_name): logger.info( f'Image [{hash_runtime_image_name}] already exists so we will reuse it.' )