github/OpenHands
1
0
Fork 0
mirror of https://github.com/OpenHands/OpenHands.git synced 2025-12-26 05:48:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Split container image build & push (#2456)

Browse Source 
* Split container image build & push

* Code cleanup

* Cleanup

* Add back useless docker_build_success step to make CI happy

* Revert "Cleanup"

This reverts commit 2a260791a95a110a54335141d017a418397eecf9.

* Use fresh built sandbox image in integration test

* fix dependency

* DEBUG: only build

* Attempt to fix dependency

* Change dependency

* Combine both jobs

* Fix env

* Remove Mac integration tests as they are too unstable

* Move sandbox tests to ghcr

* Use loaded image
This commit is contained in:
Boxuan Li 2024-06-18 10:44:51 -07:00 committed by GitHub
parent 5e8ef23a53
commit c2fa99b4a1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 230 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
.github/workflows/ghcr.yml vendored
View File

@ -1,4 +1,4 @@
name: Publish Docker Image
name: Build Publish and Test Docker Image
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@ -7,7 +7,7 @@ concurrency:
on:
push:
branches:
- main
- main
tags:
- '*'
pull_request:
@ -19,19 +19,23 @@ on:
default: ''
jobs:
ghcr_build_and_push:
ghcr_build:
runs-on: ubuntu-latest
outputs:
tags: ${{ steps.capture-tags.outputs.tags }}
permissions:
contents: read
packages: write
strategy:
matrix:
image: ["app", "sandbox"]
image: ["sandbox", "opendevin"]
platform: ["amd64", "arm64"]
steps:
- name: checkout
- name: Checkout
uses: actions/checkout@v4
- name: Free Disk Space (Ubuntu)
@ -40,7 +44,6 @@ jobs:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: true
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
@ -57,26 +60,207 @@ jobs:
id: buildx
uses: docker/setup-buildx-action@v3
- name: Login to ghcr
uses: docker/login-action@v1
- name: Build and export image
id: build
run: ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} ${{ matrix.platform }}
- name: Capture tags
id: capture-tags
run: |
tags=$(cat tags.txt)
echo "tags=$tags"
echo "tags=$tags" >> $GITHUB_OUTPUT
- name: Upload Docker image as artifact
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.image }}-docker-image-${{ matrix.platform }}
path: /tmp/${{ matrix.image }}_image_${{ matrix.platform }}.tar
test-for-sandbox:
name: Test for Sandbox
runs-on: ubuntu-latest
needs: ghcr_build
env:
PERSIST_SANDBOX: "false"
steps:
- uses: actions/checkout@v4
- name: Install poetry via pipx
run: pipx install poetry
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
cache: "poetry"
- name: Install Python dependencies using Poetry
run: make install-python-dependencies
- name: Download sandbox Docker image
uses: actions/download-artifact@v4
with:
name: sandbox-docker-image-amd64
path: /tmp/
- name: Load sandbox image and run sandbox tests
run: |
# Load the Docker image and capture the output
output=$(docker load -i /tmp/sandbox_image_amd64.tar)
# Extract the image name from the output
image_name=$(echo "$output" | grep -oP 'Loaded image: \K.*')
# Print the full name of the image
echo "Loaded Docker image: $image_name"
SANDBOX_CONTAINER_IMAGE=$image_name poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml -s ./tests/unit/test_sandbox.py
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
integration-tests-on-linux:
name: Integration Tests on Linux
runs-on: ubuntu-latest
needs: ghcr_build
env:
PERSIST_SANDBOX: "false"
strategy:
fail-fast: false
matrix:
python-version: ["3.11"]
sandbox: ["ssh", "exec", "local"]
steps:
- uses: actions/checkout@v4
- name: Install poetry via pipx
run: pipx install poetry
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'
- name: Install Python dependencies using Poetry
run: make install-python-dependencies
- name: Download sandbox Docker image
uses: actions/download-artifact@v4
with:
name: sandbox-docker-image-amd64
path: /tmp/
- name: Load sandbox image and run integration tests
env:
SANDBOX_TYPE: ${{ matrix.sandbox }}
run: |
# Load the Docker image and capture the output
output=$(docker load -i /tmp/sandbox_image_amd64.tar)
# Extract the image name from the output
image_name=$(echo "$output" | grep -oP 'Loaded image: \K.*')
# Print the full name of the image
echo "Loaded Docker image: $image_name"
SANDBOX_CONTAINER_IMAGE=$image_name TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
ghcr_push:
runs-on: ubuntu-latest
# don't push if integration tests or sandbox tests fail
needs: [ghcr_build, integration-tests-on-linux, test-for-sandbox]
if: github.ref == 'refs/heads/main'
env:
tags: ${{ needs.ghcr_build.outputs.tags }}
permissions:
contents: read
packages: write
strategy:
matrix:
image: ["sandbox", "opendevin"]
platform: ["amd64", "arm64"]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Login to GHCR
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push ${{ matrix.image }}
if: "!github.event.pull_request.head.repo.fork"
run: |
./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} --push
- name: Download Docker images
uses: actions/download-artifact@v4
with:
name: ${{ matrix.image }}-docker-image-${{ matrix.platform }}
path: /tmp/${{ matrix.platform }}
- name: Build ${{ matrix.image }}
if: "github.event.pull_request.head.repo.fork"
- name: Load images and push to registry
run: |
./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }}
mv /tmp/${{ matrix.platform }}/${{ matrix.image }}_image_${{ matrix.platform }}.tar .
loaded_image=$(docker load -i ${{ matrix.image }}_image_${{ matrix.platform }}.tar | grep "Loaded image:" | awk '{print $3}')
tags=$(echo ${tags} | tr ' ' '\n')
for tag in $tags; do
echo "tag = $tag"
docker tag $loaded_image ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_${{ matrix.platform }}
docker push ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_${{ matrix.platform }}
done
create_manifest:
runs-on: ubuntu-latest
needs: [ghcr_build, ghcr_push]
if: github.ref == 'refs/heads/main'
env:
tags: ${{ needs.ghcr_build.outputs.tags }}
strategy:
matrix:
image: ["sandbox", "opendevin"]
permissions:
contents: read
packages: write
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Login to GHCR
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create and push multi-platform manifest
run: |
tags=$(echo ${tags} | tr ' ' '\n')
for tag in $tags; do
echo 'tag = $tag'
docker buildx imagetools create --tag ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:$tag \
ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_amd64 \
ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_arm64
done
# FIXME: an admin needs to mark this as non-mandatory, and then we can remove it
docker_build_success:
name: Docker Build Success
runs-on: ubuntu-latest
needs: ghcr_build_and_push
needs: ghcr_build
steps:
- run: echo Done!

104
.github/workflows/run-integration-tests.yml vendored
View File

@ -1,104 +0,0 @@
name: Run Integration Tests
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
on:
push:
branches:
- main
paths-ignore:
- '**/*.md'
- 'frontend/**'
- 'docs/**'
- 'evaluation/**'
pull_request:
env:
PERSIST_SANDBOX : "false"
jobs:
integration-tests-on-linux:
name: Integration Tests on Linux
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.11"]
sandbox: ["ssh", "exec", "local"]
steps:
- uses: actions/checkout@v4
- name: Install poetry via pipx
run: pipx install poetry
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'
- name: Install Python dependencies using Poetry
run: poetry install
- name: Build Environment
run: make build
- name: Run Integration Tests
env:
SANDBOX_TYPE: ${{ matrix.sandbox }}
run: |
TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
integration-tests-on-mac:
name: Integration Tests on MacOS
runs-on: macos-13
if: contains(github.event.pull_request.title, 'mac') || contains(github.event.pull_request.title, 'Mac')
strategy:
fail-fast: false
matrix:
python-version: ["3.11"]
sandbox: ["ssh"]
steps:
- uses: actions/checkout@v4
- name: Install poetry via pipx
run: pipx install poetry
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'
- name: Install Python dependencies using Poetry
run: poetry install
- name: Install & Start Docker
run: |
brew install colima docker
colima start
# For testcontainers to find the Colima socket
# https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running
sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
- name: Build Environment
run: make build
- name: Run Integration Tests
env:
SANDBOX_TYPE: ${{ matrix.sandbox }}
run: |
TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

30
.github/workflows/run-unit-tests.yml vendored
View File

@ -97,33 +97,3 @@ jobs:
uses: codecov/codecov-action@v4
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test-for-sandbox:
name: Test for Sandbox
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install poetry via pipx
run: pipx install poetry
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
cache: "poetry"
- name: Install Python dependencies using Poetry
run: poetry install
- name: Build Environment
run: make build
- name: Run Integration Test for Sandbox
run: |
poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml -s ./tests/unit/test_sandbox.py
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

55
containers/build.sh
View File

@ -3,12 +3,9 @@ set -eo pipefail
image_name=$1
org_name=$2
push=0
if [[ $3 == "--push" ]]; then
push=1
fi
platform=$3
echo -e "Building: $image_name"
echo "Building: $image_name for platform: $platform"
tags=()
OPEN_DEVIN_BUILD_VERSION="dev"
@ -19,49 +16,57 @@ cache_tag="$cache_tag_base"
if [[ -n $GITHUB_REF_NAME ]]; then
# check if ref name is a version number
if [[ $GITHUB_REF_NAME =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
major_version=$(echo $GITHUB_REF_NAME | cut -d. -f1)
minor_version=$(echo $GITHUB_REF_NAME | cut -d. -f1,2)
tags+=($major_version $minor_version)
major_version=$(echo "$GITHUB_REF_NAME" | cut -d. -f1)
minor_version=$(echo "$GITHUB_REF_NAME" | cut -d. -f1,2)
tags+=("$major_version" "$minor_version")
fi
sanitized=$(echo $GITHUB_REF_NAME | sed 's/[^a-zA-Z0-9.-]\+/-/g')
sanitized=$(echo "$GITHUB_REF_NAME" | sed 's/[^a-zA-Z0-9.-]\+/-/g')
OPEN_DEVIN_BUILD_VERSION=$sanitized
cache_tag+="-${sanitized}"
tags+=($sanitized)
tags+=("$sanitized")
fi
echo "Tags: ${tags[@]}"
dir=./containers/$image_name
if [ ! -f $dir/Dockerfile ]; then
if [[ "$image_name" == "opendevin" ]]; then
dir="./containers/app"
else
dir="./containers/$image_name"
fi
if [[ ! -f "$dir/Dockerfile" ]]; then
echo "No Dockerfile found"
exit 1
fi
if [ ! -f $dir/config.sh ]; then
if [[ ! -f "$dir/config.sh" ]]; then
echo "No config.sh found for Dockerfile"
exit 1
fi
source $dir/config.sh
source "$dir/config.sh"
if [[ -n "$org_name" ]]; then
DOCKER_ORG="$org_name"
fi
DOCKER_REPOSITORY=$DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE
DOCKER_REPOSITORY="$DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE"
DOCKER_REPOSITORY=${DOCKER_REPOSITORY,,} # lowercase
echo "Repo: $DOCKER_REPOSITORY"
echo "Base dir: $DOCKER_BASE_DIR"
args=""
for tag in ${tags[@]}; do
for tag in "${tags[@]}"; do
args+=" -t $DOCKER_REPOSITORY:$tag"
done
if [[ $push -eq 1 ]]; then
args+=" --push"
args+=" --cache-to=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag,mode=max"
fi
output_image="/tmp/${image_name}_image_${platform}.tar"
docker buildx build \
$args \
--build-arg OPEN_DEVIN_BUILD_VERSION=$OPEN_DEVIN_BUILD_VERSION \
--cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag \
--cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag_base-main \
--platform linux/amd64,linux/arm64 \
--build-arg OPEN_DEVIN_BUILD_VERSION="$OPEN_DEVIN_BUILD_VERSION" \
--platform linux/$platform \
--provenance=false \
-f $dir/Dockerfile $DOCKER_BASE_DIR
-f "$dir/Dockerfile" \
--output type=docker,dest="$output_image" \
"$DOCKER_BASE_DIR"
echo "${tags[*]}" > tags.txt