Runtime build fixes for OpenHands as a python library (#3989)

.github/workflows/pypi-release.yml

@@ -26,6 +26,6 @@ jobs:
       - name: Install Poetry Dependencies
         run: poetry install --no-interaction --no-root
       - name: Build poetry project
-        run: poetry build -v
+        run: ./build.sh
       - name: publish
         run: poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }}

build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+cp pyproject.toml poetry.lock openhands
+poetry build -v

openhands/__init__.py

@@ -1,12 +1,14 @@
 import os
 
+__package_name__ = 'openhands_ai'
+
 
 def get_version():
     try:
         from importlib.metadata import PackageNotFoundError, version
 
         try:
-            return version('openhands-ai')
+            return version(__package_name__)
         except PackageNotFoundError:
             pass
     except ImportError:
@@ -16,7 +18,7 @@ def get_version():
         from pkg_resources import DistributionNotFound, get_distribution
 
         try:
-            return get_distribution('openhands-ai').version
+            return get_distribution(__package_name__).version
         except DistributionNotFound:
             pass
     except ImportError:

openhands/runtime/utils/runtime_build.py

@@ -1,8 +1,8 @@
 import argparse
 import hashlib
+import importlib.metadata
 import os
 import shutil
-import subprocess
 import tempfile
 
 import docker
@@ -10,6 +10,7 @@ from dirhash import dirhash
 from jinja2 import Environment, FileSystemLoader
 
 import openhands
+from openhands import __package_name__
 from openhands import __version__ as oh_version
 from openhands.core.logger import openhands_logger as logger
 from openhands.runtime.builder import DockerRuntimeBuilder, RuntimeBuilder
@@ -22,55 +23,50 @@ def get_runtime_image_repo():
 def _put_source_code_to_dir(temp_dir: str):
     """Builds the project source tarball directly in temp_dir and unpacks it.
     The OpenHands source code ends up in the temp_dir/code directory.
-
     Parameters:
     - temp_dir (str): The directory to put the source code in
     """
     if not os.path.isdir(temp_dir):
         raise RuntimeError(f'Temp directory {temp_dir} does not exist')
 
-    project_root = os.path.dirname(os.path.dirname(os.path.abspath(openhands.__file__)))
-    logger.info(f'Building source distribution using project root: {project_root}')
+    dest_dir = os.path.join(temp_dir, 'code')
+    openhands_dir = None
 
-    # Fetch the correct version from pyproject.toml
-    package_version = oh_version
-    tarball_filename = f'openhands_ai-{package_version}.tar.gz'
-    tarball_path = os.path.join(temp_dir, tarball_filename)
+    try:
+        # Try to get the source directory from the installed package
+        distribution = importlib.metadata.distribution(__package_name__)
+        source_dir = os.path.dirname(distribution.locate_file(__package_name__))
+        openhands_dir = os.path.join(source_dir, 'openhands')
+    except importlib.metadata.PackageNotFoundError:
+        pass
 
-    # Run "python -m build -s" on project_root to create project tarball directly in temp_dir
-    _cleaned_project_root = project_root.replace(
-        ' ', r'\ '
-    )  # escape spaces in the project root
-    result = subprocess.run(
-        f'python -m build -s -o "{temp_dir}" {_cleaned_project_root}',
-        shell=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-    logger.info(result.stdout.decode())
-    err_logs = result.stderr.decode()
-    if err_logs:
-        logger.error(err_logs)
+    if openhands_dir is not None and os.path.isdir(openhands_dir):
+        logger.info(f'Package {__package_name__} found')
+        shutil.copytree(openhands_dir, os.path.join(dest_dir, 'openhands'))
+        # note: "pyproject.toml" and "poetry.lock" are included in the openhands
+        # package, so we need to move them out to the top-level directory
+        for filename in ['pyproject.toml', 'poetry.lock']:
+            shutil.move(os.path.join(dest_dir, 'openhands', filename), dest_dir)
+    else:
+        # If package is not found, build from source code
+        project_root = os.path.dirname(
+            os.path.dirname(os.path.abspath(openhands.__file__))
+        )
+        logger.info(f'Building source distribution using project root: {project_root}')
 
-    if result.returncode != 0:
-        logger.error(f'Image build failed:\n{result}')
-        raise RuntimeError(f'Image build failed:\n{result}')
+        # Copy the 'openhands' directory
+        openhands_dir = os.path.join(project_root, 'openhands')
+        if not os.path.isdir(openhands_dir):
+            raise RuntimeError(f"'openhands' directory not found in {project_root}")
+        shutil.copytree(openhands_dir, os.path.join(dest_dir, 'openhands'))
 
-    if not os.path.exists(tarball_path):
-        logger.error(f'Source distribution not found at {tarball_path}. (Do you need to run `make build`?)')
-        raise RuntimeError(f'Source distribution not found at {tarball_path}')
-    logger.info(f'Source distribution created at {tarball_path}')
+        # Copy pyproject.toml and poetry.lock files
+        for file in ['pyproject.toml', 'poetry.lock']:
+            src_file = os.path.join(project_root, file)
+            dest_file = os.path.join(dest_dir, file)
+            shutil.copy2(src_file, dest_file)
 
-    # Unzip the tarball
-    shutil.unpack_archive(tarball_path, temp_dir)
-    # Remove the tarball
-    os.remove(tarball_path)
-    # Rename the directory containing the code to 'code'
-    os.rename(
-        os.path.join(temp_dir, f'openhands_ai-{package_version}'),
-        os.path.join(temp_dir, 'code'),
-    )
-    logger.info(f'Unpacked source code directory: {os.path.join(temp_dir, "code")}')
+    logger.info(f'Unpacked source code directory: {dest_dir}')
 
 
 def _generate_dockerfile(

pyproject.toml

@@ -6,7 +6,6 @@ authors = ["OpenHands"]
 license = "MIT"
 readme = "README.md"
 repository = "https://github.com/All-Hands-AI/OpenHands"
-include = ["poetry.lock"]
 packages = [
   { include = "openhands/**/*" }
 ]

tests/unit/test_runtime_build.py

@@ -54,14 +54,12 @@ def _check_source_code_in_dir(temp_dir):
     # check the source file is the same as the current code base
     assert os.path.exists(os.path.join(code_dir, 'pyproject.toml'))
 
-    # The source code should only include the `openhands` folder, but not the other folders
+    # The source code should only include the `openhands` folder,
+    # and pyproject.toml & poetry.lock that are needed to build the runtime image
     assert set(os.listdir(code_dir)) == {
         'openhands',
         'pyproject.toml',
         'poetry.lock',
-        'LICENSE',
-        'README.md',
-        'PKG-INFO',
     }
     assert os.path.exists(os.path.join(code_dir, 'openhands'))
     assert os.path.isdir(os.path.join(code_dir, 'openhands'))