diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml deleted file mode 100644 index e8d318c3e6..0000000000 --- a/.github/workflows/integration-runner.yml +++ /dev/null @@ -1,199 +0,0 @@ -name: Run Integration Tests - -on: - pull_request: - types: [labeled] - workflow_dispatch: - inputs: - reason: - description: 'Reason for manual trigger' - required: true - default: '' - schedule: - - cron: '30 22 * * *' # Runs at 10:30pm UTC every day - -env: - N_PROCESSES: 10 # Global configuration for number of parallel processes for evaluation - -jobs: - run-integration-tests: - if: github.event.label.name == 'integration-test' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' - runs-on: blacksmith-4vcpu-ubuntu-2204 - permissions: - contents: "read" - id-token: "write" - pull-requests: "write" - issues: "write" - strategy: - matrix: - python-version: ["3.12"] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install poetry via pipx - run: pipx install poetry - - - name: Set up Python - uses: useblacksmith/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - cache: "poetry" - - - name: Setup Node.js - uses: useblacksmith/setup-node@v5 - with: - node-version: '22.x' - - - name: Comment on PR if 'integration-test' label is present - if: github.event_name == 'pull_request' && github.event.label.name == 'integration-test' - uses: KeisukeYamashita/create-comment@v1 - with: - unique: false - comment: | - Hi! I started running the integration tests on your PR. You will receive a comment with the results shortly. - - - name: Install Python dependencies using Poetry - run: poetry install --with dev,test,runtime,evaluation - - - name: Configure config.toml for testing with Haiku - env: - LLM_MODEL: "litellm_proxy/claude-3-5-haiku-20241022" - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} - MAX_ITERATIONS: 10 - run: | - echo "[llm.eval]" > config.toml - echo "model = \"$LLM_MODEL\"" >> config.toml - echo "api_key = \"$LLM_API_KEY\"" >> config.toml - echo "base_url = \"$LLM_BASE_URL\"" >> config.toml - echo "temperature = 0.0" >> config.toml - - - name: Build environment - run: make build - - - name: Run integration test evaluation for Haiku - env: - SANDBOX_FORCE_REBUILD_RUNTIME: True - run: | - poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' 10 $N_PROCESSES '' 'haiku_run' - - # get integration tests report - REPORT_FILE_HAIKU=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/*haiku*_maxiter_10_N* -name "report.md" -type f | head -n 1) - echo "REPORT_FILE: $REPORT_FILE_HAIKU" - echo "INTEGRATION_TEST_REPORT_HAIKU<> $GITHUB_ENV - cat $REPORT_FILE_HAIKU >> $GITHUB_ENV - echo >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Wait a little bit - run: sleep 10 - - - name: Configure config.toml for testing with DeepSeek - env: - LLM_MODEL: "litellm_proxy/deepseek-chat" - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} - MAX_ITERATIONS: 10 - run: | - echo "[llm.eval]" > config.toml - echo "model = \"$LLM_MODEL\"" >> config.toml - echo "api_key = \"$LLM_API_KEY\"" >> config.toml - echo "base_url = \"$LLM_BASE_URL\"" >> config.toml - echo "temperature = 0.0" >> config.toml - - - name: Run integration test evaluation for DeepSeek - env: - SANDBOX_FORCE_REBUILD_RUNTIME: True - run: | - poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' 10 $N_PROCESSES '' 'deepseek_run' - - # get integration tests report - REPORT_FILE_DEEPSEEK=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/deepseek*_maxiter_10_N* -name "report.md" -type f | head -n 1) - echo "REPORT_FILE: $REPORT_FILE_DEEPSEEK" - echo "INTEGRATION_TEST_REPORT_DEEPSEEK<> $GITHUB_ENV - cat $REPORT_FILE_DEEPSEEK >> $GITHUB_ENV - echo >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - # ------------------------------------------------------------- - # Run VisualBrowsingAgent tests for DeepSeek, limited to t05 and t06 - - name: Wait a little bit (again) - run: sleep 5 - - - name: Configure config.toml for testing VisualBrowsingAgent (DeepSeek) - env: - LLM_MODEL: "litellm_proxy/deepseek-chat" - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} - MAX_ITERATIONS: 15 - run: | - echo "[llm.eval]" > config.toml - echo "model = \"$LLM_MODEL\"" >> config.toml - echo "api_key = \"$LLM_API_KEY\"" >> config.toml - echo "base_url = \"$LLM_BASE_URL\"" >> config.toml - echo "temperature = 0.0" >> config.toml - - name: Run integration test evaluation for VisualBrowsingAgent (DeepSeek) - env: - SANDBOX_FORCE_REBUILD_RUNTIME: True - run: | - poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD VisualBrowsingAgent '' 15 $N_PROCESSES "t05_simple_browsing,t06_github_pr_browsing.py" 'visualbrowsing_deepseek_run' - - # Find and export the visual browsing agent test results - REPORT_FILE_VISUALBROWSING_DEEPSEEK=$(find evaluation/evaluation_outputs/outputs/integration_tests/VisualBrowsingAgent/deepseek*_maxiter_15_N* -name "report.md" -type f | head -n 1) - echo "REPORT_FILE_VISUALBROWSING_DEEPSEEK: $REPORT_FILE_VISUALBROWSING_DEEPSEEK" - echo "INTEGRATION_TEST_REPORT_VISUALBROWSING_DEEPSEEK<> $GITHUB_ENV - cat $REPORT_FILE_VISUALBROWSING_DEEPSEEK >> $GITHUB_ENV - echo >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Create archive of evaluation outputs - run: | - TIMESTAMP=$(date +'%y-%m-%d-%H-%M') - cd evaluation/evaluation_outputs/outputs # Change to the outputs directory - tar -czvf ../../../integration_tests_${TIMESTAMP}.tar.gz integration_tests/CodeActAgent/* integration_tests/VisualBrowsingAgent/* # Only include the actual result directories - - - name: Upload evaluation results as artifact - uses: actions/upload-artifact@v4 - id: upload_results_artifact - with: - name: integration-test-outputs-${{ github.run_id }}-${{ github.run_attempt }} - path: integration_tests_*.tar.gz - - - name: Get artifact URLs - run: | - echo "ARTIFACT_URL=${{ steps.upload_results_artifact.outputs.artifact-url }}" >> $GITHUB_ENV - - - name: Set timestamp and trigger reason - run: | - echo "TIMESTAMP=$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_ENV - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "TRIGGER_REASON=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV - elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "TRIGGER_REASON=manual-${{ github.event.inputs.reason }}" >> $GITHUB_ENV - else - echo "TRIGGER_REASON=nightly-scheduled" >> $GITHUB_ENV - fi - - - name: Comment with results and artifact link - id: create_comment - uses: KeisukeYamashita/create-comment@v1 - with: - # if triggered by PR, use PR number, otherwise use 9745 as fallback issue number for manual triggers - number: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || 9745 }} - unique: false - comment: | - Trigger by: ${{ github.event_name == 'pull_request' && format('Pull Request (integration-test label on PR #{0})', github.event.pull_request.number) || (github.event_name == 'workflow_dispatch' && format('Manual Trigger: {0}', github.event.inputs.reason)) || 'Nightly Scheduled Run' }} - Commit: ${{ github.sha }} - **Integration Tests Report (Haiku)** - Haiku LLM Test Results: - ${{ env.INTEGRATION_TEST_REPORT_HAIKU }} - --- - **Integration Tests Report (DeepSeek)** - DeepSeek LLM Test Results: - ${{ env.INTEGRATION_TEST_REPORT_DEEPSEEK }} - --- - **Integration Tests Report VisualBrowsing (DeepSeek)** - ${{ env.INTEGRATION_TEST_REPORT_VISUALBROWSING_DEEPSEEK }} - --- - Download testing outputs (includes both Haiku and DeepSeek results): [Download](${{ steps.upload_results_artifact.outputs.artifact-url }}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a605abaf64..e2338202cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ by implementing the [interface specified here](https://github.com/OpenHands/Open #### Testing When you write code, it is also good to write tests. Please navigate to the [`./tests`](./tests) folder to see existing test suites. -At the moment, we have two kinds of tests: [`unit`](./tests/unit) and [`integration`](./evaluation/integration_tests). Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project. +At the moment, we have these kinds of tests: [`unit`](./tests/unit), [`runtime`](./tests/runtime), and [`end-to-end (e2e)`](./tests/e2e). Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project. ## Sending Pull Requests to OpenHands diff --git a/evaluation/integration_tests/README.md b/evaluation/integration_tests/README.md deleted file mode 100644 index afe48d70f4..0000000000 --- a/evaluation/integration_tests/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Integration tests - -This directory implements integration tests that [was running in CI](https://github.com/OpenHands/OpenHands/tree/23d3becf1d6f5d07e592f7345750c314a826b4e9/tests/integration). - -[PR 3985](https://github.com/OpenHands/OpenHands/pull/3985) introduce LLM-based editing, which requires access to LLM to perform edit. Hence, we remove integration tests from CI and intend to run them as nightly evaluation to ensure the quality of OpenHands softwares. - -## To add new tests - -Each test is a file named like `tXX_testname.py` where `XX` is a number. -Make sure to name the file for each test to start with `t` and ends with `.py`. - -Each test should be structured as a subclass of [`BaseIntegrationTest`](./tests/base.py), where you need to implement `initialize_runtime` that setup the runtime enviornment before test, and `verify_result` that takes in a `Runtime` and history of `Event` and return a `TestResult`. See [t01_fix_simple_typo.py](./tests/t01_fix_simple_typo.py) and [t05_simple_browsing.py](./tests/t05_simple_browsing.py) for two representative examples. - -```python -class TestResult(BaseModel): - success: bool - reason: str | None = None - - -class BaseIntegrationTest(ABC): - """Base class for integration tests.""" - - INSTRUCTION: str - - @classmethod - @abstractmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - """Initialize the runtime for the test to run.""" - pass - - @classmethod - @abstractmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - """Verify the result of the test. - - This method will be called after the agent performs the task on the runtime. - """ - pass -``` - - -## Setup Environment and LLM Configuration - -Please follow instruction [here](../README.md#setup) to setup your local -development environment and LLM. - -## Start the evaluation - -```bash -./evaluation/integration_tests/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids] -``` - -- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for - your LLM settings, as defined in your `config.toml`. -- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version - you would like to evaluate. It could also be a release tag like `0.9.0`. -- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, - defaulting to `CodeActAgent`. -- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` - instances. By default, the script evaluates the entire Exercism test set - (133 issues). Note: in order to use `eval_limit`, you must also set `agent`. -- `eval-num-workers`: the number of workers to use for evaluation. Default: `1`. -- `eval_ids`, e.g. `"1,3,10"`, limits the evaluation to instances with the - given IDs (comma separated). - -Example: -```bash -./evaluation/integration_tests/scripts/run_infer.sh llm.claude-35-sonnet-eval HEAD CodeActAgent -``` diff --git a/evaluation/integration_tests/__init__.py b/evaluation/integration_tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/evaluation/integration_tests/run_infer.py b/evaluation/integration_tests/run_infer.py deleted file mode 100644 index 88d49d4055..0000000000 --- a/evaluation/integration_tests/run_infer.py +++ /dev/null @@ -1,251 +0,0 @@ -import asyncio -import importlib.util -import os - -import pandas as pd - -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import ( - EvalMetadata, - EvalOutput, - get_default_sandbox_config_for_eval, - get_metrics, - get_openhands_config_for_eval, - make_metadata, - prepare_dataset, - reset_logger_for_multiprocessing, - run_evaluation, - update_llm_config_for_completions_logging, -) -from evaluation.utils.shared import ( - codeact_user_response as fake_user_response, -) -from openhands.controller.state.state import State -from openhands.core.config import ( - AgentConfig, - OpenHandsConfig, - get_evaluation_parser, - get_llm_config_arg, -) -from openhands.core.logger import openhands_logger as logger -from openhands.core.main import create_runtime, run_controller -from openhands.events.action import MessageAction -from openhands.events.serialization.event import event_to_dict -from openhands.runtime.base import Runtime -from openhands.utils.async_utils import call_async_from_sync - -FAKE_RESPONSES = { - 'CodeActAgent': fake_user_response, - 'VisualBrowsingAgent': fake_user_response, -} - - -def get_config( - metadata: EvalMetadata, - instance_id: str, -) -> OpenHandsConfig: - sandbox_config = get_default_sandbox_config_for_eval() - sandbox_config.platform = 'linux/amd64' - config = get_openhands_config_for_eval( - metadata=metadata, - runtime=os.environ.get('RUNTIME', 'docker'), - sandbox_config=sandbox_config, - ) - config.debug = True - config.set_llm_config( - update_llm_config_for_completions_logging( - metadata.llm_config, metadata.eval_output_dir, instance_id - ) - ) - agent_config = AgentConfig( - enable_jupyter=True, - enable_browsing=True, - enable_llm_editor=False, - ) - config.set_agent_config(agent_config) - return config - - -def process_instance( - instance: pd.Series, - metadata: EvalMetadata, - reset_logger: bool = True, -) -> EvalOutput: - config = get_config(metadata, instance.instance_id) - - # Setup the logger properly, so you can run multi-processing to parallelize the evaluation - if reset_logger: - log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs') - reset_logger_for_multiprocessing(logger, str(instance.instance_id), log_dir) - else: - logger.info( - f'\nStarting evaluation for instance {str(instance.instance_id)}.\n' - ) - - # ============================================= - # import test instance - # ============================================= - instance_id = instance.instance_id - spec = importlib.util.spec_from_file_location(instance_id, instance.file_path) - test_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(test_module) - assert hasattr(test_module, 'Test'), ( - f'Test module {instance_id} does not have a Test class' - ) - - test_class: type[BaseIntegrationTest] = test_module.Test - assert issubclass(test_class, BaseIntegrationTest), ( - f'Test class {instance_id} does not inherit from BaseIntegrationTest' - ) - - instruction = test_class.INSTRUCTION - - # ============================================= - # create sandbox and run the agent - # ============================================= - runtime: Runtime = create_runtime(config) - call_async_from_sync(runtime.connect) - try: - test_class.initialize_runtime(runtime) - - # Here's how you can run the agent (similar to the `main` function) and get the final task state - state: State | None = asyncio.run( - run_controller( - config=config, - initial_user_action=MessageAction(content=instruction), - runtime=runtime, - fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class], - ) - ) - if state is None: - raise ValueError('State should not be None.') - - # # ============================================= - # # result evaluation - # # ============================================= - - histories = state.history - - # some basic check - logger.info(f'Total events in history: {len(histories)}') - assert len(histories) > 0, 'History should not be empty' - - test_result: TestResult = test_class.verify_result(runtime, histories) - metrics = get_metrics(state) - finally: - runtime.close() - - # Save the output - output = EvalOutput( - instance_id=str(instance.instance_id), - instance=instance.to_dict(), - instruction=instruction, - metadata=metadata, - history=[event_to_dict(event) for event in histories], - metrics=metrics, - error=state.last_error if state and state.last_error else None, - test_result=test_result.model_dump(), - ) - return output - - -def load_integration_tests() -> pd.DataFrame: - """Load tests from python files under ./tests""" - cur_dir = os.path.dirname(os.path.abspath(__file__)) - test_dir = os.path.join(cur_dir, 'tests') - test_files = [ - os.path.join(test_dir, f) - for f in os.listdir(test_dir) - if f.startswith('t') and f.endswith('.py') - ] - df = pd.DataFrame(test_files, columns=['file_path']) - df['instance_id'] = df['file_path'].apply( - lambda x: os.path.basename(x).rstrip('.py') - ) - return df - - -if __name__ == '__main__': - parser = get_evaluation_parser() - args, _ = parser.parse_known_args() - integration_tests = load_integration_tests() - - llm_config = None - if args.llm_config: - llm_config = get_llm_config_arg(args.llm_config) - - if llm_config is None: - raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}') - - metadata = make_metadata( - llm_config, - 'integration_tests', - args.agent_cls, - args.max_iterations, - args.eval_note, - args.eval_output_dir, - ) - output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl') - - # Parse dataset IDs if provided - eval_ids = None - if args.eval_ids: - eval_ids = str(args.eval_ids).split(',') - logger.info(f'\nUsing specific dataset IDs: {eval_ids}\n') - - instances = prepare_dataset( - integration_tests, - output_file, - args.eval_n_limit, - eval_ids=eval_ids, - ) - - run_evaluation( - instances, - metadata, - output_file, - args.eval_num_workers, - process_instance, - ) - - df = pd.read_json(output_file, lines=True, orient='records') - - # record success and reason - df['success'] = df['test_result'].apply(lambda x: x['success']) - df['reason'] = df['test_result'].apply(lambda x: x['reason']) - logger.info('-' * 100) - logger.info( - f'Success rate: {df["success"].mean():.2%} ({df["success"].sum()}/{len(df)})' - ) - logger.info( - '\nEvaluation Results:' - + '\n' - + df[['instance_id', 'success', 'reason']].to_string(index=False) - ) - logger.info('-' * 100) - - # record cost for each instance, with 3 decimal places - # we sum up all the "costs" from the metrics array - df['cost'] = df['metrics'].apply( - lambda m: round(sum(c['cost'] for c in m['costs']), 3) - if m and 'costs' in m - else 0.0 - ) - - # capture the top-level error if present, per instance - df['error_message'] = df.get('error', None) - - logger.info(f'Total cost: USD {df["cost"].sum():.2f}') - - report_file = os.path.join(metadata.eval_output_dir, 'report.md') - with open(report_file, 'w') as f: - f.write( - f'Success rate: {df["success"].mean():.2%}' - f' ({df["success"].sum()}/{len(df)})\n' - ) - f.write(f'\nTotal cost: USD {df["cost"].sum():.2f}\n') - f.write( - df[ - ['instance_id', 'success', 'reason', 'cost', 'error_message'] - ].to_markdown(index=False) - ) diff --git a/evaluation/integration_tests/scripts/run_infer.sh b/evaluation/integration_tests/scripts/run_infer.sh deleted file mode 100755 index 5696a46e62..0000000000 --- a/evaluation/integration_tests/scripts/run_infer.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -source "evaluation/utils/version_control.sh" - -MODEL_CONFIG=$1 -COMMIT_HASH=$2 -AGENT=$3 -EVAL_LIMIT=$4 -MAX_ITERATIONS=$5 -NUM_WORKERS=$6 -EVAL_IDS=$7 - -if [ -z "$NUM_WORKERS" ]; then - NUM_WORKERS=1 - echo "Number of workers not specified, use default $NUM_WORKERS" -fi -checkout_eval_branch - -if [ -z "$AGENT" ]; then - echo "Agent not specified, use default CodeActAgent" - AGENT="CodeActAgent" -fi - -get_openhands_version - -echo "AGENT: $AGENT" -echo "OPENHANDS_VERSION: $OPENHANDS_VERSION" -echo "MODEL_CONFIG: $MODEL_CONFIG" - -EVAL_NOTE=$OPENHANDS_VERSION - -# Default to NOT use unit tests. -if [ -z "$USE_UNIT_TESTS" ]; then - export USE_UNIT_TESTS=false -fi -echo "USE_UNIT_TESTS: $USE_UNIT_TESTS" -# If use unit tests, set EVAL_NOTE to the commit hash -if [ "$USE_UNIT_TESTS" = true ]; then - EVAL_NOTE=$EVAL_NOTE-w-test -fi - -# export PYTHONPATH=evaluation/integration_tests:\$PYTHONPATH -COMMAND="poetry run python evaluation/integration_tests/run_infer.py \ - --agent-cls $AGENT \ - --llm-config $MODEL_CONFIG \ - --max-iterations ${MAX_ITERATIONS:-10} \ - --eval-num-workers $NUM_WORKERS \ - --eval-note $EVAL_NOTE" - -if [ -n "$EVAL_LIMIT" ]; then - echo "EVAL_LIMIT: $EVAL_LIMIT" - COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT" -fi - -if [ -n "$EVAL_IDS" ]; then - echo "EVAL_IDS: $EVAL_IDS" - COMMAND="$COMMAND --eval-ids $EVAL_IDS" -fi - -# Run the command -eval $COMMAND diff --git a/evaluation/integration_tests/tests/__init__.py b/evaluation/integration_tests/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/evaluation/integration_tests/tests/base.py b/evaluation/integration_tests/tests/base.py deleted file mode 100644 index bc98b884d2..0000000000 --- a/evaluation/integration_tests/tests/base.py +++ /dev/null @@ -1,32 +0,0 @@ -from abc import ABC, abstractmethod - -from pydantic import BaseModel - -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class TestResult(BaseModel): - success: bool - reason: str | None = None - - -class BaseIntegrationTest(ABC): - """Base class for integration tests.""" - - INSTRUCTION: str - - @classmethod - @abstractmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - """Initialize the runtime for the test to run.""" - pass - - @classmethod - @abstractmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - """Verify the result of the test. - - This method will be called after the agent performs the task on the runtime. - """ - pass diff --git a/evaluation/integration_tests/tests/t01_fix_simple_typo.py b/evaluation/integration_tests/tests/t01_fix_simple_typo.py deleted file mode 100644 index 532d5d5b38..0000000000 --- a/evaluation/integration_tests/tests/t01_fix_simple_typo.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import tempfile - -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from openhands.events.action import CmdRunAction -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Fix typos in bad.txt.' - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - # create a file with a typo in /workspace/bad.txt - with tempfile.TemporaryDirectory() as temp_dir: - temp_file_path = os.path.join(temp_dir, 'bad.txt') - with open(temp_file_path, 'w') as f: - f.write('This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!') - - # Copy the file to the desired location - runtime.copy_to(temp_file_path, '/workspace') - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - # check if the file /workspace/bad.txt has been fixed - action = CmdRunAction(command='cat /workspace/bad.txt') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, reason=f'Failed to run command: {obs.content}' - ) - # check if the file /workspace/bad.txt has been fixed - if ( - obs.content.strip().replace('\r\n', '\n') - == 'This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!' - ): - return TestResult(success=True) - return TestResult(success=False, reason=f'File not fixed: {obs.content}') diff --git a/evaluation/integration_tests/tests/t02_add_bash_hello.py b/evaluation/integration_tests/tests/t02_add_bash_hello.py deleted file mode 100644 index 88384a87f2..0000000000 --- a/evaluation/integration_tests/tests/t02_add_bash_hello.py +++ /dev/null @@ -1,40 +0,0 @@ -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import assert_and_raise -from openhands.events.action import CmdRunAction -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = "Write a shell script '/workspace/hello.sh' that prints 'hello'." - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - action = CmdRunAction(command='mkdir -p /workspace') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - # check if the file /workspace/hello.sh exists - action = CmdRunAction(command='cat /workspace/hello.sh') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to cat /workspace/hello.sh: {obs.content}.', - ) - - # execute the script - action = CmdRunAction(command='bash /workspace/hello.sh') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to execute /workspace/hello.sh: {obs.content}.', - ) - if obs.content.strip() != 'hello': - return TestResult( - success=False, reason=f'Script did not print "hello": {obs.content}.' - ) - return TestResult(success=True) diff --git a/evaluation/integration_tests/tests/t03_jupyter_write_file.py b/evaluation/integration_tests/tests/t03_jupyter_write_file.py deleted file mode 100644 index 2f88e1228b..0000000000 --- a/evaluation/integration_tests/tests/t03_jupyter_write_file.py +++ /dev/null @@ -1,43 +0,0 @@ -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import assert_and_raise -from openhands.events.action import CmdRunAction -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = "Use Jupyter IPython to write a text file containing 'hello world' to '/workspace/test.txt'." - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - action = CmdRunAction(command='mkdir -p /workspace') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - # check if the file /workspace/hello.sh exists - action = CmdRunAction(command='cat /workspace/test.txt') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to cat /workspace/test.txt: {obs.content}.', - ) - - # execute the script - action = CmdRunAction(command='cat /workspace/test.txt') - obs = runtime.run_action(action) - - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to cat /workspace/test.txt: {obs.content}.', - ) - - if 'hello world' not in obs.content.strip(): - return TestResult( - success=False, - reason=f'File did not contain "hello world": {obs.content}.', - ) - return TestResult(success=True) diff --git a/evaluation/integration_tests/tests/t04_git_staging.py b/evaluation/integration_tests/tests/t04_git_staging.py deleted file mode 100644 index 1c3daaba37..0000000000 --- a/evaluation/integration_tests/tests/t04_git_staging.py +++ /dev/null @@ -1,57 +0,0 @@ -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import assert_and_raise -from openhands.events.action import CmdRunAction -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Write a git commit message for the current staging area and commit the changes.' - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - action = CmdRunAction(command='mkdir -p /workspace') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - # git init - action = CmdRunAction(command='git init') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - # create file - action = CmdRunAction(command='echo \'print("hello world")\' > hello.py') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - # git add - cmd_str = 'git add hello.py' - action = CmdRunAction(command=cmd_str) - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - # check if the file /workspace/hello.py exists - action = CmdRunAction(command='cat /workspace/hello.py') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to cat /workspace/hello.py: {obs.content}.', - ) - - # check if the staging area is empty - action = CmdRunAction(command='git status') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, reason=f'Failed to git status: {obs.content}.' - ) - if 'nothing to commit, working tree clean' in obs.content.strip(): - return TestResult(success=True) - - return TestResult( - success=False, - reason=f'Failed to check for "nothing to commit, working tree clean": {obs.content}.', - ) diff --git a/evaluation/integration_tests/tests/t05_simple_browsing.py b/evaluation/integration_tests/tests/t05_simple_browsing.py deleted file mode 100644 index 8542e50d80..0000000000 --- a/evaluation/integration_tests/tests/t05_simple_browsing.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import tempfile - -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import assert_and_raise -from openhands.events.action import AgentFinishAction, CmdRunAction, MessageAction -from openhands.events.event import Event -from openhands.events.observation import AgentDelegateObservation -from openhands.runtime.base import Runtime - -HTML_FILE = """ - - - - - - The Ultimate Answer - - - -
-

The Ultimate Answer

-

Click the button to reveal the answer to life, the universe, and everything.

- -
-
- - - -""" - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Browse localhost:8000, and tell me the ultimate answer to life.' - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - action = CmdRunAction(command='mkdir -p /workspace') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - action = CmdRunAction(command='mkdir -p /tmp/server') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - # create a file with a typo in /workspace/bad.txt - with tempfile.TemporaryDirectory() as temp_dir: - temp_file_path = os.path.join(temp_dir, 'index.html') - with open(temp_file_path, 'w') as f: - f.write(HTML_FILE) - # Copy the file to the desired location - runtime.copy_to(temp_file_path, '/tmp/server') - - # create README.md - action = CmdRunAction( - command='cd /tmp/server && nohup python3 -m http.server 8000 &' - ) - obs = runtime.run_action(action) - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - from openhands.core.logger import openhands_logger as logger - - # check if the "The answer is OpenHands is all you need!" is in any message - message_actions = [ - event - for event in histories - if isinstance( - event, (MessageAction, AgentFinishAction, AgentDelegateObservation) - ) - ] - logger.debug(f'Total message-like events: {len(message_actions)}') - - for event in message_actions: - try: - if isinstance(event, AgentDelegateObservation): - content = event.content - elif isinstance(event, AgentFinishAction): - content = event.outputs.get('content', '') - elif isinstance(event, MessageAction): - content = event.content - else: - logger.warning(f'Unexpected event type: {type(event)}') - continue - - if 'OpenHands is all you need!' in content: - return TestResult(success=True) - except Exception as e: - logger.error(f'Error processing event: {e}') - - logger.debug( - f'Total messages: {len(message_actions)}. Messages: {message_actions}' - ) - return TestResult( - success=False, - reason=f'The answer is not found in any message. Total messages: {len(message_actions)}.', - ) diff --git a/evaluation/integration_tests/tests/t06_github_pr_browsing.py b/evaluation/integration_tests/tests/t06_github_pr_browsing.py deleted file mode 100644 index b85e868401..0000000000 --- a/evaluation/integration_tests/tests/t06_github_pr_browsing.py +++ /dev/null @@ -1,58 +0,0 @@ -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from openhands.events.action import AgentFinishAction, MessageAction -from openhands.events.event import Event -from openhands.events.observation import AgentDelegateObservation -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Look at https://github.com/OpenHands/OpenHands/pull/8, and tell me what is happening there and what did @asadm suggest.' - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - pass - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - from openhands.core.logger import openhands_logger as logger - - # check if the license information is in any message - message_actions = [ - event - for event in histories - if isinstance( - event, (MessageAction, AgentFinishAction, AgentDelegateObservation) - ) - ] - logger.info(f'Total message-like events: {len(message_actions)}') - - for event in message_actions: - try: - if isinstance(event, AgentDelegateObservation): - content = event.content - elif isinstance(event, AgentFinishAction): - content = event.outputs.get('content', '') - if event.thought: - content += f'\n\n{event.thought}' - elif isinstance(event, MessageAction): - content = event.content - else: - logger.warning(f'Unexpected event type: {type(event)}') - continue - - if ( - 'non-commercial' in content - or 'MIT' in content - or 'Apache 2.0' in content - ): - return TestResult(success=True) - except Exception as e: - logger.error(f'Error processing event: {e}') - - logger.debug( - f'Total messages: {len(message_actions)}. Messages: {message_actions}' - ) - return TestResult( - success=False, - reason=f'The answer is not found in any message. Total messages: {len(message_actions)}.', - ) diff --git a/evaluation/integration_tests/tests/t07_interactive_commands.py b/evaluation/integration_tests/tests/t07_interactive_commands.py deleted file mode 100644 index 24a66d3f38..0000000000 --- a/evaluation/integration_tests/tests/t07_interactive_commands.py +++ /dev/null @@ -1,73 +0,0 @@ -import hashlib - -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from openhands.events.action import ( - AgentFinishAction, - FileWriteAction, - MessageAction, -) -from openhands.events.event import Event -from openhands.events.observation import AgentDelegateObservation -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Execute the python script /workspace/python_script.py with input "John" and "25" and tell me the secret number.' - SECRET_NUMBER = int(hashlib.sha256(str(25).encode()).hexdigest()[:8], 16) % 1000 - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - from openhands.core.logger import openhands_logger as logger - - action = FileWriteAction( - path='/workspace/python_script.py', - content=( - 'name = input("Enter your name: "); age = input("Enter your age: "); ' - 'import hashlib; secret = int(hashlib.sha256(str(age).encode()).hexdigest()[:8], 16) % 1000; ' - 'print(f"Hello {name}, you are {age} years old. Tell you a secret number: {secret}")' - ), - ) - logger.info(action, extra={'msg_type': 'ACTION'}) - observation = runtime.run_action(action) - logger.info(observation, extra={'msg_type': 'OBSERVATION'}) - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - from openhands.core.logger import openhands_logger as logger - - # check if the license information is in any message - message_actions = [ - event - for event in histories - if isinstance( - event, (MessageAction, AgentFinishAction, AgentDelegateObservation) - ) - ] - logger.info(f'Total message-like events: {len(message_actions)}') - - for event in message_actions: - try: - if isinstance(event, AgentDelegateObservation): - content = event.content - elif isinstance(event, AgentFinishAction): - content = event.outputs.get('content', '') - if event.thought: - content += f'\n\n{event.thought}' - elif isinstance(event, MessageAction): - content = event.content - else: - logger.warning(f'Unexpected event type: {type(event)}') - continue - - if str(cls.SECRET_NUMBER) in content: - return TestResult(success=True) - except Exception as e: - logger.error(f'Error processing event: {e}') - - logger.debug( - f'Total messages: {len(message_actions)}. Messages: {message_actions}' - ) - return TestResult( - success=False, - reason=f'The answer is not found in any message. Total messages: {len(message_actions)}.', - )