diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 0000000000..50a641bbec --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1 @@ +This way of running OpenHands is not officially supported. It is maintained by the community. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7ee414958c..8aee64d59c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,6 +13,7 @@ - [ ] Other (dependency update, docs, typo fixes, etc.) ## Checklist + - [ ] I have read and reviewed the code and I understand what the code is doing. - [ ] I have tested the code to the best of my ability and ensured it works as expected. diff --git a/.github/workflows/check-package-versions.yml b/.github/workflows/check-package-versions.yml new file mode 100644 index 0000000000..44e680ff4b --- /dev/null +++ b/.github/workflows/check-package-versions.yml @@ -0,0 +1,65 @@ +name: Check Package Versions + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + check-package-versions: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Check for any 'rev' fields in pyproject.toml + run: | + python - <<'PY' + import sys, tomllib, pathlib + + path = pathlib.Path("pyproject.toml") + if not path.exists(): + print("āŒ ERROR: pyproject.toml not found") + sys.exit(1) + + try: + data = tomllib.loads(path.read_text(encoding="utf-8")) + except Exception as e: + print(f"āŒ ERROR: Failed to parse pyproject.toml: {e}") + sys.exit(1) + + poetry = data.get("tool", {}).get("poetry", {}) + sections = { + "dependencies": poetry.get("dependencies", {}), + } + + errors = [] + + print("šŸ” Checking for any dependencies with 'rev' fields...\n") + for section_name, deps in sections.items(): + if not isinstance(deps, dict): + continue + + for pkg_name, cfg in deps.items(): + if isinstance(cfg, dict) and "rev" in cfg: + msg = f" āœ– {pkg_name} in [{section_name}] uses rev='{cfg['rev']}' (NOT ALLOWED)" + print(msg) + errors.append(msg) + else: + print(f" ā€¢ {pkg_name}: OK") + + if errors: + print("\nāŒ FAILED: Found dependencies using 'rev' fields:\n" + "\n".join(errors)) + print("\nPlease use versioned releases instead, e.g.:") + print(' my-package = "1.0.0"') + sys.exit(1) + + print("\nāœ… SUCCESS: No 'rev' fields found. All dependencies are using proper versioned releases.") + PY diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml deleted file mode 100644 index e8d318c3e6..0000000000 --- a/.github/workflows/integration-runner.yml +++ /dev/null @@ -1,199 +0,0 @@ -name: Run Integration Tests - -on: - pull_request: - types: [labeled] - workflow_dispatch: - inputs: - reason: - description: 'Reason for manual trigger' - required: true - default: '' - schedule: - - cron: '30 22 * * *' # Runs at 10:30pm UTC every day - -env: - N_PROCESSES: 10 # Global configuration for number of parallel processes for evaluation - -jobs: - run-integration-tests: - if: github.event.label.name == 'integration-test' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' - runs-on: blacksmith-4vcpu-ubuntu-2204 - permissions: - contents: "read" - id-token: "write" - pull-requests: "write" - issues: "write" - strategy: - matrix: - python-version: ["3.12"] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install poetry via pipx - run: pipx install poetry - - - name: Set up Python - uses: useblacksmith/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - cache: "poetry" - - - name: Setup Node.js - uses: useblacksmith/setup-node@v5 - with: - node-version: '22.x' - - - name: Comment on PR if 'integration-test' label is present - if: github.event_name == 'pull_request' && github.event.label.name == 'integration-test' - uses: KeisukeYamashita/create-comment@v1 - with: - unique: false - comment: | - Hi! I started running the integration tests on your PR. You will receive a comment with the results shortly. - - - name: Install Python dependencies using Poetry - run: poetry install --with dev,test,runtime,evaluation - - - name: Configure config.toml for testing with Haiku - env: - LLM_MODEL: "litellm_proxy/claude-3-5-haiku-20241022" - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} - MAX_ITERATIONS: 10 - run: | - echo "[llm.eval]" > config.toml - echo "model = \"$LLM_MODEL\"" >> config.toml - echo "api_key = \"$LLM_API_KEY\"" >> config.toml - echo "base_url = \"$LLM_BASE_URL\"" >> config.toml - echo "temperature = 0.0" >> config.toml - - - name: Build environment - run: make build - - - name: Run integration test evaluation for Haiku - env: - SANDBOX_FORCE_REBUILD_RUNTIME: True - run: | - poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' 10 $N_PROCESSES '' 'haiku_run' - - # get integration tests report - REPORT_FILE_HAIKU=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/*haiku*_maxiter_10_N* -name "report.md" -type f | head -n 1) - echo "REPORT_FILE: $REPORT_FILE_HAIKU" - echo "INTEGRATION_TEST_REPORT_HAIKU<> $GITHUB_ENV - cat $REPORT_FILE_HAIKU >> $GITHUB_ENV - echo >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Wait a little bit - run: sleep 10 - - - name: Configure config.toml for testing with DeepSeek - env: - LLM_MODEL: "litellm_proxy/deepseek-chat" - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} - MAX_ITERATIONS: 10 - run: | - echo "[llm.eval]" > config.toml - echo "model = \"$LLM_MODEL\"" >> config.toml - echo "api_key = \"$LLM_API_KEY\"" >> config.toml - echo "base_url = \"$LLM_BASE_URL\"" >> config.toml - echo "temperature = 0.0" >> config.toml - - - name: Run integration test evaluation for DeepSeek - env: - SANDBOX_FORCE_REBUILD_RUNTIME: True - run: | - poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' 10 $N_PROCESSES '' 'deepseek_run' - - # get integration tests report - REPORT_FILE_DEEPSEEK=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/deepseek*_maxiter_10_N* -name "report.md" -type f | head -n 1) - echo "REPORT_FILE: $REPORT_FILE_DEEPSEEK" - echo "INTEGRATION_TEST_REPORT_DEEPSEEK<> $GITHUB_ENV - cat $REPORT_FILE_DEEPSEEK >> $GITHUB_ENV - echo >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - # ------------------------------------------------------------- - # Run VisualBrowsingAgent tests for DeepSeek, limited to t05 and t06 - - name: Wait a little bit (again) - run: sleep 5 - - - name: Configure config.toml for testing VisualBrowsingAgent (DeepSeek) - env: - LLM_MODEL: "litellm_proxy/deepseek-chat" - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} - MAX_ITERATIONS: 15 - run: | - echo "[llm.eval]" > config.toml - echo "model = \"$LLM_MODEL\"" >> config.toml - echo "api_key = \"$LLM_API_KEY\"" >> config.toml - echo "base_url = \"$LLM_BASE_URL\"" >> config.toml - echo "temperature = 0.0" >> config.toml - - name: Run integration test evaluation for VisualBrowsingAgent (DeepSeek) - env: - SANDBOX_FORCE_REBUILD_RUNTIME: True - run: | - poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD VisualBrowsingAgent '' 15 $N_PROCESSES "t05_simple_browsing,t06_github_pr_browsing.py" 'visualbrowsing_deepseek_run' - - # Find and export the visual browsing agent test results - REPORT_FILE_VISUALBROWSING_DEEPSEEK=$(find evaluation/evaluation_outputs/outputs/integration_tests/VisualBrowsingAgent/deepseek*_maxiter_15_N* -name "report.md" -type f | head -n 1) - echo "REPORT_FILE_VISUALBROWSING_DEEPSEEK: $REPORT_FILE_VISUALBROWSING_DEEPSEEK" - echo "INTEGRATION_TEST_REPORT_VISUALBROWSING_DEEPSEEK<> $GITHUB_ENV - cat $REPORT_FILE_VISUALBROWSING_DEEPSEEK >> $GITHUB_ENV - echo >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Create archive of evaluation outputs - run: | - TIMESTAMP=$(date +'%y-%m-%d-%H-%M') - cd evaluation/evaluation_outputs/outputs # Change to the outputs directory - tar -czvf ../../../integration_tests_${TIMESTAMP}.tar.gz integration_tests/CodeActAgent/* integration_tests/VisualBrowsingAgent/* # Only include the actual result directories - - - name: Upload evaluation results as artifact - uses: actions/upload-artifact@v4 - id: upload_results_artifact - with: - name: integration-test-outputs-${{ github.run_id }}-${{ github.run_attempt }} - path: integration_tests_*.tar.gz - - - name: Get artifact URLs - run: | - echo "ARTIFACT_URL=${{ steps.upload_results_artifact.outputs.artifact-url }}" >> $GITHUB_ENV - - - name: Set timestamp and trigger reason - run: | - echo "TIMESTAMP=$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_ENV - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "TRIGGER_REASON=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV - elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "TRIGGER_REASON=manual-${{ github.event.inputs.reason }}" >> $GITHUB_ENV - else - echo "TRIGGER_REASON=nightly-scheduled" >> $GITHUB_ENV - fi - - - name: Comment with results and artifact link - id: create_comment - uses: KeisukeYamashita/create-comment@v1 - with: - # if triggered by PR, use PR number, otherwise use 9745 as fallback issue number for manual triggers - number: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || 9745 }} - unique: false - comment: | - Trigger by: ${{ github.event_name == 'pull_request' && format('Pull Request (integration-test label on PR #{0})', github.event.pull_request.number) || (github.event_name == 'workflow_dispatch' && format('Manual Trigger: {0}', github.event.inputs.reason)) || 'Nightly Scheduled Run' }} - Commit: ${{ github.sha }} - **Integration Tests Report (Haiku)** - Haiku LLM Test Results: - ${{ env.INTEGRATION_TEST_REPORT_HAIKU }} - --- - **Integration Tests Report (DeepSeek)** - DeepSeek LLM Test Results: - ${{ env.INTEGRATION_TEST_REPORT_DEEPSEEK }} - --- - **Integration Tests Report VisualBrowsing (DeepSeek)** - ${{ env.INTEGRATION_TEST_REPORT_VISUALBROWSING_DEEPSEEK }} - --- - Download testing outputs (includes both Haiku and DeepSeek results): [Download](${{ steps.upload_results_artifact.outputs.artifact-url }}) diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml index 1bb4126e73..4506f1ea75 100644 --- a/.github/workflows/py-tests.yml +++ b/.github/workflows/py-tests.yml @@ -70,37 +70,7 @@ jobs: .coverage.${{ matrix.python_version }} .coverage.runtime.${{ matrix.python_version }} include-hidden-files: true - # Run specific Windows python tests - test-on-windows: - name: Python Tests on Windows - runs-on: windows-latest - strategy: - matrix: - python-version: ["3.12"] - steps: - - uses: actions/checkout@v4 - - name: Install pipx - run: pip install pipx - - name: Install poetry via pipx - run: pipx install poetry - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "poetry" - - name: Install Python dependencies using Poetry - run: poetry install --with dev,test,runtime - - name: Run Windows unit tests - run: poetry run pytest -svv tests/runtime//test_windows_bash.py - env: - PYTHONPATH: ".;$env:PYTHONPATH" - DEBUG: "1" - - name: Run Windows runtime tests with LocalRuntime - run: $env:TEST_RUNTIME="local"; poetry run pytest -svv tests/runtime/test_bash.py - env: - PYTHONPATH: ".;$env:PYTHONPATH" - TEST_RUNTIME: local - DEBUG: "1" + test-enterprise: name: Enterprise Python Unit Tests runs-on: blacksmith-4vcpu-ubuntu-2404 diff --git a/.gitignore b/.gitignore index 97236ca41c..6fc0934a02 100644 --- a/.gitignore +++ b/.gitignore @@ -185,6 +185,9 @@ cython_debug/ .repomix repomix-output.txt +# Emacs backup +*~ + # evaluation evaluation/evaluation_outputs evaluation/outputs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a605abaf64..e2338202cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ by implementing the [interface specified here](https://github.com/OpenHands/Open #### Testing When you write code, it is also good to write tests. Please navigate to the [`./tests`](./tests) folder to see existing test suites. -At the moment, we have two kinds of tests: [`unit`](./tests/unit) and [`integration`](./evaluation/integration_tests). Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project. +At the moment, we have these kinds of tests: [`unit`](./tests/unit), [`runtime`](./tests/runtime), and [`end-to-end (e2e)`](./tests/e2e). Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project. ## Sending Pull Requests to OpenHands diff --git a/Development.md b/Development.md index 0ab8d9a684..8b524be511 100644 --- a/Development.md +++ b/Development.md @@ -159,7 +159,7 @@ poetry run pytest ./tests/unit/test_*.py To reduce build time (e.g., if no changes were made to the client-runtime component), you can use an existing Docker container image by setting the SANDBOX_RUNTIME_CONTAINER_IMAGE environment variable to the desired Docker image. -Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/openhands/runtime:0.61-nikolaik` +Example: `export SANDBOX_RUNTIME_CONTAINER_IMAGE=ghcr.io/openhands/runtime:0.62-nikolaik` ## Develop inside Docker container diff --git a/README.md b/README.md index 5fe0edae5a..cd47210ef1 100644 --- a/README.md +++ b/README.md @@ -82,17 +82,17 @@ You'll find OpenHands running at [http://localhost:3000](http://localhost:3000) You can also run OpenHands directly with Docker: ```bash -docker pull docker.openhands.dev/openhands/runtime:0.61-nikolaik +docker pull docker.openhands.dev/openhands/runtime:0.62-nikolaik docker run -it --rm --pull=always \ - -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.openhands.dev/openhands/runtime:0.61-nikolaik \ + -e SANDBOX_RUNTIME_CONTAINER_IMAGE=docker.openhands.dev/openhands/runtime:0.62-nikolaik \ -e LOG_ALL_EVENTS=true \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/.openhands:/.openhands \ -p 3000:3000 \ --add-host host.docker.internal:host-gateway \ --name openhands-app \ - docker.openhands.dev/openhands/openhands:0.61 + docker.openhands.dev/openhands/openhands:0.62 ``` diff --git a/containers/dev/README.md b/containers/dev/README.md index 0fb1bbc3b3..4f2df5cb0b 100644 --- a/containers/dev/README.md +++ b/containers/dev/README.md @@ -1,7 +1,7 @@ # Develop in Docker > [!WARNING] -> This is not officially supported and may not work. +> This way of running OpenHands is not officially supported. It is maintained by the community and may not work. Install [Docker](https://docs.docker.com/engine/install/) on your host machine and run: diff --git a/containers/dev/compose.yml b/containers/dev/compose.yml index f001d63106..c6168b094f 100644 --- a/containers/dev/compose.yml +++ b/containers/dev/compose.yml @@ -12,7 +12,7 @@ services: - SANDBOX_API_HOSTNAME=host.docker.internal - DOCKER_HOST_ADDR=host.docker.internal # - - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/openhands/runtime:0.61-nikolaik} + - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-ghcr.io/openhands/runtime:0.62-nikolaik} - SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234} - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace} ports: diff --git a/docker-compose.yml b/docker-compose.yml index 304658fa07..b663324625 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: image: openhands:latest container_name: openhands-app-${DATE:-} environment: - - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.openhands.dev/openhands/runtime:0.61-nikolaik} + - SANDBOX_RUNTIME_CONTAINER_IMAGE=${SANDBOX_RUNTIME_CONTAINER_IMAGE:-docker.openhands.dev/openhands/runtime:0.62-nikolaik} #- SANDBOX_USER_ID=${SANDBOX_USER_ID:-1234} # enable this only if you want a specific non-root sandbox user but you will have to manually adjust permissions of ~/.openhands for this user - WORKSPACE_MOUNT_PATH=${WORKSPACE_BASE:-$PWD/workspace} ports: diff --git a/enterprise/poetry.lock b/enterprise/poetry.lock index 36c88d6bfd..cb3bfc2c3e 100644 --- a/enterprise/poetry.lock +++ b/enterprise/poetry.lock @@ -5820,13 +5820,15 @@ llama = ["llama-index (>=0.12.29,<0.13.0)", "llama-index-core (>=0.12.29,<0.13.0 [[package]] name = "openhands-agent-server" -version = "1.0.0a5" +version = "1.1.0" description = "OpenHands Agent Server - REST/WebSocket interface for OpenHands AI Agent" optional = false python-versions = ">=3.12" groups = ["main"] -files = [] -develop = false +files = [ + {file = "openhands_agent_server-1.1.0-py3-none-any.whl", hash = "sha256:59a856883df23488c0723e47655ef21649a321fcd4709a25a4690866eff6ac88"}, + {file = "openhands_agent_server-1.1.0.tar.gz", hash = "sha256:e39bebd39afd45cfcfd765005e7c4e5409e46678bd7612ae20bae79f7057b935"}, +] [package.dependencies] aiosqlite = ">=0.19" @@ -5839,16 +5841,9 @@ uvicorn = ">=0.31.1" websockets = ">=12" wsproto = ">=1.2.0" -[package.source] -type = "git" -url = "https://github.com/OpenHands/software-agent-sdk.git" -reference = "d5995c31c55e488d4ab0372d292973bc6fad71f1" -resolved_reference = "d5995c31c55e488d4ab0372d292973bc6fad71f1" -subdirectory = "openhands-agent-server" - [[package]] name = "openhands-ai" -version = "0.0.0-post.5514+7c9e66194" +version = "0.0.0-post.5525+0b6631523" description = "OpenHands: Code Less, Make More" optional = false python-versions = "^3.12,<3.14" @@ -5889,9 +5884,9 @@ memory-profiler = "^0.61.0" numpy = "*" openai = "1.99.9" openhands-aci = "0.3.2" -openhands-agent-server = {git = "https://github.com/OpenHands/software-agent-sdk.git", rev = "d5995c31c55e488d4ab0372d292973bc6fad71f1", subdirectory = "openhands-agent-server"} -openhands-sdk = {git = "https://github.com/OpenHands/software-agent-sdk.git", rev = "d5995c31c55e488d4ab0372d292973bc6fad71f1", subdirectory = "openhands-sdk"} -openhands-tools = {git = "https://github.com/OpenHands/software-agent-sdk.git", rev = "d5995c31c55e488d4ab0372d292973bc6fad71f1", subdirectory = "openhands-tools"} +openhands-agent-server = "1.1.0" +openhands-sdk = "1.1.0" +openhands-tools = "1.1.0" opentelemetry-api = "^1.33.1" opentelemetry-exporter-otlp-proto-grpc = "^1.33.1" pathspec = "^0.12.1" @@ -5947,13 +5942,15 @@ url = ".." [[package]] name = "openhands-sdk" -version = "1.0.0a5" +version = "1.1.0" description = "OpenHands SDK - Core functionality for building AI agents" optional = false python-versions = ">=3.12" groups = ["main"] -files = [] -develop = false +files = [ + {file = "openhands_sdk-1.1.0-py3-none-any.whl", hash = "sha256:4a984ce1687a48cf99a67fdf3d37b116f8b2840743d4807810b5024af6a1d57e"}, + {file = "openhands_sdk-1.1.0.tar.gz", hash = "sha256:855e0d8f3657205e4119e50520c17e65b3358b1a923f7a051a82512a54bf426c"}, +] [package.dependencies] fastmcp = ">=2.11.3" @@ -5969,22 +5966,17 @@ websockets = ">=12" [package.extras] boto3 = ["boto3 (>=1.35.0)"] -[package.source] -type = "git" -url = "https://github.com/OpenHands/software-agent-sdk.git" -reference = "d5995c31c55e488d4ab0372d292973bc6fad71f1" -resolved_reference = "d5995c31c55e488d4ab0372d292973bc6fad71f1" -subdirectory = "openhands-sdk" - [[package]] name = "openhands-tools" -version = "1.0.0a5" +version = "1.1.0" description = "OpenHands Tools - Runtime tools for AI agents" optional = false python-versions = ">=3.12" groups = ["main"] -files = [] -develop = false +files = [ + {file = "openhands_tools-1.1.0-py3-none-any.whl", hash = "sha256:767d6746f05edade49263aa24450a037485a3dc23379f56917ef19aad22033f9"}, + {file = "openhands_tools-1.1.0.tar.gz", hash = "sha256:c2fadaa4f4e16e9a3df5781ea847565dcae7171584f09ef7c0e1d97c8dfc83f6"}, +] [package.dependencies] bashlex = ">=0.18" @@ -5996,13 +5988,6 @@ libtmux = ">=0.46.2" openhands-sdk = "*" pydantic = ">=2.11.7" -[package.source] -type = "git" -url = "https://github.com/OpenHands/software-agent-sdk.git" -reference = "d5995c31c55e488d4ab0372d292973bc6fad71f1" -resolved_reference = "d5995c31c55e488d4ab0372d292973bc6fad71f1" -subdirectory = "openhands-tools" - [[package]] name = "openpyxl" version = "3.1.5" diff --git a/evaluation/benchmarks/multi_swe_bench/README.md b/evaluation/benchmarks/multi_swe_bench/README.md index 88843ca9ce..58f2221e9f 100644 --- a/evaluation/benchmarks/multi_swe_bench/README.md +++ b/evaluation/benchmarks/multi_swe_bench/README.md @@ -15,7 +15,7 @@ python evaluation/benchmarks/multi_swe_bench/scripts/data/data_change.py ## Docker image download -Please download the multi-swe-bench dokcer images from [here](https://github.com/multi-swe-bench/multi-swe-bench?tab=readme-ov-file#run-evaluation). +Please download the multi-swe-bench docker images from [here](https://github.com/multi-swe-bench/multi-swe-bench?tab=readme-ov-file#run-evaluation). ## Generate patch @@ -47,7 +47,7 @@ For debugging purposes, you can set `export EVAL_SKIP_MAXIMUM_RETRIES_EXCEEDED=t The results will be generated in evaluation/evaluation_outputs/outputs/XXX/CodeActAgent/YYY/output.jsonl, you can refer to the [example](examples/output.jsonl). -## Runing evaluation +## Running evaluation First, install [multi-swe-bench](https://github.com/multi-swe-bench/multi-swe-bench). diff --git a/evaluation/integration_tests/README.md b/evaluation/integration_tests/README.md deleted file mode 100644 index afe48d70f4..0000000000 --- a/evaluation/integration_tests/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Integration tests - -This directory implements integration tests that [was running in CI](https://github.com/OpenHands/OpenHands/tree/23d3becf1d6f5d07e592f7345750c314a826b4e9/tests/integration). - -[PR 3985](https://github.com/OpenHands/OpenHands/pull/3985) introduce LLM-based editing, which requires access to LLM to perform edit. Hence, we remove integration tests from CI and intend to run them as nightly evaluation to ensure the quality of OpenHands softwares. - -## To add new tests - -Each test is a file named like `tXX_testname.py` where `XX` is a number. -Make sure to name the file for each test to start with `t` and ends with `.py`. - -Each test should be structured as a subclass of [`BaseIntegrationTest`](./tests/base.py), where you need to implement `initialize_runtime` that setup the runtime enviornment before test, and `verify_result` that takes in a `Runtime` and history of `Event` and return a `TestResult`. See [t01_fix_simple_typo.py](./tests/t01_fix_simple_typo.py) and [t05_simple_browsing.py](./tests/t05_simple_browsing.py) for two representative examples. - -```python -class TestResult(BaseModel): - success: bool - reason: str | None = None - - -class BaseIntegrationTest(ABC): - """Base class for integration tests.""" - - INSTRUCTION: str - - @classmethod - @abstractmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - """Initialize the runtime for the test to run.""" - pass - - @classmethod - @abstractmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - """Verify the result of the test. - - This method will be called after the agent performs the task on the runtime. - """ - pass -``` - - -## Setup Environment and LLM Configuration - -Please follow instruction [here](../README.md#setup) to setup your local -development environment and LLM. - -## Start the evaluation - -```bash -./evaluation/integration_tests/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids] -``` - -- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for - your LLM settings, as defined in your `config.toml`. -- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version - you would like to evaluate. It could also be a release tag like `0.9.0`. -- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, - defaulting to `CodeActAgent`. -- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` - instances. By default, the script evaluates the entire Exercism test set - (133 issues). Note: in order to use `eval_limit`, you must also set `agent`. -- `eval-num-workers`: the number of workers to use for evaluation. Default: `1`. -- `eval_ids`, e.g. `"1,3,10"`, limits the evaluation to instances with the - given IDs (comma separated). - -Example: -```bash -./evaluation/integration_tests/scripts/run_infer.sh llm.claude-35-sonnet-eval HEAD CodeActAgent -``` diff --git a/evaluation/integration_tests/__init__.py b/evaluation/integration_tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/evaluation/integration_tests/run_infer.py b/evaluation/integration_tests/run_infer.py deleted file mode 100644 index 88d49d4055..0000000000 --- a/evaluation/integration_tests/run_infer.py +++ /dev/null @@ -1,251 +0,0 @@ -import asyncio -import importlib.util -import os - -import pandas as pd - -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import ( - EvalMetadata, - EvalOutput, - get_default_sandbox_config_for_eval, - get_metrics, - get_openhands_config_for_eval, - make_metadata, - prepare_dataset, - reset_logger_for_multiprocessing, - run_evaluation, - update_llm_config_for_completions_logging, -) -from evaluation.utils.shared import ( - codeact_user_response as fake_user_response, -) -from openhands.controller.state.state import State -from openhands.core.config import ( - AgentConfig, - OpenHandsConfig, - get_evaluation_parser, - get_llm_config_arg, -) -from openhands.core.logger import openhands_logger as logger -from openhands.core.main import create_runtime, run_controller -from openhands.events.action import MessageAction -from openhands.events.serialization.event import event_to_dict -from openhands.runtime.base import Runtime -from openhands.utils.async_utils import call_async_from_sync - -FAKE_RESPONSES = { - 'CodeActAgent': fake_user_response, - 'VisualBrowsingAgent': fake_user_response, -} - - -def get_config( - metadata: EvalMetadata, - instance_id: str, -) -> OpenHandsConfig: - sandbox_config = get_default_sandbox_config_for_eval() - sandbox_config.platform = 'linux/amd64' - config = get_openhands_config_for_eval( - metadata=metadata, - runtime=os.environ.get('RUNTIME', 'docker'), - sandbox_config=sandbox_config, - ) - config.debug = True - config.set_llm_config( - update_llm_config_for_completions_logging( - metadata.llm_config, metadata.eval_output_dir, instance_id - ) - ) - agent_config = AgentConfig( - enable_jupyter=True, - enable_browsing=True, - enable_llm_editor=False, - ) - config.set_agent_config(agent_config) - return config - - -def process_instance( - instance: pd.Series, - metadata: EvalMetadata, - reset_logger: bool = True, -) -> EvalOutput: - config = get_config(metadata, instance.instance_id) - - # Setup the logger properly, so you can run multi-processing to parallelize the evaluation - if reset_logger: - log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs') - reset_logger_for_multiprocessing(logger, str(instance.instance_id), log_dir) - else: - logger.info( - f'\nStarting evaluation for instance {str(instance.instance_id)}.\n' - ) - - # ============================================= - # import test instance - # ============================================= - instance_id = instance.instance_id - spec = importlib.util.spec_from_file_location(instance_id, instance.file_path) - test_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(test_module) - assert hasattr(test_module, 'Test'), ( - f'Test module {instance_id} does not have a Test class' - ) - - test_class: type[BaseIntegrationTest] = test_module.Test - assert issubclass(test_class, BaseIntegrationTest), ( - f'Test class {instance_id} does not inherit from BaseIntegrationTest' - ) - - instruction = test_class.INSTRUCTION - - # ============================================= - # create sandbox and run the agent - # ============================================= - runtime: Runtime = create_runtime(config) - call_async_from_sync(runtime.connect) - try: - test_class.initialize_runtime(runtime) - - # Here's how you can run the agent (similar to the `main` function) and get the final task state - state: State | None = asyncio.run( - run_controller( - config=config, - initial_user_action=MessageAction(content=instruction), - runtime=runtime, - fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class], - ) - ) - if state is None: - raise ValueError('State should not be None.') - - # # ============================================= - # # result evaluation - # # ============================================= - - histories = state.history - - # some basic check - logger.info(f'Total events in history: {len(histories)}') - assert len(histories) > 0, 'History should not be empty' - - test_result: TestResult = test_class.verify_result(runtime, histories) - metrics = get_metrics(state) - finally: - runtime.close() - - # Save the output - output = EvalOutput( - instance_id=str(instance.instance_id), - instance=instance.to_dict(), - instruction=instruction, - metadata=metadata, - history=[event_to_dict(event) for event in histories], - metrics=metrics, - error=state.last_error if state and state.last_error else None, - test_result=test_result.model_dump(), - ) - return output - - -def load_integration_tests() -> pd.DataFrame: - """Load tests from python files under ./tests""" - cur_dir = os.path.dirname(os.path.abspath(__file__)) - test_dir = os.path.join(cur_dir, 'tests') - test_files = [ - os.path.join(test_dir, f) - for f in os.listdir(test_dir) - if f.startswith('t') and f.endswith('.py') - ] - df = pd.DataFrame(test_files, columns=['file_path']) - df['instance_id'] = df['file_path'].apply( - lambda x: os.path.basename(x).rstrip('.py') - ) - return df - - -if __name__ == '__main__': - parser = get_evaluation_parser() - args, _ = parser.parse_known_args() - integration_tests = load_integration_tests() - - llm_config = None - if args.llm_config: - llm_config = get_llm_config_arg(args.llm_config) - - if llm_config is None: - raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}') - - metadata = make_metadata( - llm_config, - 'integration_tests', - args.agent_cls, - args.max_iterations, - args.eval_note, - args.eval_output_dir, - ) - output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl') - - # Parse dataset IDs if provided - eval_ids = None - if args.eval_ids: - eval_ids = str(args.eval_ids).split(',') - logger.info(f'\nUsing specific dataset IDs: {eval_ids}\n') - - instances = prepare_dataset( - integration_tests, - output_file, - args.eval_n_limit, - eval_ids=eval_ids, - ) - - run_evaluation( - instances, - metadata, - output_file, - args.eval_num_workers, - process_instance, - ) - - df = pd.read_json(output_file, lines=True, orient='records') - - # record success and reason - df['success'] = df['test_result'].apply(lambda x: x['success']) - df['reason'] = df['test_result'].apply(lambda x: x['reason']) - logger.info('-' * 100) - logger.info( - f'Success rate: {df["success"].mean():.2%} ({df["success"].sum()}/{len(df)})' - ) - logger.info( - '\nEvaluation Results:' - + '\n' - + df[['instance_id', 'success', 'reason']].to_string(index=False) - ) - logger.info('-' * 100) - - # record cost for each instance, with 3 decimal places - # we sum up all the "costs" from the metrics array - df['cost'] = df['metrics'].apply( - lambda m: round(sum(c['cost'] for c in m['costs']), 3) - if m and 'costs' in m - else 0.0 - ) - - # capture the top-level error if present, per instance - df['error_message'] = df.get('error', None) - - logger.info(f'Total cost: USD {df["cost"].sum():.2f}') - - report_file = os.path.join(metadata.eval_output_dir, 'report.md') - with open(report_file, 'w') as f: - f.write( - f'Success rate: {df["success"].mean():.2%}' - f' ({df["success"].sum()}/{len(df)})\n' - ) - f.write(f'\nTotal cost: USD {df["cost"].sum():.2f}\n') - f.write( - df[ - ['instance_id', 'success', 'reason', 'cost', 'error_message'] - ].to_markdown(index=False) - ) diff --git a/evaluation/integration_tests/scripts/run_infer.sh b/evaluation/integration_tests/scripts/run_infer.sh deleted file mode 100755 index 5696a46e62..0000000000 --- a/evaluation/integration_tests/scripts/run_infer.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -source "evaluation/utils/version_control.sh" - -MODEL_CONFIG=$1 -COMMIT_HASH=$2 -AGENT=$3 -EVAL_LIMIT=$4 -MAX_ITERATIONS=$5 -NUM_WORKERS=$6 -EVAL_IDS=$7 - -if [ -z "$NUM_WORKERS" ]; then - NUM_WORKERS=1 - echo "Number of workers not specified, use default $NUM_WORKERS" -fi -checkout_eval_branch - -if [ -z "$AGENT" ]; then - echo "Agent not specified, use default CodeActAgent" - AGENT="CodeActAgent" -fi - -get_openhands_version - -echo "AGENT: $AGENT" -echo "OPENHANDS_VERSION: $OPENHANDS_VERSION" -echo "MODEL_CONFIG: $MODEL_CONFIG" - -EVAL_NOTE=$OPENHANDS_VERSION - -# Default to NOT use unit tests. -if [ -z "$USE_UNIT_TESTS" ]; then - export USE_UNIT_TESTS=false -fi -echo "USE_UNIT_TESTS: $USE_UNIT_TESTS" -# If use unit tests, set EVAL_NOTE to the commit hash -if [ "$USE_UNIT_TESTS" = true ]; then - EVAL_NOTE=$EVAL_NOTE-w-test -fi - -# export PYTHONPATH=evaluation/integration_tests:\$PYTHONPATH -COMMAND="poetry run python evaluation/integration_tests/run_infer.py \ - --agent-cls $AGENT \ - --llm-config $MODEL_CONFIG \ - --max-iterations ${MAX_ITERATIONS:-10} \ - --eval-num-workers $NUM_WORKERS \ - --eval-note $EVAL_NOTE" - -if [ -n "$EVAL_LIMIT" ]; then - echo "EVAL_LIMIT: $EVAL_LIMIT" - COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT" -fi - -if [ -n "$EVAL_IDS" ]; then - echo "EVAL_IDS: $EVAL_IDS" - COMMAND="$COMMAND --eval-ids $EVAL_IDS" -fi - -# Run the command -eval $COMMAND diff --git a/evaluation/integration_tests/tests/__init__.py b/evaluation/integration_tests/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/evaluation/integration_tests/tests/base.py b/evaluation/integration_tests/tests/base.py deleted file mode 100644 index bc98b884d2..0000000000 --- a/evaluation/integration_tests/tests/base.py +++ /dev/null @@ -1,32 +0,0 @@ -from abc import ABC, abstractmethod - -from pydantic import BaseModel - -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class TestResult(BaseModel): - success: bool - reason: str | None = None - - -class BaseIntegrationTest(ABC): - """Base class for integration tests.""" - - INSTRUCTION: str - - @classmethod - @abstractmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - """Initialize the runtime for the test to run.""" - pass - - @classmethod - @abstractmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - """Verify the result of the test. - - This method will be called after the agent performs the task on the runtime. - """ - pass diff --git a/evaluation/integration_tests/tests/t01_fix_simple_typo.py b/evaluation/integration_tests/tests/t01_fix_simple_typo.py deleted file mode 100644 index 532d5d5b38..0000000000 --- a/evaluation/integration_tests/tests/t01_fix_simple_typo.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import tempfile - -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from openhands.events.action import CmdRunAction -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Fix typos in bad.txt.' - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - # create a file with a typo in /workspace/bad.txt - with tempfile.TemporaryDirectory() as temp_dir: - temp_file_path = os.path.join(temp_dir, 'bad.txt') - with open(temp_file_path, 'w') as f: - f.write('This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!') - - # Copy the file to the desired location - runtime.copy_to(temp_file_path, '/workspace') - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - # check if the file /workspace/bad.txt has been fixed - action = CmdRunAction(command='cat /workspace/bad.txt') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, reason=f'Failed to run command: {obs.content}' - ) - # check if the file /workspace/bad.txt has been fixed - if ( - obs.content.strip().replace('\r\n', '\n') - == 'This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!' - ): - return TestResult(success=True) - return TestResult(success=False, reason=f'File not fixed: {obs.content}') diff --git a/evaluation/integration_tests/tests/t02_add_bash_hello.py b/evaluation/integration_tests/tests/t02_add_bash_hello.py deleted file mode 100644 index 88384a87f2..0000000000 --- a/evaluation/integration_tests/tests/t02_add_bash_hello.py +++ /dev/null @@ -1,40 +0,0 @@ -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import assert_and_raise -from openhands.events.action import CmdRunAction -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = "Write a shell script '/workspace/hello.sh' that prints 'hello'." - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - action = CmdRunAction(command='mkdir -p /workspace') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - # check if the file /workspace/hello.sh exists - action = CmdRunAction(command='cat /workspace/hello.sh') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to cat /workspace/hello.sh: {obs.content}.', - ) - - # execute the script - action = CmdRunAction(command='bash /workspace/hello.sh') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to execute /workspace/hello.sh: {obs.content}.', - ) - if obs.content.strip() != 'hello': - return TestResult( - success=False, reason=f'Script did not print "hello": {obs.content}.' - ) - return TestResult(success=True) diff --git a/evaluation/integration_tests/tests/t03_jupyter_write_file.py b/evaluation/integration_tests/tests/t03_jupyter_write_file.py deleted file mode 100644 index 2f88e1228b..0000000000 --- a/evaluation/integration_tests/tests/t03_jupyter_write_file.py +++ /dev/null @@ -1,43 +0,0 @@ -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import assert_and_raise -from openhands.events.action import CmdRunAction -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = "Use Jupyter IPython to write a text file containing 'hello world' to '/workspace/test.txt'." - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - action = CmdRunAction(command='mkdir -p /workspace') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - # check if the file /workspace/hello.sh exists - action = CmdRunAction(command='cat /workspace/test.txt') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to cat /workspace/test.txt: {obs.content}.', - ) - - # execute the script - action = CmdRunAction(command='cat /workspace/test.txt') - obs = runtime.run_action(action) - - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to cat /workspace/test.txt: {obs.content}.', - ) - - if 'hello world' not in obs.content.strip(): - return TestResult( - success=False, - reason=f'File did not contain "hello world": {obs.content}.', - ) - return TestResult(success=True) diff --git a/evaluation/integration_tests/tests/t04_git_staging.py b/evaluation/integration_tests/tests/t04_git_staging.py deleted file mode 100644 index 1c3daaba37..0000000000 --- a/evaluation/integration_tests/tests/t04_git_staging.py +++ /dev/null @@ -1,57 +0,0 @@ -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import assert_and_raise -from openhands.events.action import CmdRunAction -from openhands.events.event import Event -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Write a git commit message for the current staging area and commit the changes.' - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - action = CmdRunAction(command='mkdir -p /workspace') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - # git init - action = CmdRunAction(command='git init') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - # create file - action = CmdRunAction(command='echo \'print("hello world")\' > hello.py') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - # git add - cmd_str = 'git add hello.py' - action = CmdRunAction(command=cmd_str) - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - # check if the file /workspace/hello.py exists - action = CmdRunAction(command='cat /workspace/hello.py') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, - reason=f'Failed to cat /workspace/hello.py: {obs.content}.', - ) - - # check if the staging area is empty - action = CmdRunAction(command='git status') - obs = runtime.run_action(action) - if obs.exit_code != 0: - return TestResult( - success=False, reason=f'Failed to git status: {obs.content}.' - ) - if 'nothing to commit, working tree clean' in obs.content.strip(): - return TestResult(success=True) - - return TestResult( - success=False, - reason=f'Failed to check for "nothing to commit, working tree clean": {obs.content}.', - ) diff --git a/evaluation/integration_tests/tests/t05_simple_browsing.py b/evaluation/integration_tests/tests/t05_simple_browsing.py deleted file mode 100644 index 8542e50d80..0000000000 --- a/evaluation/integration_tests/tests/t05_simple_browsing.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import tempfile - -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from evaluation.utils.shared import assert_and_raise -from openhands.events.action import AgentFinishAction, CmdRunAction, MessageAction -from openhands.events.event import Event -from openhands.events.observation import AgentDelegateObservation -from openhands.runtime.base import Runtime - -HTML_FILE = """ - - - - - - The Ultimate Answer - - - -
-

The Ultimate Answer

-

Click the button to reveal the answer to life, the universe, and everything.

- -
-
- - - -""" - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Browse localhost:8000, and tell me the ultimate answer to life.' - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - action = CmdRunAction(command='mkdir -p /workspace') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - action = CmdRunAction(command='mkdir -p /tmp/server') - obs = runtime.run_action(action) - assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}') - - # create a file with a typo in /workspace/bad.txt - with tempfile.TemporaryDirectory() as temp_dir: - temp_file_path = os.path.join(temp_dir, 'index.html') - with open(temp_file_path, 'w') as f: - f.write(HTML_FILE) - # Copy the file to the desired location - runtime.copy_to(temp_file_path, '/tmp/server') - - # create README.md - action = CmdRunAction( - command='cd /tmp/server && nohup python3 -m http.server 8000 &' - ) - obs = runtime.run_action(action) - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - from openhands.core.logger import openhands_logger as logger - - # check if the "The answer is OpenHands is all you need!" is in any message - message_actions = [ - event - for event in histories - if isinstance( - event, (MessageAction, AgentFinishAction, AgentDelegateObservation) - ) - ] - logger.debug(f'Total message-like events: {len(message_actions)}') - - for event in message_actions: - try: - if isinstance(event, AgentDelegateObservation): - content = event.content - elif isinstance(event, AgentFinishAction): - content = event.outputs.get('content', '') - elif isinstance(event, MessageAction): - content = event.content - else: - logger.warning(f'Unexpected event type: {type(event)}') - continue - - if 'OpenHands is all you need!' in content: - return TestResult(success=True) - except Exception as e: - logger.error(f'Error processing event: {e}') - - logger.debug( - f'Total messages: {len(message_actions)}. Messages: {message_actions}' - ) - return TestResult( - success=False, - reason=f'The answer is not found in any message. Total messages: {len(message_actions)}.', - ) diff --git a/evaluation/integration_tests/tests/t06_github_pr_browsing.py b/evaluation/integration_tests/tests/t06_github_pr_browsing.py deleted file mode 100644 index b85e868401..0000000000 --- a/evaluation/integration_tests/tests/t06_github_pr_browsing.py +++ /dev/null @@ -1,58 +0,0 @@ -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from openhands.events.action import AgentFinishAction, MessageAction -from openhands.events.event import Event -from openhands.events.observation import AgentDelegateObservation -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Look at https://github.com/OpenHands/OpenHands/pull/8, and tell me what is happening there and what did @asadm suggest.' - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - pass - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - from openhands.core.logger import openhands_logger as logger - - # check if the license information is in any message - message_actions = [ - event - for event in histories - if isinstance( - event, (MessageAction, AgentFinishAction, AgentDelegateObservation) - ) - ] - logger.info(f'Total message-like events: {len(message_actions)}') - - for event in message_actions: - try: - if isinstance(event, AgentDelegateObservation): - content = event.content - elif isinstance(event, AgentFinishAction): - content = event.outputs.get('content', '') - if event.thought: - content += f'\n\n{event.thought}' - elif isinstance(event, MessageAction): - content = event.content - else: - logger.warning(f'Unexpected event type: {type(event)}') - continue - - if ( - 'non-commercial' in content - or 'MIT' in content - or 'Apache 2.0' in content - ): - return TestResult(success=True) - except Exception as e: - logger.error(f'Error processing event: {e}') - - logger.debug( - f'Total messages: {len(message_actions)}. Messages: {message_actions}' - ) - return TestResult( - success=False, - reason=f'The answer is not found in any message. Total messages: {len(message_actions)}.', - ) diff --git a/evaluation/integration_tests/tests/t07_interactive_commands.py b/evaluation/integration_tests/tests/t07_interactive_commands.py deleted file mode 100644 index 24a66d3f38..0000000000 --- a/evaluation/integration_tests/tests/t07_interactive_commands.py +++ /dev/null @@ -1,73 +0,0 @@ -import hashlib - -from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult -from openhands.events.action import ( - AgentFinishAction, - FileWriteAction, - MessageAction, -) -from openhands.events.event import Event -from openhands.events.observation import AgentDelegateObservation -from openhands.runtime.base import Runtime - - -class Test(BaseIntegrationTest): - INSTRUCTION = 'Execute the python script /workspace/python_script.py with input "John" and "25" and tell me the secret number.' - SECRET_NUMBER = int(hashlib.sha256(str(25).encode()).hexdigest()[:8], 16) % 1000 - - @classmethod - def initialize_runtime(cls, runtime: Runtime) -> None: - from openhands.core.logger import openhands_logger as logger - - action = FileWriteAction( - path='/workspace/python_script.py', - content=( - 'name = input("Enter your name: "); age = input("Enter your age: "); ' - 'import hashlib; secret = int(hashlib.sha256(str(age).encode()).hexdigest()[:8], 16) % 1000; ' - 'print(f"Hello {name}, you are {age} years old. Tell you a secret number: {secret}")' - ), - ) - logger.info(action, extra={'msg_type': 'ACTION'}) - observation = runtime.run_action(action) - logger.info(observation, extra={'msg_type': 'OBSERVATION'}) - - @classmethod - def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult: - from openhands.core.logger import openhands_logger as logger - - # check if the license information is in any message - message_actions = [ - event - for event in histories - if isinstance( - event, (MessageAction, AgentFinishAction, AgentDelegateObservation) - ) - ] - logger.info(f'Total message-like events: {len(message_actions)}') - - for event in message_actions: - try: - if isinstance(event, AgentDelegateObservation): - content = event.content - elif isinstance(event, AgentFinishAction): - content = event.outputs.get('content', '') - if event.thought: - content += f'\n\n{event.thought}' - elif isinstance(event, MessageAction): - content = event.content - else: - logger.warning(f'Unexpected event type: {type(event)}') - continue - - if str(cls.SECRET_NUMBER) in content: - return TestResult(success=True) - except Exception as e: - logger.error(f'Error processing event: {e}') - - logger.debug( - f'Total messages: {len(message_actions)}. Messages: {message_actions}' - ) - return TestResult( - success=False, - reason=f'The answer is not found in any message. Total messages: {len(message_actions)}.', - ) diff --git a/frontend/__tests__/components/context-menu/account-settings-context-menu.test.tsx b/frontend/__tests__/components/context-menu/account-settings-context-menu.test.tsx index 1cfc1b8fb7..5eeafbc51d 100644 --- a/frontend/__tests__/components/context-menu/account-settings-context-menu.test.tsx +++ b/frontend/__tests__/components/context-menu/account-settings-context-menu.test.tsx @@ -33,9 +33,24 @@ describe("AccountSettingsContextMenu", () => { expect( screen.getByTestId("account-settings-context-menu"), ).toBeInTheDocument(); + expect(screen.getByText("SIDEBAR$DOCS")).toBeInTheDocument(); expect(screen.getByText("ACCOUNT_SETTINGS$LOGOUT")).toBeInTheDocument(); }); + it("should render Documentation link with correct attributes", () => { + renderWithRouter( + , + ); + + const documentationLink = screen.getByText("SIDEBAR$DOCS").closest("a"); + expect(documentationLink).toHaveAttribute("href", "https://docs.openhands.dev"); + expect(documentationLink).toHaveAttribute("target", "_blank"); + expect(documentationLink).toHaveAttribute("rel", "noopener noreferrer"); + }); + it("should call onLogout when the logout option is clicked", async () => { renderWithRouter( ({ useAuthUrl: () => "https://gitlab.com/oauth/authorize", })); +// Mock the useTracking hook +vi.mock("#/hooks/use-tracking", () => ({ + useTracking: () => ({ + trackLoginButtonClick: vi.fn(), + }), +})); + describe("AuthModal", () => { beforeEach(() => { vi.stubGlobal("location", { href: "" }); diff --git a/frontend/__tests__/components/features/microagent-management/microagent-management.test.tsx b/frontend/__tests__/components/features/microagent-management/microagent-management.test.tsx index d22825d8d1..afdb8e84ba 100644 --- a/frontend/__tests__/components/features/microagent-management/microagent-management.test.tsx +++ b/frontend/__tests__/components/features/microagent-management/microagent-management.test.tsx @@ -21,6 +21,7 @@ const mockUseConfig = vi.fn(); const mockUseRepositoryMicroagents = vi.fn(); const mockUseMicroagentManagementConversations = vi.fn(); const mockUseSearchRepositories = vi.fn(); +const mockUseCreateConversationAndSubscribeMultiple = vi.fn(); vi.mock("#/hooks/use-user-providers", () => ({ useUserProviders: () => mockUseUserProviders(), @@ -47,6 +48,17 @@ vi.mock("#/hooks/query/use-search-repositories", () => ({ useSearchRepositories: () => mockUseSearchRepositories(), })); +vi.mock("#/hooks/use-tracking", () => ({ + useTracking: () => ({ + trackEvent: vi.fn(), + }), +})); + +vi.mock("#/hooks/use-create-conversation-and-subscribe-multiple", () => ({ + useCreateConversationAndSubscribeMultiple: () => + mockUseCreateConversationAndSubscribeMultiple(), +})); + describe("MicroagentManagement", () => { const RouterStub = createRoutesStub([ { @@ -309,6 +321,16 @@ describe("MicroagentManagement", () => { isError: false, }); + mockUseCreateConversationAndSubscribeMultiple.mockReturnValue({ + createConversationAndSubscribe: vi.fn(({ onSuccessCallback }) => { + // Immediately call the success callback to close the modal + if (onSuccessCallback) { + onSuccessCallback(); + } + }), + isPending: false, + }); + // Mock the search repositories hook to return repositories with OpenHands suffixes const mockSearchResults = getRepositoriesWithOpenHandsSuffix(mockRepositories); diff --git a/frontend/__tests__/components/image-preview.test.tsx b/frontend/__tests__/components/image-preview.test.tsx index 39d2f089fb..7e40d4b925 100644 --- a/frontend/__tests__/components/image-preview.test.tsx +++ b/frontend/__tests__/components/image-preview.test.tsx @@ -30,7 +30,7 @@ describe("ImagePreview", () => { expect(onRemoveMock).toHaveBeenCalledOnce(); }); - it("shoud not display the close button when onRemove is not provided", () => { + it("should not display the close button when onRemove is not provided", () => { render(); expect(screen.queryByRole("button")).not.toBeInTheDocument(); }); diff --git a/frontend/__tests__/routes/accept-tos.test.tsx b/frontend/__tests__/routes/accept-tos.test.tsx index ce6f36793b..7b15081485 100644 --- a/frontend/__tests__/routes/accept-tos.test.tsx +++ b/frontend/__tests__/routes/accept-tos.test.tsx @@ -1,10 +1,9 @@ import { render, screen } from "@testing-library/react"; import { it, describe, expect, vi, beforeEach, afterEach } from "vitest"; import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import AcceptTOS from "#/routes/accept-tos"; import * as CaptureConsent from "#/utils/handle-capture-consent"; -import * as ToastHandlers from "#/utils/custom-toast-handlers"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { openHands } from "#/api/open-hands-axios"; // Mock the react-router hooks @@ -44,9 +43,13 @@ const createWrapper = () => { }, }); - return ({ children }: { children: React.ReactNode }) => ( - {children} - ); + function Wrapper({ children }: { children: React.ReactNode }) { + return ( + {children} + ); + } + + return Wrapper; }; describe("AcceptTOS", () => { @@ -106,7 +109,10 @@ describe("AcceptTOS", () => { // Wait for the mutation to complete await new Promise(process.nextTick); - expect(handleCaptureConsentSpy).toHaveBeenCalledWith(true); + expect(handleCaptureConsentSpy).toHaveBeenCalledWith( + expect.anything(), + true, + ); expect(openHands.post).toHaveBeenCalledWith("/api/accept_tos", { redirect_url: "/dashboard", }); diff --git a/frontend/__tests__/routes/app-settings.test.tsx b/frontend/__tests__/routes/app-settings.test.tsx index 31b0f6b829..44dacce2fb 100644 --- a/frontend/__tests__/routes/app-settings.test.tsx +++ b/frontend/__tests__/routes/app-settings.test.tsx @@ -46,6 +46,21 @@ describe("Content", () => { }); }); + it("should render analytics toggle as enabled when server returns null (opt-in by default)", async () => { + const getSettingsSpy = vi.spyOn(SettingsService, "getSettings"); + getSettingsSpy.mockResolvedValue({ + ...MOCK_DEFAULT_USER_SETTINGS, + user_consents_to_analytics: null, + }); + + renderAppSettingsScreen(); + + await waitFor(() => { + const analytics = screen.getByTestId("enable-analytics-switch"); + expect(analytics).toBeChecked(); + }); + }); + it("should render the language options", async () => { renderAppSettingsScreen(); @@ -163,7 +178,10 @@ describe("Form submission", () => { await userEvent.click(submit); await waitFor(() => - expect(handleCaptureConsentsSpy).toHaveBeenCalledWith(true), + expect(handleCaptureConsentsSpy).toHaveBeenCalledWith( + expect.anything(), + true, + ), ); }); @@ -188,7 +206,10 @@ describe("Form submission", () => { await userEvent.click(submit); await waitFor(() => - expect(handleCaptureConsentsSpy).toHaveBeenCalledWith(false), + expect(handleCaptureConsentsSpy).toHaveBeenCalledWith( + expect.anything(), + false, + ), ); }); diff --git a/frontend/__tests__/utils/error-handler.test.ts b/frontend/__tests__/utils/error-handler.test.ts index 0f1e91cae2..b0cf26bc64 100644 --- a/frontend/__tests__/utils/error-handler.test.ts +++ b/frontend/__tests__/utils/error-handler.test.ts @@ -32,6 +32,7 @@ describe("Error Handler", () => { const error = { message: "Test error", source: "test", + posthog, }; trackError(error); @@ -52,6 +53,7 @@ describe("Error Handler", () => { extra: "info", details: { foo: "bar" }, }, + posthog, }; trackError(error); @@ -73,6 +75,7 @@ describe("Error Handler", () => { const error = { message: "Toast error", source: "toast-test", + posthog, }; showErrorToast(error); @@ -94,6 +97,7 @@ describe("Error Handler", () => { message: "Toast error", source: "toast-test", metadata: { context: "testing" }, + posthog, }; showErrorToast(error); @@ -113,6 +117,7 @@ describe("Error Handler", () => { message: "Agent error", source: "agent-status", metadata: { id: "error.agent" }, + posthog, }); expect(posthog.captureException).toHaveBeenCalledWith( @@ -127,6 +132,7 @@ describe("Error Handler", () => { message: "Server error", source: "server", metadata: { error_code: 500, details: "Internal error" }, + posthog, }); expect(posthog.captureException).toHaveBeenCalledWith( @@ -145,6 +151,7 @@ describe("Error Handler", () => { message: error.message, source: "feedback", metadata: { conversationId: "123", error }, + posthog, }); expect(posthog.captureException).toHaveBeenCalledWith( @@ -164,6 +171,7 @@ describe("Error Handler", () => { message: "Chat error", source: "chat-test", msgId: "123", + posthog, }; showChatError(error); diff --git a/frontend/__tests__/utils/handle-capture-consent.test.ts b/frontend/__tests__/utils/handle-capture-consent.test.ts index 3b337424a7..0faf999c2b 100644 --- a/frontend/__tests__/utils/handle-capture-consent.test.ts +++ b/frontend/__tests__/utils/handle-capture-consent.test.ts @@ -13,14 +13,14 @@ describe("handleCaptureConsent", () => { }); it("should opt out of of capturing", () => { - handleCaptureConsent(false); + handleCaptureConsent(posthog, false); expect(optOutSpy).toHaveBeenCalled(); expect(optInSpy).not.toHaveBeenCalled(); }); it("should opt in to capturing if the user consents", () => { - handleCaptureConsent(true); + handleCaptureConsent(posthog, true); expect(optInSpy).toHaveBeenCalled(); expect(optOutSpy).not.toHaveBeenCalled(); @@ -28,7 +28,7 @@ describe("handleCaptureConsent", () => { it("should not opt in to capturing if the user is already opted in", () => { hasOptedInSpy.mockReturnValueOnce(true); - handleCaptureConsent(true); + handleCaptureConsent(posthog, true); expect(optInSpy).not.toHaveBeenCalled(); expect(optOutSpy).not.toHaveBeenCalled(); @@ -36,7 +36,7 @@ describe("handleCaptureConsent", () => { it("should not opt out of capturing if the user is already opted out", () => { hasOptedOutSpy.mockReturnValueOnce(true); - handleCaptureConsent(false); + handleCaptureConsent(posthog, false); expect(optOutSpy).not.toHaveBeenCalled(); expect(optInSpy).not.toHaveBeenCalled(); diff --git a/frontend/src/utils/__tests__/status.test.ts b/frontend/__tests__/utils/status.test.ts similarity index 86% rename from frontend/src/utils/__tests__/status.test.ts rename to frontend/__tests__/utils/status.test.ts index cca6c0efaf..66dea4c799 100644 --- a/frontend/src/utils/__tests__/status.test.ts +++ b/frontend/__tests__/utils/status.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { getStatusCode, getIndicatorColor, IndicatorColor } from "../status"; +import { getStatusCode, getIndicatorColor, IndicatorColor } from "#/utils/status"; import { AgentState } from "#/types/agent-state"; import { I18nKey } from "#/i18n/declaration"; @@ -87,6 +87,36 @@ describe("getStatusCode", () => { // Should return runtime status since no agent state expect(result).toBe("STATUS$STARTING_RUNTIME"); }); + + it("should prioritize task ERROR status over websocket CONNECTING state", () => { + // Test case: Task has errored but websocket is still trying to connect + const result = getStatusCode( + { id: "", message: "", type: "info", status_update: true }, // statusMessage + "CONNECTING", // webSocketStatus (stuck connecting) + null, // conversationStatus + null, // runtimeStatus + AgentState.LOADING, // agentState + "ERROR", // taskStatus (ERROR) + ); + + // Should return error message, not "Connecting..." + expect(result).toBe(I18nKey.AGENT_STATUS$ERROR_OCCURRED); + }); + + it("should show Connecting when task is working and websocket is connecting", () => { + // Test case: Task is in progress and websocket is connecting normally + const result = getStatusCode( + { id: "", message: "", type: "info", status_update: true }, // statusMessage + "CONNECTING", // webSocketStatus + null, // conversationStatus + null, // runtimeStatus + AgentState.LOADING, // agentState + "WORKING", // taskStatus (in progress) + ); + + // Should show connecting message since task hasn't errored + expect(result).toBe(I18nKey.CHAT_INTERFACE$CONNECTING); + }); }); describe("getIndicatorColor", () => { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index feb13dcc3a..862f0c5a00 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,17 +1,18 @@ { "name": "openhands-frontend", - "version": "0.61.0", + "version": "0.62.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openhands-frontend", - "version": "0.61.0", + "version": "0.62.0", "dependencies": { "@heroui/react": "^2.8.4", "@heroui/use-infinite-scroll": "^2.2.11", "@microlink/react-json-view": "^1.26.2", "@monaco-editor/react": "^4.7.0-rc.0", + "@posthog/react": "^1.4.0", "@react-router/node": "^7.9.3", "@react-router/serve": "^7.9.3", "@react-types/shared": "^3.32.0", @@ -38,7 +39,7 @@ "jose": "^6.1.0", "lucide-react": "^0.544.0", "monaco-editor": "^0.53.0", - "posthog-js": "^1.268.8", + "posthog-js": "^1.290.0", "react": "^19.1.1", "react-dom": "^19.1.1", "react-highlight": "^0.15.0", @@ -3511,9 +3512,29 @@ "license": "MIT" }, "node_modules/@posthog/core": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.2.2.tgz", - "integrity": "sha512-f16Ozx6LIigRG+HsJdt+7kgSxZTHeX5f1JlCGKI1lXcvlZgfsCR338FuMI2QRYXGl+jg/vYFzGOTQBxl90lnBg==" + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.5.2.tgz", + "integrity": "sha512-iedUP3EnOPPxTA2VaIrsrd29lSZnUV+ZrMnvY56timRVeZAXoYCkmjfIs3KBAsF8OUT5h1GXLSkoQdrV0r31OQ==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.6" + } + }, + "node_modules/@posthog/react": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@posthog/react/-/react-1.4.0.tgz", + "integrity": "sha512-xzPeZ753fQ0deZzdgY/0YavZvNpmdaxUzLYJYu5XjONNcZ8PwJnNLEK+7D/Cj8UM4Q8nWI7QC5mjum0uLWa4FA==", + "license": "MIT", + "peerDependencies": { + "@types/react": ">=16.8.0", + "posthog-js": ">=1.257.2", + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } }, "node_modules/@react-aria/breadcrumbs": { "version": "3.5.28", @@ -8183,7 +8204,6 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -8198,7 +8218,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" @@ -11403,7 +11422,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, "license": "ISC" }, "node_modules/istanbul-lib-coverage": { @@ -14073,7 +14091,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -14264,27 +14281,16 @@ "license": "MIT" }, "node_modules/posthog-js": { - "version": "1.268.8", - "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.268.8.tgz", - "integrity": "sha512-BJiKK4MlUvs7ybnQcy1KkwAz+SZkE/wRLotetIoank5kbqZs8FLbeyozFvmmgx4aoMmaVymYBSmYphYjYQeidw==", + "version": "1.290.0", + "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.290.0.tgz", + "integrity": "sha512-zavBwZkf+3JeiSDVE7ZDXBfzva/iOljicdhdJH+cZoqp0LsxjKxjnNhGOd3KpAhw0wqdwjhd7Lp1aJuI7DXyaw==", + "license": "SEE LICENSE IN LICENSE", "dependencies": { - "@posthog/core": "1.2.2", + "@posthog/core": "1.5.2", "core-js": "^3.38.1", "fflate": "^0.4.8", "preact": "^10.19.3", "web-vitals": "^4.2.4" - }, - "peerDependencies": { - "@rrweb/types": "2.0.0-alpha.17", - "rrweb-snapshot": "2.0.0-alpha.17" - }, - "peerDependenciesMeta": { - "@rrweb/types": { - "optional": true - }, - "rrweb-snapshot": { - "optional": true - } } }, "node_modules/posthog-js/node_modules/web-vitals": { @@ -15547,7 +15553,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -15560,7 +15565,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" diff --git a/frontend/package.json b/frontend/package.json index 5ad91c3636..46958662a0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "openhands-frontend", - "version": "0.61.0", + "version": "0.62.0", "private": true, "type": "module", "engines": { @@ -11,6 +11,7 @@ "@heroui/use-infinite-scroll": "^2.2.11", "@microlink/react-json-view": "^1.26.2", "@monaco-editor/react": "^4.7.0-rc.0", + "@posthog/react": "^1.4.0", "@react-router/node": "^7.9.3", "@react-router/serve": "^7.9.3", "@react-types/shared": "^3.32.0", @@ -37,7 +38,7 @@ "jose": "^6.1.0", "lucide-react": "^0.544.0", "monaco-editor": "^0.53.0", - "posthog-js": "^1.268.8", + "posthog-js": "^1.290.0", "react": "^19.1.1", "react-dom": "^19.1.1", "react-highlight": "^0.15.0", diff --git a/frontend/src/components/features/analytics/analytics-consent-form-modal.tsx b/frontend/src/components/features/analytics/analytics-consent-form-modal.tsx index c3ab215272..cc2f293235 100644 --- a/frontend/src/components/features/analytics/analytics-consent-form-modal.tsx +++ b/frontend/src/components/features/analytics/analytics-consent-form-modal.tsx @@ -1,4 +1,5 @@ import { useTranslation } from "react-i18next"; +import { usePostHog } from "posthog-js/react"; import { BaseModalTitle, BaseModalDescription, @@ -17,6 +18,7 @@ interface AnalyticsConsentFormModalProps { export function AnalyticsConsentFormModal({ onClose, }: AnalyticsConsentFormModalProps) { + const posthog = usePostHog(); const { t } = useTranslation(); const { mutate: saveUserSettings } = useSaveSettings(); @@ -29,7 +31,7 @@ export function AnalyticsConsentFormModal({ { user_consents_to_analytics: analytics }, { onSuccess: () => { - handleCaptureConsent(analytics); + handleCaptureConsent(posthog, analytics); onClose(); }, }, diff --git a/frontend/src/components/features/chat/change-agent-button.tsx b/frontend/src/components/features/chat/change-agent-button.tsx new file mode 100644 index 0000000000..706f582b59 --- /dev/null +++ b/frontend/src/components/features/chat/change-agent-button.tsx @@ -0,0 +1,109 @@ +import React, { useMemo, useEffect } from "react"; +import { useTranslation } from "react-i18next"; +import { Typography } from "#/ui/typography"; +import { I18nKey } from "#/i18n/declaration"; +import CodeTagIcon from "#/icons/code-tag.svg?react"; +import ChevronDownSmallIcon from "#/icons/chevron-down-small.svg?react"; +import LessonPlanIcon from "#/icons/lesson-plan.svg?react"; +import { useConversationStore } from "#/state/conversation-store"; +import { ChangeAgentContextMenu } from "./change-agent-context-menu"; +import { cn } from "#/utils/utils"; +import { USE_PLANNING_AGENT } from "#/utils/feature-flags"; +import { useAgentState } from "#/hooks/use-agent-state"; +import { AgentState } from "#/types/agent-state"; + +export function ChangeAgentButton() { + const { t } = useTranslation(); + const [contextMenuOpen, setContextMenuOpen] = React.useState(false); + + const conversationMode = useConversationStore( + (state) => state.conversationMode, + ); + + const setConversationMode = useConversationStore( + (state) => state.setConversationMode, + ); + + const shouldUsePlanningAgent = USE_PLANNING_AGENT(); + + const { curAgentState } = useAgentState(); + + const isAgentRunning = curAgentState === AgentState.RUNNING; + + // Close context menu when agent starts running + useEffect(() => { + if (isAgentRunning && contextMenuOpen) { + setContextMenuOpen(false); + } + }, [isAgentRunning, contextMenuOpen]); + + const handleButtonClick = (event: React.MouseEvent) => { + event.preventDefault(); + event.stopPropagation(); + setContextMenuOpen(!contextMenuOpen); + }; + + const handleCodeClick = (event: React.MouseEvent) => { + event.preventDefault(); + event.stopPropagation(); + setConversationMode("code"); + }; + + const handlePlanClick = (event: React.MouseEvent) => { + event.preventDefault(); + event.stopPropagation(); + setConversationMode("plan"); + }; + + const isExecutionAgent = conversationMode === "code"; + + const buttonLabel = useMemo(() => { + if (isExecutionAgent) { + return t(I18nKey.COMMON$CODE); + } + return t(I18nKey.COMMON$PLAN); + }, [isExecutionAgent, t]); + + const buttonIcon = useMemo(() => { + if (isExecutionAgent) { + return ; + } + return ; + }, [isExecutionAgent]); + + if (!shouldUsePlanningAgent) { + return null; + } + + return ( +
+ + {contextMenuOpen && ( + setContextMenuOpen(false)} + onCodeClick={handleCodeClick} + onPlanClick={handlePlanClick} + /> + )} +
+ ); +} diff --git a/frontend/src/components/features/chat/change-agent-context-menu.tsx b/frontend/src/components/features/chat/change-agent-context-menu.tsx new file mode 100644 index 0000000000..6e88ce97d4 --- /dev/null +++ b/frontend/src/components/features/chat/change-agent-context-menu.tsx @@ -0,0 +1,81 @@ +import React from "react"; +import { useTranslation } from "react-i18next"; +import { I18nKey } from "#/i18n/declaration"; +import CodeTagIcon from "#/icons/code-tag.svg?react"; +import LessonPlanIcon from "#/icons/lesson-plan.svg?react"; +import { ContextMenu } from "#/ui/context-menu"; +import { ContextMenuListItem } from "../context-menu/context-menu-list-item"; +import { ContextMenuIconText } from "../context-menu/context-menu-icon-text"; +import { useClickOutsideElement } from "#/hooks/use-click-outside-element"; +import { cn } from "#/utils/utils"; +import { CONTEXT_MENU_ICON_TEXT_CLASSNAME } from "#/utils/constants"; + +const contextMenuListItemClassName = cn( + "cursor-pointer p-0 h-auto hover:bg-transparent", + CONTEXT_MENU_ICON_TEXT_CLASSNAME, +); + +const contextMenuIconTextClassName = + "gap-2 p-2 hover:bg-[#5C5D62] rounded h-[30px]"; + +interface ChangeAgentContextMenuProps { + onClose: () => void; + onCodeClick?: (event: React.MouseEvent) => void; + onPlanClick?: (event: React.MouseEvent) => void; +} + +export function ChangeAgentContextMenu({ + onClose, + onCodeClick, + onPlanClick, +}: ChangeAgentContextMenuProps) { + const { t } = useTranslation(); + const menuRef = useClickOutsideElement(onClose); + + const handleCodeClick = (event: React.MouseEvent) => { + event.preventDefault(); + event.stopPropagation(); + onCodeClick?.(event); + onClose(); + }; + + const handlePlanClick = (event: React.MouseEvent) => { + event.preventDefault(); + event.stopPropagation(); + onPlanClick?.(event); + onClose(); + }; + + return ( + + + + + + + + + ); +} diff --git a/frontend/src/components/features/chat/chat-interface.tsx b/frontend/src/components/features/chat/chat-interface.tsx index cabf087689..f37bd59c26 100644 --- a/frontend/src/components/features/chat/chat-interface.tsx +++ b/frontend/src/components/features/chat/chat-interface.tsx @@ -1,5 +1,5 @@ import React from "react"; -import posthog from "posthog-js"; +import { usePostHog } from "posthog-js/react"; import { useParams } from "react-router"; import { useTranslation } from "react-i18next"; import { convertImageToBase64 } from "#/utils/convert-image-to-base-64"; @@ -60,6 +60,7 @@ function getEntryPoint( } export function ChatInterface() { + const posthog = usePostHog(); const { setMessageToSend } = useConversationStore(); const { data: conversation } = useActiveConversation(); const { errorMessage } = useErrorMessageStore(); diff --git a/frontend/src/components/features/chat/components/chat-input-actions.tsx b/frontend/src/components/features/chat/components/chat-input-actions.tsx index abe226520e..7683464499 100644 --- a/frontend/src/components/features/chat/components/chat-input-actions.tsx +++ b/frontend/src/components/features/chat/components/chat-input-actions.tsx @@ -8,6 +8,7 @@ import { generateAgentStateChangeEvent } from "#/services/agent-state-service"; import { AgentState } from "#/types/agent-state"; import { useV1PauseConversation } from "#/hooks/mutation/use-v1-pause-conversation"; import { useV1ResumeConversation } from "#/hooks/mutation/use-v1-resume-conversation"; +import { ChangeAgentButton } from "../change-agent-button"; interface ChatInputActionsProps { disabled: boolean; @@ -56,7 +57,10 @@ export function ChatInputActions({ return (
- +
+ + +
; @@ -20,6 +22,12 @@ export function ChatInputField({ }: ChatInputFieldProps) { const { t } = useTranslation(); + const conversationMode = useConversationStore( + (state) => state.conversationMode, + ); + + const isPlanMode = conversationMode === "plan"; + return (
void; @@ -20,6 +20,7 @@ export function GitControlBarPrButton({ isConversationReady = true, }: GitControlBarPrButtonProps) { const { t } = useTranslation(); + const { trackCreatePrButtonClick } = useTracking(); const { providers } = useUserProviders(); @@ -28,7 +29,7 @@ export function GitControlBarPrButton({ providersAreSet && hasRepository && isConversationReady; const handlePrClick = () => { - posthog.capture("create_pr_button_clicked"); + trackCreatePrButtonClick(); onSuggestionsClick(getCreatePRPrompt(currentGitProvider)); }; diff --git a/frontend/src/components/features/chat/git-control-bar-pull-button.tsx b/frontend/src/components/features/chat/git-control-bar-pull-button.tsx index 7adb9a4649..d0a1374098 100644 --- a/frontend/src/components/features/chat/git-control-bar-pull-button.tsx +++ b/frontend/src/components/features/chat/git-control-bar-pull-button.tsx @@ -1,10 +1,10 @@ import { useTranslation } from "react-i18next"; -import posthog from "posthog-js"; import ArrowDownIcon from "#/icons/u-arrow-down.svg?react"; import { cn, getGitPullPrompt } from "#/utils/utils"; import { useActiveConversation } from "#/hooks/query/use-active-conversation"; import { useUserProviders } from "#/hooks/use-user-providers"; import { I18nKey } from "#/i18n/declaration"; +import { useTracking } from "#/hooks/use-tracking"; interface GitControlBarPullButtonProps { onSuggestionsClick: (value: string) => void; @@ -16,6 +16,7 @@ export function GitControlBarPullButton({ isConversationReady = true, }: GitControlBarPullButtonProps) { const { t } = useTranslation(); + const { trackPullButtonClick } = useTracking(); const { data: conversation } = useActiveConversation(); const { providers } = useUserProviders(); @@ -26,7 +27,7 @@ export function GitControlBarPullButton({ providersAreSet && hasRepository && isConversationReady; const handlePullClick = () => { - posthog.capture("pull_button_clicked"); + trackPullButtonClick(); onSuggestionsClick(getGitPullPrompt()); }; diff --git a/frontend/src/components/features/chat/git-control-bar-push-button.tsx b/frontend/src/components/features/chat/git-control-bar-push-button.tsx index 5c40bd845f..dec4e97bed 100644 --- a/frontend/src/components/features/chat/git-control-bar-push-button.tsx +++ b/frontend/src/components/features/chat/git-control-bar-push-button.tsx @@ -1,10 +1,10 @@ import { useTranslation } from "react-i18next"; -import posthog from "posthog-js"; import ArrowUpIcon from "#/icons/u-arrow-up.svg?react"; import { cn, getGitPushPrompt } from "#/utils/utils"; import { useUserProviders } from "#/hooks/use-user-providers"; import { I18nKey } from "#/i18n/declaration"; import { Provider } from "#/types/settings"; +import { useTracking } from "#/hooks/use-tracking"; interface GitControlBarPushButtonProps { onSuggestionsClick: (value: string) => void; @@ -20,6 +20,7 @@ export function GitControlBarPushButton({ isConversationReady = true, }: GitControlBarPushButtonProps) { const { t } = useTranslation(); + const { trackPushButtonClick } = useTracking(); const { providers } = useUserProviders(); @@ -28,7 +29,7 @@ export function GitControlBarPushButton({ providersAreSet && hasRepository && isConversationReady; const handlePushClick = () => { - posthog.capture("push_button_clicked"); + trackPushButtonClick(); onSuggestionsClick(getGitPushPrompt(currentGitProvider)); }; diff --git a/frontend/src/components/features/chat/plan-preview.tsx b/frontend/src/components/features/chat/plan-preview.tsx new file mode 100644 index 0000000000..71504c54bc --- /dev/null +++ b/frontend/src/components/features/chat/plan-preview.tsx @@ -0,0 +1,82 @@ +import { useTranslation } from "react-i18next"; +import { ArrowUpRight } from "lucide-react"; +import LessonPlanIcon from "#/icons/lesson-plan.svg?react"; +import { USE_PLANNING_AGENT } from "#/utils/feature-flags"; +import { Typography } from "#/ui/typography"; +import { I18nKey } from "#/i18n/declaration"; + +interface PlanPreviewProps { + title?: string; + description?: string; + onViewClick?: () => void; + onBuildClick?: () => void; +} + +// TODO: Remove the hardcoded values and use the plan content from the conversation store +/* eslint-disable i18next/no-literal-string */ +export function PlanPreview({ + title = "Improve Developer Onboarding and Examples", + description = "Based on the analysis of Browser-Use's current documentation and examples, this plan addresses gaps in developer onboarding by creating a progressive learning path, troubleshooting resources, and practical examples that address real-world scenarios (like the LM Studio/local LLM integration issues encountered...", + onViewClick, + onBuildClick, +}: PlanPreviewProps) { + const { t } = useTranslation(); + + const shouldUsePlanningAgent = USE_PLANNING_AGENT(); + + if (!shouldUsePlanningAgent) { + return null; + } + + return ( +
+ {/* Header */} +
+ + + {t(I18nKey.COMMON$PLAN_MD)} + +
+ +
+ + {/* Content */} +
+

+ {title} +

+

+ {description} + + {t(I18nKey.COMMON$READ_MORE)} + +

+
+ + {/* Footer */} +
+ +
+
+ ); +} diff --git a/frontend/src/components/features/context-menu/account-settings-context-menu.tsx b/frontend/src/components/features/context-menu/account-settings-context-menu.tsx index 8c212dd5fc..c09920e614 100644 --- a/frontend/src/components/features/context-menu/account-settings-context-menu.tsx +++ b/frontend/src/components/features/context-menu/account-settings-context-menu.tsx @@ -8,6 +8,7 @@ import { useClickOutsideElement } from "#/hooks/use-click-outside-element"; import { useConfig } from "#/hooks/query/use-config"; import { I18nKey } from "#/i18n/declaration"; import LogOutIcon from "#/icons/log-out.svg?react"; +import DocumentIcon from "#/icons/document.svg?react"; import { SAAS_NAV_ITEMS, OSS_NAV_ITEMS } from "#/constants/settings-nav"; interface AccountSettingsContextMenuProps { @@ -58,6 +59,21 @@ export function AccountSettingsContextMenu({ + + + + {t(I18nKey.SIDEBAR$DOCS)} + + + { - setShouldShownAgentLoading(shouldShownAgentLoading); + setShouldShownAgentLoading(!!shouldShownAgentLoading); }, [shouldShownAgentLoading, setShouldShownAgentLoading]); return ( diff --git a/frontend/src/components/features/conversation-panel/conversation-card/conversation-card.tsx b/frontend/src/components/features/conversation-panel/conversation-card/conversation-card.tsx index 8c6b895eaf..fff0a0888d 100644 --- a/frontend/src/components/features/conversation-panel/conversation-card/conversation-card.tsx +++ b/frontend/src/components/features/conversation-panel/conversation-card/conversation-card.tsx @@ -1,5 +1,5 @@ import React from "react"; -import posthog from "posthog-js"; +import { usePostHog } from "posthog-js/react"; import { cn } from "#/utils/utils"; import { transformVSCodeUrl } from "#/utils/vscode-url-helper"; import ConversationService from "#/api/conversation-service/conversation-service.api"; @@ -44,6 +44,7 @@ export function ConversationCard({ contextMenuOpen = false, onContextMenuToggle, }: ConversationCardProps) { + const posthog = usePostHog(); const [titleMode, setTitleMode] = React.useState<"view" | "edit">("view"); const onTitleSave = (newTitle: string) => { diff --git a/frontend/src/components/features/markdown/headings.tsx b/frontend/src/components/features/markdown/headings.tsx new file mode 100644 index 0000000000..2e12fc7db4 --- /dev/null +++ b/frontend/src/components/features/markdown/headings.tsx @@ -0,0 +1,80 @@ +import React from "react"; +import { ExtraProps } from "react-markdown"; + +// Custom component to render

in markdown +export function h1({ + children, +}: React.ClassAttributes & + React.HTMLAttributes & + ExtraProps) { + return ( +

+ {children} +

+ ); +} + +// Custom component to render

in markdown +export function h2({ + children, +}: React.ClassAttributes & + React.HTMLAttributes & + ExtraProps) { + return ( +

+ {children} +

+ ); +} + +// Custom component to render

in markdown +export function h3({ + children, +}: React.ClassAttributes & + React.HTMLAttributes & + ExtraProps) { + return ( +

+ {children} +

+ ); +} + +// Custom component to render

in markdown +export function h4({ + children, +}: React.ClassAttributes & + React.HTMLAttributes & + ExtraProps) { + return ( +

+ {children} +

+ ); +} + +// Custom component to render
in markdown +export function h5({ + children, +}: React.ClassAttributes & + React.HTMLAttributes & + ExtraProps) { + return ( +
+ {children} +
+ ); +} + +// Custom component to render
in markdown +export function h6({ + children, +}: React.ClassAttributes & + React.HTMLAttributes & + ExtraProps) { + return ( +
+ {children} +
+ ); +} diff --git a/frontend/src/components/features/waitlist/auth-modal.tsx b/frontend/src/components/features/waitlist/auth-modal.tsx index cbc9e0db32..d20ef04a28 100644 --- a/frontend/src/components/features/waitlist/auth-modal.tsx +++ b/frontend/src/components/features/waitlist/auth-modal.tsx @@ -11,6 +11,7 @@ import BitbucketLogo from "#/assets/branding/bitbucket-logo.svg?react"; import { useAuthUrl } from "#/hooks/use-auth-url"; import { GetConfigResponse } from "#/api/option-service/option.types"; import { Provider } from "#/types/settings"; +import { useTracking } from "#/hooks/use-tracking"; interface AuthModalProps { githubAuthUrl: string | null; @@ -26,6 +27,7 @@ export function AuthModal({ providersConfigured, }: AuthModalProps) { const { t } = useTranslation(); + const { trackLoginButtonClick } = useTracking(); const gitlabAuthUrl = useAuthUrl({ appMode: appMode || null, @@ -47,6 +49,7 @@ export function AuthModal({ const handleGitHubAuth = () => { if (githubAuthUrl) { + trackLoginButtonClick({ provider: "github" }); // Always start the OIDC flow, let the backend handle TOS check window.location.href = githubAuthUrl; } @@ -54,6 +57,7 @@ export function AuthModal({ const handleGitLabAuth = () => { if (gitlabAuthUrl) { + trackLoginButtonClick({ provider: "gitlab" }); // Always start the OIDC flow, let the backend handle TOS check window.location.href = gitlabAuthUrl; } @@ -61,6 +65,7 @@ export function AuthModal({ const handleBitbucketAuth = () => { if (bitbucketAuthUrl) { + trackLoginButtonClick({ provider: "bitbucket" }); // Always start the OIDC flow, let the backend handle TOS check window.location.href = bitbucketAuthUrl; } @@ -68,6 +73,7 @@ export function AuthModal({ const handleEnterpriseSsoAuth = () => { if (enterpriseSsoUrl) { + trackLoginButtonClick({ provider: "enterprise_sso" }); // Always start the OIDC flow, let the backend handle TOS check window.location.href = enterpriseSsoUrl; } diff --git a/frontend/src/components/shared/modals/settings/settings-form.tsx b/frontend/src/components/shared/modals/settings/settings-form.tsx index 838b4f0b06..e08b59c8e0 100644 --- a/frontend/src/components/shared/modals/settings/settings-form.tsx +++ b/frontend/src/components/shared/modals/settings/settings-form.tsx @@ -1,7 +1,7 @@ import { useLocation } from "react-router"; import { useTranslation } from "react-i18next"; import React from "react"; -import posthog from "posthog-js"; +import { usePostHog } from "posthog-js/react"; import { I18nKey } from "#/i18n/declaration"; import { organizeModelsAndProviders } from "#/utils/organize-models-and-providers"; import { DangerModal } from "../confirmation-modals/danger-modal"; @@ -22,6 +22,7 @@ interface SettingsFormProps { } export function SettingsForm({ settings, models, onClose }: SettingsFormProps) { + const posthog = usePostHog(); const { mutate: saveUserSettings } = useSaveSettings(); const location = useLocation(); diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts b/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts index 39db76cd2e..ef4ffa253f 100644 --- a/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts +++ b/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts @@ -49,6 +49,10 @@ const getExecuteBashObservationContent = ( let { output } = observation; + if (!output) { + output = ""; + } + if (output.length > MAX_CONTENT_LENGTH) { output = `${output.slice(0, MAX_CONTENT_LENGTH)}...`; } @@ -136,6 +140,7 @@ const getTaskTrackerObservationContent = ( if ( "content" in observation && observation.content && + typeof observation.content === "string" && observation.content.trim() ) { content += `\n\n**Result:** ${observation.content.trim()}`; diff --git a/frontend/src/context/ws-client-provider.tsx b/frontend/src/context/ws-client-provider.tsx index 8f0a2829c0..38f390476f 100644 --- a/frontend/src/context/ws-client-provider.tsx +++ b/frontend/src/context/ws-client-provider.tsx @@ -1,6 +1,7 @@ import React from "react"; import { io, Socket } from "socket.io-client"; import { useQueryClient } from "@tanstack/react-query"; +import { usePostHog } from "posthog-js/react"; import EventLogger from "#/utils/event-logger"; import { handleAssistantMessage } from "#/services/actions"; import { showChatError, trackError } from "#/utils/error-handler"; @@ -100,7 +101,10 @@ interface ErrorArgData { msg_id: string; } -export function updateStatusWhenErrorMessagePresent(data: ErrorArg | unknown) { +export function updateStatusWhenErrorMessagePresent( + data: ErrorArg | unknown, + posthog?: ReturnType, +) { const isObject = (val: unknown): val is object => !!val && typeof val === "object"; const isString = (val: unknown): val is string => typeof val === "string"; @@ -123,6 +127,7 @@ export function updateStatusWhenErrorMessagePresent(data: ErrorArg | unknown) { source: "websocket", metadata, msgId, + posthog, }); } } @@ -131,6 +136,7 @@ export function WsClientProvider({ conversationId, children, }: React.PropsWithChildren) { + const posthog = usePostHog(); const { setErrorMessage, removeErrorMessage } = useErrorMessageStore(); const { removeOptimisticUserMessage } = useOptimisticUserMessageStore(); const { addEvent, clearEvents } = useEventStore(); @@ -178,6 +184,7 @@ export function WsClientProvider({ message: errorMessage, source: "chat", metadata: { msgId: event.id }, + posthog, }); setErrorMessage(errorMessage); @@ -193,6 +200,7 @@ export function WsClientProvider({ message: event.message, source: "chat", metadata: { msgId: event.id }, + posthog, }); } else { removeErrorMessage(); @@ -260,14 +268,14 @@ export function WsClientProvider({ sio.io.opts.query = sio.io.opts.query || {}; sio.io.opts.query.latest_event_id = lastEventRef.current?.id; - updateStatusWhenErrorMessagePresent(data); + updateStatusWhenErrorMessagePresent(data, posthog); setErrorMessage(hasValidMessageProperty(data) ? data.message : ""); } function handleError(data: unknown) { // set status setWebSocketStatus("DISCONNECTED"); - updateStatusWhenErrorMessagePresent(data); + updateStatusWhenErrorMessagePresent(data, posthog); setErrorMessage( hasValidMessageProperty(data) diff --git a/frontend/src/entry.client.tsx b/frontend/src/entry.client.tsx index dc1e2e4dd5..9fe6212d4e 100644 --- a/frontend/src/entry.client.tsx +++ b/frontend/src/entry.client.tsx @@ -8,17 +8,18 @@ import { HydratedRouter } from "react-router/dom"; import React, { startTransition, StrictMode } from "react"; import { hydrateRoot } from "react-dom/client"; -import posthog from "posthog-js"; +import { PostHogProvider } from "posthog-js/react"; import "./i18n"; import { QueryClientProvider } from "@tanstack/react-query"; import OptionService from "./api/option-service/option-service.api"; import { displayErrorToast } from "./utils/custom-toast-handlers"; import { queryClient } from "./query-client-config"; -function PosthogInit() { +function PostHogWrapper({ children }: { children: React.ReactNode }) { const [posthogClientKey, setPosthogClientKey] = React.useState( null, ); + const [isLoading, setIsLoading] = React.useState(true); React.useEffect(() => { (async () => { @@ -27,20 +28,27 @@ function PosthogInit() { setPosthogClientKey(config.POSTHOG_CLIENT_KEY); } catch { displayErrorToast("Error fetching PostHog client key"); + } finally { + setIsLoading(false); } })(); }, []); - React.useEffect(() => { - if (posthogClientKey) { - posthog.init(posthogClientKey, { + if (isLoading || !posthogClientKey) { + return children; + } + + return ( + + {children} + + ); } async function prepareApp() { @@ -62,10 +70,10 @@ prepareApp().then(() => document, - - + + + -