github/OpenHands
1
0
Fork 0
mirror of https://github.com/OpenHands/OpenHands.git synced 2025-12-26 13:52:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into openhands-fix-issue-5219-try2

Browse Source
This commit is contained in:
Engel Nyst 2024-11-30 02:25:17 +01:00 committed by GitHub
commit aef16c6f9a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
354 changed files with 8092 additions and 5637 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/dependabot.yml vendored
View File

@ -16,6 +16,9 @@ updates:
chromadb:
patterns:
- "chromadb"
browsergym:
patterns:
- "browsergym"
security-all:
applies-to: "security-updates"
patterns:

31
.github/workflows/eval-runner.yml vendored
View File

@ -1,10 +1,8 @@
name: Run Evaluation
name: Run SWE-Bench Evaluation
on:
pull_request:
types: [labeled]
schedule:
- cron: "0 1 * * *" # Run daily at 1 AM UTC
workflow_dispatch:
inputs:
reason:
@ -60,24 +58,6 @@ jobs:
echo "api_key = \"$DEEPSEEK_API_KEY\"" >> config.toml
echo "temperature = 0.0" >> config.toml
- name: Run integration test evaluation
env:
ALLHANDS_API_KEY: ${{ secrets.ALLHANDS_EVAL_RUNTIME_API_KEY }}
RUNTIME: remote
SANDBOX_REMOTE_RUNTIME_API_URL: https://runtime.eval.all-hands.dev
EVAL_DOCKER_IMAGE_PREFIX: us-central1-docker.pkg.dev/evaluation-092424/swe-bench-images
run: |
poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES
# get evaluation report
REPORT_FILE=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/deepseek-chat_maxiter_10_N* -name "report.md" -type f | head -n 1)
echo "REPORT_FILE: $REPORT_FILE"
echo "INTEGRATION_TEST_REPORT<<EOF" >> $GITHUB_ENV
cat $REPORT_FILE >> $GITHUB_ENV
echo >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- name: Run SWE-Bench evaluation
env:
ALLHANDS_API_KEY: ${{ secrets.ALLHANDS_EVAL_RUNTIME_API_KEY }}
@ -86,12 +66,12 @@ jobs:
EVAL_DOCKER_IMAGE_PREFIX: us-central1-docker.pkg.dev/evaluation-092424/swe-bench-images
run: |
poetry run ./evaluation/swe_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 300 30 $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
poetry run ./evaluation/benchmarks/swe_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 300 30 $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
OUTPUT_FOLDER=$(find evaluation/evaluation_outputs/outputs/princeton-nlp__SWE-bench_Lite-test/CodeActAgent -name "deepseek-chat_maxiter_50_N_*-no-hint-run_1" -type d | head -n 1)
echo "OUTPUT_FOLDER for SWE-bench evaluation: $OUTPUT_FOLDER"
poetry run ./evaluation/swe_bench/scripts/eval_infer_remote.sh $OUTPUT_FOLDER/output.jsonl $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
poetry run ./evaluation/benchmarks/swe_bench/scripts/eval_infer_remote.sh $OUTPUT_FOLDER/output.jsonl $N_PROCESSES "princeton-nlp/SWE-bench_Lite" test
poetry run ./evaluation/swe_bench/scripts/eval/summarize_outputs.py $OUTPUT_FOLDER/output.jsonl > summarize_outputs.log 2>&1
poetry run ./evaluation/benchmarks/swe_bench/scripts/eval/summarize_outputs.py $OUTPUT_FOLDER/output.jsonl > summarize_outputs.log 2>&1
echo "SWEBENCH_REPORT<<EOF" >> $GITHUB_ENV
cat summarize_outputs.log >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
@ -145,9 +125,6 @@ jobs:
**SWE-Bench Evaluation Report**
${{ env.SWEBENCH_REPORT }}
---
**Integration Tests Evaluation Report**
${{ env.INTEGRATION_TEST_REPORT }}
---
You can download the full evaluation outputs [here](${{ env.ARTIFACT_URL }}).
- name: Post to a Slack channel

158
.github/workflows/integration-runner.yml vendored Normal file
View File

@ -0,0 +1,158 @@
name: Run Integration Tests
on:
pull_request:
types: [labeled]
workflow_dispatch:
inputs:
reason:
description: 'Reason for manual trigger'
required: true
default: ''
schedule:
- cron: '30 22 * * *' # Runs at 10:30pm UTC every day
env:
N_PROCESSES: 10 # Global configuration for number of parallel processes for evaluation
jobs:
run-integration-tests:
if: github.event.label.name == 'integration-test' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
runs-on: ubuntu-latest
permissions:
contents: "read"
id-token: "write"
pull-requests: "write"
issues: "write"
strategy:
matrix:
python-version: ["3.12"]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install poetry via pipx
run: pipx install poetry
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "poetry"
- name: Comment on PR if 'integration-test' label is present
if: github.event_name == 'pull_request' && github.event.label.name == 'integration-test'
uses: KeisukeYamashita/create-comment@v1
with:
unique: false
comment: |
Hi! I started running the integration tests on your PR. You will receive a comment with the results shortly.
- name: Install Python dependencies using Poetry
run: poetry install --without evaluation,llama-index
- name: Configure config.toml for testing with Haiku
env:
LLM_MODEL: "litellm_proxy/claude-3-5-haiku-20241022"
LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
run: |
echo "[llm.eval]" > config.toml
echo "model = \"$LLM_MODEL\"" >> config.toml
echo "api_key = \"$LLM_API_KEY\"" >> config.toml
echo "base_url = \"$LLM_BASE_URL\"" >> config.toml
echo "temperature = 0.0" >> config.toml
- name: Build environment
run: make build
- name: Run integration test evaluation for Haiku
env:
SANDBOX_FORCE_REBUILD_RUNTIME: True
run: |
poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES '' 'haiku_run'
# get integration tests report
REPORT_FILE_HAIKU=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/*haiku*_maxiter_10_N* -name "report.md" -type f | head -n 1)
echo "REPORT_FILE: $REPORT_FILE_HAIKU"
echo "INTEGRATION_TEST_REPORT_HAIKU<<EOF" >> $GITHUB_ENV
cat $REPORT_FILE_HAIKU >> $GITHUB_ENV
echo >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- name: Wait a little bit
run: sleep 10
- name: Configure config.toml for testing with DeepSeek
env:
LLM_MODEL: "litellm_proxy/deepseek-chat"
LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
run: |
echo "[llm.eval]" > config.toml
echo "model = \"$LLM_MODEL\"" >> config.toml
echo "api_key = \"$LLM_API_KEY\"" >> config.toml
echo "base_url = \"$LLM_BASE_URL\"" >> config.toml
echo "temperature = 0.0" >> config.toml
- name: Run integration test evaluation for DeepSeek
env:
SANDBOX_FORCE_REBUILD_RUNTIME: True
run: |
poetry run ./evaluation/integration_tests/scripts/run_infer.sh llm.eval HEAD CodeActAgent '' $N_PROCESSES '' 'deepseek_run'
# get integration tests report
REPORT_FILE_DEEPSEEK=$(find evaluation/evaluation_outputs/outputs/integration_tests/CodeActAgent/deepseek*_maxiter_10_N* -name "report.md" -type f | head -n 1)
echo "REPORT_FILE: $REPORT_FILE_DEEPSEEK"
echo "INTEGRATION_TEST_REPORT_DEEPSEEK<<EOF" >> $GITHUB_ENV
cat $REPORT_FILE_DEEPSEEK >> $GITHUB_ENV
echo >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- name: Create archive of evaluation outputs
run: |
TIMESTAMP=$(date +'%y-%m-%d-%H-%M')
cd evaluation/evaluation_outputs/outputs # Change to the outputs directory
tar -czvf ../../../integration_tests_${TIMESTAMP}.tar.gz integration_tests/CodeActAgent/* # Only include the actual result directories
- name: Upload evaluation results as artifact
uses: actions/upload-artifact@v4
id: upload_results_artifact
with:
name: integration-test-outputs-${{ github.run_id }}-${{ github.run_attempt }}
path: integration_tests_*.tar.gz
- name: Get artifact URLs
run: |
echo "ARTIFACT_URL=${{ steps.upload_results_artifact.outputs.artifact-url }}" >> $GITHUB_ENV
- name: Set timestamp and trigger reason
run: |
echo "TIMESTAMP=$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_ENV
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "TRIGGER_REASON=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
echo "TRIGGER_REASON=manual-${{ github.event.inputs.reason }}" >> $GITHUB_ENV
else
echo "TRIGGER_REASON=nightly-scheduled" >> $GITHUB_ENV
fi
- name: Comment with results and artifact link
id: create_comment
uses: KeisukeYamashita/create-comment@v1
with:
# if triggered by PR, use PR number, otherwise use 5318 as fallback issue number for manual triggers
number: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || 5318 }}
unique: false
comment: |
Trigger by: ${{ github.event_name == 'pull_request' && format('Pull Request (integration-test label on PR #{0})', github.event.pull_request.number) || (github.event_name == 'workflow_dispatch' && format('Manual Trigger: {0}', github.event.inputs.reason)) || 'Nightly Scheduled Run' }}
Commit: ${{ github.sha }}
**Integration Tests Report (Haiku)**
Haiku LLM Test Results:
${{ env.INTEGRATION_TEST_REPORT_HAIKU }}
---
**Integration Tests Report (DeepSeek)**
DeepSeek LLM Test Results:
${{ env.INTEGRATION_TEST_REPORT_DEEPSEEK }}
---
Download testing outputs (includes both Haiku and DeepSeek results): [Download](${{ steps.upload_results_artifact.outputs.artifact-url }})

81
.github/workflows/review-pr.yml vendored
View File

@ -1,81 +0,0 @@
# Workflow that uses OpenHands to review a pull request. PR must be labeled 'review-this'
name: Use OpenHands to Review Pull Request
on:
pull_request:
types: [synchronize, labeled]
permissions:
contents: write
pull-requests: write
jobs:
dogfood:
if: contains(github.event.pull_request.labels.*.name, 'review-this')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: install git, github cli
run: |
sudo apt-get install -y git gh
git config --global --add safe.directory $PWD
- name: Checkout Repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.base.ref }} # check out the target branch
- name: Download Diff
run: |
curl -O "${{ github.event.pull_request.diff_url }}" -L
- name: Write Task File
run: |
echo "Your coworker wants to apply a pull request to this project." > task.txt
echo "Read and review ${{ github.event.pull_request.number }}.diff file. Create a review-${{ github.event.pull_request.number }}.txt and write your concise comments and suggestions there." >> task.txt
echo "Do not ask me for confirmation at any point." >> task.txt
echo "" >> task.txt
echo "Title" >> task.txt
echo "${{ github.event.pull_request.title }}" >> task.txt
echo "" >> task.txt
echo "Description" >> task.txt
echo "${{ github.event.pull_request.body }}" >> task.txt
echo "" >> task.txt
echo "Diff file is: ${{ github.event.pull_request.number }}.diff" >> task.txt
- name: Set up environment
run: |
curl -sSL https://install.python-poetry.org | python3 -
export PATH="/github/home/.local/bin:$PATH"
poetry install --without evaluation,llama-index
poetry run playwright install --with-deps chromium
- name: Run OpenHands
env:
LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
LLM_MODEL: ${{ vars.LLM_MODEL }}
run: |
# Append path to launch poetry
export PATH="/github/home/.local/bin:$PATH"
# Append path to correctly import package, note: must set pwd at first
export PYTHONPATH=$(pwd):$PYTHONPATH
export WORKSPACE_MOUNT_PATH=$GITHUB_WORKSPACE
export WORKSPACE_BASE=$GITHUB_WORKSPACE
echo -e "/exit\n" | poetry run python openhands/core/main.py -i 50 -f task.txt
rm task.txt
- name: Check if review file is non-empty
id: check_file
run: |
ls -la
if [[ -s review-${{ github.event.pull_request.number }}.txt ]]; then
echo "non_empty=true" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Create PR review if file is non-empty
env:
GH_TOKEN: ${{ github.token }}
if: steps.check_file.outputs.non_empty == 'true'
run: |
gh pr review ${{ github.event.pull_request.number }} --comment --body-file "review-${{ github.event.pull_request.number }}.txt"

2
docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md
View File

@ -76,7 +76,7 @@ La fonction `run_controller()` est le cœur de l'exécution d'OpenHands. Elle g
## Le moyen le plus simple de commencer : Explorer les benchmarks existants
Nous vous encourageons à examiner les différents benchmarks d'évaluation disponibles dans le [répertoire `evaluation/`](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation) de notre dépôt.
Nous vous encourageons à examiner les différents benchmarks d'évaluation disponibles dans le [répertoire `evaluation/benchmarks/`](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks) de notre dépôt.
Pour intégrer votre propre benchmark, nous vous suggérons de commencer par celui qui ressemble le plus à vos besoins. Cette approche peut considérablement rationaliser votre processus d'intégration, vous permettant de vous appuyer sur les structures existantes et de les adapter à vos exigences spécifiques.

2
docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/evaluation-harness.md
View File

@ -73,7 +73,7 @@ OpenHands 的主要入口点在 `openhands/core/main.py` 中。以下是它工
## 入门最简单的方法：探索现有基准
我们鼓励您查看我们仓库的 [`evaluation/` 目录](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation)中提供的各种评估基准。
我们鼓励您查看我们仓库的 [`evaluation/benchmarks/` 目录](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks)中提供的各种评估基准。
要集成您自己的基准，我们建议从最接近您需求的基准开始。这种方法可以显著简化您的集成过程，允许您在现有结构的基础上进行构建并使其适应您的特定要求。

465
docs/modules/usage/configuration-options.md Normal file
View File

@ -0,0 +1,465 @@
# Configuration Options
This guide details all configuration options available for OpenHands, helping you customize its behavior and integrate it with other services.
:::note
If you are running in [GUI Mode](https://docs.all-hands.dev/modules/usage/how-to/gui-mode), the settings available in the Settings UI will always
take precedence.
:::
---
# Table of Contents
1. [Core Configuration](#core-configuration)
- [API Keys](#api-keys)
- [Workspace](#workspace)
- [Debugging and Logging](#debugging-and-logging)
- [Session Management](#session-management)
- [Trajectories](#trajectories)
- [File Store](#file-store)
- [Task Management](#task-management)
- [Sandbox Configuration](#sandbox-configuration)
- [Miscellaneous](#miscellaneous)
2. [LLM Configuration](#llm-configuration)
- [AWS Credentials](#aws-credentials)
- [API Configuration](#api-configuration)
- [Custom LLM Provider](#custom-llm-provider)
- [Embeddings](#embeddings)
- [Message Handling](#message-handling)
- [Model Selection](#model-selection)
- [Retrying](#retrying)
- [Advanced Options](#advanced-options)
3. [Agent Configuration](#agent-configuration)
- [Microagent Configuration](#microagent-configuration)
- [Memory Configuration](#memory-configuration)
- [LLM Configuration](#llm-configuration-2)
- [ActionSpace Configuration](#actionspace-configuration)
- [Microagent Usage](#microagent-usage)
4. [Sandbox Configuration](#sandbox-configuration-2)
- [Execution](#execution)
- [Container Image](#container-image)
- [Networking](#networking)
- [Linting and Plugins](#linting-and-plugins)
- [Dependencies and Environment](#dependencies-and-environment)
- [Evaluation](#evaluation)
5. [Security Configuration](#security-configuration)
- [Confirmation Mode](#confirmation-mode)
- [Security Analyzer](#security-analyzer)
---
## Core Configuration
The core configuration options are defined in the `[core]` section of the `config.toml` file.
**API Keys**
- `e2b_api_key`
- Type: `str`
- Default: `""`
- Description: API key for E2B
- `modal_api_token_id`
- Type: `str`
- Default: `""`
- Description: API token ID for Modal
- `modal_api_token_secret`
- Type: `str`
- Default: `""`
- Description: API token secret for Modal
**Workspace**
- `workspace_base`
- Type: `str`
- Default: `"./workspace"`
- Description: Base path for the workspace
- `cache_dir`
- Type: `str`
- Default: `"/tmp/cache"`
- Description: Cache directory path
**Debugging and Logging**
- `debug`
- Type: `bool`
- Default: `false`
- Description: Enable debugging
- `disable_color`
- Type: `bool`
- Default: `false`
- Description: Disable color in terminal output
**Trajectories**
- `trajectories_path`
- Type: `str`
- Default: `"./trajectories"`
- Description: Path to store trajectories (can be a folder or a file). If it's a folder, the trajectories will be saved in a file named with the session id name and .json extension, in that folder.
**File Store**
- `file_store_path`
- Type: `str`
- Default: `"/tmp/file_store"`
- Description: File store path
- `file_store`
- Type: `str`
- Default: `"memory"`
- Description: File store type
- `file_uploads_allowed_extensions`
- Type: `list of str`
- Default: `[".*"]`
- Description: List of allowed file extensions for uploads
- `file_uploads_max_file_size_mb`
- Type: `int`
- Default: `0`
- Description: Maximum file size for uploads, in megabytes
- `file_uploads_restrict_file_types`
- Type: `bool`
- Default: `false`
- Description: Restrict file types for file uploads
- `file_uploads_allowed_extensions`
- Type: `list of str`
- Default: `[".*"]`
- Description: List of allowed file extensions for uploads
**Task Management**
- `max_budget_per_task`
- Type: `float`
- Default: `0.0`
- Description: Maximum budget per task (0.0 means no limit)
- `max_iterations`
- Type: `int`
- Default: `100`
- Description: Maximum number of iterations
**Sandbox Configuration**
- `workspace_mount_path_in_sandbox`
- Type: `str`
- Default: `"/workspace"`
- Description: Path to mount the workspace in the sandbox
- `workspace_mount_path`
- Type: `str`
- Default: `""`
- Description: Path to mount the workspace
- `workspace_mount_rewrite`
- Type: `str`
- Default: `""`
- Description: Path to rewrite the workspace mount path to. You can usually ignore this, it refers to special cases of running inside another container.
**Miscellaneous**
- `run_as_openhands`
- Type: `bool`
- Default: `true`
- Description: Run as OpenHands
- `runtime`
- Type: `str`
- Default: `"eventstream"`
- Description: Runtime environment
- `default_agent`
- Type: `str`
- Default: `"CodeActAgent"`
- Description: Name of the default agent
- `jwt_secret`
- Type: `str`
- Default: `uuid.uuid4().hex`
- Description: JWT secret for authentication. Please set it to your own value.
## LLM Configuration
The LLM (Large Language Model) configuration options are defined in the `[llm]` section of the `config.toml` file.
To use these with the docker command, pass in `-e LLM_<option>`. Example: `-e LLM_NUM_RETRIES`.
**AWS Credentials**
- `aws_access_key_id`
- Type: `str`
- Default: `""`
- Description: AWS access key ID
- `aws_region_name`
- Type: `str`
- Default: `""`
- Description: AWS region name
- `aws_secret_access_key`
- Type: `str`
- Default: `""`
- Description: AWS secret access key
**API Configuration**
- `api_key`
- Type: `str`
- Default: `None`
- Description: API key to use
- `base_url`
- Type: `str`
- Default: `""`
- Description: API base URL
- `api_version`
- Type: `str`
- Default: `""`
- Description: API version
- `input_cost_per_token`
- Type: `float`
- Default: `0.0`
- Description: Cost per input token
- `output_cost_per_token`
- Type: `float`
- Default: `0.0`
- Description: Cost per output token
**Custom LLM Provider**
- `custom_llm_provider`
- Type: `str`
- Default: `""`
- Description: Custom LLM provider
**Embeddings**
- `embedding_base_url`
- Type: `str`
- Default: `""`
- Description: Embedding API base URL
- `embedding_deployment_name`
- Type: `str`
- Default: `""`
- Description: Embedding deployment name
- `embedding_model`
- Type: `str`
- Default: `"local"`
- Description: Embedding model to use
**Message Handling**
- `max_message_chars`
- Type: `int`
- Default: `30000`
- Description: The approximate maximum number of characters in the content of an event included in the prompt to the LLM. Larger observations are truncated.
- `max_input_tokens`
- Type: `int`
- Default: `0`
- Description: Maximum number of input tokens
- `max_output_tokens`
- Type: `int`
- Default: `0`
- Description: Maximum number of output tokens
**Model Selection**
- `model`
- Type: `str`
- Default: `"claude-3-5-sonnet-20241022"`
- Description: Model to use
**Retrying**
- `num_retries`
- Type: `int`
- Default: `8`
- Description: Number of retries to attempt
- `retry_max_wait`
- Type: `int`
- Default: `120`
- Description: Maximum wait time (in seconds) between retry attempts
- `retry_min_wait`
- Type: `int`
- Default: `15`
- Description: Minimum wait time (in seconds) between retry attempts
- `retry_multiplier`
- Type: `float`
- Default: `2.0`
- Description: Multiplier for exponential backoff calculation
**Advanced Options**
- `drop_params`
- Type: `bool`
- Default: `false`
- Description: Drop any unmapped (unsupported) params without causing an exception
- `caching_prompt`
- Type: `bool`
- Default: `true`
- Description: Using the prompt caching feature if provided by the LLM and supported
- `ollama_base_url`
- Type: `str`
- Default: `""`
- Description: Base URL for the OLLAMA API
- `temperature`
- Type: `float`
- Default: `0.0`
- Description: Temperature for the API
- `timeout`
- Type: `int`
- Default: `0`
- Description: Timeout for the API
- `top_p`
- Type: `float`
- Default: `1.0`
- Description: Top p for the API
- `disable_vision`
- Type: `bool`
- Default: `None`
- Description: If model is vision capable, this option allows to disable image processing (useful for cost reduction)
## Agent Configuration
The agent configuration options are defined in the `[agent]` and `[agent.<agent_name>]` sections of the `config.toml` file.
**Microagent Configuration**
- `micro_agent_name`
- Type: `str`
- Default: `""`
- Description: Name of the micro agent to use for this agent
**Memory Configuration**
- `memory_enabled`
- Type: `bool`
- Default: `false`
- Description: Whether long-term memory (embeddings) is enabled
- `memory_max_threads`
- Type: `int`
- Default: `3`
- Description: The maximum number of threads indexing at the same time for embeddings
**LLM Configuration**
- `llm_config`
- Type: `str`
- Default: `'your-llm-config-group'`
- Description: The name of the LLM config to use
**ActionSpace Configuration**
- `function_calling`
- Type: `bool`
- Default: `true`
- Description: Whether function calling is enabled
- `codeact_enable_browsing`
- Type: `bool`
- Default: `false`
- Description: Whether browsing delegate is enabled in the action space (only works with function calling)
- `codeact_enable_llm_editor`
- Type: `bool`
- Default: `false`
- Description: Whether LLM editor is enabled in the action space (only works with function calling)
- `codeact_enable_jupyter`
- Type: `bool`
- Default: `false`
- Description: Whether Jupyter is enabled in the action space
**Microagent Usage**
- `use_microagents`
- Type: `bool`
- Default: `true`
- Description: Whether to use microagents at all
- `disabled_microagents`
- Type: `list of str`
- Default: `None`
- Description: A list of microagents to disable
## Sandbox Configuration
The sandbox configuration options are defined in the `[sandbox]` section of the `config.toml` file.
To use these with the docker command, pass in `-e SANDBOX_<option>`. Example: `-e SANDBOX_TIMEOUT`.
**Execution**
- `timeout`
- Type: `int`
- Default: `120`
- Description: Sandbox timeout in seconds
- `user_id`
- Type: `int`
- Default: `1000`
- Description: Sandbox user ID
**Container Image**
- `base_container_image`
- Type: `str`
- Default: `"nikolaik/python-nodejs:python3.12-nodejs22"`
- Description: Container image to use for the sandbox
**Networking**
- `use_host_network`
- Type: `bool`
- Default: `false`
- Description: Use host network
**Linting and Plugins**
- `enable_auto_lint`
- Type: `bool`
- Default: `false`
- Description: Enable auto linting after editing
- `initialize_plugins`
- Type: `bool`
- Default: `true`
- Description: Whether to initialize plugins
**Dependencies and Environment**
- `runtime_extra_deps`
- Type: `str`
- Default: `""`
- Description: Extra dependencies to install in the runtime image
- `runtime_startup_env_vars`
- Type: `dict`
- Default: `{}`
- Description: Environment variables to set at the launch of the runtime
**Evaluation**
- `browsergym_eval_env`
- Type: `str`
- Default: `""`
- Description: BrowserGym environment to use for evaluation
## Security Configuration
The security configuration options are defined in the `[security]` section of the `config.toml` file.
To use these with the docker command, pass in `-e SECURITY_<option>`. Example: `-e SECURITY_CONFIRMATION_MODE`.
**Confirmation Mode**
- `confirmation_mode`
- Type: `bool`
- Default: `false`
- Description: Enable confirmation mode
**Security Analyzer**
- `security_analyzer`
- Type: `str`
- Default: `""`
- Description: The security analyzer to use
---
> **Note**: Adjust configurations carefully, especially for memory, security, and network-related settings to ensure optimal performance and security.
Please note that the configuration options may be subject to change in future versions of OpenHands. It's recommended to refer to the official documentation for the most up-to-date information.

2
docs/modules/usage/how-to/evaluation-harness.md
View File

@ -73,7 +73,7 @@ The `run_controller()` function is the core of OpenHands's execution. It manages
## Easiest way to get started: Exploring Existing Benchmarks
We encourage you to review the various evaluation benchmarks available in the [`evaluation/` directory](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation) of our repository.
We encourage you to review the various evaluation benchmarks available in the [`evaluation/benchmarks/` directory](https://github.com/All-Hands-AI/OpenHands/blob/main/evaluation/benchmarks) of our repository.
To integrate your own benchmark, we suggest starting with the one that most closely resembles your needs. This approach can significantly streamline your integration process, allowing you to build upon existing structures and adapt them to your specific requirements.

380
docs/package-lock.json generated
View File

@ -8,10 +8,10 @@
"name": "docs",
"version": "0.0.0",
"dependencies": {
"@docusaurus/core": "^3.6.2",
"@docusaurus/plugin-content-pages": "^3.6.2",
"@docusaurus/preset-classic": "^3.6.2",
"@docusaurus/theme-mermaid": "^3.6.2",
"@docusaurus/core": "^3.6.3",
"@docusaurus/plugin-content-pages": "^3.6.3",
"@docusaurus/preset-classic": "^3.6.3",
"@docusaurus/theme-mermaid": "^3.6.3",
"@mdx-js/react": "^3.1.0",
"clsx": "^2.0.0",
"prism-react-renderer": "^2.4.0",
@ -22,7 +22,7 @@
},
"devDependencies": {
"@docusaurus/module-type-aliases": "^3.5.1",
"@docusaurus/tsconfig": "^3.6.2",
"@docusaurus/tsconfig": "^3.6.3",
"@docusaurus/types": "^3.5.1",
"typescript": "~5.6.3"
},
@ -3168,9 +3168,9 @@
}
},
"node_modules/@docusaurus/babel": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/babel/-/babel-3.6.2.tgz",
"integrity": "sha512-v8N8TWGXDsb5sxQC3Rcqb1CZr0LlU1OgqqVBUchN6cpIUr7EJuVJs5eHcIu5Ag8mwO/hWN3f7FE9uaHTMapAbg==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/babel/-/babel-3.6.3.tgz",
"integrity": "sha512-7dW9Hat9EHYCVicFXYA4hjxBY38+hPuCURL8oRF9fySRm7vzNWuEOghA1TXcykuXZp0HLG2td4RhDxCvGG7tNw==",
"dependencies": {
"@babel/core": "^7.25.9",
"@babel/generator": "^7.25.9",
@ -3182,8 +3182,8 @@
"@babel/runtime": "^7.25.9",
"@babel/runtime-corejs3": "^7.25.9",
"@babel/traverse": "^7.25.9",
"@docusaurus/logger": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/logger": "3.6.3",
"@docusaurus/utils": "3.6.3",
"babel-plugin-dynamic-import-node": "^2.3.3",
"fs-extra": "^11.1.1",
"tslib": "^2.6.0"
@ -3193,16 +3193,16 @@
}
},
"node_modules/@docusaurus/bundler": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/bundler/-/bundler-3.6.2.tgz",
"integrity": "sha512-YkEifEVs4lV931SrHBB4n6WqRowMw+aM/QPH3z8aU+5t1dWa+1p2OPqARS+tSbh3la9ns+L1zIfSbd8RHi2/PQ==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/bundler/-/bundler-3.6.3.tgz",
"integrity": "sha512-47JLuc8D4wA+6VOvmMd5fUC9rFppBQpQOnxDYiVXffm/DeV/wmm3sbpNd5Y+O+G2+nevLTRnvCm/qyancv0Y3A==",
"dependencies": {
"@babel/core": "^7.25.9",
"@docusaurus/babel": "3.6.2",
"@docusaurus/cssnano-preset": "3.6.2",
"@docusaurus/logger": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/babel": "3.6.3",
"@docusaurus/cssnano-preset": "3.6.3",
"@docusaurus/logger": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils": "3.6.3",
"babel-loader": "^9.2.1",
"clean-css": "^5.3.2",
"copy-webpack-plugin": "^11.0.0",
@ -3236,17 +3236,17 @@
}
},
"node_modules/@docusaurus/core": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/core/-/core-3.6.2.tgz",
"integrity": "sha512-irMts/mGLZv8dWcy0WUtbY/U6b5qIfHgQd1/kXMyAxUJo99fL0wFSqhMI+tcxjk0HYy427MXerLMqFJj+Arg1w==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/core/-/core-3.6.3.tgz",
"integrity": "sha512-xL7FRY9Jr5DWqB6pEnqgKqcMPJOX5V0pgWXi5lCiih11sUBmcFKM7c3+GyxcVeeWFxyYSDP3grLTWqJoP4P9Vw==",
"dependencies": {
"@docusaurus/babel": "3.6.2",
"@docusaurus/bundler": "3.6.2",
"@docusaurus/logger": "3.6.2",
"@docusaurus/mdx-loader": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-common": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/babel": "3.6.3",
"@docusaurus/bundler": "3.6.3",
"@docusaurus/logger": "3.6.3",
"@docusaurus/mdx-loader": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-common": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"boxen": "^6.2.1",
"chalk": "^4.1.2",
"chokidar": "^3.5.3",
@ -3310,9 +3310,9 @@
}
},
"node_modules/@docusaurus/cssnano-preset": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/cssnano-preset/-/cssnano-preset-3.6.2.tgz",
"integrity": "sha512-mBkVa4QMHRwCFCVLYdBlOZuAT1iVVsS7GGSgliSVAeTOagP/AbtlBsCVrBs+keEuDuRF1w/6QEcqDoZe9fa5pw==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/cssnano-preset/-/cssnano-preset-3.6.3.tgz",
"integrity": "sha512-qP7SXrwZ+23GFJdPN4aIHQrZW+oH/7tzwEuc/RNL0+BdZdmIjYQqUxdXsjE4lFxLNZjj0eUrSNYIS6xwfij+5Q==",
"dependencies": {
"cssnano-preset-advanced": "^6.1.2",
"postcss": "^8.4.38",
@ -3324,9 +3324,9 @@
}
},
"node_modules/@docusaurus/logger": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/logger/-/logger-3.6.2.tgz",
"integrity": "sha512-1p4IQhhgLyIfsey4UAdAIW69aUE1Ei6O91Nsw30ryZeDWSG5dh4o3zaRGOLxfAX69Ac/yDm6YCwJOafUxL6Vxg==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/logger/-/logger-3.6.3.tgz",
"integrity": "sha512-xSubJixcNyMV9wMV4q0s47CBz3Rlc5jbcCCuij8pfQP8qn/DIpt0ks8W6hQWzHAedg/J/EwxxUOUrnEoKzJo8g==",
"dependencies": {
"chalk": "^4.1.2",
"tslib": "^2.6.0"
@ -3336,13 +3336,13 @@
}
},
"node_modules/@docusaurus/mdx-loader": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/mdx-loader/-/mdx-loader-3.6.2.tgz",
"integrity": "sha512-7fbRmNgF3CR96Ja82Ya0/Cdu1OL9UJ/22llNMY8lr5gAbw718Y5ryXMVRIYn0JNLTiSxzgtvW4DIsUWEB8NMpw==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/mdx-loader/-/mdx-loader-3.6.3.tgz",
"integrity": "sha512-3iJdiDz9540ppBseeI93tWTDtUGVkxzh59nMq4ignylxMuXBLK8dFqVeaEor23v1vx6TrGKZ2FuLaTB+U7C0QQ==",
"dependencies": {
"@docusaurus/logger": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/logger": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"@mdx-js/mdx": "^3.0.0",
"@slorber/remark-comment": "^1.0.0",
"escape-html": "^1.0.3",
@ -3374,11 +3374,11 @@
}
},
"node_modules/@docusaurus/module-type-aliases": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/module-type-aliases/-/module-type-aliases-3.6.2.tgz",
"integrity": "sha512-NrJkL2rLTCjHtWOqUvWzwqvJrsKLj0gVJeV6q5yeKdKKgItietcTf2fTRkM9LHKSUN8CBDXxwHABeQvTahvmXQ==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/module-type-aliases/-/module-type-aliases-3.6.3.tgz",
"integrity": "sha512-MjaXX9PN/k5ugNvfRZdWyKWq4FsrhN4LEXaj0pEmMebJuBNlFeGyKQUa9DRhJHpadNaiMLrbo9m3U7Ig5YlsZg==",
"dependencies": {
"@docusaurus/types": "3.6.2",
"@docusaurus/types": "3.6.3",
"@types/history": "^4.7.11",
"@types/react": "*",
"@types/react-router-config": "*",
@ -3392,18 +3392,18 @@
}
},
"node_modules/@docusaurus/plugin-content-blog": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.6.2.tgz",
"integrity": "sha512-6bJxr6Or4NslEVH3BJuPH30kUWiqUjDRdGPhvxpHmt9W/RY2/6u72WICG3bW3dLFxJ/2uDLBU92lHnatpvo7Ew==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.6.3.tgz",
"integrity": "sha512-k0ogWwwJU3pFRFfvW1kRVHxzf2DutLGaaLjAnHVEU6ju+aRP0Z5ap/13DHyPOfHeE4WKpn/M0TqjdwZAcY3kAw==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/logger": "3.6.2",
"@docusaurus/mdx-loader": "3.6.2",
"@docusaurus/theme-common": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-common": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/logger": "3.6.3",
"@docusaurus/mdx-loader": "3.6.3",
"@docusaurus/theme-common": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-common": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"cheerio": "1.0.0-rc.12",
"feed": "^4.2.2",
"fs-extra": "^11.1.1",
@ -3425,19 +3425,19 @@
}
},
"node_modules/@docusaurus/plugin-content-docs": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.6.2.tgz",
"integrity": "sha512-e6WW1g10RIXXLN/rrtqTi/FyJ1Hj3X9Mmgz4V11/0pDCxIGGI8m4ocbAglUlLtgvbLD5viNLefl/NwbOW3JXiQ==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.6.3.tgz",
"integrity": "sha512-r2wS8y/fsaDcxkm20W5bbYJFPzdWdEaTWVYjNxlHlcmX086eqQR1Fomlg9BHTJ0dLXPzAlbC8EN4XqMr3QzNCQ==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/logger": "3.6.2",
"@docusaurus/mdx-loader": "3.6.2",
"@docusaurus/module-type-aliases": "3.6.2",
"@docusaurus/theme-common": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-common": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/logger": "3.6.3",
"@docusaurus/mdx-loader": "3.6.3",
"@docusaurus/module-type-aliases": "3.6.3",
"@docusaurus/theme-common": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-common": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"@types/react-router-config": "^5.0.7",
"combine-promises": "^1.1.0",
"fs-extra": "^11.1.1",
@ -3456,15 +3456,15 @@
}
},
"node_modules/@docusaurus/plugin-content-pages": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-pages/-/plugin-content-pages-3.6.2.tgz",
"integrity": "sha512-fo4NyGkw10lYHyHaTxE6TZLYnxNtCfRHeZkNK1N9pBYqe7TT2dBUNAEeVW2U3ed9m6YuB7JKSQsa++GGmcP+6g==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-pages/-/plugin-content-pages-3.6.3.tgz",
"integrity": "sha512-eHrmTgjgLZsuqfsYr5X2xEwyIcck0wseSofWrjTwT9FLOWp+KDmMAuVK+wRo7sFImWXZk3oV/xX/g9aZrhD7OA==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/mdx-loader": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/mdx-loader": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"fs-extra": "^11.1.1",
"tslib": "^2.6.0",
"webpack": "^5.88.1"
@ -3478,13 +3478,13 @@
}
},
"node_modules/@docusaurus/plugin-debug": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-debug/-/plugin-debug-3.6.2.tgz",
"integrity": "sha512-T/eS3VvHElpeV5S8uwp7Si4ujEynmgFtJLvA2CSa5pzQuOF1EEghF9nekAIj0cWtDHsqNUDZNr8hK1brivFXSg==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-debug/-/plugin-debug-3.6.3.tgz",
"integrity": "sha512-zB9GXfIZNPRfzKnNjU6xGVrqn9bPXuGhpjgsuc/YtcTDjnjhasg38NdYd5LEqXex5G/zIorQgWB3n6x/Ut62vQ==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils": "3.6.3",
"fs-extra": "^11.1.1",
"react-json-view-lite": "^1.2.0",
"tslib": "^2.6.0"
@ -3498,13 +3498,13 @@
}
},
"node_modules/@docusaurus/plugin-google-analytics": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-3.6.2.tgz",
"integrity": "sha512-B7ihrr3wz8e4XqW+dIAtq844u3Z83u5CeiL1xrCqzFH+vDCjUZHTamS3zKXNcgi6YVVe6hUQXPG15ltaqQaVPQ==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-3.6.3.tgz",
"integrity": "sha512-rCDNy1QW8Dag7nZq67pcum0bpFLrwvxJhYuVprhFh8BMBDxV0bY+bAkGHbSf68P3Bk9C3hNOAXX1srGLIDvcTA==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"tslib": "^2.6.0"
},
"engines": {
@ -3516,13 +3516,13 @@
}
},
"node_modules/@docusaurus/plugin-google-gtag": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-3.6.2.tgz",
"integrity": "sha512-V8ijI6qddAAkJ0vd8sjZ7S/apRTLJn9dAwvj/rSMd93witGdKINemL+9TyfLkhcXKTxyqRT8zKdu8ewjPXqKHg==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-3.6.3.tgz",
"integrity": "sha512-+OyDvhM6rqVkQOmLVkQWVJAizEEfkPzVWtIHXlWPOCFGK9X4/AWeBSrU0WG4iMg9Z4zD4YDRrU+lvI4s6DSC+w==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"@types/gtag.js": "^0.0.12",
"tslib": "^2.6.0"
},
@ -3535,13 +3535,13 @@
}
},
"node_modules/@docusaurus/plugin-google-tag-manager": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-tag-manager/-/plugin-google-tag-manager-3.6.2.tgz",
"integrity": "sha512-fnWQ5FdN9f8c8VTgjaQ98208Y+d/JjHhD506rWIIL9rt1cJOf29XElxvOeKpMJadfkgY5KLZSAiHkGt+4qgN4g==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-tag-manager/-/plugin-google-tag-manager-3.6.3.tgz",
"integrity": "sha512-1M6UPB13gWUtN2UHX083/beTn85PlRI9ABItTl/JL1FJ5dJTWWFXXsHf9WW/6hrVwthwTeV/AGbGKvLKV+IlCA==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"tslib": "^2.6.0"
},
"engines": {
@ -3553,16 +3553,16 @@
}
},
"node_modules/@docusaurus/plugin-sitemap": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-sitemap/-/plugin-sitemap-3.6.2.tgz",
"integrity": "sha512-qcAQAP1Ot0dZpeRoJ0L/Zck5FVDkll2IleVZQLzxeRVDZIw1P9/TK7/Aw1w2pmH7dmw/Cwk/cLSVRvLAmp9k7A==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/plugin-sitemap/-/plugin-sitemap-3.6.3.tgz",
"integrity": "sha512-94qOO4M9Fwv9KfVQJsgbe91k+fPJ4byf1L3Ez8TUa6TAFPo/BrLwQ80zclHkENlL1824TuxkcMKv33u6eydQCg==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/logger": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-common": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/logger": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-common": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"fs-extra": "^11.1.1",
"sitemap": "^7.1.1",
"tslib": "^2.6.0"
@ -3576,23 +3576,23 @@
}
},
"node_modules/@docusaurus/preset-classic": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/preset-classic/-/preset-classic-3.6.2.tgz",
"integrity": "sha512-r2n5eHdhiNSrJGsrrYcw+WsyStmXxe0ZG3RdA9LVyK5+jBHM8blrUWJEDugnzCNbyhUzhdtcmgCC9fhdAvKuQw==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/preset-classic/-/preset-classic-3.6.3.tgz",
"integrity": "sha512-VHSYWROT3flvNNI1SrnMOtW1EsjeHNK9dhU6s9eY5hryZe79lUqnZJyze/ymDe2LXAqzyj6y5oYvyBoZZk6ErA==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/plugin-content-blog": "3.6.2",
"@docusaurus/plugin-content-docs": "3.6.2",
"@docusaurus/plugin-content-pages": "3.6.2",
"@docusaurus/plugin-debug": "3.6.2",
"@docusaurus/plugin-google-analytics": "3.6.2",
"@docusaurus/plugin-google-gtag": "3.6.2",
"@docusaurus/plugin-google-tag-manager": "3.6.2",
"@docusaurus/plugin-sitemap": "3.6.2",
"@docusaurus/theme-classic": "3.6.2",
"@docusaurus/theme-common": "3.6.2",
"@docusaurus/theme-search-algolia": "3.6.2",
"@docusaurus/types": "3.6.2"
"@docusaurus/core": "3.6.3",
"@docusaurus/plugin-content-blog": "3.6.3",
"@docusaurus/plugin-content-docs": "3.6.3",
"@docusaurus/plugin-content-pages": "3.6.3",
"@docusaurus/plugin-debug": "3.6.3",
"@docusaurus/plugin-google-analytics": "3.6.3",
"@docusaurus/plugin-google-gtag": "3.6.3",
"@docusaurus/plugin-google-tag-manager": "3.6.3",
"@docusaurus/plugin-sitemap": "3.6.3",
"@docusaurus/theme-classic": "3.6.3",
"@docusaurus/theme-common": "3.6.3",
"@docusaurus/theme-search-algolia": "3.6.3",
"@docusaurus/types": "3.6.3"
},
"engines": {
"node": ">=18.0"
@ -3603,23 +3603,23 @@
}
},
"node_modules/@docusaurus/theme-classic": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-classic/-/theme-classic-3.6.2.tgz",
"integrity": "sha512-bCdOPqPNezhLx+hgNVO2Cf+8/1AHa9uHDOqTx/CKAx2I0J/jV9G+6JiMtpSRKGNfBoLT1O+56/7+WtkOf54xTw==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-classic/-/theme-classic-3.6.3.tgz",
"integrity": "sha512-1RRLK1tSArI2c00qugWYO3jRocjOZwGF1mBzPPylDVRwWCS/rnWWR91ChdbbaxIupRJ+hX8ZBYrwr5bbU0oztQ==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/logger": "3.6.2",
"@docusaurus/mdx-loader": "3.6.2",
"@docusaurus/module-type-aliases": "3.6.2",
"@docusaurus/plugin-content-blog": "3.6.2",
"@docusaurus/plugin-content-docs": "3.6.2",
"@docusaurus/plugin-content-pages": "3.6.2",
"@docusaurus/theme-common": "3.6.2",
"@docusaurus/theme-translations": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-common": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/logger": "3.6.3",
"@docusaurus/mdx-loader": "3.6.3",
"@docusaurus/module-type-aliases": "3.6.3",
"@docusaurus/plugin-content-blog": "3.6.3",
"@docusaurus/plugin-content-docs": "3.6.3",
"@docusaurus/plugin-content-pages": "3.6.3",
"@docusaurus/theme-common": "3.6.3",
"@docusaurus/theme-translations": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-common": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"@mdx-js/react": "^3.0.0",
"clsx": "^2.0.0",
"copy-text-to-clipboard": "^3.2.0",
@ -3643,14 +3643,14 @@
}
},
"node_modules/@docusaurus/theme-common": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-common/-/theme-common-3.6.2.tgz",
"integrity": "sha512-lfgsL064KEHpCkgGUc0OYoUPCpYfzggp6Hof8sz59UuKiLvb/Z7raewE9/NfocrJ2HZI17rLgMX3SQlRDh/5gg==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-common/-/theme-common-3.6.3.tgz",
"integrity": "sha512-b8ZkhczXHDxWWyvz+YJy4t/PlPbEogTTbgnHoflYnH7rmRtyoodTsu8WVM12la5LmlMJBclBXFl29OH8kPE7gg==",
"dependencies": {
"@docusaurus/mdx-loader": "3.6.2",
"@docusaurus/module-type-aliases": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-common": "3.6.2",
"@docusaurus/mdx-loader": "3.6.3",
"@docusaurus/module-type-aliases": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-common": "3.6.3",
"@types/history": "^4.7.11",
"@types/react": "*",
"@types/react-router-config": "*",
@ -3670,15 +3670,15 @@
}
},
"node_modules/@docusaurus/theme-mermaid": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-mermaid/-/theme-mermaid-3.6.2.tgz",
"integrity": "sha512-Ui+rBtqMPKj3RCOxNlY04i1tEjNg+fZg4URTvkHmYR07hcKaJw+vkw+wlaYjd0HFZk+3Er9vUAcwsCWuea4cVQ==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-mermaid/-/theme-mermaid-3.6.3.tgz",
"integrity": "sha512-kIqpjNCP/9R2GGf8UmiDxD3CkOAEJuJIEFlaKMgQtjVxa/vH+9PLI1+DFbArGoG4+0ENTYUq8phHPW7SeL36uQ==",
"dependencies": {
"@docusaurus/core": "3.6.2",
"@docusaurus/module-type-aliases": "3.6.2",
"@docusaurus/theme-common": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/module-type-aliases": "3.6.3",
"@docusaurus/theme-common": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"mermaid": ">=10.4",
"tslib": "^2.6.0"
},
@ -3691,18 +3691,18 @@
}
},
"node_modules/@docusaurus/theme-search-algolia": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-search-algolia/-/theme-search-algolia-3.6.2.tgz",
"integrity": "sha512-SFLS+Rq8Cg2yepnHucA9sRpIR97yHvZWlCgMzBLunV3KHbB6hD2h5HPhFV39wYHYCjJUAOH1lX9poJ1qKYuSvg==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-search-algolia/-/theme-search-algolia-3.6.3.tgz",
"integrity": "sha512-rt+MGCCpYgPyWCGXtbxlwFbTSobu15jWBTPI2LHsHNa5B0zSmOISX6FWYAPt5X1rNDOqMGM0FATnh7TBHRohVA==",
"dependencies": {
"@docsearch/react": "^3.5.2",
"@docusaurus/core": "3.6.2",
"@docusaurus/logger": "3.6.2",
"@docusaurus/plugin-content-docs": "3.6.2",
"@docusaurus/theme-common": "3.6.2",
"@docusaurus/theme-translations": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-validation": "3.6.2",
"@docusaurus/core": "3.6.3",
"@docusaurus/logger": "3.6.3",
"@docusaurus/plugin-content-docs": "3.6.3",
"@docusaurus/theme-common": "3.6.3",
"@docusaurus/theme-translations": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-validation": "3.6.3",
"algoliasearch": "^4.18.0",
"algoliasearch-helper": "^3.13.3",
"clsx": "^2.0.0",
@ -3721,9 +3721,9 @@
}
},
"node_modules/@docusaurus/theme-translations": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-translations/-/theme-translations-3.6.2.tgz",
"integrity": "sha512-LIWrYoDUsOTKmb0c7IQzawiPUTAaczBs5IOx6srxOWoTHVUMLzJCkl5Y6whfuRrnul8G05qv2vk238bN5Ko62g==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/theme-translations/-/theme-translations-3.6.3.tgz",
"integrity": "sha512-Gb0regclToVlngSIIwUCtBMQBq48qVUaN1XQNKW4XwlsgUyk0vP01LULdqbem7czSwIeBAFXFoORJ0RPX7ht/w==",
"dependencies": {
"fs-extra": "^11.1.1",
"tslib": "^2.6.0"
@ -3733,15 +3733,15 @@
}
},
"node_modules/@docusaurus/tsconfig": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/tsconfig/-/tsconfig-3.6.2.tgz",
"integrity": "sha512-TWLkyYHBYhIJNcXCEc3D1M9R8UFV4IZ82rGef5U9mE1ZrcgDUlZxYaYdoSuHrPrzPRIl3orjmpscO2FAk2gdZw==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/tsconfig/-/tsconfig-3.6.3.tgz",
"integrity": "sha512-1pT/rTrRpMV15E4tJH95W5PrjboMn5JkKF+Ys8cTjMegetiXjs0gPFOSDA5hdTlberKQLDO50xPjMJHondLuzA==",
"dev": true
},
"node_modules/@docusaurus/types": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/types/-/types-3.6.2.tgz",
"integrity": "sha512-117Wsk6xXrWEAsCYCXS3TGJv5tkdIZDcd7T/V0UJvKYmY0gyVPPcEQChy8yTdjbIkbB2q4fa7Jpox72Qv86mqQ==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/types/-/types-3.6.3.tgz",
"integrity": "sha512-xD9oTGDrouWzefkhe9ogB2fDV96/82cRpNGx2HIvI5L87JHNhQVIWimQ/3JIiiX/TEd5S9s+VO6FFguwKNRVow==",
"dependencies": {
"@mdx-js/mdx": "^3.0.0",
"@types/history": "^4.7.11",
@ -3759,13 +3759,13 @@
}
},
"node_modules/@docusaurus/utils": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/utils/-/utils-3.6.2.tgz",
"integrity": "sha512-oxnpUcFZGE3uPCDoXr8GJriB3VWM9sFjPedFidX3Fsz87l1NZNc1wtbKPfQ7GYFDMYo2IGlAv5+47Me9RkM6lg==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/utils/-/utils-3.6.3.tgz",
"integrity": "sha512-0R/FR3bKVl4yl8QwbL4TYFfR+OXBRpVUaTJdENapBGR3YMwfM6/JnhGilWQO8AOwPJGtGoDK7ib8+8UF9f3OZQ==",
"dependencies": {
"@docusaurus/logger": "3.6.2",
"@docusaurus/types": "3.6.2",
"@docusaurus/utils-common": "3.6.2",
"@docusaurus/logger": "3.6.3",
"@docusaurus/types": "3.6.3",
"@docusaurus/utils-common": "3.6.3",
"@svgr/webpack": "^8.1.0",
"escape-string-regexp": "^4.0.0",
"file-loader": "^6.2.0",
@ -3790,11 +3790,11 @@
}
},
"node_modules/@docusaurus/utils-common": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/utils-common/-/utils-common-3.6.2.tgz",
"integrity": "sha512-dr5wK+OyU2QAWxG7S5siD2bPgS7+ZeqWHfgLNHZ5yalaZf8TbeNNLqydfngukAY56BGZN0NbMkX6jGIr7ZF0sA==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/utils-common/-/utils-common-3.6.3.tgz",
"integrity": "sha512-v4nKDaANLgT3pMBewHYEMAl/ufY0LkXao1QkFWzI5huWFOmNQ2UFzv2BiKeHX5Ownis0/w6cAyoxPhVdDonlSQ==",
"dependencies": {
"@docusaurus/types": "3.6.2",
"@docusaurus/types": "3.6.3",
"tslib": "^2.6.0"
},
"engines": {
@ -3802,13 +3802,13 @@
}
},
"node_modules/@docusaurus/utils-validation": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/@docusaurus/utils-validation/-/utils-validation-3.6.2.tgz",
"integrity": "sha512-Y3EwblDz72KOcobb5t2zlhHSmrfE8EaHusPJ96Kx2JYtNXL2omqCoOb6FpaXWhES75wvjUpkFLYfiNqAqEov8g==",
"version": "3.6.3",
"resolved": "https://registry.npmjs.org/@docusaurus/utils-validation/-/utils-validation-3.6.3.tgz",
"integrity": "sha512-bhEGGiN5BE38h21vjqD70Gxg++j+PfYVddDUE5UFvLDup68QOcpD33CLr+2knPorlxRbEaNfz6HQDUMQ3HuqKw==",
"dependencies": {
"@docusaurus/logger": "3.6.2",
"@docusaurus/utils": "3.6.2",
"@docusaurus/utils-common": "3.6.2",
"@docusaurus/logger": "3.6.3",
"@docusaurus/utils": "3.6.3",
"@docusaurus/utils-common": "3.6.3",
"fs-extra": "^11.2.0",
"joi": "^17.9.2",
"js-yaml": "^4.1.0",

10
docs/package.json
View File

@ -15,10 +15,10 @@
"typecheck": "tsc"
},
"dependencies": {
"@docusaurus/core": "^3.6.2",
"@docusaurus/plugin-content-pages": "^3.6.2",
"@docusaurus/preset-classic": "^3.6.2",
"@docusaurus/theme-mermaid": "^3.6.2",
"@docusaurus/core": "^3.6.3",
"@docusaurus/plugin-content-pages": "^3.6.3",
"@docusaurus/preset-classic": "^3.6.3",
"@docusaurus/theme-mermaid": "^3.6.3",
"@mdx-js/react": "^3.1.0",
"clsx": "^2.0.0",
"prism-react-renderer": "^2.4.0",
@ -29,7 +29,7 @@
},
"devDependencies": {
"@docusaurus/module-type-aliases": "^3.5.1",
"@docusaurus/tsconfig": "^3.6.2",
"@docusaurus/tsconfig": "^3.6.3",
"@docusaurus/types": "^3.5.1",
"typescript": "~5.6.3"
},

5
docs/sidebars.ts
View File

@ -100,6 +100,11 @@ const sidebars: SidebarsConfig = {
label: 'Runtime Configuration',
id: 'usage/runtimes',
},
{
type: 'doc',
label: 'Configuration Options',
id: 'usage/configuration-options',
},
{
type: 'doc',
label: 'Custom Sandbox',

2
docs/src/components/HomepageHeader/HomepageHeader.tsx
View File

@ -23,7 +23,7 @@ export function HomepageHeader() {
<a href="https://codecov.io/github/All-Hands-AI/OpenHands?branch=main"><img alt="CodeCov" src="https://img.shields.io/codecov/c/github/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" /></a>
<a href="https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE"><img src="https://img.shields.io/github/license/All-Hands-AI/OpenHands?style=for-the-badge&color=blue" alt="MIT License" /></a>
<br/>
<a href="https://join.slack.com/t/opendevin/shared_invite/zt-2oikve2hu-UDxHeo8nsE69y6T7yFX_BA"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community" /></a>
<a href="https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw"><img src="https://img.shields.io/badge/Slack-Join%20Us-red?logo=slack&logoColor=white&style=for-the-badge" alt="Join our Slack community" /></a>
<a href="https://discord.gg/ESHStjSjD4"><img src="https://img.shields.io/badge/Discord-Join%20Us-purple?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community" /></a>
<a href="https://github.com/All-Hands-AI/OpenHands/blob/main/CREDITS.md"><img src="https://img.shields.io/badge/Project-Credits-blue?style=for-the-badge&color=FFE165&logo=github&logoColor=white" alt="Credits" /></a>
<br/>

BIN
docs/static/img/teaser.mp4 vendored
View File

Binary file not shown.

422
docs/yarn.lock
View File

@ -1550,10 +1550,10 @@
"@docsearch/css" "3.6.1"
algoliasearch "^4.19.1"
"@docusaurus/babel@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/babel/-/babel-3.6.2.tgz#c63dd2d9d7861189fe51950b3b6550477057bcee"
integrity sha512-v8N8TWGXDsb5sxQC3Rcqb1CZr0LlU1OgqqVBUchN6cpIUr7EJuVJs5eHcIu5Ag8mwO/hWN3f7FE9uaHTMapAbg==
"@docusaurus/babel@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/babel/-/babel-3.6.3.tgz#016714fe7a8807d0fc2f7180eace5e82bebbb8a6"
integrity sha512-7dW9Hat9EHYCVicFXYA4hjxBY38+hPuCURL8oRF9fySRm7vzNWuEOghA1TXcykuXZp0HLG2td4RhDxCvGG7tNw==
dependencies:
"@babel/core" "^7.25.9"
"@babel/generator" "^7.25.9"
@ -1565,23 +1565,23 @@
"@babel/runtime" "^7.25.9"
"@babel/runtime-corejs3" "^7.25.9"
"@babel/traverse" "^7.25.9"
"@docusaurus/logger" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/logger" "3.6.3"
"@docusaurus/utils" "3.6.3"
babel-plugin-dynamic-import-node "^2.3.3"
fs-extra "^11.1.1"
tslib "^2.6.0"
"@docusaurus/bundler@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/bundler/-/bundler-3.6.2.tgz#5bdd46862b40f1eea93f14714b858d07c2dd8c2f"
integrity sha512-YkEifEVs4lV931SrHBB4n6WqRowMw+aM/QPH3z8aU+5t1dWa+1p2OPqARS+tSbh3la9ns+L1zIfSbd8RHi2/PQ==
"@docusaurus/bundler@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/bundler/-/bundler-3.6.3.tgz#f09c2e29613f988b874a4be2247708e121b7fc5c"
integrity sha512-47JLuc8D4wA+6VOvmMd5fUC9rFppBQpQOnxDYiVXffm/DeV/wmm3sbpNd5Y+O+G2+nevLTRnvCm/qyancv0Y3A==
dependencies:
"@babel/core" "^7.25.9"
"@docusaurus/babel" "3.6.2"
"@docusaurus/cssnano-preset" "3.6.2"
"@docusaurus/logger" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/babel" "3.6.3"
"@docusaurus/cssnano-preset" "3.6.3"
"@docusaurus/logger" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils" "3.6.3"
babel-loader "^9.2.1"
clean-css "^5.3.2"
copy-webpack-plugin "^11.0.0"
@ -1602,18 +1602,18 @@
webpack "^5.95.0"
webpackbar "^6.0.1"
"@docusaurus/core@3.6.2", "@docusaurus/core@^3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/core/-/core-3.6.2.tgz#78628790f255555bb4c81e5952d16ea1412c4548"
integrity sha512-irMts/mGLZv8dWcy0WUtbY/U6b5qIfHgQd1/kXMyAxUJo99fL0wFSqhMI+tcxjk0HYy427MXerLMqFJj+Arg1w==
"@docusaurus/core@3.6.3", "@docusaurus/core@^3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/core/-/core-3.6.3.tgz#6bf968ee26a36d71387bab293f27ccffc0e428b6"
integrity sha512-xL7FRY9Jr5DWqB6pEnqgKqcMPJOX5V0pgWXi5lCiih11sUBmcFKM7c3+GyxcVeeWFxyYSDP3grLTWqJoP4P9Vw==
dependencies:
"@docusaurus/babel" "3.6.2"
"@docusaurus/bundler" "3.6.2"
"@docusaurus/logger" "3.6.2"
"@docusaurus/mdx-loader" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-common" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/babel" "3.6.3"
"@docusaurus/bundler" "3.6.3"
"@docusaurus/logger" "3.6.3"
"@docusaurus/mdx-loader" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-common" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
boxen "^6.2.1"
chalk "^4.1.2"
chokidar "^3.5.3"
@ -1651,32 +1651,32 @@
webpack-dev-server "^4.15.2"
webpack-merge "^6.0.1"
"@docusaurus/cssnano-preset@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/cssnano-preset/-/cssnano-preset-3.6.2.tgz#007e7150b099ea2e9e874dd48a809614c628a335"
integrity sha512-mBkVa4QMHRwCFCVLYdBlOZuAT1iVVsS7GGSgliSVAeTOagP/AbtlBsCVrBs+keEuDuRF1w/6QEcqDoZe9fa5pw==
"@docusaurus/cssnano-preset@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/cssnano-preset/-/cssnano-preset-3.6.3.tgz#ea19b307183ec20dea4927efc4ddf249150b8c6a"
integrity sha512-qP7SXrwZ+23GFJdPN4aIHQrZW+oH/7tzwEuc/RNL0+BdZdmIjYQqUxdXsjE4lFxLNZjj0eUrSNYIS6xwfij+5Q==
dependencies:
cssnano-preset-advanced "^6.1.2"
postcss "^8.4.38"
postcss-sort-media-queries "^5.2.0"
tslib "^2.6.0"
"@docusaurus/logger@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/logger/-/logger-3.6.2.tgz#4f73f82b33e1d432f3940fc208b3c0646ca5549c"
integrity sha512-1p4IQhhgLyIfsey4UAdAIW69aUE1Ei6O91Nsw30ryZeDWSG5dh4o3zaRGOLxfAX69Ac/yDm6YCwJOafUxL6Vxg==
"@docusaurus/logger@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/logger/-/logger-3.6.3.tgz#c6e514c9429487ef38be2f2129b2b842740d92fd"
integrity sha512-xSubJixcNyMV9wMV4q0s47CBz3Rlc5jbcCCuij8pfQP8qn/DIpt0ks8W6hQWzHAedg/J/EwxxUOUrnEoKzJo8g==
dependencies:
chalk "^4.1.2"
tslib "^2.6.0"
"@docusaurus/mdx-loader@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/mdx-loader/-/mdx-loader-3.6.2.tgz#d240974b0e754d5a5d8eb3f9d0a00a2055fabc68"
integrity sha512-7fbRmNgF3CR96Ja82Ya0/Cdu1OL9UJ/22llNMY8lr5gAbw718Y5ryXMVRIYn0JNLTiSxzgtvW4DIsUWEB8NMpw==
"@docusaurus/mdx-loader@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/mdx-loader/-/mdx-loader-3.6.3.tgz#127babc7cdb26d37c723bc3ae518bda17ce40160"
integrity sha512-3iJdiDz9540ppBseeI93tWTDtUGVkxzh59nMq4ignylxMuXBLK8dFqVeaEor23v1vx6TrGKZ2FuLaTB+U7C0QQ==
dependencies:
"@docusaurus/logger" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/logger" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
"@mdx-js/mdx" "^3.0.0"
"@slorber/remark-comment" "^1.0.0"
escape-html "^1.0.3"
@ -1699,12 +1699,12 @@
vfile "^6.0.1"
webpack "^5.88.1"
"@docusaurus/module-type-aliases@3.6.2", "@docusaurus/module-type-aliases@^3.5.1":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/module-type-aliases/-/module-type-aliases-3.6.2.tgz#7432618696668acc9a7cfb47de66c6987cd57680"
integrity sha512-NrJkL2rLTCjHtWOqUvWzwqvJrsKLj0gVJeV6q5yeKdKKgItietcTf2fTRkM9LHKSUN8CBDXxwHABeQvTahvmXQ==
"@docusaurus/module-type-aliases@3.6.3", "@docusaurus/module-type-aliases@^3.5.1":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/module-type-aliases/-/module-type-aliases-3.6.3.tgz#1f7030b1cf1f658cf664d41b6eadba93bbe51d87"
integrity sha512-MjaXX9PN/k5ugNvfRZdWyKWq4FsrhN4LEXaj0pEmMebJuBNlFeGyKQUa9DRhJHpadNaiMLrbo9m3U7Ig5YlsZg==
dependencies:
"@docusaurus/types" "3.6.2"
"@docusaurus/types" "3.6.3"
"@types/history" "^4.7.11"
"@types/react" "*"
"@types/react-router-config" "*"
@ -1712,19 +1712,19 @@
react-helmet-async "*"
react-loadable "npm:@docusaurus/react-loadable@6.0.0"
"@docusaurus/plugin-content-blog@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.6.2.tgz#b197dd920e380bf1394995215ba7fee8019baa82"
integrity sha512-6bJxr6Or4NslEVH3BJuPH30kUWiqUjDRdGPhvxpHmt9W/RY2/6u72WICG3bW3dLFxJ/2uDLBU92lHnatpvo7Ew==
"@docusaurus/plugin-content-blog@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.6.3.tgz#d6a597e4bfdeb3f1f6ce06d2ac86207296988cc9"
integrity sha512-k0ogWwwJU3pFRFfvW1kRVHxzf2DutLGaaLjAnHVEU6ju+aRP0Z5ap/13DHyPOfHeE4WKpn/M0TqjdwZAcY3kAw==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/logger" "3.6.2"
"@docusaurus/mdx-loader" "3.6.2"
"@docusaurus/theme-common" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-common" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/logger" "3.6.3"
"@docusaurus/mdx-loader" "3.6.3"
"@docusaurus/theme-common" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-common" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
cheerio "1.0.0-rc.12"
feed "^4.2.2"
fs-extra "^11.1.1"
@ -1736,20 +1736,20 @@
utility-types "^3.10.0"
webpack "^5.88.1"
"@docusaurus/plugin-content-docs@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.6.2.tgz#3a8b4b162a2688e5855c04ed6c4ec0b6951619a0"
integrity sha512-e6WW1g10RIXXLN/rrtqTi/FyJ1Hj3X9Mmgz4V11/0pDCxIGGI8m4ocbAglUlLtgvbLD5viNLefl/NwbOW3JXiQ==
"@docusaurus/plugin-content-docs@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.6.3.tgz#aae044d2af6996d1a6de8d815aca8a83b485e0a5"
integrity sha512-r2wS8y/fsaDcxkm20W5bbYJFPzdWdEaTWVYjNxlHlcmX086eqQR1Fomlg9BHTJ0dLXPzAlbC8EN4XqMr3QzNCQ==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/logger" "3.6.2"
"@docusaurus/mdx-loader" "3.6.2"
"@docusaurus/module-type-aliases" "3.6.2"
"@docusaurus/theme-common" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-common" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/logger" "3.6.3"
"@docusaurus/mdx-loader" "3.6.3"
"@docusaurus/module-type-aliases" "3.6.3"
"@docusaurus/theme-common" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-common" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
"@types/react-router-config" "^5.0.7"
combine-promises "^1.1.0"
fs-extra "^11.1.1"
@ -1759,115 +1759,115 @@
utility-types "^3.10.0"
webpack "^5.88.1"
"@docusaurus/plugin-content-pages@3.6.2", "@docusaurus/plugin-content-pages@^3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-pages/-/plugin-content-pages-3.6.2.tgz#49b1a033d41841f7a8bcbbe67511609b402cc80f"
integrity sha512-fo4NyGkw10lYHyHaTxE6TZLYnxNtCfRHeZkNK1N9pBYqe7TT2dBUNAEeVW2U3ed9m6YuB7JKSQsa++GGmcP+6g==
"@docusaurus/plugin-content-pages@3.6.3", "@docusaurus/plugin-content-pages@^3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-pages/-/plugin-content-pages-3.6.3.tgz#0a5a43d1677ee519f63a54634653c54ddf41f475"
integrity sha512-eHrmTgjgLZsuqfsYr5X2xEwyIcck0wseSofWrjTwT9FLOWp+KDmMAuVK+wRo7sFImWXZk3oV/xX/g9aZrhD7OA==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/mdx-loader" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/mdx-loader" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
fs-extra "^11.1.1"
tslib "^2.6.0"
webpack "^5.88.1"
"@docusaurus/plugin-debug@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-debug/-/plugin-debug-3.6.2.tgz#6983f64954fe69a51b65b2d9431bdf0b5ccf1884"
integrity sha512-T/eS3VvHElpeV5S8uwp7Si4ujEynmgFtJLvA2CSa5pzQuOF1EEghF9nekAIj0cWtDHsqNUDZNr8hK1brivFXSg==
"@docusaurus/plugin-debug@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-debug/-/plugin-debug-3.6.3.tgz#4e62ddfbae4d597b073f8e3c632cc12d012339e3"
integrity sha512-zB9GXfIZNPRfzKnNjU6xGVrqn9bPXuGhpjgsuc/YtcTDjnjhasg38NdYd5LEqXex5G/zIorQgWB3n6x/Ut62vQ==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils" "3.6.3"
fs-extra "^11.1.1"
react-json-view-lite "^1.2.0"
tslib "^2.6.0"
"@docusaurus/plugin-google-analytics@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-3.6.2.tgz#4266b4b2273998e87fa733d932d5b464c2a10b21"
integrity sha512-B7ihrr3wz8e4XqW+dIAtq844u3Z83u5CeiL1xrCqzFH+vDCjUZHTamS3zKXNcgi6YVVe6hUQXPG15ltaqQaVPQ==
"@docusaurus/plugin-google-analytics@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-3.6.3.tgz#63648d469b1e3c50fad8878e7a7db9856e503d5f"
integrity sha512-rCDNy1QW8Dag7nZq67pcum0bpFLrwvxJhYuVprhFh8BMBDxV0bY+bAkGHbSf68P3Bk9C3hNOAXX1srGLIDvcTA==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
tslib "^2.6.0"
"@docusaurus/plugin-google-gtag@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-3.6.2.tgz#23f70f7a05e61cfb9d9d7ee18dbff3ef2b129f2c"
integrity sha512-V8ijI6qddAAkJ0vd8sjZ7S/apRTLJn9dAwvj/rSMd93witGdKINemL+9TyfLkhcXKTxyqRT8zKdu8ewjPXqKHg==
"@docusaurus/plugin-google-gtag@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-3.6.3.tgz#8a1388b4123904be17e661ea7aa71d798d0c046e"
integrity sha512-+OyDvhM6rqVkQOmLVkQWVJAizEEfkPzVWtIHXlWPOCFGK9X4/AWeBSrU0WG4iMg9Z4zD4YDRrU+lvI4s6DSC+w==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
"@types/gtag.js" "^0.0.12"
tslib "^2.6.0"
"@docusaurus/plugin-google-tag-manager@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-tag-manager/-/plugin-google-tag-manager-3.6.2.tgz#36ab95fcd5c1bf96fd18c0cf9b208bb428b81242"
integrity sha512-fnWQ5FdN9f8c8VTgjaQ98208Y+d/JjHhD506rWIIL9rt1cJOf29XElxvOeKpMJadfkgY5KLZSAiHkGt+4qgN4g==
"@docusaurus/plugin-google-tag-manager@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-tag-manager/-/plugin-google-tag-manager-3.6.3.tgz#38cbe416803f29782807cebf3ebf240cb47c3c74"
integrity sha512-1M6UPB13gWUtN2UHX083/beTn85PlRI9ABItTl/JL1FJ5dJTWWFXXsHf9WW/6hrVwthwTeV/AGbGKvLKV+IlCA==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
tslib "^2.6.0"
"@docusaurus/plugin-sitemap@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-sitemap/-/plugin-sitemap-3.6.2.tgz#c8ff7cf82bd5d943a13bb8d0ae690080a025029e"
integrity sha512-qcAQAP1Ot0dZpeRoJ0L/Zck5FVDkll2IleVZQLzxeRVDZIw1P9/TK7/Aw1w2pmH7dmw/Cwk/cLSVRvLAmp9k7A==
"@docusaurus/plugin-sitemap@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/plugin-sitemap/-/plugin-sitemap-3.6.3.tgz#0458e6f7476ab6fd1466e01b153a3211d3223c53"
integrity sha512-94qOO4M9Fwv9KfVQJsgbe91k+fPJ4byf1L3Ez8TUa6TAFPo/BrLwQ80zclHkENlL1824TuxkcMKv33u6eydQCg==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/logger" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-common" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/logger" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-common" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
fs-extra "^11.1.1"
sitemap "^7.1.1"
tslib "^2.6.0"
"@docusaurus/preset-classic@^3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/preset-classic/-/preset-classic-3.6.2.tgz#5ec801fa317123ba8458af3105eca8eac78a49bc"
integrity sha512-r2n5eHdhiNSrJGsrrYcw+WsyStmXxe0ZG3RdA9LVyK5+jBHM8blrUWJEDugnzCNbyhUzhdtcmgCC9fhdAvKuQw==
"@docusaurus/preset-classic@^3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/preset-classic/-/preset-classic-3.6.3.tgz#072298b5b6d0de7d0346b1e9b550a30ef2add56d"
integrity sha512-VHSYWROT3flvNNI1SrnMOtW1EsjeHNK9dhU6s9eY5hryZe79lUqnZJyze/ymDe2LXAqzyj6y5oYvyBoZZk6ErA==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/plugin-content-blog" "3.6.2"
"@docusaurus/plugin-content-docs" "3.6.2"
"@docusaurus/plugin-content-pages" "3.6.2"
"@docusaurus/plugin-debug" "3.6.2"
"@docusaurus/plugin-google-analytics" "3.6.2"
"@docusaurus/plugin-google-gtag" "3.6.2"
"@docusaurus/plugin-google-tag-manager" "3.6.2"
"@docusaurus/plugin-sitemap" "3.6.2"
"@docusaurus/theme-classic" "3.6.2"
"@docusaurus/theme-common" "3.6.2"
"@docusaurus/theme-search-algolia" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/plugin-content-blog" "3.6.3"
"@docusaurus/plugin-content-docs" "3.6.3"
"@docusaurus/plugin-content-pages" "3.6.3"
"@docusaurus/plugin-debug" "3.6.3"
"@docusaurus/plugin-google-analytics" "3.6.3"
"@docusaurus/plugin-google-gtag" "3.6.3"
"@docusaurus/plugin-google-tag-manager" "3.6.3"
"@docusaurus/plugin-sitemap" "3.6.3"
"@docusaurus/theme-classic" "3.6.3"
"@docusaurus/theme-common" "3.6.3"
"@docusaurus/theme-search-algolia" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/theme-classic@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-classic/-/theme-classic-3.6.2.tgz#4c2770d3609176dd2462dfb0cb4d0b3d3010404b"
integrity sha512-bCdOPqPNezhLx+hgNVO2Cf+8/1AHa9uHDOqTx/CKAx2I0J/jV9G+6JiMtpSRKGNfBoLT1O+56/7+WtkOf54xTw==
"@docusaurus/theme-classic@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-classic/-/theme-classic-3.6.3.tgz#00599a9de5fd5c122fd1b8c59d3b755878f2a72c"
integrity sha512-1RRLK1tSArI2c00qugWYO3jRocjOZwGF1mBzPPylDVRwWCS/rnWWR91ChdbbaxIupRJ+hX8ZBYrwr5bbU0oztQ==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/logger" "3.6.2"
"@docusaurus/mdx-loader" "3.6.2"
"@docusaurus/module-type-aliases" "3.6.2"
"@docusaurus/plugin-content-blog" "3.6.2"
"@docusaurus/plugin-content-docs" "3.6.2"
"@docusaurus/plugin-content-pages" "3.6.2"
"@docusaurus/theme-common" "3.6.2"
"@docusaurus/theme-translations" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-common" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/logger" "3.6.3"
"@docusaurus/mdx-loader" "3.6.3"
"@docusaurus/module-type-aliases" "3.6.3"
"@docusaurus/plugin-content-blog" "3.6.3"
"@docusaurus/plugin-content-docs" "3.6.3"
"@docusaurus/plugin-content-pages" "3.6.3"
"@docusaurus/theme-common" "3.6.3"
"@docusaurus/theme-translations" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-common" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
"@mdx-js/react" "^3.0.0"
clsx "^2.0.0"
copy-text-to-clipboard "^3.2.0"
@ -1882,15 +1882,15 @@
tslib "^2.6.0"
utility-types "^3.10.0"
"@docusaurus/theme-common@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-common/-/theme-common-3.6.2.tgz#a520d9053b6ea0fa913d42898d35f73ed5ca3b9b"
integrity sha512-lfgsL064KEHpCkgGUc0OYoUPCpYfzggp6Hof8sz59UuKiLvb/Z7raewE9/NfocrJ2HZI17rLgMX3SQlRDh/5gg==
"@docusaurus/theme-common@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-common/-/theme-common-3.6.3.tgz#a8a6ebd2b0fd7a5cca4d0c6a2f9ccff905fa7438"
integrity sha512-b8ZkhczXHDxWWyvz+YJy4t/PlPbEogTTbgnHoflYnH7rmRtyoodTsu8WVM12la5LmlMJBclBXFl29OH8kPE7gg==
dependencies:
"@docusaurus/mdx-loader" "3.6.2"
"@docusaurus/module-type-aliases" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-common" "3.6.2"
"@docusaurus/mdx-loader" "3.6.3"
"@docusaurus/module-type-aliases" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-common" "3.6.3"
"@types/history" "^4.7.11"
"@types/react" "*"
"@types/react-router-config" "*"
@ -1900,32 +1900,32 @@
tslib "^2.6.0"
utility-types "^3.10.0"
"@docusaurus/theme-mermaid@^3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-mermaid/-/theme-mermaid-3.6.2.tgz#f7ad64ae5b9510ddbf77bc4264ada059c8e815b1"
integrity sha512-Ui+rBtqMPKj3RCOxNlY04i1tEjNg+fZg4URTvkHmYR07hcKaJw+vkw+wlaYjd0HFZk+3Er9vUAcwsCWuea4cVQ==
"@docusaurus/theme-mermaid@^3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-mermaid/-/theme-mermaid-3.6.3.tgz#4bb0b4940b39cc855626a23190aa45928aa168b4"
integrity sha512-kIqpjNCP/9R2GGf8UmiDxD3CkOAEJuJIEFlaKMgQtjVxa/vH+9PLI1+DFbArGoG4+0ENTYUq8phHPW7SeL36uQ==
dependencies:
"@docusaurus/core" "3.6.2"
"@docusaurus/module-type-aliases" "3.6.2"
"@docusaurus/theme-common" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/module-type-aliases" "3.6.3"
"@docusaurus/theme-common" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
mermaid ">=10.4"
tslib "^2.6.0"
"@docusaurus/theme-search-algolia@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-search-algolia/-/theme-search-algolia-3.6.2.tgz#b03b7d35a385004d089d000be764abdfb3fa5721"
integrity sha512-SFLS+Rq8Cg2yepnHucA9sRpIR97yHvZWlCgMzBLunV3KHbB6hD2h5HPhFV39wYHYCjJUAOH1lX9poJ1qKYuSvg==
"@docusaurus/theme-search-algolia@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-search-algolia/-/theme-search-algolia-3.6.3.tgz#1a3331a489f392f5b032c4efc5f431e57eddf7ce"
integrity sha512-rt+MGCCpYgPyWCGXtbxlwFbTSobu15jWBTPI2LHsHNa5B0zSmOISX6FWYAPt5X1rNDOqMGM0FATnh7TBHRohVA==
dependencies:
"@docsearch/react" "^3.5.2"
"@docusaurus/core" "3.6.2"
"@docusaurus/logger" "3.6.2"
"@docusaurus/plugin-content-docs" "3.6.2"
"@docusaurus/theme-common" "3.6.2"
"@docusaurus/theme-translations" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-validation" "3.6.2"
"@docusaurus/core" "3.6.3"
"@docusaurus/logger" "3.6.3"
"@docusaurus/plugin-content-docs" "3.6.3"
"@docusaurus/theme-common" "3.6.3"
"@docusaurus/theme-translations" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-validation" "3.6.3"
algoliasearch "^4.18.0"
algoliasearch-helper "^3.13.3"
clsx "^2.0.0"
@ -1935,23 +1935,23 @@
tslib "^2.6.0"
utility-types "^3.10.0"
"@docusaurus/theme-translations@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-translations/-/theme-translations-3.6.2.tgz#ff6d2588aa9bf9fb1e07465def067529d5668665"
integrity sha512-LIWrYoDUsOTKmb0c7IQzawiPUTAaczBs5IOx6srxOWoTHVUMLzJCkl5Y6whfuRrnul8G05qv2vk238bN5Ko62g==
"@docusaurus/theme-translations@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/theme-translations/-/theme-translations-3.6.3.tgz#6e473835ea016ce4acd7d2997f411811db8c4f6b"
integrity sha512-Gb0regclToVlngSIIwUCtBMQBq48qVUaN1XQNKW4XwlsgUyk0vP01LULdqbem7czSwIeBAFXFoORJ0RPX7ht/w==
dependencies:
fs-extra "^11.1.1"
tslib "^2.6.0"
"@docusaurus/tsconfig@^3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/tsconfig/-/tsconfig-3.6.2.tgz#a6d3fec19ae45a67da678b54c019b0e3839b6711"
integrity sha512-TWLkyYHBYhIJNcXCEc3D1M9R8UFV4IZ82rGef5U9mE1ZrcgDUlZxYaYdoSuHrPrzPRIl3orjmpscO2FAk2gdZw==
"@docusaurus/tsconfig@^3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/tsconfig/-/tsconfig-3.6.3.tgz#8af20c45f0a67e193debedcb341c0a1e78b1dd63"
integrity sha512-1pT/rTrRpMV15E4tJH95W5PrjboMn5JkKF+Ys8cTjMegetiXjs0gPFOSDA5hdTlberKQLDO50xPjMJHondLuzA==
"@docusaurus/types@3.6.2", "@docusaurus/types@^3.5.1":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/types/-/types-3.6.2.tgz#bd69c4c99b535b67f01276dc186622e0b1fc1305"
integrity sha512-117Wsk6xXrWEAsCYCXS3TGJv5tkdIZDcd7T/V0UJvKYmY0gyVPPcEQChy8yTdjbIkbB2q4fa7Jpox72Qv86mqQ==
"@docusaurus/types@3.6.3", "@docusaurus/types@^3.5.1":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/types/-/types-3.6.3.tgz#e87592e31616da1b8dc473e4c8205c61885a1518"
integrity sha512-xD9oTGDrouWzefkhe9ogB2fDV96/82cRpNGx2HIvI5L87JHNhQVIWimQ/3JIiiX/TEd5S9s+VO6FFguwKNRVow==
dependencies:
"@mdx-js/mdx" "^3.0.0"
"@types/history" "^4.7.11"
@ -1963,36 +1963,36 @@
webpack "^5.95.0"
webpack-merge "^5.9.0"
"@docusaurus/utils-common@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/utils-common/-/utils-common-3.6.2.tgz#3367572d72090b7f17e721af7f020f8e39931662"
integrity sha512-dr5wK+OyU2QAWxG7S5siD2bPgS7+ZeqWHfgLNHZ5yalaZf8TbeNNLqydfngukAY56BGZN0NbMkX6jGIr7ZF0sA==
"@docusaurus/utils-common@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/utils-common/-/utils-common-3.6.3.tgz#57f840bd6f0928cf10060198cb421f1b9212c8f5"
integrity sha512-v4nKDaANLgT3pMBewHYEMAl/ufY0LkXao1QkFWzI5huWFOmNQ2UFzv2BiKeHX5Ownis0/w6cAyoxPhVdDonlSQ==
dependencies:
"@docusaurus/types" "3.6.2"
"@docusaurus/types" "3.6.3"
tslib "^2.6.0"
"@docusaurus/utils-validation@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/utils-validation/-/utils-validation-3.6.2.tgz#62b97a0d72694c85fa63928c494dd238a84c991f"
integrity sha512-Y3EwblDz72KOcobb5t2zlhHSmrfE8EaHusPJ96Kx2JYtNXL2omqCoOb6FpaXWhES75wvjUpkFLYfiNqAqEov8g==
"@docusaurus/utils-validation@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/utils-validation/-/utils-validation-3.6.3.tgz#3eca7125235eb90983ff660b97a71f331e331f57"
integrity sha512-bhEGGiN5BE38h21vjqD70Gxg++j+PfYVddDUE5UFvLDup68QOcpD33CLr+2knPorlxRbEaNfz6HQDUMQ3HuqKw==
dependencies:
"@docusaurus/logger" "3.6.2"
"@docusaurus/utils" "3.6.2"
"@docusaurus/utils-common" "3.6.2"
"@docusaurus/logger" "3.6.3"
"@docusaurus/utils" "3.6.3"
"@docusaurus/utils-common" "3.6.3"
fs-extra "^11.2.0"
joi "^17.9.2"
js-yaml "^4.1.0"
lodash "^4.17.21"
tslib "^2.6.0"
"@docusaurus/utils@3.6.2":
version "3.6.2"
resolved "https://registry.yarnpkg.com/@docusaurus/utils/-/utils-3.6.2.tgz#727299c2051eee04c1b431bc6ccd55fd4e5a0d52"
integrity sha512-oxnpUcFZGE3uPCDoXr8GJriB3VWM9sFjPedFidX3Fsz87l1NZNc1wtbKPfQ7GYFDMYo2IGlAv5+47Me9RkM6lg==
"@docusaurus/utils@3.6.3":
version "3.6.3"
resolved "https://registry.yarnpkg.com/@docusaurus/utils/-/utils-3.6.3.tgz#8dcb1969e4011a84dfb0a031da806dadddebf0ea"
integrity sha512-0R/FR3bKVl4yl8QwbL4TYFfR+OXBRpVUaTJdENapBGR3YMwfM6/JnhGilWQO8AOwPJGtGoDK7ib8+8UF9f3OZQ==
dependencies:
"@docusaurus/logger" "3.6.2"
"@docusaurus/types" "3.6.2"
"@docusaurus/utils-common" "3.6.2"
"@docusaurus/logger" "3.6.3"
"@docusaurus/types" "3.6.3"
"@docusaurus/utils-common" "3.6.3"
"@svgr/webpack" "^8.1.0"
escape-string-regexp "^4.0.0"
file-loader "^6.2.0"

44
evaluation/README.md
View File

@ -6,9 +6,9 @@ This folder contains code and resources to run experiments and evaluations.
### Setup
Before starting evaluation, follow the instructions here [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
Before starting evaluation, follow the instructions [here](https://github.com/All-Hands-AI/OpenHands/blob/main/Development.md) to setup your local development environment and LLM.
Once you are done with setup, you can follow the benchmark-specific instructions in each subdirectory of the evaluation directory.
Once you are done with setup, you can follow the benchmark-specific instructions in each subdirectory of the [evaluation directory](#supported-benchmarks).
Generally these will involve running `run_infer.py` to perform inference with the agents.
### Implementing and Evaluating an Agent
@ -42,32 +42,36 @@ temperature = 0.0
## Supported Benchmarks
The OpenHands evaluation harness supports a wide variety of benchmarks across software engineering, web browsing, and miscellaneous assistance tasks.
The OpenHands evaluation harness supports a wide variety of benchmarks across [software engineering](#software-engineering), [web browsing](#web-browsing), and [miscellaneous assistance](#misc-assistance) tasks.
### Software Engineering
- SWE-Bench: [`evaluation/swe_bench`](./swe_bench)
- HumanEvalFix: [`evaluation/humanevalfix`](./humanevalfix)
- BIRD: [`evaluation/bird`](./bird)
- BioCoder: [`evaluation/ml_bench`](./ml_bench)
- ML-Bench: [`evaluation/ml_bench`](./ml_bench)
- APIBench: [`evaluation/gorilla`](./gorilla/)
- ToolQA: [`evaluation/toolqa`](./toolqa/)
- AiderBench: [`evaluation/aider_bench`](./aider_bench/)
- SWE-Bench: [`evaluation/benchmarks/swe_bench`](./benchmarks/swe_bench)
- HumanEvalFix: [`evaluation/benchmarks/humanevalfix`](./benchmarks/humanevalfix)
- BIRD: [`evaluation/benchmarks/bird`](./benchmarks/bird)
- BioCoder: [`evaluation/benchmarks/ml_bench`](./benchmarks/ml_bench)
- ML-Bench: [`evaluation/benchmarks/ml_bench`](./benchmarks/ml_bench)
- APIBench: [`evaluation/benchmarks/gorilla`](./benchmarks/gorilla/)
- ToolQA: [`evaluation/benchmarks/toolqa`](./benchmarks/toolqa/)
- AiderBench: [`evaluation/benchmarks/aider_bench`](./benchmarks/aider_bench/)
- Commit0: [`evaluation/benchmarks/commit0_bench`](./benchmarks/commit0_bench/)
- DiscoveryBench: [`evaluation/benchmarks/discoverybench`](./benchmarks/discoverybench/)
### Web Browsing
- WebArena: [`evaluation/webarena`](./webarena/)
- MiniWob++: [`evaluation/miniwob`](./miniwob/)
- WebArena: [`evaluation/benchmarks/webarena`](./benchmarks/webarena/)
- MiniWob++: [`evaluation/benchmarks/miniwob`](./benchmarks/miniwob/)
- Browsing Delegation: [`evaluation/benchmarks/browsing_delegation`](./benchmarks/browsing_delegation/)
### Misc. Assistance
- GAIA: [`evaluation/gaia`](./gaia)
- GPQA: [`evaluation/gpqa`](./gpqa)
- AgentBench: [`evaluation/agent_bench`](./agent_bench)
- MINT: [`evaluation/mint`](./mint)
- Entity deduction Arena (EDA): [`evaluation/EDA`](./EDA)
- ProofWriter: [`evaluation/logic_reasoning`](./logic_reasoning)
- GAIA: [`evaluation/benchmarks/gaia`](./benchmarks/gaia)
- GPQA: [`evaluation/benchmarks/gpqa`](./benchmarks/gpqa)
- AgentBench: [`evaluation/benchmarks/agent_bench`](./benchmarks/agent_bench)
- MINT: [`evaluation/benchmarks/mint`](./benchmarks/mint)
- Entity deduction Arena (EDA): [`evaluation/benchmarks/EDA`](./benchmarks/EDA)
- ProofWriter: [`evaluation/benchmarks/logic_reasoning`](./benchmarks/logic_reasoning)
- ScienceAgentBench: [`evaluation/benchmarks/scienceagentbench`](./benchmarks/scienceagentbench)
## Result Visualization
@ -79,7 +83,7 @@ You can start your own fork of [our huggingface evaluation outputs](https://hugg
To learn more about how to integrate your benchmark into OpenHands, check out [tutorial here](https://docs.all-hands.dev/modules/usage/how-to/evaluation-harness). Briefly,
- Each subfolder contains a specific benchmark or experiment. For example, `evaluation/swe_bench` should contain
- Each subfolder contains a specific benchmark or experiment. For example, [`evaluation/benchmarks/swe_bench`](./benchmarks/swe_bench) should contain
all the preprocessing/evaluation/analysis scripts.
- Raw data and experimental records should not be stored within this repo.
- For model outputs, they should be stored at [this huggingface space](https://huggingface.co/spaces/OpenHands/evaluation) for visualization.

4
evaluation/EDA/README.md → evaluation/benchmarks/EDA/README.md
View File

@ -12,7 +12,7 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
```bash
export OPENAI_API_KEY="sk-XXX"; # This is required for evaluation (to simulate another party of conversation)
./evaluation/EDA/scripts/run_infer.sh [model_config] [git-version] [agent] [dataset] [eval_limit]
./evaluation/benchmarks/EDA/scripts/run_infer.sh [model_config] [git-version] [agent] [dataset] [eval_limit]
```
where `model_config` is mandatory, while `git-version`, `agent`, `dataset` and `eval_limit` are optional.
@ -33,7 +33,7 @@ to `CodeActAgent`.
For example,
```bash
./evaluation/EDA/scripts/run_infer.sh eval_gpt4o_2024_05_13 0.6.2 CodeActAgent things
./evaluation/benchmarks/EDA/scripts/run_infer.sh eval_gpt4o_2024_05_13 0.6.2 CodeActAgent things
```
## Reference

0
evaluation/EDA/game.py → evaluation/benchmarks/EDA/game.py
View File

2
evaluation/EDA/run_infer.py → evaluation/benchmarks/EDA/run_infer.py
View File

@ -4,7 +4,7 @@ import os
import pandas as pd
from datasets import load_dataset
from evaluation.EDA.game import Q20Game, Q20GameCelebrity
from evaluation.benchmarks.EDA.game import Q20Game, Q20GameCelebrity
from evaluation.utils.shared import (
EvalMetadata,
EvalOutput,

2
evaluation/EDA/scripts/run_infer.sh → evaluation/benchmarks/EDA/scripts/run_infer.sh
View File

@ -43,7 +43,7 @@ echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
echo "DATASET: $DATASET"
COMMAND="poetry run python evaluation/EDA/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/EDA/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--dataset $DATASET \

24
evaluation/agent_bench/README.md → evaluation/benchmarks/agent_bench/README.md
View File

@ -9,7 +9,7 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
## Start the evaluation
```bash
./evaluation/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
./evaluation/benchmarks/agent_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
```
- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your
@ -25,7 +25,7 @@ in order to use `eval_limit`, you must also set `agent`.
Following is the basic command to start the evaluation.
You can update the arguments in the script `evaluation/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`, `--eval-num-workers` and so on.
You can update the arguments in the script `evaluation/benchmarks/agent_bench/scripts/run_infer.sh`, such as `--max-iterations`, `--eval-num-workers` and so on.
- `--agent-cls`, the agent to use. For example, `CodeActAgent`.
- `--llm-config`: the LLM configuration to use. For example, `eval_gpt4_1106_preview`.
@ -34,5 +34,23 @@ You can update the arguments in the script `evaluation/agent_bench/scripts/run_i
- `--eval-n-limit`: the number of examples to evaluate. For example, `100`.
```bash
./evaluation/agent_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
./evaluation/benchmarks/agent_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 1
```
## Run with Remote Runtime (experimental)
You can run the evaluation using a remote runtime instead of a local Docker container. This is useful when you want to run the evaluation in a cloud environment or when you don't have Docker installed locally.
To use the remote runtime, set the following environment variables:
```bash
# Required environment variables
export ALLHANDS_API_KEY="your-api-key" # Contact the team to get an API key
export RUNTIME=remote
export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
# Run the evaluation
./evaluation/benchmarks/agent_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview HEAD CodeActAgent 1
```
The remote runtime will build a container image and run the evaluation in a cloud environment. The results will be saved locally in the same way as when running with a local runtime.

0
evaluation/agent_bench/__init__.py → evaluation/benchmarks/agent_bench/__init__.py
View File

0
evaluation/agent_bench/helper.py → evaluation/benchmarks/agent_bench/helper.py
View File

10
evaluation/agent_bench/run_infer.py → evaluation/benchmarks/agent_bench/run_infer.py
View File

@ -7,7 +7,7 @@ from typing import Any
import pandas as pd
from datasets import load_dataset
from evaluation.agent_bench.helper import (
from evaluation.benchmarks.agent_bench.helper import (
FAKE_RESPONSES,
INST_SUFFIXES,
compare_results,
@ -43,12 +43,16 @@ def get_config(
config = AppConfig(
default_agent=metadata.agent_class,
run_as_openhands=False,
runtime='eventstream',
runtime=os.environ.get('RUNTIME', 'eventstream'),
max_iterations=metadata.max_iterations,
sandbox=SandboxConfig(
base_container_image='python:3.12-bookworm',
base_container_image='python:3.12-slim',
enable_auto_lint=True,
use_host_network=False,
api_key=os.environ.get('ALLHANDS_API_KEY', None),
remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'),
keep_runtime_alive=False,
remote_runtime_init_timeout=3600,
),
# do not mount workspace
workspace_base=None,

2
evaluation/agent_bench/scripts/run_infer.sh → evaluation/benchmarks/agent_bench/scripts/run_infer.sh
View File

@ -26,7 +26,7 @@ echo "AGENT: $AGENT"
echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
COMMAND="export PYTHONPATH=evaluation/agent_bench:\$PYTHONPATH && poetry run python evaluation/agent_bench/run_infer.py \
COMMAND="export PYTHONPATH=evaluation/benchmarks/agent_bench:\$PYTHONPATH && poetry run python evaluation/benchmarks/agent_bench/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 30 \

0
evaluation/agent_bench/scripts/summarise_results.py → evaluation/benchmarks/agent_bench/scripts/summarise_results.py
View File

14
evaluation/aider_bench/README.md → evaluation/benchmarks/aider_bench/README.md
View File

@ -16,7 +16,7 @@ development environment and LLM.
## Start the evaluation
```bash
./evaluation/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
```
- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for
@ -42,7 +42,7 @@ export SKIP_NUM=12 # skip the first 12 instances from the dataset
Following is the basic command to start the evaluation.
You can update the arguments in the script
`evaluation/aider_bench/scripts/run_infer.sh`, such as `--max-iterations`,
`evaluation/benchmarks/aider_bench/scripts/run_infer.sh`, such as `--max-iterations`,
`--eval-num-workers` and so on:
- `--agent-cls`, the agent to use. For example, `CodeActAgent`.
@ -53,7 +53,7 @@ You can update the arguments in the script
- `--eval-ids`: the IDs of the examples to evaluate (comma separated). For example, `"1,3,10"`.
```bash
./evaluation/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
./evaluation/benchmarks/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10"
```
### Run Inference on `RemoteRuntime` (experimental)
@ -61,25 +61,25 @@ You can update the arguments in the script
This is in limited beta. Contact Xingyao over slack if you want to try this out!
```bash
./evaluation/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids]
# Example - This runs evaluation on CodeActAgent for 133 instances on aider_bench test set, with 2 workers running in parallel
export ALLHANDS_API_KEY="YOUR-API-KEY"
export RUNTIME=remote
export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
./evaluation/aider_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 133 2
./evaluation/benchmarks/aider_bench/scripts/run_infer.sh llm.eval HEAD CodeActAgent 133 2
```
## Summarize Results
```bash
poetry run python ./evaluation/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file]
poetry run python ./evaluation/benchmarks/aider_bench/scripts/summarize_results.py [path_to_output_jsonl_file]
```
Full example:
```bash
poetry run python ./evaluation/aider_bench/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl
poetry run python ./evaluation/benchmarks/aider_bench/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/AiderBench/CodeActAgent/claude-3-5-sonnet@20240620_maxiter_30_N_v1.9/output.jsonl
```
This will list the instances that passed and the instances that failed. For each

0
evaluation/aider_bench/create_dataset.py → evaluation/benchmarks/aider_bench/create_dataset.py
View File

0
evaluation/aider_bench/helper.py → evaluation/benchmarks/aider_bench/helper.py
View File

2
evaluation/aider_bench/run_infer.py → evaluation/benchmarks/aider_bench/run_infer.py
View File

@ -7,7 +7,7 @@ from typing import Any
import pandas as pd
from datasets import load_dataset
from evaluation.aider_bench.helper import (
from evaluation.benchmarks.aider_bench.helper import (
FAKE_RESPONSES,
INST_SUFFIXES,
INSTRUCTIONS_ADDENDUM,

2
evaluation/aider_bench/scripts/run_infer.sh → evaluation/benchmarks/aider_bench/scripts/run_infer.sh
View File

@ -39,7 +39,7 @@ if [ "$USE_UNIT_TESTS" = true ]; then
EVAL_NOTE=$EVAL_NOTE-w-test
fi
COMMAND="export PYTHONPATH=evaluation/aider_bench:\$PYTHONPATH && poetry run python evaluation/aider_bench/run_infer.py \
COMMAND="export PYTHONPATH=evaluation/benchmarks/aider_bench:\$PYTHONPATH && poetry run python evaluation/benchmarks/aider_bench/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 30 \

0
evaluation/aider_bench/scripts/summarize_results.py → evaluation/benchmarks/aider_bench/scripts/summarize_results.py
View File

4
evaluation/biocoder/README.md → evaluation/benchmarks/biocoder/README.md
View File

@ -21,7 +21,7 @@ To reproduce this image, please see the Dockerfile_Openopenhands in the `biocode
```bash
./evaluation/biocoder/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
./evaluation/benchmarks/biocoder/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
```
where `model_config` is mandatory, while `git-version`, `agent`, `dataset` and `eval_limit` are optional.
@ -43,7 +43,7 @@ with current OpenHands version, then your command would be:
## Examples
```bash
./evaluation/biocoder/scripts/run_infer.sh eval_gpt4o_2024_05_13 HEAD CodeActAgent 1
./evaluation/benchmarks/biocoder/scripts/run_infer.sh eval_gpt4o_2024_05_13 HEAD CodeActAgent 1
```
## Reference

2
evaluation/biocoder/run_infer.py → evaluation/benchmarks/biocoder/run_infer.py
View File

@ -8,7 +8,7 @@ from typing import Any
import pandas as pd
from datasets import load_dataset
from evaluation.biocoder.utils import BiocoderData
from evaluation.benchmarks.biocoder.utils import BiocoderData
from evaluation.utils.shared import (
EvalMetadata,
EvalOutput,

2
evaluation/biocoder/scripts/run_infer.sh → evaluation/benchmarks/biocoder/scripts/run_infer.sh
View File

@ -28,7 +28,7 @@ echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
echo "DATASET: $DATASET"
COMMAND="poetry run python evaluation/biocoder/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/biocoder/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 10 \

0
evaluation/biocoder/scripts/setup/copy_changed_code.py → evaluation/benchmarks/biocoder/scripts/setup/copy_changed_code.py
View File

0
evaluation/biocoder/scripts/setup/remove_code.py → evaluation/benchmarks/biocoder/scripts/setup/remove_code.py
View File

0
evaluation/biocoder/utils.py → evaluation/benchmarks/biocoder/utils.py
View File

2
evaluation/bird/README.md → evaluation/benchmarks/bird/README.md
View File

@ -9,7 +9,7 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
## Run Inference on Bird
```bash
./evaluation/bird/scripts/run_infer.sh [model_config] [git-version]
./evaluation/benchmarks/bird/scripts/run_infer.sh [model_config] [git-version]
```
- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your

0
evaluation/bird/__init__.py → evaluation/benchmarks/bird/__init__.py
View File

0
evaluation/bird/run_infer.py → evaluation/benchmarks/bird/run_infer.py
View File

2
evaluation/bird/scripts/run_infer.sh → evaluation/benchmarks/bird/scripts/run_infer.sh
View File

@ -26,7 +26,7 @@ echo "AGENT: $AGENT"
echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
COMMAND="poetry run python evaluation/bird/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/bird/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 5 \

2
evaluation/browsing_delegation/README.md → evaluation/benchmarks/browsing_delegation/README.md
View File

@ -12,7 +12,7 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
## Run Inference
```bash
./evaluation/browsing_delegation/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
./evaluation/benchmarks/browsing_delegation/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit]
# e.g., ./evaluation/swe_bench/scripts/run_infer.sh llm.eval_gpt4_1106_preview_llm HEAD CodeActAgent 300
```

0
evaluation/browsing_delegation/run_infer.py → evaluation/benchmarks/browsing_delegation/run_infer.py
View File

2
evaluation/browsing_delegation/scripts/run_infer.sh → evaluation/benchmarks/browsing_delegation/scripts/run_infer.sh
View File

@ -28,7 +28,7 @@ echo "MODEL_CONFIG: $MODEL_CONFIG"
EVAL_NOTE="$AGENT_VERSION"
COMMAND="poetry run python evaluation/browsing_delegation/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/browsing_delegation/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 1 \

12
evaluation/commit0_bench/README.md → evaluation/benchmarks/commit0_bench/README.md
View File

@ -24,10 +24,10 @@ Make sure your Docker daemon is running, and you have ample disk space (at least
When the `run_infer.sh` script is started, it will automatically pull the `lite` split in Commit0. For example, for instance ID `commit-0/minitorch`, it will try to pull our pre-build docker image `wentingzhao/minitorch` from DockerHub. This image will be used create an OpenHands runtime image where the agent will operate on.
```bash
./evaluation/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
# Example
./evaluation/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 16 100 8 wentingzhao/commit0_combined test
./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 16 100 8 wentingzhao/commit0_combined test
```
where `model_config` is mandatory, and the rest are optional.
@ -56,7 +56,7 @@ Let's say you'd like to run 10 instances using `llm.eval_sonnet` and CodeActAgen
then your command would be:
```bash
./evaluation/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
```
### Run Inference on `RemoteRuntime` (experimental)
@ -64,17 +64,17 @@ then your command would be:
This is in limited beta. Contact Xingyao over slack if you want to try this out!
```bash
./evaluation/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh [repo_split] [model_config] [git-version] [agent] [eval_limit] [max_iter] [num_workers] [dataset] [dataset_split]
# Example - This runs evaluation on CodeActAgent for 10 instances on "wentingzhao/commit0_combined"'s test set, with max 30 iteration per instances, with 1 number of workers running in parallel
ALLHANDS_API_KEY="YOUR-API-KEY" RUNTIME=remote SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev" EVAL_DOCKER_IMAGE_PREFIX="docker.io/wentingzhao" \
./evaluation/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
./evaluation/benchmarks/commit0_bench/scripts/run_infer.sh lite llm.eval_sonnet HEAD CodeActAgent 10 30 1 wentingzhao/commit0_combined test
```
To clean-up all existing runtime you've already started, run:
```bash
ALLHANDS_API_KEY="YOUR-API-KEY" ./evaluation/commit0_bench/scripts/cleanup_remote_runtime.sh
ALLHANDS_API_KEY="YOUR-API-KEY" ./evaluation/benchmarks/commit0_bench/scripts/cleanup_remote_runtime.sh
```
### Specify a subset of tasks to run infer

0
evaluation/commit0_bench/run_infer.py → evaluation/benchmarks/commit0_bench/run_infer.py
View File

0
evaluation/commit0_bench/scripts/cleanup_remote_runtime.sh → evaluation/benchmarks/commit0_bench/scripts/cleanup_remote_runtime.sh
View File

2
evaluation/commit0_bench/scripts/run_infer.sh → evaluation/benchmarks/commit0_bench/scripts/run_infer.sh
View File

@ -91,7 +91,7 @@ fi
function run_eval() {
local eval_note=$1
COMMAND="poetry run python evaluation/commit0_bench/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/commit0_bench/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations $MAX_ITER \

4
evaluation/discoverybench/README.md → evaluation/benchmarks/discoverybench/README.md
View File

@ -16,7 +16,7 @@
2. Execute the bash script to start DiscoveryBench Evaluation
```
./evaluation/discoverybench/scripts/run_infer.sh [YOUR MODEL CONFIG]
./evaluation/benchmarks/discoverybench/scripts/run_infer.sh [YOUR MODEL CONFIG]
```
Replace `[YOUR MODEL CONFIG]` with any model the model that you have set up in `config.toml`
@ -27,7 +27,7 @@ When the `run_infer.sh` script is started, it will automatically pull the latest
```
./evaluation/discoverybench/scripts/run_infer.sh [MODEL_CONFIG] [GIT_COMMIT] [AGENT] [EVAL_LIMIT] [NUM_WORKERS]
./evaluation/benchmarks/discoverybench/scripts/run_infer.sh [MODEL_CONFIG] [GIT_COMMIT] [AGENT] [EVAL_LIMIT] [NUM_WORKERS]
```
- `MODEL_CONFIG`: Name of the model you want to evaluate with

0
evaluation/discoverybench/eval_utils/README.md → evaluation/benchmarks/discoverybench/eval_utils/README.md
View File

0
evaluation/discoverybench/eval_utils/__init__.py → evaluation/benchmarks/discoverybench/eval_utils/__init__.py
View File

0
evaluation/discoverybench/eval_utils/eval_w_subhypo_gen.py → evaluation/benchmarks/discoverybench/eval_utils/eval_w_subhypo_gen.py
View File

0
evaluation/discoverybench/eval_utils/lm_utils.py → evaluation/benchmarks/discoverybench/eval_utils/lm_utils.py
View File

0
evaluation/discoverybench/eval_utils/openai_helpers.py → evaluation/benchmarks/discoverybench/eval_utils/openai_helpers.py
View File

0
evaluation/discoverybench/eval_utils/openai_semantic_gen_prompts.py → evaluation/benchmarks/discoverybench/eval_utils/openai_semantic_gen_prompts.py
View File

0
evaluation/discoverybench/eval_utils/response_parser.py → evaluation/benchmarks/discoverybench/eval_utils/response_parser.py
View File

4
evaluation/discoverybench/run_infer.py → evaluation/benchmarks/discoverybench/run_infer.py
View File

@ -5,10 +5,10 @@ import os
import git
import pandas as pd
from evaluation.discoverybench.eval_utils.eval_w_subhypo_gen import (
from evaluation.benchmarks.discoverybench.eval_utils.eval_w_subhypo_gen import (
run_eval_gold_vs_gen_NL_hypo_workflow,
)
from evaluation.discoverybench.eval_utils.response_parser import (
from evaluation.benchmarks.discoverybench.eval_utils.response_parser import (
extract_gen_hypo_from_logs,
)
from evaluation.utils.shared import (

2
evaluation/discoverybench/scripts/run_infer.sh → evaluation/benchmarks/discoverybench/scripts/run_infer.sh
View File

@ -29,7 +29,7 @@ echo "AGENT: $AGENT"
echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
COMMAND="poetry run python evaluation/discoverybench/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/discoverybench/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 10 \

10
evaluation/gaia/README.md → evaluation/benchmarks/gaia/README.md
View File

@ -10,11 +10,11 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
We are using the GAIA dataset hosted on [Hugging Face](https://huggingface.co/datasets/gaia-benchmark/GAIA).
Please accept the terms and make sure to have logged in on your computer by `huggingface-cli login` before running the evaluation.
Following is the basic command to start the evaluation. Here we are evaluating on the validation set for the `2023_all` split. You can adjust `./evaluation/gaia/scripts/run_infer.sh` to change the subset you want to evaluate on.
Following is the basic command to start the evaluation. Here we are evaluating on the validation set for the `2023_all` split. You can adjust `./evaluation/benchmarks/gaia/scripts/run_infer.sh` to change the subset you want to evaluate on.
```bash
./evaluation/gaia/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [gaia_subset]
# e.g., ./evaluation/gaia/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 CodeActAgent 300
./evaluation/benchmarks/gaia/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [gaia_subset]
# e.g., ./evaluation/benchmarks/gaia/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 CodeActAgent 300
```
where `model_config` is mandatory, while `git-version`, `agent`, `eval_limit` and `gaia_subset` are optional.
@ -35,13 +35,13 @@ to `CodeActAgent`.
For example,
```bash
./evaluation/gaia/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 CodeActAgent 10
./evaluation/benchmarks/gaia/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 CodeActAgent 10
```
## Get score
Then you can get stats by running the following command:
```bash
python ./evaluation/gaia/get_score.py \
python ./evaluation/benchmarks/gaia/get_score.py \
--file <path_to/output.json>
```

0
evaluation/gaia/get_score.py → evaluation/benchmarks/gaia/get_score.py
View File

2
evaluation/gaia/run_infer.py → evaluation/benchmarks/gaia/run_infer.py
View File

@ -7,7 +7,7 @@ import huggingface_hub
import pandas as pd
from datasets import load_dataset
from evaluation.gaia.scorer import question_scorer
from evaluation.benchmarks.gaia.scorer import question_scorer
from evaluation.utils.shared import (
EvalMetadata,
EvalOutput,

0
evaluation/gaia/scorer.py → evaluation/benchmarks/gaia/scorer.py
View File

2
evaluation/gaia/scripts/run_infer.sh → evaluation/benchmarks/gaia/scripts/run_infer.sh
View File

@ -35,7 +35,7 @@ echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
echo "LEVELS: $LEVELS"
COMMAND="poetry run python ./evaluation/gaia/run_infer.py \
COMMAND="poetry run python ./evaluation/benchmarks/gaia/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 30 \

4
evaluation/gorilla/README.md → evaluation/benchmarks/gorilla/README.md
View File

@ -11,7 +11,7 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
Make sure your Docker daemon is running, then run this bash script:
```bash
./evaluation/gorilla/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [hubs]
./evaluation/benchmarks/gorilla/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [hubs]
```
where `model_config` is mandatory, while all other arguments are optional.
@ -35,5 +35,5 @@ Note: in order to use `eval_limit`, you must also set `agent`; in order to use `
For example,
```bash
./evaluation/gorilla/scripts/run_infer.sh llm 0.6.2 CodeActAgent 10 th
./evaluation/benchmarks/gorilla/scripts/run_infer.sh llm 0.6.2 CodeActAgent 10 th
```

0
evaluation/gorilla/ast_eval_hf.py → evaluation/benchmarks/gorilla/ast_eval_hf.py
View File

0
evaluation/gorilla/ast_eval_tf.py → evaluation/benchmarks/gorilla/ast_eval_tf.py
View File

0
evaluation/gorilla/ast_eval_th.py → evaluation/benchmarks/gorilla/ast_eval_th.py
View File

2
evaluation/gorilla/run_infer.py → evaluation/benchmarks/gorilla/run_infer.py
View File

@ -5,7 +5,7 @@ import os
import pandas as pd
import requests
from evaluation.gorilla.utils import encode_question, get_data_for_hub
from evaluation.benchmarks.gorilla.utils import encode_question, get_data_for_hub
from evaluation.utils.shared import (
EvalMetadata,
EvalOutput,

2
evaluation/gorilla/scripts/run_infer.sh → evaluation/benchmarks/gorilla/scripts/run_infer.sh
View File

@ -33,7 +33,7 @@ echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
echo "HUBS: $HUBS"
COMMAND="poetry run python evaluation/gorilla/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/gorilla/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 30 \

0
evaluation/gorilla/utils.py → evaluation/benchmarks/gorilla/utils.py
View File

2
evaluation/gpqa/README.md → evaluation/benchmarks/gpqa/README.md
View File

@ -23,7 +23,7 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
'gpqa_main', 'gqpa_diamond', 'gpqa_experts', 'gpqa_extended' -- data split options
From the root of the OpenHands repo, run the following command:
```bash
./evaluation/gpqa/scripts/run_infer.sh [model_config_name] [git-version] [num_samples_eval] [data_split] [AgentClass]
./evaluation/benchmarks/gpqa/scripts/run_infer.sh [model_config_name] [git-version] [num_samples_eval] [data_split] [AgentClass]
```
You can replace `model_config_name` with any model you set up in `config.toml`.

0
evaluation/gpqa/__init__.py → evaluation/benchmarks/gpqa/__init__.py
View File

0
evaluation/gpqa/run_infer.py → evaluation/benchmarks/gpqa/run_infer.py
View File

2
evaluation/gpqa/scripts/run_infer.sh → evaluation/benchmarks/gpqa/scripts/run_infer.sh
View File

@ -33,7 +33,7 @@ echo "AGENT: $AGENT"
echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
COMMAND="poetry run python evaluation/gpqa/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/gpqa/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 10 \

2
evaluation/humanevalfix/README.md → evaluation/benchmarks/humanevalfix/README.md
View File

@ -9,7 +9,7 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
## Run Inference on HumanEvalFix
```bash
./evaluation/humanevalfix/scripts/run_infer.sh eval_gpt4_1106_preview
./evaluation/benchmarks/humanevalfix/scripts/run_infer.sh eval_gpt4_1106_preview
```
You can replace `eval_gpt4_1106_preview` with any model you set up in `config.toml`.

0
evaluation/humanevalfix/__init__.py → evaluation/benchmarks/humanevalfix/__init__.py
View File

0
evaluation/humanevalfix/run_infer.py → evaluation/benchmarks/humanevalfix/run_infer.py
View File

2
evaluation/humanevalfix/scripts/run_infer.sh → evaluation/benchmarks/humanevalfix/scripts/run_infer.sh
View File

@ -64,7 +64,7 @@ echo "AGENT: $AGENT"
echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
COMMAND="poetry run python evaluation/humanevalfix/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/humanevalfix/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 10 \

0
evaluation/logic_reasoning/.cache_program/facts.kfb → evaluation/benchmarks/logic_reasoning/.cache_program/facts.kfb
View File

0
evaluation/logic_reasoning/.cache_program/rules.krb → evaluation/benchmarks/logic_reasoning/.cache_program/rules.krb
View File

0
evaluation/logic_reasoning/Dockerfile → evaluation/benchmarks/logic_reasoning/Dockerfile
View File

2
evaluation/logic_reasoning/README.md → evaluation/benchmarks/logic_reasoning/README.md
View File

@ -10,5 +10,5 @@ Please follow instruction [here](../README.md#setup) to setup your local develop
The following code will run inference on the first example of the ProofWriter dataset,
```bash
./evaluation/logic_reasoning/scripts/run_infer.sh eval_gpt4_1106_preview_llm ProofWriter
./evaluation/benchmarks/logic_reasoning/scripts/run_infer.sh eval_gpt4_1106_preview_llm ProofWriter
```

0
evaluation/logic_reasoning/__init__.py → evaluation/benchmarks/logic_reasoning/__init__.py
View File

0
evaluation/logic_reasoning/instruction.txt → evaluation/benchmarks/logic_reasoning/instruction.txt
View File

0
evaluation/logic_reasoning/logic_inference.py → evaluation/benchmarks/logic_reasoning/logic_inference.py
View File

0
evaluation/logic_reasoning/run_infer.py → evaluation/benchmarks/logic_reasoning/run_infer.py
View File

2
evaluation/logic_reasoning/scripts/run_infer.sh → evaluation/benchmarks/logic_reasoning/scripts/run_infer.sh
View File

@ -34,7 +34,7 @@ echo "AGENT: $AGENT"
echo "AGENT_VERSION: $AGENT_VERSION"
echo "MODEL_CONFIG: $MODEL_CONFIG"
COMMAND="poetry run python evaluation/logic_reasoning/run_infer.py \
COMMAND="poetry run python evaluation/benchmarks/logic_reasoning/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--dataset $DATASET \

0
evaluation/miniwob/Dockerfile → evaluation/benchmarks/miniwob/Dockerfile
View File

8
evaluation/miniwob/README.md → evaluation/benchmarks/miniwob/README.md
View File

@ -13,7 +13,7 @@ Access with browser the above MiniWoB URLs and see if they load correctly.
## Run Evaluation
```sh
./evaluation/miniwob/scripts/run_infer.sh llm.claude-35-sonnet-eval
./evaluation/benchmarks/miniwob/scripts/run_infer.sh llm.claude-35-sonnet-eval
```
### Run Inference on `RemoteRuntime` (experimental)
@ -21,13 +21,13 @@ Access with browser the above MiniWoB URLs and see if they load correctly.
This is in limited beta. Contact Xingyao over slack if you want to try this out!
```bash
./evaluation/miniwob/scripts/run_infer.sh [model_config] [git-version] [agent] [note] [eval_limit] [num_workers]
./evaluation/benchmarks/miniwob/scripts/run_infer.sh [model_config] [git-version] [agent] [note] [eval_limit] [num_workers]
# Example - This runs evaluation on BrowsingAgent for 125 instances on miniwob, with 2 workers running in parallel
export ALLHANDS_API_KEY="YOUR-API-KEY"
export RUNTIME=remote
export SANDBOX_REMOTE_RUNTIME_API_URL="https://runtime.eval.all-hands.dev"
./evaluation/miniwob/scripts/run_infer.sh llm.eval HEAD BrowsingAgent "" 125 2
./evaluation/benchmarks/miniwob/scripts/run_infer.sh llm.eval HEAD BrowsingAgent "" 125 2
```
Results will be in `evaluation/evaluation_outputs/outputs/miniwob/`
@ -35,7 +35,7 @@ Results will be in `evaluation/evaluation_outputs/outputs/miniwob/`
To calculate the average reward, run:
```sh
poetry run python evaluation/miniwob/get_success_rate.py evaluation/evaluation_outputs/outputs/miniwob/SOME_AGENT/EXP_NAME/output.jsonl
poetry run python evaluation/benchmarks/miniwob/get_success_rate.py evaluation/evaluation_outputs/outputs/miniwob/SOME_AGENT/EXP_NAME/output.jsonl
```
## Submit your evaluation results

0
evaluation/miniwob/get_avg_reward.py → evaluation/benchmarks/miniwob/get_avg_reward.py
View File

0
evaluation/miniwob/run_infer.py → evaluation/benchmarks/miniwob/run_infer.py
View File

2
evaluation/miniwob/scripts/run_infer.sh → evaluation/benchmarks/miniwob/scripts/run_infer.sh
View File

@ -33,7 +33,7 @@ echo "MODEL_CONFIG: $MODEL_CONFIG"
EVAL_NOTE="${AGENT_VERSION}_${NOTE}"
COMMAND="export PYTHONPATH=evaluation/miniwob:\$PYTHONPATH && poetry run python evaluation/miniwob/run_infer.py \
COMMAND="export PYTHONPATH=evaluation/benchmarks/miniwob:\$PYTHONPATH && poetry run python evaluation/benchmarks/miniwob/run_infer.py \
--agent-cls $AGENT \
--llm-config $MODEL_CONFIG \
--max-iterations 10 \

0
evaluation/mint/.gitignore → evaluation/benchmarks/mint/.gitignore vendored
View File

0
evaluation/mint/Dockerfile → evaluation/benchmarks/mint/Dockerfile
View File

6
evaluation/mint/README.md → evaluation/benchmarks/mint/README.md
View File

@ -6,7 +6,7 @@ We support evaluation of the [Eurus subset focus on math and code reasoning](htt
## Setup Environment and LLM Configuration
Please follow instruction [here](../README.md#setup) to setup your local development environment and LLM.
Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM.
## Start the evaluation
@ -15,7 +15,7 @@ We are using the MINT dataset hosted on [Hugging Face](https://huggingface.co/da
Following is the basic command to start the evaluation. Currently, the only agent supported with MINT is `CodeActAgent`.
```bash
./evaluation/mint/scripts/run_infer.sh [model_config] [git-version] [subset] [eval_limit]
./evaluation/benchmarks/mint/scripts/run_infer.sh [model_config] [git-version] [subset] [eval_limit]
```
where `model_config` is mandatory, while others are optional.
@ -34,7 +34,7 @@ Note: in order to use `eval_limit`, you must also set `subset`.
For example,
```bash
./evaluation/swe_bench/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 gsm8k 3
./evaluation/benchmarks/mint/scripts/run_infer.sh eval_gpt4_1106_preview 0.6.2 gsm8k 3
```
## Reference

0
evaluation/mint/config_variables.py → evaluation/benchmarks/mint/config_variables.py
View File

0
evaluation/mint/datatypes.py → evaluation/benchmarks/mint/datatypes.py
View File

0
evaluation/mint/env.py → evaluation/benchmarks/mint/env.py
View File

0
evaluation/mint/prompts/__init__.py → evaluation/benchmarks/mint/prompts/__init__.py
View File

0
evaluation/mint/prompts/template_with_tool.txt → evaluation/benchmarks/mint/prompts/template_with_tool.txt
View File

Some files were not shown because too many files have changed in this diff Show More