mirror of
https://github.com/OpenHands/OpenHands.git
synced 2025-12-26 05:48:36 +08:00
563 lines
20 KiB
Python
563 lines
20 KiB
Python
import os
|
|
|
|
import pytest
|
|
|
|
from openhands.core.config import (
|
|
AgentConfig,
|
|
AppConfig,
|
|
LLMConfig,
|
|
finalize_config,
|
|
get_llm_config_arg,
|
|
load_from_env,
|
|
load_from_toml,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def setup_env():
|
|
# Create old-style and new-style TOML files
|
|
with open('old_style_config.toml', 'w') as f:
|
|
f.write('[default]\nLLM_MODEL="GPT-4"\n')
|
|
|
|
with open('new_style_config.toml', 'w') as f:
|
|
f.write('[app]\nLLM_MODEL="GPT-3"\n')
|
|
|
|
yield
|
|
|
|
# Cleanup TOML files after the test
|
|
os.remove('old_style_config.toml')
|
|
os.remove('new_style_config.toml')
|
|
|
|
|
|
@pytest.fixture
|
|
def temp_toml_file(tmp_path):
|
|
# Fixture to create a temporary directory and TOML file for testing
|
|
tmp_toml_file = os.path.join(tmp_path, 'config.toml')
|
|
yield tmp_toml_file
|
|
|
|
|
|
@pytest.fixture
|
|
def default_config(monkeypatch):
|
|
# Fixture to provide a default AppConfig instance
|
|
yield AppConfig()
|
|
|
|
|
|
def test_compat_env_to_config(monkeypatch, setup_env):
|
|
# Use `monkeypatch` to set environment variables for this specific test
|
|
monkeypatch.setenv('WORKSPACE_BASE', '/repos/openhands/workspace')
|
|
monkeypatch.setenv('LLM_API_KEY', 'sk-proj-rgMV0...')
|
|
monkeypatch.setenv('LLM_MODEL', 'gpt-4o')
|
|
monkeypatch.setenv('AGENT_MEMORY_MAX_THREADS', '4')
|
|
monkeypatch.setenv('AGENT_MEMORY_ENABLED', 'True')
|
|
monkeypatch.setenv('DEFAULT_AGENT', 'CodeActAgent')
|
|
monkeypatch.setenv('SANDBOX_TIMEOUT', '10')
|
|
|
|
config = AppConfig()
|
|
load_from_env(config, os.environ)
|
|
|
|
assert config.workspace_base == '/repos/openhands/workspace'
|
|
assert isinstance(config.get_llm_config(), LLMConfig)
|
|
assert config.get_llm_config().api_key == 'sk-proj-rgMV0...'
|
|
assert config.get_llm_config().model == 'gpt-4o'
|
|
assert isinstance(config.get_agent_config(), AgentConfig)
|
|
assert isinstance(config.get_agent_config().memory_max_threads, int)
|
|
assert config.get_agent_config().memory_max_threads == 4
|
|
assert config.get_agent_config().memory_enabled is True
|
|
assert config.default_agent == 'CodeActAgent'
|
|
assert config.sandbox.timeout == 10
|
|
|
|
|
|
def test_load_from_old_style_env(monkeypatch, default_config):
|
|
# Test loading configuration from old-style environment variables using monkeypatch
|
|
monkeypatch.setenv('LLM_API_KEY', 'test-api-key')
|
|
monkeypatch.setenv('AGENT_MEMORY_ENABLED', 'True')
|
|
monkeypatch.setenv('DEFAULT_AGENT', 'PlannerAgent')
|
|
monkeypatch.setenv('WORKSPACE_BASE', '/opt/files/workspace')
|
|
monkeypatch.setenv('SANDBOX_BASE_CONTAINER_IMAGE', 'custom_image')
|
|
|
|
load_from_env(default_config, os.environ)
|
|
|
|
assert default_config.get_llm_config().api_key == 'test-api-key'
|
|
assert default_config.get_agent_config().memory_enabled is True
|
|
assert default_config.default_agent == 'PlannerAgent'
|
|
assert default_config.workspace_base == '/opt/files/workspace'
|
|
assert default_config.workspace_mount_path is None # before finalize_config
|
|
assert default_config.workspace_mount_path_in_sandbox is not None
|
|
assert default_config.sandbox.base_container_image == 'custom_image'
|
|
|
|
|
|
def test_load_from_new_style_toml(default_config, temp_toml_file):
|
|
# Test loading configuration from a new-style TOML file
|
|
with open(temp_toml_file, 'w', encoding='utf-8') as toml_file:
|
|
toml_file.write(
|
|
"""
|
|
[llm]
|
|
model = "test-model"
|
|
api_key = "toml-api-key"
|
|
|
|
[llm.cheap]
|
|
model = "some-cheap-model"
|
|
api_key = "cheap-model-api-key"
|
|
|
|
[agent]
|
|
memory_enabled = true
|
|
|
|
[agent.BrowsingAgent]
|
|
llm_config = "cheap"
|
|
memory_enabled = false
|
|
|
|
[sandbox]
|
|
timeout = 1
|
|
|
|
[core]
|
|
workspace_base = "/opt/files2/workspace"
|
|
default_agent = "TestAgent"
|
|
"""
|
|
)
|
|
|
|
load_from_toml(default_config, temp_toml_file)
|
|
|
|
# default llm & agent configs
|
|
assert default_config.default_agent == 'TestAgent'
|
|
assert default_config.get_llm_config().model == 'test-model'
|
|
assert default_config.get_llm_config().api_key == 'toml-api-key'
|
|
assert default_config.get_agent_config().memory_enabled is True
|
|
|
|
# undefined agent config inherits default ones
|
|
assert (
|
|
default_config.get_llm_config_from_agent('CodeActAgent')
|
|
== default_config.get_llm_config()
|
|
)
|
|
assert default_config.get_agent_config('CodeActAgent').memory_enabled is True
|
|
|
|
# defined agent config overrides default ones
|
|
assert default_config.get_llm_config_from_agent(
|
|
'BrowsingAgent'
|
|
) == default_config.get_llm_config('cheap')
|
|
assert (
|
|
default_config.get_llm_config_from_agent('BrowsingAgent').model
|
|
== 'some-cheap-model'
|
|
)
|
|
assert default_config.get_agent_config('BrowsingAgent').memory_enabled is False
|
|
|
|
assert default_config.workspace_base == '/opt/files2/workspace'
|
|
assert default_config.sandbox.timeout == 1
|
|
|
|
assert default_config.workspace_mount_path is None
|
|
assert default_config.workspace_mount_path_in_sandbox is not None
|
|
assert default_config.workspace_mount_path_in_sandbox == '/workspace'
|
|
|
|
finalize_config(default_config)
|
|
|
|
# after finalize_config, workspace_mount_path is set to the absolute path of workspace_base
|
|
# if it was undefined
|
|
assert default_config.workspace_mount_path == '/opt/files2/workspace'
|
|
|
|
|
|
def test_compat_load_sandbox_from_toml(default_config: AppConfig, temp_toml_file: str):
|
|
# test loading configuration from a new-style TOML file
|
|
# uses a toml file with sandbox_vars instead of a sandbox section
|
|
with open(temp_toml_file, 'w', encoding='utf-8') as toml_file:
|
|
toml_file.write(
|
|
"""
|
|
[llm]
|
|
model = "test-model"
|
|
|
|
[agent]
|
|
memory_enabled = true
|
|
|
|
[core]
|
|
workspace_base = "/opt/files2/workspace"
|
|
sandbox_timeout = 500
|
|
sandbox_base_container_image = "node:14"
|
|
sandbox_user_id = 1001
|
|
default_agent = "TestAgent"
|
|
"""
|
|
)
|
|
|
|
load_from_toml(default_config, temp_toml_file)
|
|
|
|
assert default_config.get_llm_config().model == 'test-model'
|
|
assert default_config.get_llm_config_from_agent().model == 'test-model'
|
|
assert default_config.default_agent == 'TestAgent'
|
|
assert default_config.get_agent_config().memory_enabled is True
|
|
assert default_config.workspace_base == '/opt/files2/workspace'
|
|
assert default_config.sandbox.timeout == 500
|
|
assert default_config.sandbox.base_container_image == 'node:14'
|
|
assert default_config.sandbox.user_id == 1001
|
|
assert default_config.workspace_mount_path_in_sandbox == '/workspace'
|
|
|
|
finalize_config(default_config)
|
|
|
|
# app config doesn't have fields sandbox_*
|
|
assert not hasattr(default_config, 'sandbox_timeout')
|
|
assert not hasattr(default_config, 'sandbox_base_container_image')
|
|
assert not hasattr(default_config, 'sandbox_user_id')
|
|
|
|
# after finalize_config, workspace_mount_path is set to the absolute path of workspace_base
|
|
# if it was undefined
|
|
assert default_config.workspace_mount_path == '/opt/files2/workspace'
|
|
|
|
|
|
def test_env_overrides_compat_toml(monkeypatch, default_config, temp_toml_file):
|
|
# test that environment variables override TOML values using monkeypatch
|
|
# uses a toml file with sandbox_vars instead of a sandbox section
|
|
with open(temp_toml_file, 'w', encoding='utf-8') as toml_file:
|
|
toml_file.write("""
|
|
[llm]
|
|
model = "test-model"
|
|
api_key = "toml-api-key"
|
|
|
|
[core]
|
|
workspace_base = "/opt/files3/workspace"
|
|
disable_color = true
|
|
sandbox_timeout = 500
|
|
sandbox_user_id = 1001
|
|
""")
|
|
|
|
monkeypatch.setenv('LLM_API_KEY', 'env-api-key')
|
|
monkeypatch.setenv('WORKSPACE_BASE', 'UNDEFINED')
|
|
monkeypatch.setenv('SANDBOX_TIMEOUT', '1000')
|
|
monkeypatch.setenv('SANDBOX_USER_ID', '1002')
|
|
monkeypatch.delenv('LLM_MODEL', raising=False)
|
|
|
|
load_from_toml(default_config, temp_toml_file)
|
|
|
|
assert default_config.workspace_mount_path is None
|
|
|
|
load_from_env(default_config, os.environ)
|
|
|
|
assert os.environ.get('LLM_MODEL') is None
|
|
assert default_config.get_llm_config().model == 'test-model'
|
|
assert default_config.get_llm_config('llm').model == 'test-model'
|
|
assert default_config.get_llm_config_from_agent().model == 'test-model'
|
|
assert default_config.get_llm_config().api_key == 'env-api-key'
|
|
|
|
# after we set workspace_base to 'UNDEFINED' in the environment,
|
|
# workspace_base should be set to that
|
|
assert default_config.workspace_base is not None
|
|
assert default_config.workspace_base == 'UNDEFINED'
|
|
assert default_config.workspace_mount_path is None
|
|
|
|
assert default_config.disable_color is True
|
|
assert default_config.sandbox.timeout == 1000
|
|
assert default_config.sandbox.user_id == 1002
|
|
|
|
finalize_config(default_config)
|
|
# after finalize_config, workspace_mount_path is set to absolute path of workspace_base if it was undefined
|
|
assert default_config.workspace_mount_path == os.getcwd() + '/UNDEFINED'
|
|
|
|
|
|
def test_env_overrides_sandbox_toml(monkeypatch, default_config, temp_toml_file):
|
|
# test that environment variables override TOML values using monkeypatch
|
|
# uses a toml file with a sandbox section
|
|
with open(temp_toml_file, 'w', encoding='utf-8') as toml_file:
|
|
toml_file.write("""
|
|
[llm]
|
|
model = "test-model"
|
|
api_key = "toml-api-key"
|
|
|
|
[core]
|
|
workspace_base = "/opt/files3/workspace"
|
|
|
|
[sandbox]
|
|
timeout = 500
|
|
user_id = 1001
|
|
""")
|
|
|
|
monkeypatch.setenv('LLM_API_KEY', 'env-api-key')
|
|
monkeypatch.setenv('WORKSPACE_BASE', 'UNDEFINED')
|
|
monkeypatch.setenv('SANDBOX_TIMEOUT', '1000')
|
|
monkeypatch.setenv('SANDBOX_USER_ID', '1002')
|
|
monkeypatch.delenv('LLM_MODEL', raising=False)
|
|
|
|
load_from_toml(default_config, temp_toml_file)
|
|
|
|
assert default_config.workspace_mount_path is None
|
|
|
|
# before load_from_env, values are set to the values from the toml file
|
|
assert default_config.get_llm_config().api_key == 'toml-api-key'
|
|
assert default_config.sandbox.timeout == 500
|
|
assert default_config.sandbox.user_id == 1001
|
|
|
|
load_from_env(default_config, os.environ)
|
|
|
|
# values from env override values from toml
|
|
assert os.environ.get('LLM_MODEL') is None
|
|
assert default_config.get_llm_config().model == 'test-model'
|
|
assert default_config.get_llm_config().api_key == 'env-api-key'
|
|
|
|
assert default_config.sandbox.timeout == 1000
|
|
assert default_config.sandbox.user_id == 1002
|
|
|
|
finalize_config(default_config)
|
|
# after finalize_config, workspace_mount_path is set to absolute path of workspace_base if it was undefined
|
|
assert default_config.workspace_mount_path == os.getcwd() + '/UNDEFINED'
|
|
|
|
|
|
def test_sandbox_config_from_toml(monkeypatch, default_config, temp_toml_file):
|
|
# Test loading configuration from a new-style TOML file
|
|
with open(temp_toml_file, 'w', encoding='utf-8') as toml_file:
|
|
toml_file.write(
|
|
"""
|
|
[core]
|
|
workspace_base = "/opt/files/workspace"
|
|
|
|
[llm]
|
|
model = "test-model"
|
|
|
|
[sandbox]
|
|
timeout = 1
|
|
base_container_image = "custom_image"
|
|
user_id = 1001
|
|
"""
|
|
)
|
|
monkeypatch.setattr(os, 'environ', {})
|
|
load_from_toml(default_config, temp_toml_file)
|
|
load_from_env(default_config, os.environ)
|
|
finalize_config(default_config)
|
|
|
|
assert default_config.get_llm_config().model == 'test-model'
|
|
assert default_config.sandbox.timeout == 1
|
|
assert default_config.sandbox.base_container_image == 'custom_image'
|
|
assert default_config.sandbox.user_id == 1001
|
|
|
|
|
|
def test_defaults_dict_after_updates(default_config):
|
|
# Test that `defaults_dict` retains initial values after updates.
|
|
initial_defaults = default_config.defaults_dict
|
|
assert initial_defaults['workspace_mount_path']['default'] is None
|
|
assert initial_defaults['default_agent']['default'] == 'CodeActAgent'
|
|
|
|
updated_config = AppConfig()
|
|
updated_config.get_llm_config().api_key = 'updated-api-key'
|
|
updated_config.get_llm_config('llm').api_key = 'updated-api-key'
|
|
updated_config.get_llm_config_from_agent('agent').api_key = 'updated-api-key'
|
|
updated_config.get_llm_config_from_agent('PlannerAgent').api_key = 'updated-api-key'
|
|
updated_config.default_agent = 'PlannerAgent'
|
|
|
|
defaults_after_updates = updated_config.defaults_dict
|
|
assert defaults_after_updates['default_agent']['default'] == 'CodeActAgent'
|
|
assert defaults_after_updates['workspace_mount_path']['default'] is None
|
|
assert defaults_after_updates['sandbox']['timeout']['default'] == 120
|
|
assert (
|
|
defaults_after_updates['sandbox']['base_container_image']['default']
|
|
== 'nikolaik/python-nodejs:python3.12-nodejs22'
|
|
)
|
|
assert defaults_after_updates == initial_defaults
|
|
|
|
|
|
def test_invalid_toml_format(monkeypatch, temp_toml_file, default_config):
|
|
# Invalid TOML format doesn't break the configuration
|
|
monkeypatch.setenv('LLM_MODEL', 'gpt-5-turbo-1106')
|
|
monkeypatch.setenv('WORKSPACE_MOUNT_PATH', '/home/user/project')
|
|
monkeypatch.delenv('LLM_API_KEY', raising=False)
|
|
with open(temp_toml_file, 'w', encoding='utf-8') as toml_file:
|
|
toml_file.write('INVALID TOML CONTENT')
|
|
|
|
load_from_toml(default_config)
|
|
load_from_env(default_config, os.environ)
|
|
default_config.jwt_secret = None # prevent leak
|
|
for llm in default_config.llms.values():
|
|
llm.api_key = None # prevent leak
|
|
assert default_config.get_llm_config().model == 'gpt-5-turbo-1106'
|
|
assert default_config.get_llm_config().custom_llm_provider is None
|
|
assert default_config.workspace_mount_path == '/home/user/project'
|
|
|
|
|
|
def test_finalize_config(default_config):
|
|
# Test finalize config
|
|
assert default_config.workspace_mount_path is None
|
|
default_config.workspace_base = None
|
|
finalize_config(default_config)
|
|
|
|
assert default_config.workspace_mount_path is None
|
|
|
|
|
|
def test_workspace_mount_path_default(default_config):
|
|
assert default_config.workspace_mount_path is None
|
|
default_config.workspace_base = '/home/user/project'
|
|
finalize_config(default_config)
|
|
assert default_config.workspace_mount_path == os.path.abspath(
|
|
default_config.workspace_base
|
|
)
|
|
|
|
|
|
def test_workspace_mount_rewrite(default_config, monkeypatch):
|
|
default_config.workspace_base = '/home/user/project'
|
|
default_config.workspace_mount_rewrite = '/home/user:/sandbox'
|
|
monkeypatch.setattr('os.getcwd', lambda: '/current/working/directory')
|
|
finalize_config(default_config)
|
|
assert default_config.workspace_mount_path == '/sandbox/project'
|
|
|
|
|
|
def test_embedding_base_url_default(default_config):
|
|
default_config.get_llm_config().base_url = 'https://api.exampleapi.com'
|
|
finalize_config(default_config)
|
|
assert (
|
|
default_config.get_llm_config().embedding_base_url
|
|
== 'https://api.exampleapi.com'
|
|
)
|
|
|
|
|
|
def test_cache_dir_creation(default_config, tmpdir):
|
|
default_config.cache_dir = str(tmpdir.join('test_cache'))
|
|
finalize_config(default_config)
|
|
assert os.path.exists(default_config.cache_dir)
|
|
|
|
|
|
def test_api_keys_repr_str():
|
|
# Test LLMConfig
|
|
llm_config = LLMConfig(
|
|
api_key='my_api_key',
|
|
aws_access_key_id='my_access_key',
|
|
aws_secret_access_key='my_secret_key',
|
|
)
|
|
assert "api_key='******'" in repr(llm_config)
|
|
assert "aws_access_key_id='******'" in repr(llm_config)
|
|
assert "aws_secret_access_key='******'" in repr(llm_config)
|
|
assert "api_key='******'" in str(llm_config)
|
|
assert "aws_access_key_id='******'" in str(llm_config)
|
|
assert "aws_secret_access_key='******'" in str(llm_config)
|
|
|
|
# Check that no other attrs in LLMConfig have 'key' or 'token' in their name
|
|
# This will fail when new attrs are added, and attract attention
|
|
known_key_token_attrs_llm = [
|
|
'api_key',
|
|
'aws_access_key_id',
|
|
'aws_secret_access_key',
|
|
'input_cost_per_token',
|
|
'output_cost_per_token',
|
|
'custom_tokenizer',
|
|
]
|
|
for attr_name in dir(LLMConfig):
|
|
if (
|
|
not attr_name.startswith('__')
|
|
and attr_name not in known_key_token_attrs_llm
|
|
):
|
|
assert (
|
|
'key' not in attr_name.lower()
|
|
), f"Unexpected attribute '{attr_name}' contains 'key' in LLMConfig"
|
|
assert (
|
|
'token' not in attr_name.lower() or 'tokens' in attr_name.lower()
|
|
), f"Unexpected attribute '{attr_name}' contains 'token' in LLMConfig"
|
|
|
|
# Test AgentConfig
|
|
# No attrs in AgentConfig have 'key' or 'token' in their name
|
|
agent_config = AgentConfig(memory_enabled=True, memory_max_threads=4)
|
|
for attr_name in dir(AgentConfig):
|
|
if not attr_name.startswith('__'):
|
|
assert (
|
|
'key' not in attr_name.lower()
|
|
), f"Unexpected attribute '{attr_name}' contains 'key' in AgentConfig"
|
|
assert (
|
|
'token' not in attr_name.lower() or 'tokens' in attr_name.lower()
|
|
), f"Unexpected attribute '{attr_name}' contains 'token' in AgentConfig"
|
|
|
|
# Test AppConfig
|
|
app_config = AppConfig(
|
|
llms={'llm': llm_config},
|
|
agents={'agent': agent_config},
|
|
e2b_api_key='my_e2b_api_key',
|
|
jwt_secret='my_jwt_secret',
|
|
modal_api_token_id='my_modal_api_token_id',
|
|
modal_api_token_secret='my_modal_api_token_secret',
|
|
runloop_api_key='my_runloop_api_key',
|
|
)
|
|
assert "e2b_api_key='******'" in repr(app_config)
|
|
assert "e2b_api_key='******'" in str(app_config)
|
|
assert "jwt_secret='******'" in repr(app_config)
|
|
assert "jwt_secret='******'" in str(app_config)
|
|
assert "modal_api_token_id='******'" in repr(app_config)
|
|
assert "modal_api_token_id='******'" in str(app_config)
|
|
assert "modal_api_token_secret='******'" in repr(app_config)
|
|
assert "modal_api_token_secret='******'" in str(app_config)
|
|
assert "runloop_api_key='******'" in repr(app_config)
|
|
assert "runloop_api_key='******'" in str(app_config)
|
|
|
|
# Check that no other attrs in AppConfig have 'key' or 'token' in their name
|
|
# This will fail when new attrs are added, and attract attention
|
|
known_key_token_attrs_app = [
|
|
'e2b_api_key',
|
|
'modal_api_token_id',
|
|
'modal_api_token_secret',
|
|
'runloop_api_key',
|
|
]
|
|
for attr_name in dir(AppConfig):
|
|
if (
|
|
not attr_name.startswith('__')
|
|
and attr_name not in known_key_token_attrs_app
|
|
):
|
|
assert (
|
|
'key' not in attr_name.lower()
|
|
), f"Unexpected attribute '{attr_name}' contains 'key' in AppConfig"
|
|
assert (
|
|
'token' not in attr_name.lower() or 'tokens' in attr_name.lower()
|
|
), f"Unexpected attribute '{attr_name}' contains 'token' in AppConfig"
|
|
|
|
|
|
def test_max_iterations_and_max_budget_per_task_from_toml(temp_toml_file):
|
|
temp_toml = """
|
|
[core]
|
|
max_iterations = 42
|
|
max_budget_per_task = 4.7
|
|
"""
|
|
|
|
config = AppConfig()
|
|
with open(temp_toml_file, 'w') as f:
|
|
f.write(temp_toml)
|
|
|
|
load_from_toml(config, temp_toml_file)
|
|
|
|
assert config.max_iterations == 42
|
|
assert config.max_budget_per_task == 4.7
|
|
|
|
|
|
def test_get_llm_config_arg(temp_toml_file):
|
|
temp_toml = """
|
|
[core]
|
|
max_iterations = 100
|
|
max_budget_per_task = 4.0
|
|
|
|
[llm.gpt3]
|
|
model="gpt-3.5-turbo"
|
|
api_key="redacted"
|
|
embedding_model="openai"
|
|
|
|
[llm.gpt4o]
|
|
model="gpt-4o"
|
|
api_key="redacted"
|
|
embedding_model="openai"
|
|
"""
|
|
|
|
with open(temp_toml_file, 'w') as f:
|
|
f.write(temp_toml)
|
|
|
|
llm_config = get_llm_config_arg('gpt3', temp_toml_file)
|
|
assert llm_config.model == 'gpt-3.5-turbo'
|
|
assert llm_config.embedding_model == 'openai'
|
|
|
|
|
|
def test_get_agent_configs(default_config, temp_toml_file):
|
|
temp_toml = """
|
|
[core]
|
|
max_iterations = 100
|
|
max_budget_per_task = 4.0
|
|
|
|
[agent.CodeActAgent]
|
|
memory_enabled = true
|
|
|
|
[agent.PlannerAgent]
|
|
memory_max_threads = 10
|
|
"""
|
|
|
|
with open(temp_toml_file, 'w') as f:
|
|
f.write(temp_toml)
|
|
|
|
load_from_toml(default_config, temp_toml_file)
|
|
|
|
codeact_config = default_config.get_agent_configs().get('CodeActAgent')
|
|
assert codeact_config.memory_enabled is True
|
|
planner_config = default_config.get_agent_configs().get('PlannerAgent')
|
|
assert planner_config.memory_max_threads == 10
|