From 305caf125719910e86a38f1fffc47c88ff6a71d9 Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Mon, 18 Aug 2025 07:44:07 -0400 Subject: [PATCH] Implement configurable base URL for E2E tests (#10394) Co-authored-by: openhands --- tests/e2e/README.md | 40 +++++++++++++++++++++++++++++----- tests/e2e/test_conversation.py | 10 ++++++--- tests/e2e/test_settings.py | 18 +++++++++------ 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index f52b948f38..77c2857f19 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -23,19 +23,44 @@ Optional environment variables: - `LLM_BASE_URL`: The base URL for the LLM API (if using a custom endpoint) +### Configuration Options + +The E2E tests support several command-line options: + +- `--base-url`: Specify the base URL of the OpenHands instance under test (default: `http://localhost:12000`) +- `--headless`: Run browser in headless mode (default: `true`) +- `--no-headless`: Run browser in non-headless mode to watch the browser interactions +- `--slow-mo`: Add delay between actions in milliseconds (default: `0`) + ### Running Locally To run the full end-to-end test suite locally: ```bash cd tests/e2e -poetry run pytest test_e2e_workflow.py -v +poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v ``` This runs all tests in sequence: 1. GitHub token configuration 2. Conversation start +#### Specifying a Custom Base URL + +By default, the tests run against `http://localhost:12000`. You can specify a different OpenHands instance URL using the `--base-url` option: + +```bash +cd tests/e2e +# Run against a remote instance +poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=https://my-openhands-instance.com + +# Run against a CI instance +poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=http://ci-instance:8080 + +# Run against localhost with a different port +poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=http://localhost:3000 +``` + ### Running Individual Tests You can run individual tests directly: @@ -43,11 +68,13 @@ You can run individual tests directly: ```bash cd tests/e2e # Run the GitHub token configuration test -poetry run pytest test_e2e_workflow.py::test_github_token_configuration -v +poetry run pytest test_settings.py::test_github_token_configuration -v # Run the conversation start test -poetry run pytest test_e2e_workflow.py::test_conversation_start -v +poetry run pytest test_conversation.py::test_conversation_start -v +# Run individual tests with custom base URL +poetry run pytest test_settings.py::test_github_token_configuration -v --base-url=https://my-instance.com ``` @@ -57,8 +84,11 @@ To run the tests with a visible browser (non-headless mode) so you can watch the ```bash cd tests/e2e -poetry run pytest test_e2e_workflow.py::test_github_token_configuration -v --no-headless --slow-mo=50 -poetry run pytest test_e2e_workflow.py::test_conversation_start -v --no-headless --slow-mo=50 +poetry run pytest test_settings.py::test_github_token_configuration -v --no-headless --slow-mo=50 +poetry run pytest test_conversation.py::test_conversation_start -v --no-headless --slow-mo=50 + +# Combine with custom base URL +poetry run pytest test_settings.py::test_github_token_configuration -v --no-headless --slow-mo=50 --base-url=https://my-instance.com ``` ### GitHub Workflow diff --git a/tests/e2e/test_conversation.py b/tests/e2e/test_conversation.py index 9aa31d837a..550d2a08de 100644 --- a/tests/e2e/test_conversation.py +++ b/tests/e2e/test_conversation.py @@ -31,7 +31,7 @@ def get_readme_line_count(): return 0 -def test_conversation_start(page: Page): +def test_conversation_start(page: Page, base_url: str): """ Test starting a conversation with the OpenHands agent: 1. Navigate to OpenHands (assumes GitHub token is already configured) @@ -44,12 +44,16 @@ def test_conversation_start(page: Page): # Create test-results directory if it doesn't exist os.makedirs('test-results', exist_ok=True) + # Use default URL if base_url is not provided + if not base_url: + base_url = 'http://localhost:12000' + expected_line_count = get_readme_line_count() print(f'Expected README.md line count: {expected_line_count}') # Navigate to the OpenHands application - print('Step 1: Navigating to OpenHands application...') - page.goto('http://localhost:12000') + print(f'Step 1: Navigating to OpenHands application at {base_url}...') + page.goto(base_url) page.wait_for_load_state('networkidle', timeout=30000) # Take initial screenshot diff --git a/tests/e2e/test_settings.py b/tests/e2e/test_settings.py index 465267f11e..7c9386135a 100644 --- a/tests/e2e/test_settings.py +++ b/tests/e2e/test_settings.py @@ -11,7 +11,7 @@ import os from playwright.sync_api import Page, expect -def test_github_token_configuration(page: Page): +def test_github_token_configuration(page: Page, base_url: str): """ Test the GitHub token configuration flow: 1. Navigate to OpenHands @@ -23,9 +23,13 @@ def test_github_token_configuration(page: Page): # Create test-results directory if it doesn't exist os.makedirs('test-results', exist_ok=True) + # Use default URL if base_url is not provided + if not base_url: + base_url = 'http://localhost:12000' + # Navigate to the OpenHands application - print('Step 1: Navigating to OpenHands application...') - page.goto('http://localhost:12000') + print(f'Step 1: Navigating to OpenHands application at {base_url}...') + page.goto(base_url) page.wait_for_load_state('networkidle', timeout=30000) # Take initial screenshot @@ -170,7 +174,7 @@ def test_github_token_configuration(page: Page): # Navigate back to home page after successful save print('Navigating back to home page...') - page.goto('http://localhost:12000') + page.goto(base_url) page.wait_for_load_state('networkidle') page.wait_for_timeout( 5000 @@ -245,18 +249,18 @@ def test_github_token_configuration(page: Page): # Navigate back to home page print('Navigating back to home page...') - page.goto('http://localhost:12000') + page.goto(base_url) page.wait_for_load_state('networkidle') page.wait_for_timeout(3000) else: print( 'GitHub token input field not found, going back to home page' ) - page.goto('http://localhost:12000') + page.goto(base_url) page.wait_for_load_state('networkidle') else: print('Integrations tab not found, going back to home page') - page.goto('http://localhost:12000') + page.goto(base_url) page.wait_for_load_state('networkidle') else: print('Settings button not found, continuing with existing token')