Implement configurable base URL for E2E tests (#10394)

Co-authored-by: openhands <openhands@all-hands.dev>

tests/e2e/README.md

@@ -23,19 +23,44 @@ Optional environment variables:
 
 - `LLM_BASE_URL`: The base URL for the LLM API (if using a custom endpoint)
 
+### Configuration Options
+
+The E2E tests support several command-line options:
+
+- `--base-url`: Specify the base URL of the OpenHands instance under test (default: `http://localhost:12000`)
+- `--headless`: Run browser in headless mode (default: `true`)
+- `--no-headless`: Run browser in non-headless mode to watch the browser interactions
+- `--slow-mo`: Add delay between actions in milliseconds (default: `0`)
+
 ### Running Locally
 
 To run the full end-to-end test suite locally:
 
 ```bash
 cd tests/e2e
-poetry run pytest test_e2e_workflow.py -v
+poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v
 ```
 
 This runs all tests in sequence:
 1. GitHub token configuration
 2. Conversation start
 
+#### Specifying a Custom Base URL
+
+By default, the tests run against `http://localhost:12000`. You can specify a different OpenHands instance URL using the `--base-url` option:
+
+```bash
+cd tests/e2e
+# Run against a remote instance
+poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=https://my-openhands-instance.com
+
+# Run against a CI instance
+poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=http://ci-instance:8080
+
+# Run against localhost with a different port
+poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=http://localhost:3000
+```
+
 ### Running Individual Tests
 
 You can run individual tests directly:
@@ -43,11 +68,13 @@ You can run individual tests directly:
 ```bash
 cd tests/e2e
 # Run the GitHub token configuration test
-poetry run pytest test_e2e_workflow.py::test_github_token_configuration -v
+poetry run pytest test_settings.py::test_github_token_configuration -v
 
 # Run the conversation start test
-poetry run pytest test_e2e_workflow.py::test_conversation_start -v
+poetry run pytest test_conversation.py::test_conversation_start -v
 
+# Run individual tests with custom base URL
+poetry run pytest test_settings.py::test_github_token_configuration -v --base-url=https://my-instance.com
 
 ```
 
@@ -57,8 +84,11 @@ To run the tests with a visible browser (non-headless mode) so you can watch the
 
 ```bash
 cd tests/e2e
-poetry run pytest test_e2e_workflow.py::test_github_token_configuration -v --no-headless --slow-mo=50
-poetry run pytest test_e2e_workflow.py::test_conversation_start -v --no-headless --slow-mo=50
+poetry run pytest test_settings.py::test_github_token_configuration -v --no-headless --slow-mo=50
+poetry run pytest test_conversation.py::test_conversation_start -v --no-headless --slow-mo=50
+
+# Combine with custom base URL
+poetry run pytest test_settings.py::test_github_token_configuration -v --no-headless --slow-mo=50 --base-url=https://my-instance.com
 ```
 
 ### GitHub Workflow

tests/e2e/test_conversation.py

@@ -31,7 +31,7 @@ def get_readme_line_count():
         return 0
 
 
-def test_conversation_start(page: Page):
+def test_conversation_start(page: Page, base_url: str):
     """
     Test starting a conversation with the OpenHands agent:
     1. Navigate to OpenHands (assumes GitHub token is already configured)
@@ -44,12 +44,16 @@ def test_conversation_start(page: Page):
     # Create test-results directory if it doesn't exist
     os.makedirs('test-results', exist_ok=True)
 
+    # Use default URL if base_url is not provided
+    if not base_url:
+        base_url = 'http://localhost:12000'
+
     expected_line_count = get_readme_line_count()
     print(f'Expected README.md line count: {expected_line_count}')
 
     # Navigate to the OpenHands application
-    print('Step 1: Navigating to OpenHands application...')
-    page.goto('http://localhost:12000')
+    print(f'Step 1: Navigating to OpenHands application at {base_url}...')
+    page.goto(base_url)
     page.wait_for_load_state('networkidle', timeout=30000)
 
     # Take initial screenshot

tests/e2e/test_settings.py

@@ -11,7 +11,7 @@ import os
 from playwright.sync_api import Page, expect
 
 
-def test_github_token_configuration(page: Page):
+def test_github_token_configuration(page: Page, base_url: str):
     """
     Test the GitHub token configuration flow:
     1. Navigate to OpenHands
@@ -23,9 +23,13 @@ def test_github_token_configuration(page: Page):
     # Create test-results directory if it doesn't exist
     os.makedirs('test-results', exist_ok=True)
 
+    # Use default URL if base_url is not provided
+    if not base_url:
+        base_url = 'http://localhost:12000'
+
     # Navigate to the OpenHands application
-    print('Step 1: Navigating to OpenHands application...')
-    page.goto('http://localhost:12000')
+    print(f'Step 1: Navigating to OpenHands application at {base_url}...')
+    page.goto(base_url)
     page.wait_for_load_state('networkidle', timeout=30000)
 
     # Take initial screenshot
@@ -170,7 +174,7 @@ def test_github_token_configuration(page: Page):
 
                                     # Navigate back to home page after successful save
                                     print('Navigating back to home page...')
-                                    page.goto('http://localhost:12000')
+                                    page.goto(base_url)
                                     page.wait_for_load_state('networkidle')
                                     page.wait_for_timeout(
                                         5000
@@ -245,18 +249,18 @@ def test_github_token_configuration(page: Page):
 
                                 # Navigate back to home page
                                 print('Navigating back to home page...')
-                                page.goto('http://localhost:12000')
+                                page.goto(base_url)
                                 page.wait_for_load_state('networkidle')
                                 page.wait_for_timeout(3000)
                         else:
                             print(
                                 'GitHub token input field not found, going back to home page'
                             )
-                            page.goto('http://localhost:12000')
+                            page.goto(base_url)
                             page.wait_for_load_state('networkidle')
                     else:
                         print('Integrations tab not found, going back to home page')
-                        page.goto('http://localhost:12000')
+                        page.goto(base_url)
                         page.wait_for_load_state('networkidle')
                 else:
                     print('Settings button not found, continuing with existing token')