mirror of
https://github.com/OpenHands/OpenHands.git
synced 2025-12-26 05:48:36 +08:00
Co-authored-by: openhands <openhands@all-hands.dev>
This commit is contained in:
Graham Neubig
GitHub
parent
822ce86150
commit
ad85e3249a
6
.github/workflows/e2e-tests.yml
vendored
6
.github/workflows/e2e-tests.yml
vendored
@ -183,7 +183,11 @@ jobs:
|
||||
|
||||
# Run the tests with detailed output
|
||||
cd tests/e2e
|
||||
poetry run python -m pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --no-header --capture=no --timeout=600
|
||||
poetry run python -m pytest \
|
||||
test_settings.py::test_github_token_configuration \
|
||||
test_conversation.py::test_conversation_start \
|
||||
test_browsing_catchphrase.py::test_browsing_catchphrase \
|
||||
-v --no-header --capture=no --timeout=900
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
|
||||
291
tests/e2e/test_browsing_catchphrase.py
Normal file
291
tests/e2e/test_browsing_catchphrase.py
Normal file
@ -0,0 +1,291 @@
|
||||
"""
|
||||
E2E: Web browsing catchphrase test (Issue #10378)
|
||||
|
||||
Goal: In a new conversation, instruct the agent to browse to all-hands.dev and
|
||||
return the page's main catchphrase. We assert that a browsing action/observation
|
||||
is emitted and that the agent returns the expected catchphrase.
|
||||
|
||||
This follows existing patterns from tests/e2e/test_conversation.py and
|
||||
uses robust waits and screenshots.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from playwright.sync_api import Page, expect
|
||||
|
||||
CATCHPHRASE_PATTERNS = [
|
||||
r'\bcode\s*less\W*make\s*more\b',
|
||||
]
|
||||
|
||||
|
||||
def _screenshot(page: Page, name: str) -> None:
|
||||
os.makedirs('test-results', exist_ok=True)
|
||||
page.screenshot(path=f'test-results/browse_{name}.png')
|
||||
|
||||
|
||||
def _wait_for_home_and_repo_selection(page: Page) -> None:
|
||||
# Wait for home screen
|
||||
home_screen = page.locator('[data-testid="home-screen"]')
|
||||
expect(home_screen).to_be_visible(timeout=30000)
|
||||
|
||||
# Ensure repo dropdown is visible
|
||||
repo_dropdown = page.locator('[data-testid="repo-dropdown"]')
|
||||
expect(repo_dropdown).to_be_visible(timeout=30000)
|
||||
|
||||
# Open dropdown and type repo name
|
||||
repo_dropdown.click()
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
# Try to search and pick the official repo
|
||||
try:
|
||||
page.keyboard.press('Control+a')
|
||||
page.keyboard.type('openhands-agent/OpenHands')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
# Try multiple selectors for the option
|
||||
option_selectors = [
|
||||
'[data-testid="repo-dropdown"] [role="option"]:has-text("openhands-agent/OpenHands")',
|
||||
'[data-testid="repo-dropdown"] [role="option"]:has-text("OpenHands")',
|
||||
'[role="option"]:has-text("openhands-agent/OpenHands")',
|
||||
'[role="option"]:has-text("OpenHands")',
|
||||
'div:has-text("openhands-agent/OpenHands"):not([id="aria-results"])',
|
||||
'div:has-text("OpenHands"):not([id="aria-results"])',
|
||||
]
|
||||
|
||||
for selector in option_selectors:
|
||||
try:
|
||||
option = page.locator(selector).first
|
||||
if option.is_visible(timeout=3000):
|
||||
option.click(force=True)
|
||||
page.wait_for_timeout(1000)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
|
||||
def _launch_conversation(page: Page) -> None:
|
||||
launch_button = page.locator('[data-testid="repo-launch-button"]')
|
||||
expect(launch_button).to_be_visible(timeout=30000)
|
||||
|
||||
# Wait until enabled
|
||||
start = time.time()
|
||||
while time.time() - start < 120:
|
||||
try:
|
||||
if not launch_button.is_disabled():
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
try:
|
||||
if launch_button.is_disabled():
|
||||
# Force-enable and click via JS as fallback
|
||||
page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const btn = document.querySelector('[data-testid="repo-launch-button"]');
|
||||
if (btn) { btn.removeAttribute('disabled'); btn.click(); return true; }
|
||||
return false;
|
||||
}
|
||||
"""
|
||||
)
|
||||
else:
|
||||
launch_button.click()
|
||||
except Exception:
|
||||
# Last resort: try pressing Enter
|
||||
try:
|
||||
launch_button.focus()
|
||||
page.keyboard.press('Enter')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_screenshot(page, 'after_launch_click')
|
||||
|
||||
# Wait for conversation route
|
||||
# Also wait for possible loading indicators to disappear
|
||||
loading_selectors = [
|
||||
'[data-testid="loading-indicator"]',
|
||||
'[data-testid="loading-spinner"]',
|
||||
'.loading-spinner',
|
||||
'.spinner',
|
||||
'div:has-text("Loading...")',
|
||||
'div:has-text("Initializing...")',
|
||||
'div:has-text("Please wait...")',
|
||||
]
|
||||
for selector in loading_selectors:
|
||||
try:
|
||||
loading = page.locator(selector)
|
||||
if loading.is_visible(timeout=3000):
|
||||
expect(loading).not_to_be_visible(timeout=120000)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Confirm chat input is present
|
||||
chat_input = page.locator('[data-testid="chat-input"]')
|
||||
expect(chat_input).to_be_visible(timeout=120000)
|
||||
|
||||
# Give UI extra time to settle
|
||||
page.wait_for_timeout(5000)
|
||||
|
||||
|
||||
def _send_prompt(page: Page, prompt: str) -> None:
|
||||
# Find input
|
||||
selectors = [
|
||||
'[data-testid="chat-input"] textarea',
|
||||
'[data-testid="message-input"]',
|
||||
'textarea',
|
||||
'form textarea',
|
||||
]
|
||||
message_input = None
|
||||
for sel in selectors:
|
||||
try:
|
||||
el = page.locator(sel)
|
||||
if el.is_visible(timeout=5000):
|
||||
message_input = el
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not message_input:
|
||||
raise AssertionError('Message input not found')
|
||||
|
||||
message_input.fill(prompt)
|
||||
|
||||
# Submit
|
||||
submit_selectors = [
|
||||
'[data-testid="chat-input"] button[type="submit"]',
|
||||
'button[type="submit"]',
|
||||
'button:has-text("Send")',
|
||||
]
|
||||
submitted = False
|
||||
for sel in submit_selectors:
|
||||
try:
|
||||
btn = page.locator(sel)
|
||||
if btn.is_visible(timeout=3000):
|
||||
# wait until enabled
|
||||
start = time.time()
|
||||
while time.time() - start < 60:
|
||||
try:
|
||||
if not btn.is_disabled():
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(1000)
|
||||
try:
|
||||
btn.click()
|
||||
submitted = True
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not submitted:
|
||||
message_input.press('Enter')
|
||||
|
||||
_screenshot(page, 'prompt_sent')
|
||||
|
||||
|
||||
def _wait_for_browsing_event(page: Page, timeout_s: int = 240) -> None:
|
||||
start = time.time()
|
||||
browse_indicators = [
|
||||
'Interactive browsing in progress',
|
||||
'Browsing the web',
|
||||
'Browsing completed',
|
||||
]
|
||||
|
||||
while time.time() - start < timeout_s:
|
||||
for text in browse_indicators:
|
||||
try:
|
||||
# Use partial match for robustness
|
||||
if page.get_by_text(text, exact=False).is_visible(timeout=2000):
|
||||
_screenshot(page, 'browsing_event_seen')
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Also look for generic observation text that hints at web browsing
|
||||
try:
|
||||
if page.get_by_text('Current URL:', exact=False).is_visible(timeout=1000):
|
||||
_screenshot(page, 'browsing_url_seen')
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
raise AssertionError('Did not observe a browsing action/observation in time')
|
||||
|
||||
|
||||
def _wait_for_catchphrase(page: Page, timeout_s: int = 300) -> None:
|
||||
start = time.time()
|
||||
pattern = re.compile('|'.join(CATCHPHRASE_PATTERNS), re.IGNORECASE)
|
||||
|
||||
while time.time() - start < timeout_s:
|
||||
try:
|
||||
messages = page.locator('[data-testid="agent-message"]').all()
|
||||
for i, msg in enumerate(messages):
|
||||
try:
|
||||
content = msg.text_content() or ''
|
||||
if pattern.search(content):
|
||||
_screenshot(page, f'catchphrase_found_{i}')
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# also search globally on page for the phrase in case rendering differs
|
||||
try:
|
||||
if page.get_by_text('Code Less, Make More', exact=False).is_visible(
|
||||
timeout=1000
|
||||
):
|
||||
_screenshot(page, 'catchphrase_found_global')
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
raise AssertionError(
|
||||
'Agent did not return the expected catchphrase within time limit'
|
||||
)
|
||||
|
||||
|
||||
def test_browsing_catchphrase(page: Page):
|
||||
os.makedirs('test-results', exist_ok=True)
|
||||
|
||||
# 1) Navigate to app
|
||||
page.goto('http://localhost:12000')
|
||||
page.wait_for_load_state('networkidle', timeout=30000)
|
||||
_screenshot(page, 'initial_load')
|
||||
|
||||
# If we land on home, proceed with selection and launch
|
||||
_wait_for_home_and_repo_selection(page)
|
||||
_screenshot(page, 'home_ready')
|
||||
|
||||
# 2) Launch conversation
|
||||
_launch_conversation(page)
|
||||
_screenshot(page, 'conversation_loaded')
|
||||
|
||||
# 3) Send browsing instruction
|
||||
prompt = (
|
||||
'Use the web-browsing tool to navigate to https://www.all-hands.dev and '
|
||||
'tell me the main catchphrase displayed on the page. Do not answer from '
|
||||
'memory; perform the browsing action and respond with only the exact catchphrase.'
|
||||
)
|
||||
_send_prompt(page, prompt)
|
||||
|
||||
# 4) Wait for browsing action or observation to appear
|
||||
_wait_for_browsing_event(page)
|
||||
|
||||
# 5) Wait for agent final answer containing the catchphrase
|
||||
_wait_for_catchphrase(page)
|
||||
|
||||
_screenshot(page, 'final_state')
|
||||
Loading…
x
Reference in New Issue
Block a user