mirror of
https://github.com/OpenHands/OpenHands.git
synced 2025-12-26 13:52:43 +08:00
901 lines
36 KiB
Python
901 lines
36 KiB
Python
"""
|
|
E2E: Multi-conversation resume test
|
|
|
|
This test verifies that a user can resume an older conversation and continue it:
|
|
1. Start a conversation and ask a question
|
|
2. Get a response from the agent
|
|
3. Navigate away/close the conversation
|
|
4. Resume the same conversation later
|
|
5. Ask a follow-up question that requires context from the previous interaction
|
|
6. Verify the agent remembers the previous context and responds appropriately
|
|
|
|
This test assumes the GitHub token has already been configured (by the settings test).
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
from playwright.sync_api import Page, expect
|
|
|
|
|
|
def test_multi_conversation_resume(page: Page):
|
|
"""
|
|
Test resuming an older conversation and continuing it:
|
|
1. Navigate to OpenHands (assumes GitHub token is already configured)
|
|
2. Select the OpenHands repository
|
|
3. Start a conversation and ask about a specific file
|
|
4. Wait for agent response
|
|
5. Navigate away from the conversation
|
|
6. Resume the same conversation
|
|
7. Ask a follow-up question that requires context from the first interaction
|
|
8. Verify the agent remembers the previous context
|
|
"""
|
|
# Create test-results directory if it doesn't exist
|
|
os.makedirs('test-results', exist_ok=True)
|
|
|
|
# Navigate to the OpenHands application
|
|
print('Step 1: Navigating to OpenHands application...')
|
|
page.goto('http://localhost:12000')
|
|
page.wait_for_load_state('networkidle', timeout=30000)
|
|
|
|
# Take initial screenshot
|
|
page.screenshot(path='test-results/multi_conv_01_initial_load.png')
|
|
print('Screenshot saved: multi_conv_01_initial_load.png')
|
|
|
|
# Step 2: Select the OpenHands repository
|
|
print('Step 2: Selecting openhands-agent/OpenHands repository...')
|
|
|
|
# Wait for the home screen to load
|
|
home_screen = page.locator('[data-testid="home-screen"]')
|
|
expect(home_screen).to_be_visible(timeout=15000)
|
|
print('Home screen is visible')
|
|
|
|
# Look for the repository dropdown/selector
|
|
repo_dropdown = page.locator('[data-testid="repo-dropdown"]')
|
|
expect(repo_dropdown).to_be_visible(timeout=15000)
|
|
print('Repository dropdown is visible')
|
|
|
|
# Click on the repository input to open dropdown
|
|
repo_dropdown.click()
|
|
page.wait_for_timeout(1000)
|
|
|
|
# Type the repository name
|
|
try:
|
|
page.keyboard.press('Control+a') # Select all
|
|
page.keyboard.type('openhands-agent/OpenHands')
|
|
print('Used keyboard.type() for React Select component')
|
|
except Exception as e:
|
|
print(f'Keyboard input failed: {e}')
|
|
|
|
page.wait_for_timeout(2000) # Wait for search results
|
|
|
|
# Try to find and click the repository option
|
|
option_selectors = [
|
|
'[data-testid="repo-dropdown"] [role="option"]:has-text("openhands-agent/OpenHands")',
|
|
'[data-testid="repo-dropdown"] [role="option"]:has-text("OpenHands")',
|
|
'[data-testid="repo-dropdown"] div[id*="option"]:has-text("openhands-agent/OpenHands")',
|
|
'[data-testid="repo-dropdown"] div[id*="option"]:has-text("OpenHands")',
|
|
'[role="option"]:has-text("openhands-agent/OpenHands")',
|
|
'[role="option"]:has-text("OpenHands")',
|
|
'div:has-text("openhands-agent/OpenHands"):not([id="aria-results"])',
|
|
'div:has-text("OpenHands"):not([id="aria-results"])',
|
|
]
|
|
|
|
option_found = False
|
|
for selector in option_selectors:
|
|
try:
|
|
option = page.locator(selector).first
|
|
if option.is_visible(timeout=3000):
|
|
print(f'Found repository option with selector: {selector}')
|
|
try:
|
|
option.click(force=True)
|
|
print('Successfully clicked option with force=True')
|
|
option_found = True
|
|
page.wait_for_timeout(2000)
|
|
break
|
|
except Exception:
|
|
continue
|
|
except Exception:
|
|
continue
|
|
|
|
if not option_found:
|
|
print(
|
|
'Could not find repository option in dropdown, trying keyboard navigation'
|
|
)
|
|
page.keyboard.press('ArrowDown')
|
|
page.wait_for_timeout(500)
|
|
page.keyboard.press('Enter')
|
|
print('Used keyboard navigation to select option')
|
|
|
|
page.screenshot(path='test-results/multi_conv_02_repo_selected.png')
|
|
print('Screenshot saved: multi_conv_02_repo_selected.png')
|
|
|
|
# Step 3: Click Launch button
|
|
print('Step 3: Clicking Launch button...')
|
|
|
|
launch_button = page.locator('[data-testid="repo-launch-button"]')
|
|
expect(launch_button).to_be_visible(timeout=10000)
|
|
|
|
# Wait for the button to be enabled (not disabled)
|
|
max_wait_attempts = 30
|
|
button_enabled = False
|
|
for attempt in range(max_wait_attempts):
|
|
try:
|
|
is_disabled = launch_button.is_disabled()
|
|
if not is_disabled:
|
|
print(
|
|
f'Repository Launch button is now enabled (attempt {attempt + 1})'
|
|
)
|
|
button_enabled = True
|
|
break
|
|
else:
|
|
print(
|
|
f'Launch button still disabled, waiting... (attempt {attempt + 1}/{max_wait_attempts})'
|
|
)
|
|
page.wait_for_timeout(2000)
|
|
except Exception as e:
|
|
print(f'Error checking button state (attempt {attempt + 1}): {e}')
|
|
page.wait_for_timeout(2000)
|
|
|
|
try:
|
|
if button_enabled:
|
|
launch_button.click()
|
|
print('Launch button clicked normally')
|
|
else:
|
|
print('Launch button still disabled, trying JavaScript force click...')
|
|
result = page.evaluate("""() => {
|
|
const button = document.querySelector('[data-testid="repo-launch-button"]');
|
|
if (button) {
|
|
console.log('Found button, removing disabled attribute');
|
|
button.removeAttribute('disabled');
|
|
console.log('Clicking button');
|
|
button.click();
|
|
return true;
|
|
}
|
|
return false;
|
|
}""")
|
|
if result:
|
|
print('Successfully force-clicked Launch button with JavaScript')
|
|
else:
|
|
print('JavaScript could not find the Launch button')
|
|
except Exception as e:
|
|
print(f'Error clicking Launch button: {e}')
|
|
page.screenshot(path='test-results/multi_conv_03_launch_error.png')
|
|
print('Screenshot saved: multi_conv_03_launch_error.png')
|
|
raise
|
|
|
|
# Step 4: Wait for conversation interface to load
|
|
print('Step 4: Waiting for conversation interface to load...')
|
|
|
|
navigation_timeout = 300000 # 5 minutes
|
|
check_interval = 10000 # 10 seconds
|
|
|
|
page.screenshot(path='test-results/multi_conv_04_after_launch.png')
|
|
print('Screenshot saved: multi_conv_04_after_launch.png')
|
|
|
|
# Wait for loading to complete
|
|
loading_selectors = [
|
|
'[data-testid="loading-indicator"]',
|
|
'[data-testid="loading-spinner"]',
|
|
'.loading-spinner',
|
|
'.spinner',
|
|
'div:has-text("Loading...")',
|
|
'div:has-text("Initializing...")',
|
|
'div:has-text("Please wait...")',
|
|
]
|
|
|
|
for selector in loading_selectors:
|
|
try:
|
|
loading = page.locator(selector)
|
|
if loading.is_visible(timeout=5000):
|
|
print(f'Found loading indicator with selector: {selector}')
|
|
print('Waiting for loading to complete...')
|
|
expect(loading).not_to_be_visible(timeout=120000)
|
|
print('Loading completed')
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
# Wait for conversation interface to be ready
|
|
start_time = time.time()
|
|
conversation_loaded = False
|
|
while time.time() - start_time < navigation_timeout / 1000:
|
|
try:
|
|
selectors = [
|
|
'.scrollbar.flex.flex-col.grow',
|
|
'[data-testid="chat-input"]',
|
|
'[data-testid="app-route"]',
|
|
'[data-testid="conversation-screen"]',
|
|
'[data-testid="message-input"]',
|
|
'.conversation-container',
|
|
'.chat-container',
|
|
'textarea',
|
|
'form textarea',
|
|
'div[role="main"]',
|
|
'main',
|
|
]
|
|
|
|
for selector in selectors:
|
|
try:
|
|
element = page.locator(selector)
|
|
if element.is_visible(timeout=2000):
|
|
print(
|
|
f'Found conversation interface element with selector: {selector}'
|
|
)
|
|
conversation_loaded = True
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if conversation_loaded:
|
|
break
|
|
|
|
if (time.time() - start_time) % (check_interval / 1000) < 1:
|
|
elapsed = int(time.time() - start_time)
|
|
page.screenshot(
|
|
path=f'test-results/multi_conv_05_waiting_{elapsed}s.png'
|
|
)
|
|
print(f'Screenshot saved: multi_conv_05_waiting_{elapsed}s.png')
|
|
|
|
page.wait_for_timeout(5000)
|
|
except Exception as e:
|
|
print(f'Error checking for conversation interface: {e}')
|
|
page.wait_for_timeout(5000)
|
|
|
|
if not conversation_loaded:
|
|
print('Timed out waiting for conversation interface to load')
|
|
page.screenshot(path='test-results/multi_conv_06_timeout.png')
|
|
print('Screenshot saved: multi_conv_06_timeout.png')
|
|
raise TimeoutError('Timed out waiting for conversation interface to load')
|
|
|
|
# Step 5: Wait for agent to be ready
|
|
print('Step 5: Waiting for agent to be ready for input...')
|
|
|
|
max_wait_time = 480
|
|
start_time = time.time()
|
|
agent_ready = False
|
|
print(f'Waiting up to {max_wait_time} seconds for agent to be ready...')
|
|
|
|
while time.time() - start_time < max_wait_time:
|
|
elapsed = int(time.time() - start_time)
|
|
if elapsed % 30 == 0 and elapsed > 0:
|
|
page.screenshot(path=f'test-results/multi_conv_waiting_{elapsed}s.png')
|
|
print(
|
|
f'Screenshot saved: multi_conv_waiting_{elapsed}s.png (waiting {elapsed}s)'
|
|
)
|
|
|
|
try:
|
|
# Check if input field and submit button are ready
|
|
input_ready = False
|
|
submit_ready = False
|
|
try:
|
|
input_field = page.locator('[data-testid="chat-input"] textarea')
|
|
submit_button = page.locator(
|
|
'[data-testid="chat-input"] button[type="submit"]'
|
|
)
|
|
if (
|
|
input_field.is_visible(timeout=2000)
|
|
and input_field.is_enabled(timeout=2000)
|
|
and submit_button.is_visible(timeout=2000)
|
|
and submit_button.is_enabled(timeout=2000)
|
|
):
|
|
print(
|
|
'Chat input field and submit button are both visible and enabled'
|
|
)
|
|
input_ready = True
|
|
submit_ready = True
|
|
except Exception:
|
|
pass
|
|
|
|
if input_ready and submit_ready:
|
|
print(
|
|
'✅ Agent is ready for user input - input field and submit button are enabled'
|
|
)
|
|
agent_ready = True
|
|
break
|
|
except Exception as e:
|
|
print(f'Error checking agent ready state: {e}')
|
|
|
|
page.wait_for_timeout(2000)
|
|
|
|
if not agent_ready:
|
|
page.screenshot(path='test-results/multi_conv_timeout_waiting_for_agent.png')
|
|
raise AssertionError(
|
|
f'Agent did not become ready for input within {max_wait_time} seconds'
|
|
)
|
|
|
|
# Step 6: Ask the first question about a specific file
|
|
print('Step 6: Asking first question about pyproject.toml file...')
|
|
|
|
# Find the message input
|
|
input_selectors = [
|
|
'[data-testid="chat-input"] textarea',
|
|
'[data-testid="message-input"]',
|
|
'textarea',
|
|
'form textarea',
|
|
'input[type="text"]',
|
|
'[placeholder*="message"]',
|
|
'[placeholder*="question"]',
|
|
'[placeholder*="ask"]',
|
|
'[contenteditable="true"]',
|
|
]
|
|
|
|
message_input = None
|
|
for selector in input_selectors:
|
|
try:
|
|
input_element = page.locator(selector)
|
|
if input_element.is_visible(timeout=5000):
|
|
print(f'Found message input with selector: {selector}')
|
|
message_input = input_element
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if not message_input:
|
|
page.screenshot(path='test-results/multi_conv_07_no_input_found.png')
|
|
print('Screenshot saved: multi_conv_07_no_input_found.png')
|
|
raise AssertionError('Could not find message input field')
|
|
|
|
# Ask about the pyproject.toml file
|
|
first_question = 'What is the name of the project defined in the pyproject.toml file? Please check the file and tell me the exact project name.'
|
|
message_input.fill(first_question)
|
|
print('Entered first question about pyproject.toml')
|
|
|
|
# Find and click submit button
|
|
submit_selectors = [
|
|
'[data-testid="chat-input"] button[type="submit"]',
|
|
'button[type="submit"]',
|
|
'button:has-text("Send")',
|
|
'button:has-text("Submit")',
|
|
'button svg[data-testid="send-icon"]',
|
|
'button.send-button',
|
|
'form button',
|
|
'button:right-of(textarea)',
|
|
'button:right-of(input[type="text"])',
|
|
]
|
|
|
|
submit_button = None
|
|
for selector in submit_selectors:
|
|
try:
|
|
button_element = page.locator(selector)
|
|
if button_element.is_visible(timeout=5000):
|
|
print(f'Found submit button with selector: {selector}')
|
|
submit_button = button_element
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if submit_button and not submit_button.is_disabled():
|
|
submit_button.click()
|
|
print('Clicked submit button')
|
|
else:
|
|
# Try pressing Enter as fallback
|
|
message_input.press('Enter')
|
|
print('Pressed Enter key to submit')
|
|
|
|
page.screenshot(path='test-results/multi_conv_08_first_question_sent.png')
|
|
print('Screenshot saved: multi_conv_08_first_question_sent.png')
|
|
|
|
# Step 7: Wait for agent response to first question
|
|
print('Step 7: Waiting for agent response to first question...')
|
|
|
|
response_wait_time = 180
|
|
response_start_time = time.time()
|
|
first_response_found = False
|
|
project_name = None
|
|
|
|
while time.time() - response_start_time < response_wait_time:
|
|
elapsed = int(time.time() - response_start_time)
|
|
|
|
if elapsed % 30 == 0 and elapsed > 0:
|
|
page.screenshot(
|
|
path=f'test-results/multi_conv_first_response_wait_{elapsed}s.png'
|
|
)
|
|
print(
|
|
f'Screenshot saved: multi_conv_first_response_wait_{elapsed}s.png (waiting {elapsed}s for first response)'
|
|
)
|
|
|
|
try:
|
|
agent_messages = page.locator('[data-testid="agent-message"]').all()
|
|
if elapsed % 30 == 0:
|
|
print(f'Found {len(agent_messages)} agent messages')
|
|
|
|
for i, msg in enumerate(agent_messages):
|
|
try:
|
|
content = msg.text_content()
|
|
if content and len(content.strip()) > 10:
|
|
content_lower = content.lower()
|
|
# Look for project name in the response
|
|
if (
|
|
'pyproject' in content_lower
|
|
and ('name' in content_lower or 'project' in content_lower)
|
|
and (
|
|
'openhands' in content_lower
|
|
or 'openhands-ai' in content_lower
|
|
)
|
|
):
|
|
print(
|
|
'✅ Found agent response about pyproject.toml with project name!'
|
|
)
|
|
# Extract project name from response
|
|
name_match = re.search(
|
|
r'name.*?["\']([^"\']+)["\']', content, re.IGNORECASE
|
|
)
|
|
if name_match:
|
|
project_name = name_match.group(1)
|
|
print(f'Extracted project name: {project_name}')
|
|
else:
|
|
# Fallback: look for "openhands" variations in the content
|
|
if 'openhands-ai' in content_lower:
|
|
project_name = 'openhands-ai'
|
|
elif 'openhands' in content_lower:
|
|
project_name = 'openhands'
|
|
print(f'Fallback project name: {project_name}')
|
|
|
|
first_response_found = True
|
|
page.screenshot(
|
|
path='test-results/multi_conv_09_first_response.png'
|
|
)
|
|
print('Screenshot saved: multi_conv_09_first_response.png')
|
|
break
|
|
except Exception as e:
|
|
print(f'Error processing agent message {i}: {e}')
|
|
continue
|
|
|
|
if first_response_found:
|
|
break
|
|
except Exception as e:
|
|
print(f'Error checking for agent messages: {e}')
|
|
|
|
page.wait_for_timeout(5000)
|
|
|
|
if not first_response_found:
|
|
print('❌ Did not find agent response about pyproject.toml within time limit')
|
|
page.screenshot(path='test-results/multi_conv_09_first_response_timeout.png')
|
|
print('Screenshot saved: multi_conv_09_first_response_timeout.png')
|
|
raise AssertionError(
|
|
'Agent response did not include pyproject.toml project name within time limit'
|
|
)
|
|
|
|
# Step 8: Store conversation ID and navigate away
|
|
print('Step 8: Storing conversation ID and navigating away...')
|
|
|
|
# Get the current URL to extract conversation ID
|
|
current_url = page.url
|
|
print(f'Current URL: {current_url}')
|
|
|
|
# Extract conversation ID from URL
|
|
conversation_id_match = re.search(r'/conversations?/([a-f0-9]+)', current_url)
|
|
if not conversation_id_match:
|
|
# Try alternative URL patterns
|
|
conversation_id_match = re.search(r'/chat/([a-f0-9]+)', current_url)
|
|
|
|
if not conversation_id_match:
|
|
print(
|
|
'Could not extract conversation ID from URL, trying to find it in the page'
|
|
)
|
|
# Try to find conversation ID in page elements or local storage
|
|
conversation_id = page.evaluate("""() => {
|
|
// Try to get conversation ID from various sources
|
|
const url = window.location.href;
|
|
const match = url.match(/\\/(?:conversations?|chat)\\/([a-f0-9]+)/);
|
|
if (match) return match[1];
|
|
|
|
// Try localStorage
|
|
const stored = localStorage.getItem('currentConversationId');
|
|
if (stored) return stored;
|
|
|
|
// Try sessionStorage
|
|
const sessionStored = sessionStorage.getItem('conversationId');
|
|
if (sessionStored) return sessionStored;
|
|
|
|
return null;
|
|
}""")
|
|
|
|
if not conversation_id:
|
|
page.screenshot(path='test-results/multi_conv_10_no_conversation_id.png')
|
|
print('Screenshot saved: multi_conv_10_no_conversation_id.png')
|
|
raise AssertionError('Could not extract conversation ID')
|
|
else:
|
|
conversation_id = conversation_id_match.group(1)
|
|
|
|
print(f'Extracted conversation ID: {conversation_id}')
|
|
|
|
# Navigate to home page to "leave" the conversation
|
|
page.goto('http://localhost:12000')
|
|
page.wait_for_load_state('networkidle', timeout=30000)
|
|
|
|
page.screenshot(path='test-results/multi_conv_11_navigated_home.png')
|
|
print('Screenshot saved: multi_conv_11_navigated_home.png')
|
|
|
|
# Wait a bit to simulate time passing
|
|
print('Waiting 10 seconds to simulate time passing...')
|
|
page.wait_for_timeout(10000)
|
|
|
|
# Step 9: Resume the conversation via conversation panel
|
|
print('Step 9: Resuming the previous conversation via conversation panel...')
|
|
|
|
# Click the conversation panel button (the "sandwich button")
|
|
conversation_panel_button = page.locator(
|
|
'[data-testid="toggle-conversation-panel"]'
|
|
)
|
|
|
|
conversations_found = False
|
|
try:
|
|
if conversation_panel_button.is_visible(timeout=10000):
|
|
print(
|
|
'Found conversation panel button, clicking to open conversations list'
|
|
)
|
|
conversation_panel_button.click()
|
|
conversations_found = True
|
|
page.wait_for_timeout(3000) # Wait for panel to open
|
|
else:
|
|
print('Conversation panel button not visible')
|
|
except Exception as e:
|
|
print(f'Error clicking conversation panel button: {e}')
|
|
|
|
if not conversations_found:
|
|
print(
|
|
'Could not find conversation panel button, will try direct navigation fallback'
|
|
)
|
|
# Fallback will be handled in the conversation finding section below
|
|
|
|
page.screenshot(path='test-results/multi_conv_12_conversations_list.png')
|
|
print('Screenshot saved: multi_conv_12_conversations_list.png')
|
|
|
|
# Look for the specific conversation in the list
|
|
print(f'Looking for conversation {conversation_id} in the list...')
|
|
|
|
# Try different selectors to find the conversation in the panel
|
|
conversation_selectors = [
|
|
'[data-testid="conversation-card"]', # Main conversation card selector
|
|
f'a[href*="{conversation_id}"]', # Link containing conversation ID
|
|
f'div:has-text("{conversation_id}")', # Any div containing the ID
|
|
'a[href*="/conversations/"]', # Any conversation link (note: plural)
|
|
]
|
|
|
|
conversation_link_found = False
|
|
for selector in conversation_selectors:
|
|
try:
|
|
conversation_elements = page.locator(selector).all()
|
|
for element in conversation_elements:
|
|
try:
|
|
# Check if this element contains our conversation ID or is the right conversation
|
|
element_text = element.text_content() or ''
|
|
element_href = element.get_attribute('href') or ''
|
|
|
|
if (
|
|
conversation_id in element_href
|
|
or conversation_id in element_text
|
|
):
|
|
print(f'Found conversation link with selector: {selector}')
|
|
element.click()
|
|
conversation_link_found = True
|
|
page.wait_for_timeout(2000)
|
|
break
|
|
# Also try clicking the first conversation if we can't find the specific one
|
|
elif (
|
|
selector == 'a[href*="/conversations/"]'
|
|
and not conversation_link_found
|
|
):
|
|
print(
|
|
f'Clicking first conversation found with selector: {selector}'
|
|
)
|
|
element.click()
|
|
conversation_link_found = True
|
|
page.wait_for_timeout(2000)
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if conversation_link_found:
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if not conversation_link_found:
|
|
print(
|
|
'Could not find conversation in list, navigating directly to conversation URL as fallback'
|
|
)
|
|
# Fallback to direct navigation (use plural 'conversations' to match actual URL pattern)
|
|
conversation_url = f'http://localhost:12000/conversations/{conversation_id}'
|
|
print(f'Navigating to conversation URL: {conversation_url}')
|
|
page.goto(conversation_url)
|
|
page.wait_for_load_state('networkidle', timeout=30000)
|
|
|
|
page.screenshot(path='test-results/multi_conv_13_resumed_conversation.png')
|
|
print('Screenshot saved: multi_conv_13_resumed_conversation.png')
|
|
|
|
# Wait for the conversation to load and agent to be ready again
|
|
print('Waiting for resumed conversation to be ready...')
|
|
start_time = time.time()
|
|
agent_ready = False
|
|
max_wait_time = 120 # Shorter wait time for resume
|
|
|
|
while time.time() - start_time < max_wait_time:
|
|
try:
|
|
input_field = page.locator('[data-testid="chat-input"] textarea')
|
|
submit_button = page.locator(
|
|
'[data-testid="chat-input"] button[type="submit"]'
|
|
)
|
|
if (
|
|
input_field.is_visible(timeout=2000)
|
|
and input_field.is_enabled(timeout=2000)
|
|
and submit_button.is_visible(timeout=2000)
|
|
and submit_button.is_enabled(timeout=2000)
|
|
):
|
|
print('Resumed conversation is ready for input')
|
|
agent_ready = True
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
page.wait_for_timeout(2000)
|
|
|
|
if not agent_ready:
|
|
page.screenshot(path='test-results/multi_conv_14_resume_timeout.png')
|
|
print('Screenshot saved: multi_conv_14_resume_timeout.png')
|
|
raise AssertionError('Resumed conversation did not become ready for input')
|
|
|
|
# Step 10: Verify conversation history is preserved
|
|
print('Step 10: Verifying conversation history is preserved...')
|
|
|
|
# Check if the previous messages are visible
|
|
try:
|
|
# Look for the first question in the conversation history
|
|
user_messages = page.locator('[data-testid="user-message"]').all()
|
|
agent_messages = page.locator('[data-testid="agent-message"]').all()
|
|
|
|
print(
|
|
f'Found {len(user_messages)} user messages and {len(agent_messages)} agent messages'
|
|
)
|
|
|
|
# Verify we have at least one user message and one agent message
|
|
if len(user_messages) == 0 or len(agent_messages) == 0:
|
|
page.screenshot(path='test-results/multi_conv_15_no_history.png')
|
|
print('Screenshot saved: multi_conv_15_no_history.png')
|
|
raise AssertionError(
|
|
'Conversation history not preserved - no previous messages found'
|
|
)
|
|
|
|
# Check if the first question is in the history
|
|
first_question_found = False
|
|
for msg in user_messages:
|
|
content = msg.text_content()
|
|
if content and 'pyproject.toml' in content.lower():
|
|
first_question_found = True
|
|
print('✅ Found first question in conversation history')
|
|
break
|
|
|
|
if not first_question_found:
|
|
print('⚠️ First question not found in visible history, but continuing test')
|
|
|
|
except Exception as e:
|
|
print(f'Error checking conversation history: {e}')
|
|
|
|
# Step 11: Ask a follow-up question that requires context
|
|
print(
|
|
'Step 11: Asking follow-up question that requires context from first interaction...'
|
|
)
|
|
|
|
# Find the message input again
|
|
message_input = None
|
|
for selector in input_selectors:
|
|
try:
|
|
input_element = page.locator(selector)
|
|
if input_element.is_visible(timeout=5000):
|
|
print(f'Found message input with selector: {selector}')
|
|
message_input = input_element
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if not message_input:
|
|
page.screenshot(path='test-results/multi_conv_16_no_input_found.png')
|
|
print('Screenshot saved: multi_conv_16_no_input_found.png')
|
|
raise AssertionError(
|
|
'Could not find message input field in resumed conversation'
|
|
)
|
|
|
|
# Ask a follow-up question that references the previous interaction
|
|
if project_name:
|
|
follow_up_question = f'Based on the project name you just told me ({project_name}), can you tell me what type of project this is? Is it a Python package, web application, or something else?'
|
|
else:
|
|
follow_up_question = 'Based on the project name you just told me from the pyproject.toml file, can you tell me what type of project this is? Is it a Python package, web application, or something else?'
|
|
|
|
message_input.fill(follow_up_question)
|
|
print('Entered follow-up question that requires context from first interaction')
|
|
|
|
# Find and click submit button
|
|
submit_button = None
|
|
for selector in submit_selectors:
|
|
try:
|
|
button_element = page.locator(selector)
|
|
if button_element.is_visible(timeout=5000):
|
|
print(f'Found submit button with selector: {selector}')
|
|
submit_button = button_element
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if submit_button and not submit_button.is_disabled():
|
|
submit_button.click()
|
|
print('Clicked submit button for follow-up question')
|
|
else:
|
|
# Try pressing Enter as fallback
|
|
message_input.press('Enter')
|
|
print('Pressed Enter key to submit follow-up question')
|
|
|
|
page.screenshot(path='test-results/multi_conv_17_followup_question_sent.png')
|
|
print('Screenshot saved: multi_conv_17_followup_question_sent.png')
|
|
|
|
# Step 12: Wait for agent response to follow-up question
|
|
print('Step 12: Waiting for agent response to follow-up question...')
|
|
|
|
response_wait_time = 300 # Increased to 5 minutes for complete response
|
|
response_start_time = time.time()
|
|
followup_response_found = False
|
|
agent_completed = False
|
|
|
|
while time.time() - response_start_time < response_wait_time:
|
|
elapsed = int(time.time() - response_start_time)
|
|
|
|
if elapsed % 30 == 0 and elapsed > 0:
|
|
page.screenshot(
|
|
path=f'test-results/multi_conv_followup_response_wait_{elapsed}s.png'
|
|
)
|
|
print(
|
|
f'Screenshot saved: multi_conv_followup_response_wait_{elapsed}s.png (waiting {elapsed}s for follow-up response)'
|
|
)
|
|
|
|
try:
|
|
# First check if agent has completed its response
|
|
agent_status_indicators = [
|
|
'text="Agent is awaiting user input"',
|
|
'text="Agent is ready"',
|
|
'[data-testid="agent-status"]:has-text("awaiting")',
|
|
'[data-testid="agent-status"]:has-text("ready")',
|
|
]
|
|
|
|
# Also check if the agent is no longer showing "running task"
|
|
running_indicators = [
|
|
'text="Agent is running task"',
|
|
'text="Agent is working"',
|
|
'[data-testid="agent-status"]:has-text("running")',
|
|
'[data-testid="agent-status"]:has-text("working")',
|
|
]
|
|
|
|
# Check if agent is still running
|
|
agent_still_running = False
|
|
for indicator in running_indicators:
|
|
try:
|
|
if page.locator(indicator).is_visible(timeout=1000):
|
|
agent_still_running = True
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
# If agent is not running, check for completion status
|
|
if not agent_still_running:
|
|
for indicator in agent_status_indicators:
|
|
try:
|
|
if page.locator(indicator).is_visible(timeout=1000):
|
|
agent_completed = True
|
|
print('✅ Agent has completed its response')
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
# If we can't find explicit completion status, check if input is enabled
|
|
if not agent_completed:
|
|
try:
|
|
input_field = page.locator(
|
|
'[data-testid="chat-input"] textarea'
|
|
)
|
|
submit_button = page.locator(
|
|
'[data-testid="chat-input"] button[type="submit"]'
|
|
)
|
|
if (
|
|
input_field.is_enabled(timeout=1000)
|
|
and submit_button.is_enabled(timeout=1000)
|
|
and not submit_button.is_disabled()
|
|
):
|
|
agent_completed = True
|
|
print(
|
|
'✅ Agent appears to have completed (input field is enabled)'
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
# Only check for response content if agent has completed or we're getting close to timeout
|
|
if (
|
|
agent_completed or elapsed > 240
|
|
): # Check content after 4 minutes or when completed
|
|
agent_messages = page.locator('[data-testid="agent-message"]').all()
|
|
if elapsed % 30 == 0:
|
|
print(f'Found {len(agent_messages)} agent messages')
|
|
|
|
# Look at the most recent agent messages for the follow-up response
|
|
for i, msg in enumerate(agent_messages[-3:]): # Check last 3 messages
|
|
try:
|
|
content = msg.text_content()
|
|
if content and len(content.strip()) > 10:
|
|
content_lower = content.lower()
|
|
# Look for response that shows context awareness
|
|
context_indicators = [
|
|
'based on',
|
|
'as i mentioned',
|
|
'from what i told you',
|
|
'the project name',
|
|
'python',
|
|
'package',
|
|
'application',
|
|
'software',
|
|
'ai',
|
|
'openhands',
|
|
]
|
|
|
|
if any(
|
|
indicator in content_lower
|
|
for indicator in context_indicators
|
|
):
|
|
print(
|
|
'✅ Found agent response to follow-up question with context awareness!'
|
|
)
|
|
followup_response_found = True
|
|
|
|
# Only break if agent has completed, otherwise keep waiting
|
|
if agent_completed:
|
|
page.screenshot(
|
|
path='test-results/multi_conv_18_followup_response.png'
|
|
)
|
|
print(
|
|
'Screenshot saved: multi_conv_18_followup_response.png'
|
|
)
|
|
break
|
|
else:
|
|
print(
|
|
'Found response content but agent still processing, continuing to wait...'
|
|
)
|
|
except Exception as e:
|
|
print(f'Error processing agent message {i}: {e}')
|
|
continue
|
|
|
|
if followup_response_found and agent_completed:
|
|
break
|
|
except Exception as e:
|
|
print(f'Error checking for agent messages: {e}')
|
|
|
|
page.wait_for_timeout(5000)
|
|
|
|
# Take final screenshot
|
|
page.screenshot(path='test-results/multi_conv_19_final_state.png')
|
|
print('Screenshot saved: multi_conv_19_final_state.png')
|
|
|
|
if not followup_response_found:
|
|
print('❌ Did not find agent response to follow-up question within time limit')
|
|
page.screenshot(path='test-results/multi_conv_18_followup_response_timeout.png')
|
|
print('Screenshot saved: multi_conv_18_followup_response_timeout.png')
|
|
raise AssertionError(
|
|
'Agent response to follow-up question not found within time limit'
|
|
)
|
|
|
|
if not agent_completed:
|
|
print('⚠️ Found response content but agent may not have completed processing')
|
|
print('This could indicate the agent is still working on the response')
|
|
|
|
print(
|
|
'✅ Test completed successfully - agent resumed conversation and maintained context!'
|
|
)
|
|
print('Multi-conversation resume test passed:')
|
|
print('1. ✅ Started conversation and asked about pyproject.toml')
|
|
print('2. ✅ Received response with project name')
|
|
print('3. ✅ Successfully navigated away from conversation')
|
|
print('4. ✅ Successfully resumed the same conversation via conversation list')
|
|
print('5. ✅ Conversation history was preserved')
|
|
print('6. ✅ Asked follow-up question requiring context from first interaction')
|
|
print(
|
|
'7. ✅ Agent responded with context awareness, showing conversation continuity'
|
|
)
|