From 2a7f926591e607843f3dd4aaa0af255ed793973c Mon Sep 17 00:00:00 2001 From: Engel Nyst Date: Thu, 13 Mar 2025 15:32:01 +0100 Subject: [PATCH] Detect condensation loops at 10 repetitions, not 3 (#7237) --- openhands/controller/stuck.py | 13 ++-- tests/unit/test_is_stuck.py | 111 +++++++++++++++------------------- 2 files changed, 55 insertions(+), 69 deletions(-) diff --git a/openhands/controller/stuck.py b/openhands/controller/stuck.py index 99aa09bc75..373a95abfb 100644 --- a/openhands/controller/stuck.py +++ b/openhands/controller/stuck.py @@ -103,8 +103,9 @@ class StuckDetector: return True # scenario 5: context window error loop - if self._is_stuck_context_window_error(filtered_history): - return True + if len(filtered_history) >= 10: + if self._is_stuck_context_window_error(filtered_history): + return True return False @@ -333,12 +334,12 @@ class StuckDetector: if isinstance(event, AgentCondensationObservation) ] - # Need at least 3 condensation events to detect a loop - if len(condensation_events) < 3: + # Need at least 10 condensation events to detect a loop + if len(condensation_events) < 10: return False - # Get the last 3 condensation events - last_condensation_events = condensation_events[-3:] + # Get the last 10 condensation events + last_condensation_events = condensation_events[-10:] # Check if there are any non-condensation events between them for i in range(len(last_condensation_events) - 1): diff --git a/tests/unit/test_is_stuck.py b/tests/unit/test_is_stuck.py index f2ac3f0773..1c0a40a725 100644 --- a/tests/unit/test_is_stuck.py +++ b/tests/unit/test_is_stuck.py @@ -614,8 +614,8 @@ class TestStuckDetector: message_observation = NullObservation(content='') state.history.append(message_observation) - # Add three consecutive condensation events (should detect as stuck) - for _ in range(3): + # Add ten consecutive condensation events (should detect as stuck) + for _ in range(10): condensation = AgentCondensationObservation( content='Trimming prompt to meet context window limitations' ) @@ -638,42 +638,39 @@ class TestStuckDetector: message_observation = NullObservation(content='') state.history.append(message_observation) - # Add condensation events with other events between them - condensation1 = AgentCondensationObservation( - content='Trimming prompt to meet context window limitations' - ) - state.history.append(condensation1) + # Add 10 condensation events with other events between them + for i in range(10): + # Add a condensation event + condensation = AgentCondensationObservation( + content='Trimming prompt to meet context window limitations' + ) + state.history.append(condensation) - # Add some other events between condensation events - cmd_action = CmdRunAction(command='ls') - state.history.append(cmd_action) - cmd_observation = CmdOutputObservation( - command='ls', content='file1.txt\nfile2.txt' - ) - state.history.append(cmd_observation) + # Add some other events between condensation events (except after the last one) + if i < 9: + # Add a command action and observation + cmd_action = CmdRunAction(command=f'ls {i}') + state.history.append(cmd_action) + cmd_observation = CmdOutputObservation( + command=f'ls {i}', content='file1.txt\nfile2.txt' + ) + state.history.append(cmd_observation) - condensation2 = AgentCondensationObservation( - content='Trimming prompt to meet context window limitations' - ) - state.history.append(condensation2) - - # Add more other events - read_action = FileReadAction(path='file1.txt') - state.history.append(read_action) - read_observation = FileReadObservation(content='File content', path='file1.txt') - state.history.append(read_observation) - - condensation3 = AgentCondensationObservation( - content='Trimming prompt to meet context window limitations' - ) - state.history.append(condensation3) + # Add a file read action and observation for even iterations + if i % 2 == 0: + read_action = FileReadAction(path=f'file{i}.txt') + state.history.append(read_action) + read_observation = FileReadObservation( + content=f'File content {i}', path=f'file{i}.txt' + ) + state.history.append(read_observation) with patch('logging.Logger.warning') as mock_warning: assert stuck_detector.is_stuck(headless_mode=True) is False mock_warning.assert_not_called() - def test_is_not_stuck_context_window_error_less_than_three(self, stuck_detector): - """Test that we don't detect a loop with less than three condensation events.""" + def test_is_not_stuck_context_window_error_less_than_ten(self, stuck_detector): + """Test that we don't detect a loop with less than ten condensation events.""" state = stuck_detector.state # Add some initial events @@ -683,8 +680,8 @@ class TestStuckDetector: message_observation = NullObservation(content='') state.history.append(message_observation) - # Add only two condensation events (should not detect as stuck) - for _ in range(2): + # Add only nine condensation events (should not detect as stuck) + for _ in range(9): condensation = AgentCondensationObservation( content='Trimming prompt to meet context window limitations' ) @@ -695,7 +692,7 @@ class TestStuckDetector: mock_warning.assert_not_called() def test_is_stuck_context_window_error_with_user_messages(self, stuck_detector): - """Test that we still detect a loop even with user messages between condensation events. + """Test that we still detect a loop even with user messages between condensation events in headless mode. User messages are filtered out in the stuck detection logic, so they shouldn't prevent us from detecting a loop of condensation events. @@ -709,35 +706,23 @@ class TestStuckDetector: message_observation = NullObservation(content='') state.history.append(message_observation) - # Add condensation events with user messages between them - condensation1 = AgentCondensationObservation( - content='Trimming prompt to meet context window limitations' - ) - state.history.append(condensation1) + # Add condensation events with user messages between them (total of 10) + for i in range(10): + # Add a condensation event + condensation = AgentCondensationObservation( + content='Trimming prompt to meet context window limitations' + ) + state.history.append(condensation) - # Add user message between condensation events - user_message = MessageAction(content='Please continue', wait_for_response=False) - user_message._source = EventSource.USER - state.history.append(user_message) - user_observation = NullObservation(content='') - state.history.append(user_observation) - - condensation2 = AgentCondensationObservation( - content='Trimming prompt to meet context window limitations' - ) - state.history.append(condensation2) - - # Add another user message - user_message2 = MessageAction(content='Keep going', wait_for_response=False) - user_message2._source = EventSource.USER - state.history.append(user_message2) - user_observation2 = NullObservation(content='') - state.history.append(user_observation2) - - condensation3 = AgentCondensationObservation( - content='Trimming prompt to meet context window limitations' - ) - state.history.append(condensation3) + # Add user message between condensation events (except after the last one) + if i < 9: + user_message = MessageAction( + content=f'Please continue {i}', wait_for_response=False + ) + user_message._source = EventSource.USER + state.history.append(user_message) + user_observation = NullObservation(content='') + state.history.append(user_observation) with patch('logging.Logger.warning') as mock_warning: assert stuck_detector.is_stuck(headless_mode=True) is True @@ -754,7 +739,7 @@ class TestStuckDetector: state = stuck_detector.state # Add condensation events first - for _ in range(3): + for _ in range(10): condensation = AgentCondensationObservation( content='Trimming prompt to meet context window limitations' )