From 6714000b2ce9fcfd1ed9006052eb31c28ed28d7d Mon Sep 17 00:00:00 2001 From: Boxuan Li Date: Tue, 14 May 2024 22:48:47 -0700 Subject: [PATCH] CodeActAgent: Fix iteration reminder (#1803) This PR includes three changes: 1) Iteration reminder should start with MAX_ITERATIONS from config rather than default value 100 2) In the first prompt, we should tell the LLM it has `MAX_ITERATIONS - 1` turns left, rather than `MAX_ITERATIONS - 2` 3) Remove legacy ITERATION_REMINDER config --- .github/workflows/run-integration-tests.yml | 1 - agenthub/codeact_agent/codeact_agent.py | 2 +- opendevin/controller/agent_controller.py | 2 +- opendevin/core/schema/config.py | 1 - .../mock/CodeActAgent/test_edits/prompt_001.log | 2 +- .../mock/CodeActAgent/test_edits/prompt_002.log | 4 ++-- .../mock/CodeActAgent/test_edits/prompt_003.log | 6 +++--- .../mock/CodeActAgent/test_ipython/prompt_001.log | 2 +- .../mock/CodeActAgent/test_ipython/prompt_002.log | 4 ++-- .../test_write_simple_script/prompt_001.log | 2 +- .../test_write_simple_script/prompt_002.log | 4 ++-- tests/integration/regenerate.sh | 10 ---------- 12 files changed, 14 insertions(+), 26 deletions(-) diff --git a/.github/workflows/run-integration-tests.yml b/.github/workflows/run-integration-tests.yml index 76c6fa4620..82b4a9355a 100644 --- a/.github/workflows/run-integration-tests.yml +++ b/.github/workflows/run-integration-tests.yml @@ -57,7 +57,6 @@ jobs: AGENT: ${{ matrix.agent }} MAX_ITERATIONS: 10 LLM_EMBEDDING_MODEL: ${{ matrix.embedding-model }} - REMIND_ITERATIONS: ${{ (matrix.agent == 'CodeActAgent') && 'true' || 'false' }} run: | rm -rf workspace mkdir workspace diff --git a/agenthub/codeact_agent/codeact_agent.py b/agenthub/codeact_agent/codeact_agent.py index 46a25e285d..3273e6d745 100644 --- a/agenthub/codeact_agent/codeact_agent.py +++ b/agenthub/codeact_agent/codeact_agent.py @@ -209,7 +209,7 @@ class CodeActAgent(Agent): latest_user_message = [m for m in self.messages if m['role'] == 'user'][-1] if latest_user_message: latest_user_message['content'] += ( - f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration - 1} turns left to complete the task.' + f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task.' ) response = self.llm.completion( diff --git a/opendevin/controller/agent_controller.py b/opendevin/controller/agent_controller.py index 479955496f..42ad4f509b 100644 --- a/opendevin/controller/agent_controller.py +++ b/opendevin/controller/agent_controller.py @@ -68,7 +68,7 @@ class AgentController: """ self.id = sid self.agent = agent - self.state = State(inputs=inputs or {}) + self.state = State(inputs=inputs or {}, max_iterations=max_iterations) self.event_stream = event_stream self.event_stream.subscribe( EventStreamSubscriber.AGENT_CONTROLLER, self.on_event diff --git a/opendevin/core/schema/config.py b/opendevin/core/schema/config.py index a58ef8ade4..f409b9940a 100644 --- a/opendevin/core/schema/config.py +++ b/opendevin/core/schema/config.py @@ -31,7 +31,6 @@ class ConfigType(str, Enum): AGENT_MEMORY_MAX_THREADS = 'AGENT_MEMORY_MAX_THREADS' AGENT_MEMORY_ENABLED = 'AGENT_MEMORY_ENABLED' MAX_ITERATIONS = 'MAX_ITERATIONS' - REMIND_ITERATIONS = 'REMIND_ITERATIONS' MAX_CHARS = 'MAX_CHARS' AGENT = 'AGENT' E2B_API_KEY = 'E2B_API_KEY' diff --git a/tests/integration/mock/CodeActAgent/test_edits/prompt_001.log b/tests/integration/mock/CodeActAgent/test_edits/prompt_001.log index 22c72ff477..0c16e702dd 100644 --- a/tests/integration/mock/CodeActAgent/test_edits/prompt_001.log +++ b/tests/integration/mock/CodeActAgent/test_edits/prompt_001.log @@ -224,4 +224,4 @@ NOW, LET'S START! Fix typos in bad.txt. Do not ask me for confirmation at any point. -ENVIRONMENT REMINDER: You have 98 turns left to complete the task. +ENVIRONMENT REMINDER: You have 9 turns left to complete the task. diff --git a/tests/integration/mock/CodeActAgent/test_edits/prompt_002.log b/tests/integration/mock/CodeActAgent/test_edits/prompt_002.log index 912b44ca85..91e702a602 100644 --- a/tests/integration/mock/CodeActAgent/test_edits/prompt_002.log +++ b/tests/integration/mock/CodeActAgent/test_edits/prompt_002.log @@ -224,7 +224,7 @@ NOW, LET'S START! Fix typos in bad.txt. Do not ask me for confirmation at any point. -ENVIRONMENT REMINDER: You have 98 turns left to complete the task. +ENVIRONMENT REMINDER: You have 9 turns left to complete the task. ---------- @@ -244,4 +244,4 @@ OBSERVATION: 4:Enjoy! [Command -1 finished with exit code 0]] -ENVIRONMENT REMINDER: You have 97 turns left to complete the task. +ENVIRONMENT REMINDER: You have 8 turns left to complete the task. diff --git a/tests/integration/mock/CodeActAgent/test_edits/prompt_003.log b/tests/integration/mock/CodeActAgent/test_edits/prompt_003.log index c0581fd310..163b657ee0 100644 --- a/tests/integration/mock/CodeActAgent/test_edits/prompt_003.log +++ b/tests/integration/mock/CodeActAgent/test_edits/prompt_003.log @@ -224,7 +224,7 @@ NOW, LET'S START! Fix typos in bad.txt. Do not ask me for confirmation at any point. -ENVIRONMENT REMINDER: You have 98 turns left to complete the task. +ENVIRONMENT REMINDER: You have 9 turns left to complete the task. ---------- @@ -244,7 +244,7 @@ OBSERVATION: 4:Enjoy! [Command -1 finished with exit code 0]] -ENVIRONMENT REMINDER: You have 97 turns left to complete the task. +ENVIRONMENT REMINDER: You have 8 turns left to complete the task. ---------- @@ -276,4 +276,4 @@ File updated. Please review the changes and make sure they are correct (correct File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary. [Command -1 finished with exit code 0]] -ENVIRONMENT REMINDER: You have 96 turns left to complete the task. +ENVIRONMENT REMINDER: You have 7 turns left to complete the task. diff --git a/tests/integration/mock/CodeActAgent/test_ipython/prompt_001.log b/tests/integration/mock/CodeActAgent/test_ipython/prompt_001.log index c278b62882..094adb0963 100644 --- a/tests/integration/mock/CodeActAgent/test_ipython/prompt_001.log +++ b/tests/integration/mock/CodeActAgent/test_ipython/prompt_001.log @@ -224,4 +224,4 @@ NOW, LET'S START! Use Jupyter IPython to write a text file containing 'hello world' to '/workspace/test.txt'. Do not ask me for confirmation at any point. -ENVIRONMENT REMINDER: You have 98 turns left to complete the task. +ENVIRONMENT REMINDER: You have 9 turns left to complete the task. diff --git a/tests/integration/mock/CodeActAgent/test_ipython/prompt_002.log b/tests/integration/mock/CodeActAgent/test_ipython/prompt_002.log index 1e2108e22f..857e6c825c 100644 --- a/tests/integration/mock/CodeActAgent/test_ipython/prompt_002.log +++ b/tests/integration/mock/CodeActAgent/test_ipython/prompt_002.log @@ -224,7 +224,7 @@ NOW, LET'S START! Use Jupyter IPython to write a text file containing 'hello world' to '/workspace/test.txt'. Do not ask me for confirmation at any point. -ENVIRONMENT REMINDER: You have 98 turns left to complete the task. +ENVIRONMENT REMINDER: You have 9 turns left to complete the task. ---------- @@ -241,4 +241,4 @@ with open('/workspace/test.txt', 'w') as file: OBSERVATION: [Code executed successfully with no output] -ENVIRONMENT REMINDER: You have 97 turns left to complete the task. +ENVIRONMENT REMINDER: You have 8 turns left to complete the task. diff --git a/tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_001.log b/tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_001.log index e8af801abf..c0d9297fa8 100644 --- a/tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_001.log +++ b/tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_001.log @@ -224,4 +224,4 @@ NOW, LET'S START! Write a shell script 'hello.sh' that prints 'hello'. Do not ask me for confirmation at any point. -ENVIRONMENT REMINDER: You have 98 turns left to complete the task. +ENVIRONMENT REMINDER: You have 9 turns left to complete the task. diff --git a/tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_002.log b/tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_002.log index 8d71bcb66b..8347f7f305 100644 --- a/tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_002.log +++ b/tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_002.log @@ -224,7 +224,7 @@ NOW, LET'S START! Write a shell script 'hello.sh' that prints 'hello'. Do not ask me for confirmation at any point. -ENVIRONMENT REMINDER: You have 98 turns left to complete the task. +ENVIRONMENT REMINDER: You have 9 turns left to complete the task. ---------- @@ -240,4 +240,4 @@ OBSERVATION: [Command -1 finished with exit code 0]] -ENVIRONMENT REMINDER: You have 97 turns left to complete the task. +ENVIRONMENT REMINDER: You have 8 turns left to complete the task. diff --git a/tests/integration/regenerate.sh b/tests/integration/regenerate.sh index 1cb26d27e6..a0f05debcc 100755 --- a/tests/integration/regenerate.sh +++ b/tests/integration/regenerate.sh @@ -4,7 +4,6 @@ set -eo pipefail run_test() { SANDBOX_TYPE=$SANDBOX_TYPE \ WORKSPACE_BASE=$WORKSPACE_BASE \ - REMIND_ITERATIONS=$remind_iterations \ MAX_ITERATIONS=$MAX_ITERATIONS \ WORKSPACE_MOUNT_PATH=$WORKSPACE_MOUNT_PATH \ AGENT=$agent \ @@ -27,7 +26,6 @@ SANDBOX_TYPE="ssh" MAX_ITERATIONS=10 agents=("MonologueAgent" "CodeActAgent" "PlannerAgent" "SWEAgent") -remind_iterations_config=(false true false false) tasks=( "Fix typos in bad.txt." "Write a shell script 'hello.sh' that prints 'hello'." @@ -42,11 +40,6 @@ test_names=( num_of_tests=${#test_names[@]} num_of_agents=${#agents[@]} -if [ "$num_of_agents" -ne "${#remind_iterations_config[@]}" ]; then - echo "Every agent must have its own remind_iterations_config" - exit 1 -fi - if [ "$num_of_tests" -ne "${#test_names[@]}" ]; then echo "Every task must correspond to one test case" exit 1 @@ -109,9 +102,6 @@ for ((i = 0; i < num_of_tests; i++)); do WORKSPACE_BASE=$WORKSPACE_BASE \ DEBUG=true \ WORKSPACE_MOUNT_PATH=$WORKSPACE_MOUNT_PATH AGENT=$agent \ - REMIND_ITERATIONS=$remind_iterations \ - WORKSPACE_MOUNT_PATH=$WORKSPACE_MOUNT_PATH \ - AGENT=$agent \ poetry run python ./opendevin/core/main.py \ -i $MAX_ITERATIONS \ -t "$task Do not ask me for confirmation at any point." \