From a9a2f10170cc01a684fe4befcbcf391b8afc20de Mon Sep 17 00:00:00 2001 From: Boxuan Li Date: Sat, 8 Jun 2024 23:12:30 -0700 Subject: [PATCH] Revamp AgentRejectAction and allow ManagerAgent to handle rejection (#1735) * Fix AgentRejectAction handling * Add ManagerAgent to integration tests * Fix regenerate.sh * Fix merge * Update README for micro-agents * Add test reject to regenerate.sh * regenerate.sh: Add support for running a specific test and/or agent * Refine reject schema, and allow ManagerAgent to handle reject * Add test artifacts for test_simple_task_rejection * Fix manager agent tests * Fix README * test_simple_task_rejection: check final agent state * Integration test: exit if mock prompt not found * Update test_simple_task_rejection tests * Fix test_edits test artifacts after prompt update * Fix ManagerAgent test_edits * WIP * Fix tests * update test_edits for ManagerAgent * Skip local sandbox for reject test * Fix test comparison --- agenthub/micro/README.md | 3 + .../micro/_instructions/actions/reject.md | 2 +- agenthub/micro/commit_writer/agent.yaml | 1 + agenthub/micro/commit_writer/prompt.md | 2 +- agenthub/micro/manager/agent.yaml | 4 +- agenthub/micro/manager/prompt.md | 13 +++ opendevin/controller/agent_controller.py | 6 +- opendevin/core/main.py | 1 + opendevin/core/schema/agent.py | 4 + tests/integration/README.md | 9 ++ tests/integration/conftest.py | 15 ++- .../ManagerAgent/test_edits/prompt_001.log | 15 +++ .../ManagerAgent/test_edits/prompt_004.log | 2 +- .../ManagerAgent/test_edits/prompt_005.log | 2 +- .../ManagerAgent/test_edits/prompt_006.log | 17 +++- .../ManagerAgent/test_edits/response_003.log | 2 +- .../ManagerAgent/test_edits/response_005.log | 6 +- .../ManagerAgent/test_edits/response_006.log | 6 +- .../test_simple_task_rejection/prompt_001.log | 94 +++++++++++++++++++ .../test_simple_task_rejection/prompt_002.log | 50 ++++++++++ .../test_simple_task_rejection/prompt_003.log | 50 ++++++++++ .../test_simple_task_rejection/prompt_004.log | 50 ++++++++++ .../test_simple_task_rejection/prompt_005.log | 50 ++++++++++ .../test_simple_task_rejection/prompt_006.log | 50 ++++++++++ .../test_simple_task_rejection/prompt_007.log | 50 ++++++++++ .../test_simple_task_rejection/prompt_008.log | 94 +++++++++++++++++++ .../response_001.log | 9 ++ .../response_002.log | 8 ++ .../response_003.log | 9 ++ .../response_004.log | 9 ++ .../response_005.log | 8 ++ .../response_006.log | 8 ++ .../response_007.log | 10 ++ .../response_008.log | 10 ++ tests/integration/regenerate.sh | 2 + tests/integration/test_agent.py | 22 +++++ 36 files changed, 675 insertions(+), 18 deletions(-) create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_001.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_002.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_003.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_004.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_005.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_006.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_007.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_008.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_001.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_002.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_003.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_004.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_005.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_006.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_007.log create mode 100644 tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_008.log diff --git a/agenthub/micro/README.md b/agenthub/micro/README.md index 68be0a8a9a..959a8aad25 100644 --- a/agenthub/micro/README.md +++ b/agenthub/micro/README.md @@ -12,3 +12,6 @@ in the following structure: Note that `prompt.md` could use jinja2 template syntax. During runtime, `prompt.md` is loaded and rendered, and used together with `agent.yaml` to initialize a micro-agent. + +Micro-agents can be used independently. You can also use `ManagerAgent` which knows +how to coordinate the agents and collaboratively finish a task. diff --git a/agenthub/micro/_instructions/actions/reject.md b/agenthub/micro/_instructions/actions/reject.md index e4cb730635..8b89a9a20c 100644 --- a/agenthub/micro/_instructions/actions/reject.md +++ b/agenthub/micro/_instructions/actions/reject.md @@ -1,2 +1,2 @@ * `reject` - reject the task. Arguments: - * `outputs` - a dictionary representing the outputs of your task, if any + * `outputs` - a dictionary with only a `reason` attribute diff --git a/agenthub/micro/commit_writer/agent.yaml b/agenthub/micro/commit_writer/agent.yaml index 107924964d..1228838c9e 100644 --- a/agenthub/micro/commit_writer/agent.yaml +++ b/agenthub/micro/commit_writer/agent.yaml @@ -3,3 +3,4 @@ description: "Write a git commit message for files in the git staging area" inputs: {} outputs: answer: string + reason: string diff --git a/agenthub/micro/commit_writer/prompt.md b/agenthub/micro/commit_writer/prompt.md index 38322ba077..a99e4e44bc 100644 --- a/agenthub/micro/commit_writer/prompt.md +++ b/agenthub/micro/commit_writer/prompt.md @@ -14,7 +14,7 @@ changes. The commit message should include: You should find the diff using `git diff --cached`, compile a commit message, and call the `finish` action with `outputs.answer` set to the answer. If current repo is not a valid git repo, or there is no diff in the staging area, please call -the `reject` action with `outputs.answer` set to the reason. +the `reject` action. ## History {{ instructions.history_truncated }} diff --git a/agenthub/micro/manager/agent.yaml b/agenthub/micro/manager/agent.yaml index 5980f6f03f..d4e956618e 100644 --- a/agenthub/micro/manager/agent.yaml +++ b/agenthub/micro/manager/agent.yaml @@ -3,4 +3,6 @@ description: Delegates tasks to microagents based on their area of expertise generates: Action inputs: task: string -outputs: {} +outputs: + summary: string # if finished + reason: string # if rejected diff --git a/agenthub/micro/manager/prompt.md b/agenthub/micro/manager/prompt.md index 27080ccf04..0660ec3b2a 100644 --- a/agenthub/micro/manager/prompt.md +++ b/agenthub/micro/manager/prompt.md @@ -7,6 +7,15 @@ can do the actual work. A description of each agent is provided below. You MUST select one of the delegates below to move towards accomplishing the task, and you MUST provide the correct inputs for the delegate you select. +Note: the delegated agent either returns "finish" or "reject". +- If the action is "finish", but the full task is not done yet, you should +continue to delegate to one of the agents below to until the full task is finished. +- If the action is "reject", it means the delegated agent is not capable of the +task you send to. You should revisit the input you send to the delegate, and consider +whether any other delegate would be able to solve the task. If you cannot find +a proper delegate agent, or the delegate attempts keep failing, call the `reject` +action. + ## Agents {% for name, details in delegates.items() %} ### {{ name }} @@ -19,9 +28,13 @@ provide the correct inputs for the delegate you select. {{ instructions.history_truncated }} {{ history_to_json(state.history[-10:]) }} +If the last item in the history is an error, you should try to fix it. If you +cannot fix it, call the `reject` action. + ## Available Actions {{ instructions.actions.delegate }} {{ instructions.actions.finish }} +{{ instructions.actions.reject }} ## Format {{ instructions.format.action }} diff --git a/opendevin/controller/agent_controller.py b/opendevin/controller/agent_controller.py index 7b1ea34193..663ef80824 100644 --- a/opendevin/controller/agent_controller.py +++ b/opendevin/controller/agent_controller.py @@ -19,6 +19,7 @@ from opendevin.events.action import ( AddTaskAction, AgentDelegateAction, AgentFinishAction, + AgentRejectAction, ChangeAgentStateAction, MessageAction, ModifyTaskAction, @@ -164,6 +165,9 @@ class AgentController: elif isinstance(event, AgentFinishAction): self.state.outputs = event.outputs # type: ignore[attr-defined] await self.set_agent_state_to(AgentState.FINISHED) + elif isinstance(event, AgentRejectAction): + self.state.outputs = event.outputs # type: ignore[attr-defined] + await self.set_agent_state_to(AgentState.REJECTED) elif isinstance(event, Observation): if self._pending_action and self._pending_action.id == event.cause: await self.add_history(self._pending_action, event) @@ -252,7 +256,7 @@ class AgentController: # propagate error state until an agent or user can handle it await self.set_agent_state_to(AgentState.ERROR) return - delegate_done = delegate_state == AgentState.FINISHED + delegate_done = delegate_state in (AgentState.FINISHED, AgentState.REJECTED) if delegate_done: logger.info( f'[Agent Controller {self.id}] Delegate agent has finished execution' diff --git a/opendevin/core/main.py b/opendevin/core/main.py index af07025839..5322e6c0c4 100644 --- a/opendevin/core/main.py +++ b/opendevin/core/main.py @@ -127,6 +127,7 @@ async def main( event_stream.subscribe(EventStreamSubscriber.MAIN, on_event) while controller.get_agent_state() not in [ AgentState.FINISHED, + AgentState.REJECTED, AgentState.ERROR, AgentState.PAUSED, AgentState.STOPPED, diff --git a/opendevin/core/schema/agent.py b/opendevin/core/schema/agent.py index c9e699e9ba..64742b8f2a 100644 --- a/opendevin/core/schema/agent.py +++ b/opendevin/core/schema/agent.py @@ -30,6 +30,10 @@ class AgentState(str, Enum): """The agent is finished with the current task. """ + REJECTED = 'rejected' + """The agent rejects the task. + """ + ERROR = 'error' """An error occurred during the task. """ diff --git a/tests/integration/README.md b/tests/integration/README.md index a94d876fd4..073769b402 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -55,6 +55,15 @@ TEST_ONLY=true ./tests/integration/regenerate.sh to run all integration tests until the first failure. +If you only want to run a specific test, set environment variable +`ONLY_TEST_NAME` to the test name. If you only want to run a specific agent, +set environment variable `ONLY_TEST_AGENT` to the agent. You could also use both, +e.g. + +```bash +TEST_ONLY=true ONLY_TEST_NAME="test_simple_task_rejection" ONLY_TEST_AGENT="ManagerAgent" ./tests/integration/regenerate.sh +``` + ## Regenerate Integration Tests When you make changes to an agent's prompt, the integration tests will fail. You'll need to regenerate them diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 488ac3d10c..fc8f2b5a93 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,9 +1,9 @@ import io import os import re +import subprocess import sys import tempfile -import subprocess from functools import partial from http.server import HTTPServer, SimpleHTTPRequestHandler from threading import Thread @@ -18,7 +18,8 @@ workspace_path = os.getenv('WORKSPACE_BASE') def filter_out_symbols(input): - return ' '.join([char for char in input if char.isalnum()]) + input = re.sub(r'\\n|\\r\\n|\\r|\s+', '', input) + return input def get_log_id(prompt_log_name): @@ -84,13 +85,19 @@ def get_mock_response(test_name: str, messages: str, id: int) -> str: print('Mismatched Prompt File path', prompt_file_path) print('---' * 10) # Create a temporary file to store messages - with tempfile.NamedTemporaryFile(delete=False, mode='w', encoding='utf-8') as tmp_file: + with tempfile.NamedTemporaryFile( + delete=False, mode='w', encoding='utf-8' + ) as tmp_file: tmp_file_path = tmp_file.name tmp_file.write(messages) try: # Use diff command to compare files and capture the output - result = subprocess.run(['diff', '-u', prompt_file_path, tmp_file_path], capture_output=True, text=True) + result = subprocess.run( + ['diff', '-u', prompt_file_path, tmp_file_path], + capture_output=True, + text=True, + ) if result.returncode != 0: print('Diff:') print(result.stdout) diff --git a/tests/integration/mock/ManagerAgent/test_edits/prompt_001.log b/tests/integration/mock/ManagerAgent/test_edits/prompt_001.log index ebd8e635b1..0969847375 100644 --- a/tests/integration/mock/ManagerAgent/test_edits/prompt_001.log +++ b/tests/integration/mock/ManagerAgent/test_edits/prompt_001.log @@ -11,6 +11,15 @@ can do the actual work. A description of each agent is provided below. You MUST select one of the delegates below to move towards accomplishing the task, and you MUST provide the correct inputs for the delegate you select. +Note: the delegated agent either returns "finish" or "reject". +- If the action is "finish", but the full task is not done yet, you should +continue to delegate to one of the agents below to until the full task is finished. +- If the action is "reject", it means the delegated agent is not capable of the +task you send to. You should revisit the input you send to the delegate, and consider +whether any other delegate would be able to solve the task. If you cannot find +a proper delegate agent, or the delegate attempts keep failing, call the `reject` +action. + ## Agents ### CoderAgent @@ -62,6 +71,9 @@ They are time-ordered, with your most recent action at the bottom. [[{"source": "user", "action": "message", "args": {"content": "Fix typos in bad.txt. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}]] +If the last item in the history is an error, you should try to fix it. If you +cannot fix it, call the `reject` action. + ## Available Actions * `delegate` - send a task to another agent from the list provided. Arguments: * `agent` - the agent to which the task is delegated. MUST match a name in the list of agents provided. @@ -70,6 +82,9 @@ They are time-ordered, with your most recent action at the bottom. * `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: * `outputs` - a dictionary representing the outputs of your task, if any +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + ## Format Your response MUST be in JSON format. It must be an object, and it must contain two fields: diff --git a/tests/integration/mock/ManagerAgent/test_edits/prompt_004.log b/tests/integration/mock/ManagerAgent/test_edits/prompt_004.log index 9ac378a76c..f19d42787e 100644 --- a/tests/integration/mock/ManagerAgent/test_edits/prompt_004.log +++ b/tests/integration/mock/ManagerAgent/test_edits/prompt_004.log @@ -52,7 +52,7 @@ as well as observations you've made. This only includes the MOST RECENT actions and observations--more may have happened before that. They are time-ordered, with your most recent action at the bottom. -[[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}], [{"source": "agent", "action": "message", "args": {"content": "The content of 'bad.txt' has been read. The next step is to identify and fix the typos.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}]] +[[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}], [{"source": "agent", "action": "message", "args": {"content": "The content of 'bad.txt' has been read. The following typos have been identified:\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\nI will now proceed to fix these typos and overwrite the file with the corrected content.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}]] ## Format Your response MUST be in JSON format. It must be an object, and it must contain two fields: diff --git a/tests/integration/mock/ManagerAgent/test_edits/prompt_005.log b/tests/integration/mock/ManagerAgent/test_edits/prompt_005.log index bba53b817c..cda3633d92 100644 --- a/tests/integration/mock/ManagerAgent/test_edits/prompt_005.log +++ b/tests/integration/mock/ManagerAgent/test_edits/prompt_005.log @@ -52,7 +52,7 @@ as well as observations you've made. This only includes the MOST RECENT actions and observations--more may have happened before that. They are time-ordered, with your most recent action at the bottom. -[[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}], [{"source": "agent", "action": "message", "args": {"content": "The content of 'bad.txt' has been read. The next step is to identify and fix the typos.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}], [{"source": "agent", "action": "write", "args": {"path": "./bad.txt", "content": "This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!\n", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "./bad.txt"}}]] +[[{"source": "agent", "action": "read", "args": {"path": "./bad.txt", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "read", "content": "This is a stupid typoo.\nReally?\nNo mor typos!\nEnjoy!\n", "extras": {"path": "./bad.txt"}}], [{"source": "agent", "action": "message", "args": {"content": "The content of 'bad.txt' has been read. The following typos have been identified:\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\nI will now proceed to fix these typos and overwrite the file with the corrected content.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}], [{"source": "agent", "action": "write", "args": {"path": "./bad.txt", "content": "This is a stupid typo.\nReally?\nNo more typos!\nEnjoy!\n", "start": 0, "end": -1, "thought": ""}}, {"source": "agent", "observation": "write", "content": "", "extras": {"path": "./bad.txt"}}]] ## Format Your response MUST be in JSON format. It must be an object, and it must contain two fields: diff --git a/tests/integration/mock/ManagerAgent/test_edits/prompt_006.log b/tests/integration/mock/ManagerAgent/test_edits/prompt_006.log index b64083b4fc..14dbab3c5b 100644 --- a/tests/integration/mock/ManagerAgent/test_edits/prompt_006.log +++ b/tests/integration/mock/ManagerAgent/test_edits/prompt_006.log @@ -11,6 +11,15 @@ can do the actual work. A description of each agent is provided below. You MUST select one of the delegates below to move towards accomplishing the task, and you MUST provide the correct inputs for the delegate you select. +Note: the delegated agent either returns "finish" or "reject". +- If the action is "finish", but the full task is not done yet, you should +continue to delegate to one of the agents below to until the full task is finished. +- If the action is "reject", it means the delegated agent is not capable of the +task you send to. You should revisit the input you send to the delegate, and consider +whether any other delegate would be able to solve the task. If you cannot find +a proper delegate agent, or the delegate attempts keep failing, call the `reject` +action. + ## Agents ### CoderAgent @@ -60,7 +69,10 @@ as well as observations you've made. This only includes the MOST RECENT actions and observations--more may have happened before that. They are time-ordered, with your most recent action at the bottom. -[[{"source": "user", "action": "message", "args": {"content": "Fix typos in bad.txt. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}], [{"source": "agent", "action": "delegate", "args": {"agent": "TypoFixerAgent", "inputs": {"task": "Fix typos in bad.txt"}, "thought": ""}}, {"observation": "null", "content": "", "extras": {}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "delegate", "content": "", "extras": {"outputs": {"summary": {"file": "./bad.txt", "typos_fixed": [{"original": "typoo", "fixed": "typo"}, {"original": "mor", "fixed": "more"}]}}}}]] +[[{"source": "user", "action": "message", "args": {"content": "Fix typos in bad.txt. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}], [{"source": "agent", "action": "delegate", "args": {"agent": "TypoFixerAgent", "inputs": {"task": "Fix typos in bad.txt"}, "thought": ""}}, {"observation": "null", "content": "", "extras": {}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "delegate", "content": "", "extras": {"outputs": {"summary": {"file": "bad.txt", "typos_fixed": [{"original": "typoo", "corrected": "typo"}, {"original": "mor", "corrected": "more"}]}}}}]] + +If the last item in the history is an error, you should try to fix it. If you +cannot fix it, call the `reject` action. ## Available Actions * `delegate` - send a task to another agent from the list provided. Arguments: @@ -70,6 +82,9 @@ They are time-ordered, with your most recent action at the bottom. * `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: * `outputs` - a dictionary representing the outputs of your task, if any +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + ## Format Your response MUST be in JSON format. It must be an object, and it must contain two fields: diff --git a/tests/integration/mock/ManagerAgent/test_edits/response_003.log b/tests/integration/mock/ManagerAgent/test_edits/response_003.log index cad32c39c8..4e15b5ebdd 100644 --- a/tests/integration/mock/ManagerAgent/test_edits/response_003.log +++ b/tests/integration/mock/ManagerAgent/test_edits/response_003.log @@ -1,7 +1,7 @@ { "action": "message", "args": { - "content": "The content of 'bad.txt' has been read. The next step is to identify and fix the typos.", + "content": "The content of 'bad.txt' has been read. The following typos have been identified:\n1. 'typoo' should be 'typo'\n2. 'mor' should be 'more'\nI will now proceed to fix these typos and overwrite the file with the corrected content.", "wait_for_response": false } } diff --git a/tests/integration/mock/ManagerAgent/test_edits/response_005.log b/tests/integration/mock/ManagerAgent/test_edits/response_005.log index 9cdbe74465..b414957c52 100644 --- a/tests/integration/mock/ManagerAgent/test_edits/response_005.log +++ b/tests/integration/mock/ManagerAgent/test_edits/response_005.log @@ -3,15 +3,15 @@ "args": { "outputs": { "summary": { - "file": "./bad.txt", + "file": "bad.txt", "typos_fixed": [ { "original": "typoo", - "fixed": "typo" + "corrected": "typo" }, { "original": "mor", - "fixed": "more" + "corrected": "more" } ] } diff --git a/tests/integration/mock/ManagerAgent/test_edits/response_006.log b/tests/integration/mock/ManagerAgent/test_edits/response_006.log index 633517fbfe..4afbff8867 100644 --- a/tests/integration/mock/ManagerAgent/test_edits/response_006.log +++ b/tests/integration/mock/ManagerAgent/test_edits/response_006.log @@ -3,15 +3,15 @@ "action": "finish", "args": { "outputs": { - "file": "./bad.txt", + "file": "bad.txt", "typos_fixed": [ { "original": "typoo", - "fixed": "typo" + "corrected": "typo" }, { "original": "mor", - "fixed": "more" + "corrected": "more" } ] } diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_001.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_001.log new file mode 100644 index 0000000000..7e063d1b72 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_001.log @@ -0,0 +1,94 @@ + + +---------- + +# Task +You are in charge of accomplishing the following task: +Write a git commit message for the current staging area. Do not ask me for confirmation at any point. + +In order to accomplish this goal, you must delegate tasks to one or more agents, who +can do the actual work. A description of each agent is provided below. You MUST +select one of the delegates below to move towards accomplishing the task, and you MUST +provide the correct inputs for the delegate you select. + +Note: the delegated agent either returns "finish" or "reject". +- If the action is "finish", but the full task is not done yet, you should +continue to delegate to one of the agents below to until the full task is finished. +- If the action is "reject", it means the delegated agent is not capable of the +task you send to. You should revisit the input you send to the delegate, and consider +whether any other delegate would be able to solve the task. If you cannot find +a proper delegate agent, or the delegate attempts keep failing, call the `reject` +action. + +## Agents + +### CoderAgent +Given a particular task, and a detailed description of the codebase, accomplishes the task +#### Inputs +{"task": "string", "summary": "string"} + +### CommitWriterAgent +Write a git commit message for files in the git staging area +#### Inputs +{} + +### MathAgent +Solves simple and complex math problems using python +#### Inputs +{"task": "string"} + +### PostgresAgent +Writes and maintains PostgreSQL migrations +#### Inputs +{"task": "string"} + +### RepoExplorerAgent +Generates a detailed summary of an existing codebase +#### Inputs +{} + +### StudyRepoForTaskAgent +Given a particular task, finds and describes all relevant parts of the codebase +#### Inputs +{"task": "string"} + +### TypoFixerAgent +Fixes typos in files in the current working directory +#### Inputs +{"task": "string"} + +### VerifierAgent +Given a particular task, verifies that the task has been completed +#### Inputs +{"task": "string"} + + +## History +Here is a recent history of actions you've taken in service of this plan, +as well as observations you've made. This only includes the MOST RECENT +actions and observations--more may have happened before that. +They are time-ordered, with your most recent action at the bottom. + +[[{"source": "user", "action": "message", "args": {"content": "Write a git commit message for the current staging area. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}]] + +If the last item in the history is an error, you should try to fix it. If you +cannot fix it, call the `reject` action. + +## Available Actions +* `delegate` - send a task to another agent from the list provided. Arguments: + * `agent` - the agent to which the task is delegated. MUST match a name in the list of agents provided. + * `inputs` - a dictionary of input parameters to the agent, as specified in the list + +* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: + * `outputs` - a dictionary representing the outputs of your task, if any + +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + + +## Format +Your response MUST be in JSON format. It must be an object, and it must contain two fields: +* `action`, which is one of the actions specified here +* `args`, which is a map of key-value pairs, specifying the arguments for that action + +You MUST NOT include any other text besides the JSON response diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_002.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_002.log new file mode 100644 index 0000000000..9b7b6459a6 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_002.log @@ -0,0 +1,50 @@ + + +---------- + +# Task +You are a responsible software engineer and always write good commit messages. + +Please analyze the diff in the staging area, understand the context and content +of the updates from the diff only. Identify key elements like: +- Which files are affected? +- What types of changes were made (e.g., new features, bug fixes, refactoring, documentation, testing)? + +Then you should generate a commit message that succinctly summarizes the staged +changes. The commit message should include: +- A summary line that clearly states the purpose of the changes. +- Optionally, a detailed description if the changes are complex or need further explanation. + +You should find the diff using `git diff --cached`, compile a commit message, +and call the `finish` action with `outputs.answer` set to the answer. If current +repo is not a valid git repo, or there is no diff in the staging area, please call +the `reject` action. + +## History +Here is a recent history of actions you've taken in service of this plan, +as well as observations you've made. This only includes the MOST RECENT +actions and observations--more may have happened before that. +They are time-ordered, with your most recent action at the bottom. + +[] + +If the last item in the history is an error, you should try to fix it. + +## Available Actions +* `run` - runs a command on the command line in a Linux shell. Arguments: + * `command` - the command to run + * `background` - if true, run the command in the background, so that other commands can be run concurrently. Useful for e.g. starting a server. You won't be able to see the logs. You don't need to end the command with `&`, just set this to true. + +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + +* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: + * `outputs` - a dictionary representing the outputs of your task, if any + + +## Format +Your response MUST be in JSON format. It must be an object, and it must contain two fields: +* `action`, which is one of the actions specified here +* `args`, which is a map of key-value pairs, specifying the arguments for that action + +You MUST NOT include any other text besides the JSON response diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_003.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_003.log new file mode 100644 index 0000000000..e359bedf35 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_003.log @@ -0,0 +1,50 @@ + + +---------- + +# Task +You are a responsible software engineer and always write good commit messages. + +Please analyze the diff in the staging area, understand the context and content +of the updates from the diff only. Identify key elements like: +- Which files are affected? +- What types of changes were made (e.g., new features, bug fixes, refactoring, documentation, testing)? + +Then you should generate a commit message that succinctly summarizes the staged +changes. The commit message should include: +- A summary line that clearly states the purpose of the changes. +- Optionally, a detailed description if the changes are complex or need further explanation. + +You should find the diff using `git diff --cached`, compile a commit message, +and call the `finish` action with `outputs.answer` set to the answer. If current +repo is not a valid git repo, or there is no diff in the staging area, please call +the `reject` action. + +## History +Here is a recent history of actions you've taken in service of this plan, +as well as observations you've made. This only includes the MOST RECENT +actions and observations--more may have happened before that. +They are time-ordered, with your most recent action at the bottom. + +[[{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}]] + +If the last item in the history is an error, you should try to fix it. + +## Available Actions +* `run` - runs a command on the command line in a Linux shell. Arguments: + * `command` - the command to run + * `background` - if true, run the command in the background, so that other commands can be run concurrently. Useful for e.g. starting a server. You won't be able to see the logs. You don't need to end the command with `&`, just set this to true. + +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + +* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: + * `outputs` - a dictionary representing the outputs of your task, if any + + +## Format +Your response MUST be in JSON format. It must be an object, and it must contain two fields: +* `action`, which is one of the actions specified here +* `args`, which is a map of key-value pairs, specifying the arguments for that action + +You MUST NOT include any other text besides the JSON response diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_004.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_004.log new file mode 100644 index 0000000000..8ad9f831a7 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_004.log @@ -0,0 +1,50 @@ + + +---------- + +# Task +You are a responsible software engineer and always write good commit messages. + +Please analyze the diff in the staging area, understand the context and content +of the updates from the diff only. Identify key elements like: +- Which files are affected? +- What types of changes were made (e.g., new features, bug fixes, refactoring, documentation, testing)? + +Then you should generate a commit message that succinctly summarizes the staged +changes. The commit message should include: +- A summary line that clearly states the purpose of the changes. +- Optionally, a detailed description if the changes are complex or need further explanation. + +You should find the diff using `git diff --cached`, compile a commit message, +and call the `finish` action with `outputs.answer` set to the answer. If current +repo is not a valid git repo, or there is no diff in the staging area, please call +the `reject` action. + +## History +Here is a recent history of actions you've taken in service of this plan, +as well as observations you've made. This only includes the MOST RECENT +actions and observations--more may have happened before that. +They are time-ordered, with your most recent action at the bottom. + +[[{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}], [{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}]] + +If the last item in the history is an error, you should try to fix it. + +## Available Actions +* `run` - runs a command on the command line in a Linux shell. Arguments: + * `command` - the command to run + * `background` - if true, run the command in the background, so that other commands can be run concurrently. Useful for e.g. starting a server. You won't be able to see the logs. You don't need to end the command with `&`, just set this to true. + +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + +* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: + * `outputs` - a dictionary representing the outputs of your task, if any + + +## Format +Your response MUST be in JSON format. It must be an object, and it must contain two fields: +* `action`, which is one of the actions specified here +* `args`, which is a map of key-value pairs, specifying the arguments for that action + +You MUST NOT include any other text besides the JSON response diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_005.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_005.log new file mode 100644 index 0000000000..0ca8752399 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_005.log @@ -0,0 +1,50 @@ + + +---------- + +# Task +You are a responsible software engineer and always write good commit messages. + +Please analyze the diff in the staging area, understand the context and content +of the updates from the diff only. Identify key elements like: +- Which files are affected? +- What types of changes were made (e.g., new features, bug fixes, refactoring, documentation, testing)? + +Then you should generate a commit message that succinctly summarizes the staged +changes. The commit message should include: +- A summary line that clearly states the purpose of the changes. +- Optionally, a detailed description if the changes are complex or need further explanation. + +You should find the diff using `git diff --cached`, compile a commit message, +and call the `finish` action with `outputs.answer` set to the answer. If current +repo is not a valid git repo, or there is no diff in the staging area, please call +the `reject` action. + +## History +Here is a recent history of actions you've taken in service of this plan, +as well as observations you've made. This only includes the MOST RECENT +actions and observations--more may have happened before that. +They are time-ordered, with your most recent action at the bottom. + +[[{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}], [{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}], [{"source": "agent", "action": "run", "args": {"command": "git diff --staged", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `staged'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --staged", "exit_code": 129}}]] + +If the last item in the history is an error, you should try to fix it. + +## Available Actions +* `run` - runs a command on the command line in a Linux shell. Arguments: + * `command` - the command to run + * `background` - if true, run the command in the background, so that other commands can be run concurrently. Useful for e.g. starting a server. You won't be able to see the logs. You don't need to end the command with `&`, just set this to true. + +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + +* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: + * `outputs` - a dictionary representing the outputs of your task, if any + + +## Format +Your response MUST be in JSON format. It must be an object, and it must contain two fields: +* `action`, which is one of the actions specified here +* `args`, which is a map of key-value pairs, specifying the arguments for that action + +You MUST NOT include any other text besides the JSON response diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_006.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_006.log new file mode 100644 index 0000000000..0ca8752399 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_006.log @@ -0,0 +1,50 @@ + + +---------- + +# Task +You are a responsible software engineer and always write good commit messages. + +Please analyze the diff in the staging area, understand the context and content +of the updates from the diff only. Identify key elements like: +- Which files are affected? +- What types of changes were made (e.g., new features, bug fixes, refactoring, documentation, testing)? + +Then you should generate a commit message that succinctly summarizes the staged +changes. The commit message should include: +- A summary line that clearly states the purpose of the changes. +- Optionally, a detailed description if the changes are complex or need further explanation. + +You should find the diff using `git diff --cached`, compile a commit message, +and call the `finish` action with `outputs.answer` set to the answer. If current +repo is not a valid git repo, or there is no diff in the staging area, please call +the `reject` action. + +## History +Here is a recent history of actions you've taken in service of this plan, +as well as observations you've made. This only includes the MOST RECENT +actions and observations--more may have happened before that. +They are time-ordered, with your most recent action at the bottom. + +[[{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}], [{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}], [{"source": "agent", "action": "run", "args": {"command": "git diff --staged", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `staged'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --staged", "exit_code": 129}}]] + +If the last item in the history is an error, you should try to fix it. + +## Available Actions +* `run` - runs a command on the command line in a Linux shell. Arguments: + * `command` - the command to run + * `background` - if true, run the command in the background, so that other commands can be run concurrently. Useful for e.g. starting a server. You won't be able to see the logs. You don't need to end the command with `&`, just set this to true. + +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + +* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: + * `outputs` - a dictionary representing the outputs of your task, if any + + +## Format +Your response MUST be in JSON format. It must be an object, and it must contain two fields: +* `action`, which is one of the actions specified here +* `args`, which is a map of key-value pairs, specifying the arguments for that action + +You MUST NOT include any other text besides the JSON response diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_007.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_007.log new file mode 100644 index 0000000000..0ca8752399 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_007.log @@ -0,0 +1,50 @@ + + +---------- + +# Task +You are a responsible software engineer and always write good commit messages. + +Please analyze the diff in the staging area, understand the context and content +of the updates from the diff only. Identify key elements like: +- Which files are affected? +- What types of changes were made (e.g., new features, bug fixes, refactoring, documentation, testing)? + +Then you should generate a commit message that succinctly summarizes the staged +changes. The commit message should include: +- A summary line that clearly states the purpose of the changes. +- Optionally, a detailed description if the changes are complex or need further explanation. + +You should find the diff using `git diff --cached`, compile a commit message, +and call the `finish` action with `outputs.answer` set to the answer. If current +repo is not a valid git repo, or there is no diff in the staging area, please call +the `reject` action. + +## History +Here is a recent history of actions you've taken in service of this plan, +as well as observations you've made. This only includes the MOST RECENT +actions and observations--more may have happened before that. +They are time-ordered, with your most recent action at the bottom. + +[[{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}], [{"source": "agent", "action": "run", "args": {"command": "git diff --cached", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `cached'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --cached", "exit_code": 129}}], [{"source": "agent", "action": "run", "args": {"command": "git diff --staged", "background": false, "thought": ""}}, {"source": "agent", "observation": "run", "content": "error: unknown option `staged'\r\nusage: git diff --no-index [] \r\n\r\nDiff output format options\r\n -p, --patch generate patch\r\n -s, --no-patch suppress diff output\r\n -u generate patch\r\n -U, --unified[=] generate diffs with lines context\r\n -W, --function-context\r\n generate diffs with lines context\r\n --raw generate the diff in raw format\r\n --patch-with-raw synonym for '-p --raw'\r\n --patch-with-stat synonym for '-p --stat'\r\n --numstat machine friendly --stat\r\n --shortstat output only the last line of --stat\r\n -X, --dirstat[=...]\r\n output the distribution of relative amount of changes for each sub-directory\r\n --cumulative synonym for --dirstat=cumulative\r\n --dirstat-by-file[=...]\r\n synonym for --dirstat=files,param1,param2...\r\n --check warn if changes introduce conflict markers or whitespace errors\r\n --summary condensed summary such as creations, renames and mode changes\r\n --name-only show only names of changed files\r\n --name-status show only names and status of changed files\r\n --stat[=[,[,]]]\r\n generate diffstat\r\n --stat-width generate diffstat with a given width\r\n --stat-name-width \r\n generate diffstat with a given name width\r\n --stat-graph-width \r\n generate diffstat with a given graph width\r\n --stat-count generate diffstat with limited lines\r\n --compact-summary generate compact summary in diffstat\r\n --binary output a binary diff that can be applied\r\n --full-index show full pre- and post-image object names on the \"index\" lines\r\n --color[=] show colored diff\r\n --ws-error-highlight \r\n highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff\r\n -z do not munge pathnames and use NULs as output field terminators in --raw or --numstat\r\n --abbrev[=] use digits to display object names\r\n --src-prefix \r\n show the given source prefix instead of \"a/\"\r\n --dst-prefix \r\n show the given destination prefix instead of \"b/\"\r\n --line-prefix \r\n prepend an additional prefix to every line of output\r\n --no-prefix do not show any source or destination prefix\r\n --inter-hunk-context \r\n show context between diff hunks up to the specified number of lines\r\n --output-indicator-new \r\n specify the character to indicate a new line instead of '+'\r\n --output-indicator-old \r\n specify the character to indicate an old line instead of '-'\r\n --output-indicator-context \r\n specify the character to indicate a context instead of ' '\r\n\r\nDiff rename options\r\n -B, --break-rewrites[=[/]]\r\n break complete rewrite changes into pairs of delete and create\r\n -M, --find-renames[=]\r\n detect renames\r\n -D, --irreversible-delete\r\n omit the preimage for deletes\r\n -C, --find-copies[=]\r\n detect copies\r\n --find-copies-harder use unmodified files as source to find copies\r\n --no-renames disable rename detection\r\n --rename-empty use empty blobs as rename source\r\n --follow continue listing the history of a file beyond renames\r\n -l prevent rename/copy detection if the number of rename/copy targets exceeds given limit\r\n\r\nDiff algorithm options\r\n --minimal produce the smallest possible diff\r\n -w, --ignore-all-space\r\n ignore whitespace when comparing lines\r\n -b, --ignore-space-change\r\n ignore changes in amount of whitespace\r\n --ignore-space-at-eol\r\n ignore changes in whitespace at EOL\r\n --ignore-cr-at-eol ignore carrier-return at the end of line\r\n --ignore-blank-lines ignore changes whose lines are all blank\r\n -I, --ignore-matching-lines \r\n ignore changes whose all lines match \r\n --indent-heuristic heuristic to shift diff hunk boundaries for easy reading\r\n --patience generate diff using the \"patience diff\" algorithm\r\n --histogram generate diff using the \"histogram diff\" algorithm\r\n --diff-algorithm \r\n choose a diff algorithm\r\n --anchored generate diff using the \"anchored diff\" algorithm\r\n --word-diff[=] show word diff, using to delimit changed words\r\n --word-diff-regex \r\n use to decide what a word is\r\n --color-words[=]\r\n equivalent to --word-diff=color --word-diff-regex=\r\n --color-moved[=]\r\n moved lines of code are colored differently\r\n --color-moved-ws \r\n how white spaces are ignored in --color-moved\r\n\r\nOther diff options\r\n --relative[=]\r\n when run from subdir, exclude changes outside and show relative paths\r\n -a, --text treat all files as text\r\n -R swap two inputs, reverse the diff\r\n --exit-code exit with 1 if there were differences, 0 otherwise\r\n --quiet disable all output of the program\r\n --ext-diff allow an external diff helper to be executed\r\n --textconv run external text conversion filters when comparing binary files\r\n --ignore-submodules[=]\r\n ignore changes to submodules in the diff generation\r\n --submodule[=]\r\n specify how differences in submodules are shown\r\n --ita-invisible-in-index\r\n hide 'git add -N' entries from the index\r\n --ita-visible-in-index\r\n treat 'git add -N' entries as real in the index\r\n -S look for differences that change the number of occurrences of the specified string\r\n -G look for differences that change the number of occurrences of the specified regex\r\n --pickaxe-all show all changes in the changeset with -S or -G\r\n --pickaxe-regex treat in -S as extended POSIX regular expression\r\n -O control the order in which files appear in the output\r\n --rotate-to show the change in the specified path first\r\n --skip-to skip the output to the specified path\r\n --find-object \r\n look for differences that change the number of occurrences of the specified object\r\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\r\n select files by diff type\r\n --output Output to a specific file\r\n", "extras": {"command_id": -1, "command": "git diff --staged", "exit_code": 129}}]] + +If the last item in the history is an error, you should try to fix it. + +## Available Actions +* `run` - runs a command on the command line in a Linux shell. Arguments: + * `command` - the command to run + * `background` - if true, run the command in the background, so that other commands can be run concurrently. Useful for e.g. starting a server. You won't be able to see the logs. You don't need to end the command with `&`, just set this to true. + +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + +* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: + * `outputs` - a dictionary representing the outputs of your task, if any + + +## Format +Your response MUST be in JSON format. It must be an object, and it must contain two fields: +* `action`, which is one of the actions specified here +* `args`, which is a map of key-value pairs, specifying the arguments for that action + +You MUST NOT include any other text besides the JSON response diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_008.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_008.log new file mode 100644 index 0000000000..d2fe15e6f5 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/prompt_008.log @@ -0,0 +1,94 @@ + + +---------- + +# Task +You are in charge of accomplishing the following task: +Write a git commit message for the current staging area. Do not ask me for confirmation at any point. + +In order to accomplish this goal, you must delegate tasks to one or more agents, who +can do the actual work. A description of each agent is provided below. You MUST +select one of the delegates below to move towards accomplishing the task, and you MUST +provide the correct inputs for the delegate you select. + +Note: the delegated agent either returns "finish" or "reject". +- If the action is "finish", but the full task is not done yet, you should +continue to delegate to one of the agents below to until the full task is finished. +- If the action is "reject", it means the delegated agent is not capable of the +task you send to. You should revisit the input you send to the delegate, and consider +whether any other delegate would be able to solve the task. If you cannot find +a proper delegate agent, or the delegate attempts keep failing, call the `reject` +action. + +## Agents + +### CoderAgent +Given a particular task, and a detailed description of the codebase, accomplishes the task +#### Inputs +{"task": "string", "summary": "string"} + +### CommitWriterAgent +Write a git commit message for files in the git staging area +#### Inputs +{} + +### MathAgent +Solves simple and complex math problems using python +#### Inputs +{"task": "string"} + +### PostgresAgent +Writes and maintains PostgreSQL migrations +#### Inputs +{"task": "string"} + +### RepoExplorerAgent +Generates a detailed summary of an existing codebase +#### Inputs +{} + +### StudyRepoForTaskAgent +Given a particular task, finds and describes all relevant parts of the codebase +#### Inputs +{"task": "string"} + +### TypoFixerAgent +Fixes typos in files in the current working directory +#### Inputs +{"task": "string"} + +### VerifierAgent +Given a particular task, verifies that the task has been completed +#### Inputs +{"task": "string"} + + +## History +Here is a recent history of actions you've taken in service of this plan, +as well as observations you've made. This only includes the MOST RECENT +actions and observations--more may have happened before that. +They are time-ordered, with your most recent action at the bottom. + +[[{"source": "user", "action": "message", "args": {"content": "Write a git commit message for the current staging area. Do not ask me for confirmation at any point.", "wait_for_response": false}}, {"observation": "null", "content": "", "extras": {}}], [{"source": "agent", "action": "delegate", "args": {"agent": "CommitWriterAgent", "inputs": {}, "thought": ""}}, {"observation": "null", "content": "", "extras": {}}], [{"action": "null", "args": {}}, {"source": "agent", "observation": "delegate", "content": "", "extras": {"outputs": {"reason": "The current repository is not a valid git repository or there is no diff in the staging area."}}}]] + +If the last item in the history is an error, you should try to fix it. If you +cannot fix it, call the `reject` action. + +## Available Actions +* `delegate` - send a task to another agent from the list provided. Arguments: + * `agent` - the agent to which the task is delegated. MUST match a name in the list of agents provided. + * `inputs` - a dictionary of input parameters to the agent, as specified in the list + +* `finish` - if you're absolutely certain that you've completed your task, use the finish action to stop working. Arguments: + * `outputs` - a dictionary representing the outputs of your task, if any + +* `reject` - reject the task. Arguments: + * `outputs` - a dictionary with only a `reason` attribute + + +## Format +Your response MUST be in JSON format. It must be an object, and it must contain two fields: +* `action`, which is one of the actions specified here +* `args`, which is a map of key-value pairs, specifying the arguments for that action + +You MUST NOT include any other text besides the JSON response diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_001.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_001.log new file mode 100644 index 0000000000..2b662799ee --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_001.log @@ -0,0 +1,9 @@ +```json +{ + "action": "delegate", + "args": { + "agent": "CommitWriterAgent", + "inputs": {} + } +} +``` diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_002.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_002.log new file mode 100644 index 0000000000..1e935ca967 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_002.log @@ -0,0 +1,8 @@ +```json +{ + "action": "run", + "args": { + "command": "git diff --cached" + } +} +``` diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_003.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_003.log new file mode 100644 index 0000000000..fc4fd5be94 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_003.log @@ -0,0 +1,9 @@ +```json +{ + "action": "run", + "args": { + "command": "git diff --cached", + "background": false + } +} +``` diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_004.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_004.log new file mode 100644 index 0000000000..6f06e92d25 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_004.log @@ -0,0 +1,9 @@ +```json +{ + "action": "run", + "args": { + "command": "git diff --staged", + "background": false + } +} +``` diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_005.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_005.log new file mode 100644 index 0000000000..b0621d76c7 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_005.log @@ -0,0 +1,8 @@ +```json +{ + "action": "reject", + "args": { + "reason": "The current repository is not a valid git repository or there is no diff in the staging area." + } +} +``` diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_006.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_006.log new file mode 100644 index 0000000000..b0621d76c7 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_006.log @@ -0,0 +1,8 @@ +```json +{ + "action": "reject", + "args": { + "reason": "The current repository is not a valid git repository or there is no diff in the staging area." + } +} +``` diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_007.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_007.log new file mode 100644 index 0000000000..6da659ad79 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_007.log @@ -0,0 +1,10 @@ +```json +{ + "action": "reject", + "args": { + "outputs": { + "reason": "The current repository is not a valid git repository or there is no diff in the staging area." + } + } +} +``` diff --git a/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_008.log b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_008.log new file mode 100644 index 0000000000..6da659ad79 --- /dev/null +++ b/tests/integration/mock/ManagerAgent/test_simple_task_rejection/response_008.log @@ -0,0 +1,10 @@ +```json +{ + "action": "reject", + "args": { + "outputs": { + "reason": "The current repository is not a valid git repository or there is no diff in the staging area." + } + } +} +``` diff --git a/tests/integration/regenerate.sh b/tests/integration/regenerate.sh index eeba358e24..dae9fd3d15 100755 --- a/tests/integration/regenerate.sh +++ b/tests/integration/regenerate.sh @@ -25,6 +25,7 @@ tasks=( "Fix typos in bad.txt." "Write a shell script 'hello.sh' that prints 'hello'." "Use Jupyter IPython to write a text file containing 'hello world' to '/workspace/test.txt'." + "Write a git commit message for the current staging area." "Install and import pymsgbox==1.0.9 and print it's version in /workspace/test.txt." "Browse localhost:8000, and tell me the ultimate answer to life." ) @@ -32,6 +33,7 @@ test_names=( "test_edits" "test_write_simple_script" "test_ipython" + "test_simple_task_rejection" "test_ipython_module" "test_browse_internet" ) diff --git a/tests/integration/test_agent.py b/tests/integration/test_agent.py index f4b3943370..80407aa4e5 100644 --- a/tests/integration/test_agent.py +++ b/tests/integration/test_agent.py @@ -10,6 +10,7 @@ from opendevin.core.main import main from opendevin.core.schema import AgentState from opendevin.events.action import ( AgentFinishAction, + AgentRejectAction, ) workspace_base = os.getenv('WORKSPACE_BASE') @@ -23,6 +24,10 @@ workspace_base = os.getenv('WORKSPACE_BASE') os.getenv('AGENT') == 'CodeActAgent' and os.getenv('SANDBOX_TYPE').lower() != 'ssh', reason='CodeActAgent only supports ssh sandbox which is stateful', ) +@pytest.mark.skipif( + os.getenv('AGENT') == 'ManagerAgent', + reason='Manager agent is not capable of finishing this in reasonable steps yet', +) def test_write_simple_script(): task = "Write a shell script 'hello.sh' that prints 'hello'. Do not ask me for confirmation at any point." final_state: State = asyncio.run(main(task, exit_on_message=True)) @@ -109,6 +114,23 @@ def test_ipython(): ), f'Expected content "hello world", but got "{content.strip()}"' +@pytest.mark.skipif( + os.getenv('AGENT') != 'ManagerAgent', + reason='Currently, only ManagerAgent supports task rejection', +) +@pytest.mark.skipif( + os.getenv('SANDBOX_TYPE') == 'local', + reason='FIXME: local sandbox does not capture stderr', +) +def test_simple_task_rejection(): + # Give an impossible task to do: cannot write a commit message because + # the workspace is not a git repo + task = 'Write a git commit message for the current staging area. Do not ask me for confirmation at any point.' + final_state: State = asyncio.run(main(task)) + assert final_state.agent_state == AgentState.STOPPED + assert isinstance(final_state.history[-1][0], AgentRejectAction) + + @pytest.mark.skipif( os.getenv('AGENT') != 'CodeActAgent', reason='currently only CodeActAgent defaults to have IPython (Jupyter) execution',