github/OpenHands
1
0
Fork 0
mirror of https://github.com/OpenHands/OpenHands.git synced 2025-12-26 05:48:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Align codeact with swebench eval (#1612)

Browse Source 
* align codeact agent with the slight adjustment on eval branch

* update integration test for new prompt

* Regenerate test artifacts for CodeActAgent

---------

Co-authored-by: Boxuan Li <liboxuan@connect.hku.hk>
This commit is contained in:
Xingyao Wang 2024-05-09 15:42:07 +08:00 committed by GitHub
parent 45d1b6969a
commit 21fe8dc1eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 57 additions and 797 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
agenthub/codeact_agent/codeact_agent.py
View File

@ -4,6 +4,7 @@ from typing import List, Mapping
from agenthub.codeact_agent.prompt import EXAMPLES, SYSTEM_MESSAGE
from opendevin.controller.agent import Agent
from opendevin.controller.state.state import State
from opendevin.core.logger import opendevin_logger as logger
from opendevin.events.action import (
Action,
AgentFinishAction,
@ -17,7 +18,7 @@ from opendevin.events.observation import (
IPythonRunCellObservation,
NullObservation,
)
from opendevin.llm.llm import LLM
from opendevin.llm.llm import LLM, completion_cost
from opendevin.runtime.plugins import (
JupyterRequirement,
PluginRequirement,
@ -33,7 +34,7 @@ def parse_response(response) -> str:
return action
def truncate_observation(observation: str, max_chars: int = 5000) -> str:
def truncate_observation(observation: str, max_chars: int = 10_000) -> str:
"""
Truncate the middle of the observation if it is too long.
"""
@ -47,6 +48,37 @@ def truncate_observation(observation: str, max_chars: int = 5000) -> str:
)
def swe_agent_edit_hack(bash_command: str) -> str:
"""
Hack to handle the SWE-agent edit command. The vanilla edit command will hang the SSHBox.
REPLACE THIS:
edit 683:693
try:
return list(urlsplit(url))
except ValueError:
raise ValidationError(self.error_messages['invalid'], code='invalid')
end_of_edit
WITH THIS:
edit 683:693 <<EOF
try:
return list(urlsplit(url))
except ValueError:
raise ValidationError(self.error_messages['invalid'], code='invalid')s
EOF
"""
if 'edit' in bash_command:
# edit\s(\d+):(\d+)([\s\S]*)end_of_edit
# replace
bash_command = re.sub(
r'edit\s(\d+):(\d+)([\s\S]*)end_of_edit',
r'edit \1:\2 <<EOF\3EOF',
bash_command,
)
return bash_command
class CodeActAgent(Agent):
VERSION = '1.1'
"""
@ -113,6 +145,7 @@ class CodeActAgent(Agent):
"""
super().__init__(llm)
self.messages: List[Mapping[str, str]] = []
self.cost_accumulator = 0
def step(self, state: State) -> Action:
"""
@ -195,6 +228,11 @@ class CodeActAgent(Agent):
],
temperature=0.0,
)
cur_cost = completion_cost(completion_response=response)
self.cost_accumulator += cur_cost
logger.info(
f'Cost: {cur_cost:.2f} USD | Accumulated Cost: {self.cost_accumulator:.2f} USD'
)
action_str: str = parse_response(response)
state.num_of_chars += sum(
len(message['content']) for message in self.messages
@ -208,6 +246,8 @@ class CodeActAgent(Agent):
thought = action_str.replace(bash_command.group(0), '').strip()
# a command was found
command_group = bash_command.group(1).strip()
command_group = swe_agent_edit_hack(command_group)
if command_group.strip() == 'exit':
return AgentFinishAction()
return CmdRunAction(command=command_group, thought=thought)

3
agenthub/codeact_agent/prompt.py
View File

@ -184,8 +184,7 @@ USER:
11: app.run(port=5000)
ASSISTANT:
I should edit the file to display the numbers in a table format. Let me do that for you:
I should edit the file to display the numbers in a table format. I should include correct indentation. Let me update the file:
<execute_bash>
edit 8:8 <<EOF
return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'

4
opendevin/llm/llm.py
View File

@ -2,6 +2,7 @@ from functools import partial
import litellm
from litellm import completion as litellm_completion
from litellm import completion_cost
from litellm.exceptions import (
APIConnectionError,
RateLimitError,
@ -19,6 +20,8 @@ from opendevin.core.logger import llm_prompt_logger, llm_response_logger
from opendevin.core.logger import opendevin_logger as logger
from opendevin.core.schema import ConfigType
__all__ = ['LLM', 'completion_cost']
DEFAULT_API_KEY = config.get(ConfigType.LLM_API_KEY)
DEFAULT_BASE_URL = config.get(ConfigType.LLM_BASE_URL)
DEFAULT_MODEL_NAME = config.get(ConfigType.LLM_MODEL)
@ -68,6 +71,7 @@ class LLM:
max_output_tokens (int, optional): The maximum number of tokens to receive from the LLM per task. Defaults to LLM_MAX_OUTPUT_TOKENS.
custom_llm_provider (str, optional): A custom LLM provider. Defaults to LLM_CUSTOM_LLM_PROVIDER.
llm_timeout (int, optional): The maximum time to wait for a response in seconds. Defaults to LLM_TIMEOUT.
llm_temperature (float, optional): The temperature for LLM sampling. Defaults to LLM_TEMPERATURE.
Attributes:
model_name (str): The name of the language model.

3
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_001.log
View File

@ -175,8 +175,7 @@ USER:
11: app.run(port=5000)
ASSISTANT:
I should edit the file to display the numbers in a table format. Let me do that for you:
I should edit the file to display the numbers in a table format. I should include correct indentation. Let me update the file:
<execute_bash>
edit 8:8 <<EOF
return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'

10
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_002.log
View File

@ -175,8 +175,7 @@ USER:
11: app.run(port=5000)
ASSISTANT:
I should edit the file to display the numbers in a table format. Let me do that for you:
I should edit the file to display the numbers in a table format. I should include correct indentation. Let me update the file:
<execute_bash>
edit 8:8 <<EOF
return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
@ -217,14 +216,15 @@ The server is running on port 5000 with PID 126. You can access the list of numb
NOW, LET'S START!
Write a shell script 'hello.sh' that prints 'hello'.
Write a shell script 'hello.sh' that prints 'hello'. Do not ask me for confirmation at any point.
----------
Certainly! I will create a shell script named `hello.sh` that contains the command to print 'hello'.
I'll create and write the shell script `hello.sh` that simply prints "hello". Let's proceed with that.
<execute_bash>
echo "echo 'hello'" > hello.sh && chmod +x hello.sh
echo "echo 'hello'" > hello.sh
chmod +x hello.sh
</execute_bash>
----------

248
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_003.log
View File

@ -1,248 +0,0 @@
----------
A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
The assistant can interact with an interactive Python (Jupyter Notebook) environment and receive the corresponding output when needed. The code should be enclosed using "<execute_ipython>" tag, for example:
<execute_ipython>
print("Hello World!")
</execute_ipython>
The assistant can execute bash commands on behalf of the user by wrapping them with <execute_bash> and </execute_bash>.
For example, you can list the files in the current directory by <execute_bash> ls </execute_bash>.
The assistant should attempt fewer things at a time instead of putting too much commands OR code in one "execute" block.
The assistant can install Python packages through bash by <execute_bash> pip install [package needed] </execute_bash> and should always import packages and define variables before starting to use them.
The assistant should stop <execute> and provide an answer when they have already obtained the answer from the execution result.
If the assistant encounters an import error in IPython for a newly installed package, they should try to restart the kernel and import the package again. IPython kernel can be re-started by:
<execute_ipython>
import IPython
IPython.Application.instance().kernel.do_shutdown(True) # Restart the kernel
</execute_ipython>
Apart from the standard bash commands, you can also use the following special commands in <execute_bash> environment:
open <path> [<line_number>] - opens the file at the given path in the editor. If line_number is provided, the window will be move to include that line
goto <line_number> - moves the window to show <line_number>
scroll_down - moves the window down {WINDOW} lines
scroll_up - moves the window down {WINDOW} lines
create <filename> - creates and opens a new file with the given name
search_dir <search_term> [<dir>] - searches for search_term in all files in dir. If dir is not provided, searches in the current directory
search_file <search_term> [<file>] - searches for search_term in file. If file is not provided, searches in the current open file
find_file <file_name> [<dir>] - finds all files with the given name in dir. If dir is not provided, searches in the current directory
edit <start_line>:<end_line>
<replacement_text>
end_of_edit - replaces lines <start_line> through <end_line> (inclusive) with the given text in the open file. The replacement text is terminated by a line with only end_of_edit on it. All of the <replacement text> will be entered, so make sure your indentation is formatted properly. Python files will be checked for syntax errors after the edit. If the system detects a syntax error, the edit will not be executed. Simply try to edit the file again, but make sure to read the error message and modify the edit command you issue accordingly. Issuing the same command a second time will just lead to the same error message again.
Please note that THE EDIT COMMAND REQUIRES PROPER INDENTATION. If you'd like to add the line ' print(x)' you must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run.
The assistant's response should be concise, but do express their thoughts.
Try to include one of <execute_ipython> or <execute_bash> in each of your responses, unless it is a direct answer to a question OR a message to the user.
IMPORTANT: Whenever possible, execute the code for the user using <execute_ipython> or <execute_bash> instead of providing it.
----------
Here is an example of how you can interact with the environment for task solving:
--- START OF EXAMPLE ---
USER: Can you create a list of numbers from 1 to 10, and create a web page to display them at port 5000?
ASSISTANT:
Sure! Let me write the Python code for starting a web server and save it to a file `app.py`:
<execute_ipython>
CODE='''
from flask import Flask
app = Flask(__name__)
@app.route('/')
def index():
numbers = list(range(1, 11))
return str(numbers)
if __name__ == '__main__':
app.run(port=5000)
'''
with open('app.py', 'w') as f:
f.write(CODE)
</execute_ipython>
USER:
Observation:
[Code executed successfully with no output]
ASSISTANT:
I have created a Python file `app.py` that will display a list of numbers from 1 to 10 when you run it. Let me run the Python file for you:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 121[1]+ Exit 1 python3 app.py > server.log 2>&1
ASSISTANT: Looks like the server is running with PID 121 then crashed. Let me check the server log:
<execute_bash>
cat server.log
</execute_bash>
USER:
Observation:
Traceback (most recent call last):
File "/workspace/app.py", line 2, in <module>
from flask import Flask
ModuleNotFoundError: No module named 'flask'
ASSISTANT:
It seems that Flask is not installed. Let me install Flask for you:
<execute_bash>
pip install flask
</execute_bash>
USER:
Observation:
Defaulting to user installation because normal site-packages is not writeable
Collecting flask
Using cached flask-3.0.3-py3-none-any.whl (101 kB)
Collecting blinker>=1.6.2
Using cached blinker-1.7.0-py3-none-any.whl (13 kB)
Collecting Werkzeug>=3.0.0
Using cached werkzeug-3.0.2-py3-none-any.whl (226 kB)
Collecting click>=8.1.3
Using cached click-8.1.7-py3-none-any.whl (97 kB)
Collecting itsdangerous>=2.1.2
Using cached itsdangerous-2.2.0-py3-none-any.whl (16 kB)
Requirement already satisfied: Jinja2>=3.1.2 in /home/opendevin/.local/lib/python3.10/site-packages (from flask) (3.1.3)
Requirement already satisfied: MarkupSafe>=2.0 in /home/opendevin/.local/lib/python3.10/site-packages (from Jinja2>=3.1.2->flask) (2.1.5)
Installing collected packages: Werkzeug, itsdangerous, click, blinker, flask
Successfully installed Werkzeug-3.0.2 blinker-1.7.0 click-8.1.7 flask-3.0.3 itsdangerous-2.2.0
ASSISTANT:
Now that Flask is installed, let me run the Python file again:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 124
ASSISTANT:
Let me check the server log again:
<execute_bash>
cat server.log
</execute_bash>
USER:
Observation:
* Serving Flask app 'app'
* Debug mode: off
WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
* Running on http://127.0.0.1:5000
Press CTRL+C to quit
ASSISTANT:
The server is running on port 5000 with PID 124. You can access the list of numbers by visiting http://127.0.0.1:5000. If you have any further questions, feel free to ask!
USER: Now kill the server, make it display the numbers in a table format.
ASSISTANT:
Sure! Let me stop the server first:
<execute_bash>
kill 124
</execute_bash>
USER:
Observation:
[1]+ Terminated python3 app.py > server.log 2>&1
ASSISTANT:
The server has been stopped. Let me open the Python file and modify it.
<execute_bash>
open app.py
</execute_bash>
USER:
[File: /workspace/app.py (11 lines total)]
1:
2:from flask import Flask
3:app = Flask(__name__)
4:
5:@app.route('/')
6:def index():
7: numbers = list(range(1, 11))
8: return str(numbers)
9:
10:if __name__ == '__main__':
11: app.run(port=5000)
ASSISTANT:
I should edit the file to display the numbers in a table format. Let me do that for you:
<execute_bash>
edit 8:8 <<EOF
return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
EOF
</execute_bash>
USER:
Observation:
[File: /workspace/app.py (11 lines total)]
1:
2:from flask import Flask
3:app = Flask(__name__)
4:
5:@app.route('/')
6:def index():
7: numbers = list(range(1, 11))
8: return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
9:
10:if __name__ == '__main__':
11: app.run(port=5000)
File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.
ASSISTANT:
The file has been updated. Let me run the Python file again with the new changes:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 126
ASSISTANT:
The server is running on port 5000 with PID 126. You can access the list of numbers in a table format by visiting http://127.0.0.1:5000. Free free to let me know if you have any further requests!
--- END OF EXAMPLE ---
NOW, LET'S START!
Write a shell script 'hello.sh' that prints 'hello'.
----------
Certainly! I will create a shell script named `hello.sh` that contains the command to print 'hello'.
<execute_bash>
echo "echo 'hello'" > hello.sh && chmod +x hello.sh
</execute_bash>
----------
OBSERVATION:
[Command -1 finished with exit code 0]]
----------
The shell script `hello.sh` has been created and made executable. Now, let's run the script to see the output.
<execute_bash>
./hello.sh
</execute_bash>
----------
OBSERVATION:
hello
[Command -1 finished with exit code 0]]

256
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_004.log
View File

@ -1,256 +0,0 @@
----------
A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
The assistant can interact with an interactive Python (Jupyter Notebook) environment and receive the corresponding output when needed. The code should be enclosed using "<execute_ipython>" tag, for example:
<execute_ipython>
print("Hello World!")
</execute_ipython>
The assistant can execute bash commands on behalf of the user by wrapping them with <execute_bash> and </execute_bash>.
For example, you can list the files in the current directory by <execute_bash> ls </execute_bash>.
The assistant should attempt fewer things at a time instead of putting too much commands OR code in one "execute" block.
The assistant can install Python packages through bash by <execute_bash> pip install [package needed] </execute_bash> and should always import packages and define variables before starting to use them.
The assistant should stop <execute> and provide an answer when they have already obtained the answer from the execution result.
If the assistant encounters an import error in IPython for a newly installed package, they should try to restart the kernel and import the package again. IPython kernel can be re-started by:
<execute_ipython>
import IPython
IPython.Application.instance().kernel.do_shutdown(True) # Restart the kernel
</execute_ipython>
Apart from the standard bash commands, you can also use the following special commands in <execute_bash> environment:
open <path> [<line_number>] - opens the file at the given path in the editor. If line_number is provided, the window will be move to include that line
goto <line_number> - moves the window to show <line_number>
scroll_down - moves the window down {WINDOW} lines
scroll_up - moves the window down {WINDOW} lines
create <filename> - creates and opens a new file with the given name
search_dir <search_term> [<dir>] - searches for search_term in all files in dir. If dir is not provided, searches in the current directory
search_file <search_term> [<file>] - searches for search_term in file. If file is not provided, searches in the current open file
find_file <file_name> [<dir>] - finds all files with the given name in dir. If dir is not provided, searches in the current directory
edit <start_line>:<end_line>
<replacement_text>
end_of_edit - replaces lines <start_line> through <end_line> (inclusive) with the given text in the open file. The replacement text is terminated by a line with only end_of_edit on it. All of the <replacement text> will be entered, so make sure your indentation is formatted properly. Python files will be checked for syntax errors after the edit. If the system detects a syntax error, the edit will not be executed. Simply try to edit the file again, but make sure to read the error message and modify the edit command you issue accordingly. Issuing the same command a second time will just lead to the same error message again.
Please note that THE EDIT COMMAND REQUIRES PROPER INDENTATION. If you'd like to add the line ' print(x)' you must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run.
The assistant's response should be concise, but do express their thoughts.
Try to include one of <execute_ipython> or <execute_bash> in each of your responses, unless it is a direct answer to a question OR a message to the user.
IMPORTANT: Whenever possible, execute the code for the user using <execute_ipython> or <execute_bash> instead of providing it.
----------
Here is an example of how you can interact with the environment for task solving:
--- START OF EXAMPLE ---
USER: Can you create a list of numbers from 1 to 10, and create a web page to display them at port 5000?
ASSISTANT:
Sure! Let me write the Python code for starting a web server and save it to a file `app.py`:
<execute_ipython>
CODE='''
from flask import Flask
app = Flask(__name__)
@app.route('/')
def index():
numbers = list(range(1, 11))
return str(numbers)
if __name__ == '__main__':
app.run(port=5000)
'''
with open('app.py', 'w') as f:
f.write(CODE)
</execute_ipython>
USER:
Observation:
[Code executed successfully with no output]
ASSISTANT:
I have created a Python file `app.py` that will display a list of numbers from 1 to 10 when you run it. Let me run the Python file for you:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 121[1]+ Exit 1 python3 app.py > server.log 2>&1
ASSISTANT: Looks like the server is running with PID 121 then crashed. Let me check the server log:
<execute_bash>
cat server.log
</execute_bash>
USER:
Observation:
Traceback (most recent call last):
File "/workspace/app.py", line 2, in <module>
from flask import Flask
ModuleNotFoundError: No module named 'flask'
ASSISTANT:
It seems that Flask is not installed. Let me install Flask for you:
<execute_bash>
pip install flask
</execute_bash>
USER:
Observation:
Defaulting to user installation because normal site-packages is not writeable
Collecting flask
Using cached flask-3.0.3-py3-none-any.whl (101 kB)
Collecting blinker>=1.6.2
Using cached blinker-1.7.0-py3-none-any.whl (13 kB)
Collecting Werkzeug>=3.0.0
Using cached werkzeug-3.0.2-py3-none-any.whl (226 kB)
Collecting click>=8.1.3
Using cached click-8.1.7-py3-none-any.whl (97 kB)
Collecting itsdangerous>=2.1.2
Using cached itsdangerous-2.2.0-py3-none-any.whl (16 kB)
Requirement already satisfied: Jinja2>=3.1.2 in /home/opendevin/.local/lib/python3.10/site-packages (from flask) (3.1.3)
Requirement already satisfied: MarkupSafe>=2.0 in /home/opendevin/.local/lib/python3.10/site-packages (from Jinja2>=3.1.2->flask) (2.1.5)
Installing collected packages: Werkzeug, itsdangerous, click, blinker, flask
Successfully installed Werkzeug-3.0.2 blinker-1.7.0 click-8.1.7 flask-3.0.3 itsdangerous-2.2.0
ASSISTANT:
Now that Flask is installed, let me run the Python file again:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 124
ASSISTANT:
Let me check the server log again:
<execute_bash>
cat server.log
</execute_bash>
USER:
Observation:
* Serving Flask app 'app'
* Debug mode: off
WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
* Running on http://127.0.0.1:5000
Press CTRL+C to quit
ASSISTANT:
The server is running on port 5000 with PID 124. You can access the list of numbers by visiting http://127.0.0.1:5000. If you have any further questions, feel free to ask!
USER: Now kill the server, make it display the numbers in a table format.
ASSISTANT:
Sure! Let me stop the server first:
<execute_bash>
kill 124
</execute_bash>
USER:
Observation:
[1]+ Terminated python3 app.py > server.log 2>&1
ASSISTANT:
The server has been stopped. Let me open the Python file and modify it.
<execute_bash>
open app.py
</execute_bash>
USER:
[File: /workspace/app.py (11 lines total)]
1:
2:from flask import Flask
3:app = Flask(__name__)
4:
5:@app.route('/')
6:def index():
7: numbers = list(range(1, 11))
8: return str(numbers)
9:
10:if __name__ == '__main__':
11: app.run(port=5000)
ASSISTANT:
I should edit the file to display the numbers in a table format. Let me do that for you:
<execute_bash>
edit 8:8 <<EOF
return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
EOF
</execute_bash>
USER:
Observation:
[File: /workspace/app.py (11 lines total)]
1:
2:from flask import Flask
3:app = Flask(__name__)
4:
5:@app.route('/')
6:def index():
7: numbers = list(range(1, 11))
8: return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
9:
10:if __name__ == '__main__':
11: app.run(port=5000)
File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.
ASSISTANT:
The file has been updated. Let me run the Python file again with the new changes:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 126
ASSISTANT:
The server is running on port 5000 with PID 126. You can access the list of numbers in a table format by visiting http://127.0.0.1:5000. Free free to let me know if you have any further requests!
--- END OF EXAMPLE ---
NOW, LET'S START!
Write a shell script 'hello.sh' that prints 'hello'.
----------
Certainly! I will create a shell script named `hello.sh` that contains the command to print 'hello'.
<execute_bash>
echo "echo 'hello'" > hello.sh && chmod +x hello.sh
</execute_bash>
----------
OBSERVATION:
[Command -1 finished with exit code 0]]
----------
The shell script `hello.sh` has been created and made executable. Now, let's run the script to see the output.
<execute_bash>
./hello.sh
</execute_bash>
----------
OBSERVATION:
hello
[Command -1 finished with exit code 0]]
----------
The script executed successfully and printed "hello" as expected. If you have any more tasks or questions, feel free to ask!
----------
can you run it to make sure it is working?

270
tests/integration/mock/CodeActAgent/test_write_simple_script/prompt_005.log
View File

@ -1,270 +0,0 @@
----------
A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
The assistant can interact with an interactive Python (Jupyter Notebook) environment and receive the corresponding output when needed. The code should be enclosed using "<execute_ipython>" tag, for example:
<execute_ipython>
print("Hello World!")
</execute_ipython>
The assistant can execute bash commands on behalf of the user by wrapping them with <execute_bash> and </execute_bash>.
For example, you can list the files in the current directory by <execute_bash> ls </execute_bash>.
The assistant should attempt fewer things at a time instead of putting too much commands OR code in one "execute" block.
The assistant can install Python packages through bash by <execute_bash> pip install [package needed] </execute_bash> and should always import packages and define variables before starting to use them.
The assistant should stop <execute> and provide an answer when they have already obtained the answer from the execution result.
If the assistant encounters an import error in IPython for a newly installed package, they should try to restart the kernel and import the package again. IPython kernel can be re-started by:
<execute_ipython>
import IPython
IPython.Application.instance().kernel.do_shutdown(True) # Restart the kernel
</execute_ipython>
Apart from the standard bash commands, you can also use the following special commands in <execute_bash> environment:
open <path> [<line_number>] - opens the file at the given path in the editor. If line_number is provided, the window will be move to include that line
goto <line_number> - moves the window to show <line_number>
scroll_down - moves the window down {WINDOW} lines
scroll_up - moves the window down {WINDOW} lines
create <filename> - creates and opens a new file with the given name
search_dir <search_term> [<dir>] - searches for search_term in all files in dir. If dir is not provided, searches in the current directory
search_file <search_term> [<file>] - searches for search_term in file. If file is not provided, searches in the current open file
find_file <file_name> [<dir>] - finds all files with the given name in dir. If dir is not provided, searches in the current directory
edit <start_line>:<end_line>
<replacement_text>
end_of_edit - replaces lines <start_line> through <end_line> (inclusive) with the given text in the open file. The replacement text is terminated by a line with only end_of_edit on it. All of the <replacement text> will be entered, so make sure your indentation is formatted properly. Python files will be checked for syntax errors after the edit. If the system detects a syntax error, the edit will not be executed. Simply try to edit the file again, but make sure to read the error message and modify the edit command you issue accordingly. Issuing the same command a second time will just lead to the same error message again.
Please note that THE EDIT COMMAND REQUIRES PROPER INDENTATION. If you'd like to add the line ' print(x)' you must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run.
The assistant's response should be concise, but do express their thoughts.
Try to include one of <execute_ipython> or <execute_bash> in each of your responses, unless it is a direct answer to a question OR a message to the user.
IMPORTANT: Whenever possible, execute the code for the user using <execute_ipython> or <execute_bash> instead of providing it.
----------
Here is an example of how you can interact with the environment for task solving:
--- START OF EXAMPLE ---
USER: Can you create a list of numbers from 1 to 10, and create a web page to display them at port 5000?
ASSISTANT:
Sure! Let me write the Python code for starting a web server and save it to a file `app.py`:
<execute_ipython>
CODE='''
from flask import Flask
app = Flask(__name__)
@app.route('/')
def index():
numbers = list(range(1, 11))
return str(numbers)
if __name__ == '__main__':
app.run(port=5000)
'''
with open('app.py', 'w') as f:
f.write(CODE)
</execute_ipython>
USER:
Observation:
[Code executed successfully with no output]
ASSISTANT:
I have created a Python file `app.py` that will display a list of numbers from 1 to 10 when you run it. Let me run the Python file for you:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 121[1]+ Exit 1 python3 app.py > server.log 2>&1
ASSISTANT: Looks like the server is running with PID 121 then crashed. Let me check the server log:
<execute_bash>
cat server.log
</execute_bash>
USER:
Observation:
Traceback (most recent call last):
File "/workspace/app.py", line 2, in <module>
from flask import Flask
ModuleNotFoundError: No module named 'flask'
ASSISTANT:
It seems that Flask is not installed. Let me install Flask for you:
<execute_bash>
pip install flask
</execute_bash>
USER:
Observation:
Defaulting to user installation because normal site-packages is not writeable
Collecting flask
Using cached flask-3.0.3-py3-none-any.whl (101 kB)
Collecting blinker>=1.6.2
Using cached blinker-1.7.0-py3-none-any.whl (13 kB)
Collecting Werkzeug>=3.0.0
Using cached werkzeug-3.0.2-py3-none-any.whl (226 kB)
Collecting click>=8.1.3
Using cached click-8.1.7-py3-none-any.whl (97 kB)
Collecting itsdangerous>=2.1.2
Using cached itsdangerous-2.2.0-py3-none-any.whl (16 kB)
Requirement already satisfied: Jinja2>=3.1.2 in /home/opendevin/.local/lib/python3.10/site-packages (from flask) (3.1.3)
Requirement already satisfied: MarkupSafe>=2.0 in /home/opendevin/.local/lib/python3.10/site-packages (from Jinja2>=3.1.2->flask) (2.1.5)
Installing collected packages: Werkzeug, itsdangerous, click, blinker, flask
Successfully installed Werkzeug-3.0.2 blinker-1.7.0 click-8.1.7 flask-3.0.3 itsdangerous-2.2.0
ASSISTANT:
Now that Flask is installed, let me run the Python file again:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 124
ASSISTANT:
Let me check the server log again:
<execute_bash>
cat server.log
</execute_bash>
USER:
Observation:
* Serving Flask app 'app'
* Debug mode: off
WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
* Running on http://127.0.0.1:5000
Press CTRL+C to quit
ASSISTANT:
The server is running on port 5000 with PID 124. You can access the list of numbers by visiting http://127.0.0.1:5000. If you have any further questions, feel free to ask!
USER: Now kill the server, make it display the numbers in a table format.
ASSISTANT:
Sure! Let me stop the server first:
<execute_bash>
kill 124
</execute_bash>
USER:
Observation:
[1]+ Terminated python3 app.py > server.log 2>&1
ASSISTANT:
The server has been stopped. Let me open the Python file and modify it.
<execute_bash>
open app.py
</execute_bash>
USER:
[File: /workspace/app.py (11 lines total)]
1:
2:from flask import Flask
3:app = Flask(__name__)
4:
5:@app.route('/')
6:def index():
7: numbers = list(range(1, 11))
8: return str(numbers)
9:
10:if __name__ == '__main__':
11: app.run(port=5000)
ASSISTANT:
I should edit the file to display the numbers in a table format. Let me do that for you:
<execute_bash>
edit 8:8 <<EOF
return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
EOF
</execute_bash>
USER:
Observation:
[File: /workspace/app.py (11 lines total)]
1:
2:from flask import Flask
3:app = Flask(__name__)
4:
5:@app.route('/')
6:def index():
7: numbers = list(range(1, 11))
8: return '<table>' + ''.join([f'<tr><td>{i}</td></tr>' for i in numbers]) + '</table>'
9:
10:if __name__ == '__main__':
11: app.run(port=5000)
File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary.
ASSISTANT:
The file has been updated. Let me run the Python file again with the new changes:
<execute_bash>
python3 app.py > server.log 2>&1 &
</execute_bash>
USER:
Observation:
[1] 126
ASSISTANT:
The server is running on port 5000 with PID 126. You can access the list of numbers in a table format by visiting http://127.0.0.1:5000. Free free to let me know if you have any further requests!
--- END OF EXAMPLE ---
NOW, LET'S START!
Write a shell script 'hello.sh' that prints 'hello'.
----------
Certainly! I will create a shell script named `hello.sh` that contains the command to print 'hello'.
<execute_bash>
echo "echo 'hello'" > hello.sh && chmod +x hello.sh
</execute_bash>
----------
OBSERVATION:
[Command -1 finished with exit code 0]]
----------
The shell script `hello.sh` has been created and made executable. Now, let's run the script to see the output.
<execute_bash>
./hello.sh
</execute_bash>
----------
OBSERVATION:
hello
[Command -1 finished with exit code 0]]
----------
The script executed successfully and printed "hello" as expected. If you have any more tasks or questions, feel free to ask!
----------
can you run it to make sure it is working?
----------
Certainly! I will run the `hello.sh` script again to confirm that it is working as intended.
<execute_bash>
./hello.sh
</execute_bash>
----------
OBSERVATION:
hello
[Command -1 finished with exit code 0]]

5
tests/integration/mock/CodeActAgent/test_write_simple_script/response_001.log
View File

@ -1,4 +1,5 @@
Certainly! I will create a shell script named `hello.sh` that contains the command to print 'hello'.
I'll create and write the shell script `hello.sh` that simply prints "hello". Let's proceed with that.
<execute_bash>
echo "echo 'hello'" > hello.sh && chmod +x hello.sh
echo "echo 'hello'" > hello.sh
chmod +x hello.sh

5
tests/integration/mock/CodeActAgent/test_write_simple_script/response_002.log
View File

@ -1,4 +1 @@
The shell script `hello.sh` has been created and made executable. Now, let's run the script to see the output.
<execute_bash>
./hello.sh
The shell script `hello.sh` has been successfully created and made executable. You can run it to see the output "hello". If you need anything else, feel free to ask!

1
tests/integration/mock/CodeActAgent/test_write_simple_script/response_003.log
View File

@ -1 +0,0 @@
The script executed successfully and printed "hello" as expected. If you have any more tasks or questions, feel free to ask!

4
tests/integration/mock/CodeActAgent/test_write_simple_script/response_004.log
View File

@ -1,4 +0,0 @@
Certainly! I will run the `hello.sh` script again to confirm that it is working as intended.
<execute_bash>
./hello.sh

1
tests/integration/mock/CodeActAgent/test_write_simple_script/response_005.log
View File

@ -1 +0,0 @@
The script `hello.sh` is indeed working correctly, as it printed "hello" once more. If you need any further assistance or have another task in mind, just let me know!