diff --git a/.github/workflows/dummy-agent-test.yml b/.github/workflows/dummy-agent-test.yml index baefd93ab8..aae2646795 100644 --- a/.github/workflows/dummy-agent-test.yml +++ b/.github/workflows/dummy-agent-test.yml @@ -27,4 +27,4 @@ jobs: wget https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/1_Pooling/config.json -P /tmp/llama_index/models--BAAI--bge-small-en-v1.5/snapshots/5c38ec7c405ec4b44b94cc5a9bb96e735b38267a/1_Pooling/ - name: Run tests run: | - poetry run python opendevin/main.py -t "do a flip" -m ollama/not-a-model -d ./workspace/ -c DummyAgent + poetry run python opendevin/core/main.py -t "do a flip" -m ollama/not-a-model -d ./workspace/ -c DummyAgent diff --git a/.github/workflows/review-pr.yml b/.github/workflows/review-pr.yml index 740d3ae36e..28eca24faa 100644 --- a/.github/workflows/review-pr.yml +++ b/.github/workflows/review-pr.yml @@ -49,7 +49,7 @@ jobs: LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} SANDBOX_TYPE: exec run: | - WORKSPACE_MOUNT_PATH=$GITHUB_WORKSPACE python ./opendevin/main.py -i 50 -f task.txt -d $GITHUB_WORKSPACE + WORKSPACE_MOUNT_PATH=$GITHUB_WORKSPACE python ./opendevin/core/main.py -i 50 -f task.txt -d $GITHUB_WORKSPACE rm task.txt - name: Check if review file is non-empty diff --git a/.github/workflows/solve-issue.yml b/.github/workflows/solve-issue.yml index b9553babfd..a193075a2f 100644 --- a/.github/workflows/solve-issue.yml +++ b/.github/workflows/solve-issue.yml @@ -43,7 +43,7 @@ jobs: LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} SANDBOX_TYPE: exec run: | - WORKSPACE_MOUNT_PATH=$GITHUB_WORKSPACE python ./opendevin/main.py -i 50 -f task.txt -d $GITHUB_WORKSPACE + WORKSPACE_MOUNT_PATH=$GITHUB_WORKSPACE python ./opendevin/core/main.py -i 50 -f task.txt -d $GITHUB_WORKSPACE rm task.txt - name: Setup Git, Create Branch, and Commit Changes diff --git a/agenthub/SWE_agent/__init__.py b/agenthub/SWE_agent/__init__.py index 58e5f72038..69a7688082 100644 --- a/agenthub/SWE_agent/__init__.py +++ b/agenthub/SWE_agent/__init__.py @@ -1,4 +1,4 @@ -from opendevin.agent import Agent +from opendevin.controller.agent import Agent from .agent import SWEAgent diff --git a/agenthub/SWE_agent/agent.py b/agenthub/SWE_agent/agent.py index b09e1a5fb1..26eb65b14c 100644 --- a/agenthub/SWE_agent/agent.py +++ b/agenthub/SWE_agent/agent.py @@ -1,6 +1,7 @@ from typing import List -from opendevin.agent import Agent +from opendevin.controller.agent import Agent +from opendevin.controller.state.state import State from opendevin.events.action import ( Action, AgentThinkAction, @@ -9,7 +10,6 @@ from opendevin.events.action import ( ) from opendevin.events.observation import Observation from opendevin.llm.llm import LLM -from opendevin.state import State from .parser import parse_command from .prompts import ( @@ -48,9 +48,12 @@ class SWEAgent(Agent): ) action_resp = resp['choices'][0]['message']['content'] print(f"\033[1m\033[91m{resp['usage']}\033[0m") - print('\n==== RAW OUTPUT ====', - f'\033[96m{action_resp}\033[0m', - '==== END RAW ====\n', sep='\n') + print( + '\n==== RAW OUTPUT ====', + f'\033[96m{action_resp}\033[0m', + '==== END RAW ====\n', + sep='\n', + ) return parse_command(action_resp, self.cur_file, self.cur_line) def _update(self, action: Action) -> None: @@ -68,22 +71,15 @@ class SWEAgent(Agent): for prev_action, obs in state.updated_info: self._remember(prev_action, obs) - prompt = STEP_PROMPT( - state.plan.main_goal, - self.cur_file, - self.cur_line - ) + prompt = STEP_PROMPT(state.plan.main_goal, self.cur_file, self.cur_line) msgs = [ {'content': SYSTEM_MESSAGE, 'role': 'system'}, - {'content': prompt, 'role': 'user'} + {'content': prompt, 'role': 'user'}, ] if len(self.running_memory) > 0: - context = CONTEXT_PROMPT( - self.running_memory, - self.memory_window - ) + context = CONTEXT_PROMPT(self.running_memory, self.memory_window) msgs.insert(1, {'content': context, 'role': 'user'}) # clrs = [''] * (len(msgs)-2) + ['\033[0;36m', '\033[0;35m'] # print('\n\n'.join([c+m['content']+'\033[0m' for c, m in zip(clrs, msgs)])) diff --git a/agenthub/__init__.py b/agenthub/__init__.py index 8c7c4027e0..f329a6c4bc 100644 --- a/agenthub/__init__.py +++ b/agenthub/__init__.py @@ -1,6 +1,6 @@ from dotenv import load_dotenv -from opendevin.agent import Agent +from opendevin.controller.agent import Agent from .micro.agent import MicroAgent from .micro.registry import all_microagents @@ -8,7 +8,6 @@ from .micro.registry import all_microagents load_dotenv() - from . import ( # noqa: E402 SWE_agent, codeact_agent, @@ -18,18 +17,26 @@ from . import ( # noqa: E402 planner_agent, ) -__all__ = ['monologue_agent', 'codeact_agent', - 'planner_agent', 'SWE_agent', - 'delegator_agent', - 'dummy_agent'] +__all__ = [ + 'monologue_agent', + 'codeact_agent', + 'planner_agent', + 'SWE_agent', + 'delegator_agent', + 'dummy_agent', +] for agent in all_microagents.values(): name = agent['name'] prompt = agent['prompt'] - anon_class = type(name, (MicroAgent,), { - 'prompt': prompt, - 'agent_definition': agent, - }) + anon_class = type( + name, + (MicroAgent,), + { + 'prompt': prompt, + 'agent_definition': agent, + }, + ) Agent.register(name, anon_class) diff --git a/agenthub/codeact_agent/README.md b/agenthub/codeact_agent/README.md index 22b7ef6e55..d84331a604 100644 --- a/agenthub/codeact_agent/README.md +++ b/agenthub/codeact_agent/README.md @@ -9,7 +9,7 @@ This folder implements the [CodeAct idea](https://arxiv.org/abs/2402.13463) that ```bash mkdir workspace -PYTHONPATH=`pwd`:$PYTHONPATH python3 opendevin/main.py -d ./workspace -c CodeActAgent -t "Please write a flask app that returns 'Hello, World\!' at the root URL, then start the app on port 5000. python3 has already been installed for you." +PYTHONPATH=`pwd`:$PYTHONPATH python3 opendevin/core/main.py -d ./workspace -c CodeActAgent -t "Please write a flask app that returns 'Hello, World\!' at the root URL, then start the app on port 5000. python3 has already been installed for you." ``` Example: prompts `gpt-4-0125-preview` to write a flask server, install `flask` library, and start the server. diff --git a/agenthub/codeact_agent/__init__.py b/agenthub/codeact_agent/__init__.py index c8d08d364d..9f877f5475 100644 --- a/agenthub/codeact_agent/__init__.py +++ b/agenthub/codeact_agent/__init__.py @@ -1,4 +1,4 @@ -from opendevin.agent import Agent +from opendevin.controller.agent import Agent from .codeact_agent import CodeActAgent diff --git a/agenthub/codeact_agent/codeact_agent.py b/agenthub/codeact_agent/codeact_agent.py index fc95261253..9d8dc345ed 100644 --- a/agenthub/codeact_agent/codeact_agent.py +++ b/agenthub/codeact_agent/codeact_agent.py @@ -2,7 +2,8 @@ import re from typing import List, Mapping from agenthub.codeact_agent.prompt import EXAMPLES, SYSTEM_MESSAGE -from opendevin.agent import Agent +from opendevin.controller.agent import Agent +from opendevin.controller.state.state import State from opendevin.events.action import ( Action, AgentEchoAction, @@ -19,12 +20,11 @@ from opendevin.events.observation import ( UserMessageObservation, ) from opendevin.llm.llm import LLM -from opendevin.sandbox.plugins import ( +from opendevin.runtime.plugins import ( JupyterRequirement, PluginRequirement, SWEAgentCommandsRequirement, ) -from opendevin.state import State def parse_response(response) -> str: @@ -34,14 +34,20 @@ def parse_response(response) -> str: action += f'' return action -def truncate_observation(observation: str, max_chars: int=5000) -> str: + +def truncate_observation(observation: str, max_chars: int = 5000) -> str: """ Truncate the middle of the observation if it is too long. """ if len(observation) <= max_chars: return observation half = max_chars // 2 - return observation[:half] + '\n[... Observation truncated due to length ...]\n' + observation[-half:] + return ( + observation[:half] + + '\n[... Observation truncated due to length ...]\n' + + observation[-half:] + ) + class CodeActAgent(Agent): """ @@ -49,19 +55,22 @@ class CodeActAgent(Agent): The agent works by passing the model a list of action-observation pairs and prompting the model to take the next step. """ - sandbox_plugins: List[PluginRequirement] = [JupyterRequirement(), SWEAgentCommandsRequirement()] + sandbox_plugins: List[PluginRequirement] = [ + JupyterRequirement(), + SWEAgentCommandsRequirement(), + ] SUPPORTED_ACTIONS = ( CmdRunAction, IPythonRunCellAction, AgentEchoAction, AgentTalkAction, - NullAction + NullAction, ) SUPPORTED_OBSERVATIONS = ( AgentMessageObservation, UserMessageObservation, CmdOutputObservation, - IPythonRunCellObservation + IPythonRunCellObservation, ) def __init__( @@ -102,7 +111,7 @@ class CodeActAgent(Agent): 'content': ( f'Here is an example of how you can interact with the environment for task solving:\n{EXAMPLES}\n\n' f"NOW, LET'S START!\n\n{state.plan.main_goal}" - ) + ), }, ] updated_info = state.updated_info @@ -118,8 +127,7 @@ class CodeActAgent(Agent): obs, self.SUPPORTED_OBSERVATIONS ), f'{obs.__class__} is not supported (supported: {self.SUPPORTED_OBSERVATIONS})' if isinstance(obs, (AgentMessageObservation, UserMessageObservation)): - self.messages.append( - {'role': 'user', 'content': obs.content}) + self.messages.append({'role': 'user', 'content': obs.content}) # User wants to exit if obs.content.strip() == '/exit': @@ -135,7 +143,9 @@ class CodeActAgent(Agent): splited = content.split('\n') for i, line in enumerate(splited): if '![image](data:image/png;base64,' in line: - splited[i] = '![image](data:image/png;base64, ...) already displayed to user' + splited[i] = ( + '![image](data:image/png;base64, ...) already displayed to user' + ) content = '\n'.join(splited) content = truncate_observation(content) self.messages.append({'role': 'user', 'content': content}) @@ -150,7 +160,7 @@ class CodeActAgent(Agent): '', '', ], - temperature=0.0 + temperature=0.0, ) action_str: str = parse_response(response) state.num_of_chars += sum( @@ -158,7 +168,9 @@ class CodeActAgent(Agent): ) + len(action_str) self.messages.append({'role': 'assistant', 'content': action_str}) - if bash_command := re.search(r'(.*)', action_str, re.DOTALL): + if bash_command := re.search( + r'(.*)', action_str, re.DOTALL + ): # remove the command from the action string to get thought thought = action_str.replace(bash_command.group(0), '').strip() # a command was found @@ -166,7 +178,9 @@ class CodeActAgent(Agent): if command_group.strip() == 'exit': return AgentFinishAction() return CmdRunAction(command=command_group, thought=thought) - elif python_code := re.search(r'(.*)', action_str, re.DOTALL): + elif python_code := re.search( + r'(.*)', action_str, re.DOTALL + ): # a code block was found code_group = python_code.group(1).strip() thought = action_str.replace(python_code.group(0), '').strip() diff --git a/agenthub/codeact_agent/prompt.py b/agenthub/codeact_agent/prompt.py index d6c3c54a21..ec8d71ba31 100644 --- a/agenthub/codeact_agent/prompt.py +++ b/agenthub/codeact_agent/prompt.py @@ -1,9 +1,9 @@ -from opendevin.sandbox.plugins import SWEAgentCommandsRequirement +from opendevin.runtime.plugins import SWEAgentCommandsRequirement _SWEAGENT_BASH_DOCS = '\n'.join( filter( lambda x: not x.startswith('submit'), - SWEAgentCommandsRequirement.documentation.split('\n') + SWEAgentCommandsRequirement.documentation.split('\n'), ) ) # _SWEAGENT_BASH_DOCS content below: diff --git a/agenthub/delegator_agent/__init__.py b/agenthub/delegator_agent/__init__.py index d25d295e3a..dbae30af72 100644 --- a/agenthub/delegator_agent/__init__.py +++ b/agenthub/delegator_agent/__init__.py @@ -1,4 +1,4 @@ -from opendevin.agent import Agent +from opendevin.controller.agent import Agent from .agent import DelegatorAgent diff --git a/agenthub/delegator_agent/agent.py b/agenthub/delegator_agent/agent.py index 0c42948afb..6becb99afc 100644 --- a/agenthub/delegator_agent/agent.py +++ b/agenthub/delegator_agent/agent.py @@ -1,10 +1,10 @@ from typing import List -from opendevin.agent import Agent +from opendevin.controller.agent import Agent +from opendevin.controller.state.state import State from opendevin.events.action import Action, AgentDelegateAction, AgentFinishAction from opendevin.events.observation import AgentDelegateObservation from opendevin.llm.llm import LLM -from opendevin.state import State class DelegatorAgent(Agent): @@ -12,6 +12,7 @@ class DelegatorAgent(Agent): The planner agent utilizes a special prompting strategy to create long term plans for solving problems. The agent is given its previous action-observation pairs, current task, and hint based on last action taken at every step. """ + current_delegate: str = '' def __init__(self, llm: LLM): @@ -37,9 +38,9 @@ class DelegatorAgent(Agent): """ if self.current_delegate == '': self.current_delegate = 'study' - return AgentDelegateAction(agent='StudyRepoForTaskAgent', inputs={ - 'task': state.plan.main_goal - }) + return AgentDelegateAction( + agent='StudyRepoForTaskAgent', inputs={'task': state.plan.main_goal} + ) lastObservation = state.history[-1][1] if not isinstance(lastObservation, AgentDelegateObservation): @@ -47,24 +48,36 @@ class DelegatorAgent(Agent): if self.current_delegate == 'study': self.current_delegate = 'coder' - return AgentDelegateAction(agent='Coder', inputs={ - 'task': state.plan.main_goal, - 'summary': lastObservation.outputs['summary'], - }) + return AgentDelegateAction( + agent='Coder', + inputs={ + 'task': state.plan.main_goal, + 'summary': lastObservation.outputs['summary'], + }, + ) elif self.current_delegate == 'coder': self.current_delegate = 'verifier' - return AgentDelegateAction(agent='Verifier', inputs={ - 'task': state.plan.main_goal, - }) + return AgentDelegateAction( + agent='Verifier', + inputs={ + 'task': state.plan.main_goal, + }, + ) elif self.current_delegate == 'verifier': - if 'completed' in lastObservation.outputs and lastObservation.outputs['completed']: + if ( + 'completed' in lastObservation.outputs + and lastObservation.outputs['completed'] + ): return AgentFinishAction() else: self.current_delegate = 'coder' - return AgentDelegateAction(agent='Coder', inputs={ - 'task': state.plan.main_goal, - 'summary': lastObservation.outputs['summary'], - }) + return AgentDelegateAction( + agent='Coder', + inputs={ + 'task': state.plan.main_goal, + 'summary': lastObservation.outputs['summary'], + }, + ) else: raise Exception('Invalid delegate state') diff --git a/agenthub/dummy_agent/__init__.py b/agenthub/dummy_agent/__init__.py index 1c8698ccd1..831f488224 100644 --- a/agenthub/dummy_agent/__init__.py +++ b/agenthub/dummy_agent/__init__.py @@ -1,4 +1,4 @@ -from opendevin.agent import Agent +from opendevin.controller.agent import Agent from .agent import DummyAgent diff --git a/agenthub/dummy_agent/agent.py b/agenthub/dummy_agent/agent.py index f1c7016e15..357f12ff60 100644 --- a/agenthub/dummy_agent/agent.py +++ b/agenthub/dummy_agent/agent.py @@ -1,7 +1,8 @@ import time from typing import List, TypedDict -from opendevin.agent import Agent +from opendevin.controller.agent import Agent +from opendevin.controller.state.state import State from opendevin.events.action import ( Action, AddTaskAction, @@ -23,7 +24,6 @@ from opendevin.events.observation import ( Observation, ) from opendevin.llm.llm import LLM -from opendevin.state import State """ FIXME: There are a few problems this surfaced @@ -33,7 +33,9 @@ FIXME: There are a few problems this surfaced * Browser not working """ -ActionObs = TypedDict('ActionObs', {'action': Action, 'observations': List[Observation]}) +ActionObs = TypedDict( + 'ActionObs', {'action': Action, 'observations': List[Observation]} +) BACKGROUND_CMD = 'echo "This is in the background" && sleep .1 && echo "This too"' @@ -46,51 +48,82 @@ class DummyAgent(Agent): def __init__(self, llm: LLM): super().__init__(llm) - self.steps: List[ActionObs] = [{ - 'action': AddTaskAction(parent='0', goal='check the current directory'), - 'observations': [NullObservation('')], - }, { - 'action': AddTaskAction(parent='0.0', goal='run ls'), - 'observations': [NullObservation('')], - }, { - 'action': ModifyTaskAction(id='0.0', state='in_progress'), - 'observations': [NullObservation('')], - }, { - 'action': AgentThinkAction(thought='Time to get started!'), - 'observations': [NullObservation('')], - }, { - 'action': CmdRunAction(command='echo "foo"'), - 'observations': [CmdOutputObservation('foo', command_id=-1, command='echo "foo"')], - }, { - 'action': FileWriteAction(content='echo "Hello, World!"', path='hello.sh'), - 'observations': [FileWriteObservation('', path='hello.sh')], - }, { - 'action': FileReadAction(path='hello.sh'), - 'observations': [FileReadObservation('echo "Hello, World!"\n', path='hello.sh')], - }, { - 'action': CmdRunAction(command='bash hello.sh'), - 'observations': [CmdOutputObservation('Hello, World!', command_id=-1, command='bash hello.sh')], - }, { - 'action': CmdRunAction(command=BACKGROUND_CMD, background=True), - 'observations': [ - CmdOutputObservation('Background command started. To stop it, send a `kill` action with id 42', command_id='42', command=BACKGROUND_CMD), # type: ignore[arg-type] - CmdOutputObservation('This is in the background\nThis too\n', command_id='42', command=BACKGROUND_CMD), # type: ignore[arg-type] - ] - }, { - 'action': AgentRecallAction(query='who am I?'), - 'observations': [ - AgentRecallObservation('', memories=['I am a computer.']), - # CmdOutputObservation('This too\n', command_id='42', command=BACKGROUND_CMD), - ], - }, { - 'action': BrowseURLAction(url='https://google.com'), - 'observations': [ - # BrowserOutputObservation('', url='https://google.com', screenshot=""), - ], - }, { - 'action': AgentFinishAction(), - 'observations': [], - }] + self.steps: List[ActionObs] = [ + { + 'action': AddTaskAction(parent='0', goal='check the current directory'), + 'observations': [NullObservation('')], + }, + { + 'action': AddTaskAction(parent='0.0', goal='run ls'), + 'observations': [NullObservation('')], + }, + { + 'action': ModifyTaskAction(id='0.0', state='in_progress'), + 'observations': [NullObservation('')], + }, + { + 'action': AgentThinkAction(thought='Time to get started!'), + 'observations': [NullObservation('')], + }, + { + 'action': CmdRunAction(command='echo "foo"'), + 'observations': [ + CmdOutputObservation('foo', command_id=-1, command='echo "foo"') + ], + }, + { + 'action': FileWriteAction( + content='echo "Hello, World!"', path='hello.sh' + ), + 'observations': [FileWriteObservation('', path='hello.sh')], + }, + { + 'action': FileReadAction(path='hello.sh'), + 'observations': [ + FileReadObservation('echo "Hello, World!"\n', path='hello.sh') + ], + }, + { + 'action': CmdRunAction(command='bash hello.sh'), + 'observations': [ + CmdOutputObservation( + 'Hello, World!', command_id=-1, command='bash hello.sh' + ) + ], + }, + { + 'action': CmdRunAction(command=BACKGROUND_CMD, background=True), + 'observations': [ + CmdOutputObservation( + 'Background command started. To stop it, send a `kill` action with id 42', + command_id='42', # type: ignore[arg-type] + command=BACKGROUND_CMD, + ), + CmdOutputObservation( + 'This is in the background\nThis too\n', + command_id='42', # type: ignore[arg-type] + command=BACKGROUND_CMD, + ), + ], + }, + { + 'action': AgentRecallAction(query='who am I?'), + 'observations': [ + AgentRecallObservation('', memories=['I am a computer.']), + # CmdOutputObservation('This too\n', command_id='42', command=BACKGROUND_CMD), + ], + }, + { + 'action': BrowseURLAction(url='https://google.com'), + 'observations': [ + # BrowserOutputObservation('', url='https://google.com', screenshot=""), + ], + }, + { + 'action': AgentFinishAction(), + 'observations': [], + }, + ] def step(self, state: State) -> Action: time.sleep(0.1) @@ -102,16 +135,24 @@ class DummyAgent(Agent): for i in range(len(expected_observations)): hist_obs = state.history[hist_start + i][1].to_dict() expected_obs = expected_observations[i].to_dict() - if 'command_id' in hist_obs['extras'] and hist_obs['extras']['command_id'] != -1: + if ( + 'command_id' in hist_obs['extras'] + and hist_obs['extras']['command_id'] != -1 + ): del hist_obs['extras']['command_id'] hist_obs['content'] = '' - if 'command_id' in expected_obs['extras'] and expected_obs['extras']['command_id'] != -1: + if ( + 'command_id' in expected_obs['extras'] + and expected_obs['extras']['command_id'] != -1 + ): del expected_obs['extras']['command_id'] expected_obs['content'] = '' if hist_obs != expected_obs: print('\nactual', hist_obs) print('\nexpect', expected_obs) - assert hist_obs == expected_obs, f'Expected observation {expected_obs}, got {hist_obs}' + assert ( + hist_obs == expected_obs + ), f'Expected observation {expected_obs}, got {hist_obs}' return self.steps[state.iteration]['action'] def search_memory(self, query: str) -> List[str]: diff --git a/agenthub/micro/agent.py b/agenthub/micro/agent.py index 32df7f5ac3..4050929838 100644 --- a/agenthub/micro/agent.py +++ b/agenthub/micro/agent.py @@ -3,11 +3,11 @@ from typing import Dict, List from jinja2 import BaseLoader, Environment -from opendevin.agent import Agent +from opendevin.controller.agent import Agent +from opendevin.controller.state.state import State +from opendevin.core.exceptions import LLMOutputError from opendevin.events.action import Action, action_from_dict -from opendevin.exceptions import LLMOutputError from opendevin.llm.llm import LLM -from opendevin.state import State from .instructions import instructions from .registry import all_microagents @@ -65,7 +65,8 @@ class MicroAgent(Agent): state=state, instructions=instructions, to_json=to_json, - delegates=self.delegates) + delegates=self.delegates, + ) messages = [{'content': prompt, 'role': 'user'}] resp = self.llm.completion(messages=messages) action_resp = resp['choices'][0]['message']['content'] diff --git a/agenthub/micro/commit_writer/README.md b/agenthub/micro/commit_writer/README.md index 9e672b6edc..6c33eabe4c 100644 --- a/agenthub/micro/commit_writer/README.md +++ b/agenthub/micro/commit_writer/README.md @@ -4,7 +4,7 @@ CommitWriterAgent can help write git commit message. Example: ```bash WORKSPACE_MOUNT_PATH="`PWD`" SANDBOX_TYPE="exec" \ - poetry run python opendevin/main.py -t "dummy task" -c CommitWriterAgent -d ./ + poetry run python opendevin/core/main.py -t "dummy task" -c CommitWriterAgent -d ./ ``` This agent is special in the sense that it doesn't need a task. Once called, diff --git a/agenthub/monologue_agent/__init__.py b/agenthub/monologue_agent/__init__.py index b60cb48bb5..b6f326089c 100644 --- a/agenthub/monologue_agent/__init__.py +++ b/agenthub/monologue_agent/__init__.py @@ -1,4 +1,4 @@ -from opendevin.agent import Agent +from opendevin.controller.agent import Agent from .agent import MonologueAgent diff --git a/agenthub/monologue_agent/agent.py b/agenthub/monologue_agent/agent.py index fd3a701362..7d2f00e6e1 100644 --- a/agenthub/monologue_agent/agent.py +++ b/agenthub/monologue_agent/agent.py @@ -2,8 +2,12 @@ from typing import List import agenthub.monologue_agent.utils.prompts as prompts from agenthub.monologue_agent.utils.monologue import Monologue -from opendevin import config -from opendevin.agent import Agent +from opendevin.controller.agent import Agent +from opendevin.controller.state.state import State +from opendevin.core import config +from opendevin.core.exceptions import AgentNoInstructionError +from opendevin.core.schema import ActionType +from opendevin.core.schema.config import ConfigType from opendevin.events.action import ( Action, AgentRecallAction, @@ -23,11 +27,7 @@ from opendevin.events.observation import ( NullObservation, Observation, ) -from opendevin.exceptions import AgentNoInstructionError from opendevin.llm.llm import LLM -from opendevin.schema import ActionType -from opendevin.schema.config import ConfigType -from opendevin.state import State if config.get(ConfigType.AGENT_MEMORY_ENABLED): from agenthub.monologue_agent.utils.memory import LongTermMemory @@ -56,7 +56,7 @@ INITIAL_THOUGHTS = [ 'RUN echo "hello world"', 'hello world', 'Cool! I bet I can write files too using the write action.', - "WRITE echo \"console.log('hello world')\" > test.js", + 'WRITE echo "console.log(\'hello world\')" > test.js', '', "I just created test.js. I'll try and run it now.", 'RUN node test.js', @@ -173,8 +173,7 @@ class MonologueAgent(Agent): elif previous_action == ActionType.READ: observation = FileReadObservation(content=thought, path='') elif previous_action == ActionType.RECALL: - observation = AgentRecallObservation( - content=thought, memories=[]) + observation = AgentRecallObservation(content=thought, memories=[]) elif previous_action == ActionType.BROWSE: observation = BrowserOutputObservation( content=thought, url='', screenshot='' diff --git a/agenthub/monologue_agent/utils/memory.py b/agenthub/monologue_agent/utils/memory.py index 9f5433ff73..d0d16e5a0f 100644 --- a/agenthub/monologue_agent/utils/memory.py +++ b/agenthub/monologue_agent/utils/memory.py @@ -13,9 +13,9 @@ from tenacity import ( wait_random_exponential, ) -from opendevin import config -from opendevin.logger import opendevin_logger as logger -from opendevin.schema.config import ConfigType +from opendevin.core import config +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema.config import ConfigType from . import json @@ -37,15 +37,22 @@ else: def attempt_on_error(retry_state): - logger.error(f'{retry_state.outcome.exception()}. Attempt #{retry_state.attempt_number} | You can customize these settings in the configuration.', exc_info=False) + logger.error( + f'{retry_state.outcome.exception()}. Attempt #{retry_state.attempt_number} | You can customize these settings in the configuration.', + exc_info=False, + ) return True -@retry(reraise=True, - stop=stop_after_attempt(num_retries), - wait=wait_random_exponential(min=retry_min_wait, max=retry_max_wait), - retry=retry_if_exception_type((RateLimitError, APIConnectionError, InternalServerError)), - after=attempt_on_error) +@retry( + reraise=True, + stop=stop_after_attempt(num_retries), + wait=wait_random_exponential(min=retry_min_wait, max=retry_max_wait), + retry=retry_if_exception_type( + (RateLimitError, APIConnectionError, InternalServerError) + ), + after=attempt_on_error, +) def wrapper_get_embeddings(*args, **kwargs): return original_get_embeddings(*args, **kwargs) @@ -56,9 +63,16 @@ embedding_strategy = config.get(ConfigType.LLM_EMBEDDING_MODEL) # TODO: More embeddings: https://docs.llamaindex.ai/en/stable/examples/embeddings/OpenAI/ # There's probably a more programmatic way to do this. -supported_ollama_embed_models = ['llama2', 'mxbai-embed-large', 'nomic-embed-text', 'all-minilm', 'stable-code'] +supported_ollama_embed_models = [ + 'llama2', + 'mxbai-embed-large', + 'nomic-embed-text', + 'all-minilm', + 'stable-code', +] if embedding_strategy in supported_ollama_embed_models: from llama_index.embeddings.ollama import OllamaEmbedding + embed_model = OllamaEmbedding( model_name=embedding_strategy, base_url=config.get(ConfigType.LLM_EMBEDDING_BASE_URL, required=True), @@ -66,16 +80,20 @@ if embedding_strategy in supported_ollama_embed_models: ) elif embedding_strategy == 'openai': from llama_index.embeddings.openai import OpenAIEmbedding + embed_model = OpenAIEmbedding( model='text-embedding-ada-002', - api_key=config.get(ConfigType.LLM_API_KEY, required=True) + api_key=config.get(ConfigType.LLM_API_KEY, required=True), ) elif embedding_strategy == 'azureopenai': # Need to instruct to set these env variables in documentation from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding + embed_model = AzureOpenAIEmbedding( model='text-embedding-ada-002', - deployment_name=config.get(ConfigType.LLM_EMBEDDING_DEPLOYMENT_NAME, required=True), + deployment_name=config.get( + ConfigType.LLM_EMBEDDING_DEPLOYMENT_NAME, required=True + ), api_key=config.get(ConfigType.LLM_API_KEY, required=True), azure_endpoint=config.get(ConfigType.LLM_BASE_URL, required=True), api_version=config.get(ConfigType.LLM_API_VERSION, required=True), @@ -87,9 +105,8 @@ elif (embedding_strategy is not None) and (embedding_strategy.lower() == 'none') embed_model = None else: from llama_index.embeddings.huggingface import HuggingFaceEmbedding - embed_model = HuggingFaceEmbedding( - model_name='BAAI/bge-small-en-v1.5' - ) + + embed_model = HuggingFaceEmbedding(model_name='BAAI/bge-small-en-v1.5') sema = threading.Semaphore(value=config.get(ConfigType.AGENT_MEMORY_MAX_THREADS)) @@ -109,7 +126,8 @@ class LongTermMemory: self.collection = db.get_or_create_collection(name='memories') vector_store = ChromaVectorStore(chroma_collection=self.collection) self.index = VectorStoreIndex.from_vector_store( - vector_store, embed_model=embed_model) + vector_store, embed_model=embed_model + ) self.thought_idx = 0 self._add_threads = [] diff --git a/agenthub/monologue_agent/utils/monologue.py b/agenthub/monologue_agent/utils/monologue.py index 545498d7b5..6008789abb 100644 --- a/agenthub/monologue_agent/utils/monologue.py +++ b/agenthub/monologue_agent/utils/monologue.py @@ -1,9 +1,8 @@ - import agenthub.monologue_agent.utils.json as json import agenthub.monologue_agent.utils.prompts as prompts -from opendevin.exceptions import AgentEventTypeError +from opendevin.core.exceptions import AgentEventTypeError +from opendevin.core.logger import opendevin_logger as logger from opendevin.llm.llm import LLM -from opendevin.logger import opendevin_logger as logger class Monologue: diff --git a/agenthub/monologue_agent/utils/prompts.py b/agenthub/monologue_agent/utils/prompts.py index 95fbff85b6..a543e6223a 100644 --- a/agenthub/monologue_agent/utils/prompts.py +++ b/agenthub/monologue_agent/utils/prompts.py @@ -2,7 +2,9 @@ import re from json import JSONDecodeError from typing import List -from opendevin import config +from opendevin.core import config +from opendevin.core.exceptions import LLMOutputError +from opendevin.core.schema.config import ConfigType from opendevin.events.action import ( Action, action_from_dict, @@ -10,8 +12,6 @@ from opendevin.events.action import ( from opendevin.events.observation import ( CmdOutputObservation, ) -from opendevin.exceptions import LLMOutputError -from opendevin.schema.config import ConfigType from . import json @@ -158,7 +158,9 @@ def get_request_action_prompt( 'hint': hint, 'user': user, 'timeout': config.get(ConfigType.SANDBOX_TIMEOUT), - 'WORKSPACE_MOUNT_PATH_IN_SANDBOX': config.get(ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX), + 'WORKSPACE_MOUNT_PATH_IN_SANDBOX': config.get( + ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX + ), } @@ -178,12 +180,18 @@ def parse_action_response(response: str) -> Action: # Find response-looking json in the output and use the more promising one. Helps with weak llms response_json_matches = re.finditer( r"""{\s*\"action\":\s?\"(\w+)\"(?:,?|,\s*\"args\":\s?{((?:.|\s)*?)})\s*}""", - response) # Find all response-looking strings + response, + ) # Find all response-looking strings def rank(match): - return len(match[2]) if match[1] == 'think' else 130 # Crudely rank multiple responses by length + return ( + len(match[2]) if match[1] == 'think' else 130 + ) # Crudely rank multiple responses by length + try: - action_dict = json.loads(max(response_json_matches, key=rank)[0]) # Use the highest ranked response + action_dict = json.loads( + max(response_json_matches, key=rank)[0] + ) # Use the highest ranked response except (ValueError, JSONDecodeError): raise LLMOutputError( 'Invalid JSON, the response must be well-formed JSON as specified in the prompt.' diff --git a/agenthub/planner_agent/__init__.py b/agenthub/planner_agent/__init__.py index d81ba6cc26..e447765391 100644 --- a/agenthub/planner_agent/__init__.py +++ b/agenthub/planner_agent/__init__.py @@ -1,4 +1,4 @@ -from opendevin.agent import Agent +from opendevin.controller.agent import Agent from .agent import PlannerAgent diff --git a/agenthub/planner_agent/agent.py b/agenthub/planner_agent/agent.py index 43623a157a..f2b518964f 100644 --- a/agenthub/planner_agent/agent.py +++ b/agenthub/planner_agent/agent.py @@ -1,9 +1,9 @@ from typing import List -from opendevin.agent import Agent +from opendevin.controller.agent import Agent +from opendevin.controller.state.state import State from opendevin.events.action import Action, AgentFinishAction from opendevin.llm.llm import LLM -from opendevin.state import State from .prompt import get_prompt, parse_response diff --git a/agenthub/planner_agent/prompt.py b/agenthub/planner_agent/prompt.py index 282280ed8e..3d17f5df98 100644 --- a/agenthub/planner_agent/prompt.py +++ b/agenthub/planner_agent/prompt.py @@ -1,6 +1,9 @@ import json from typing import Dict, List, Tuple, Type +from opendevin.controller.state.plan import Plan +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema import ActionType from opendevin.events.action import ( Action, AddTaskAction, @@ -21,9 +24,6 @@ from opendevin.events.observation import ( NullObservation, Observation, ) -from opendevin.logger import opendevin_logger as logger -from opendevin.plan import Plan -from opendevin.schema import ActionType ACTION_TYPE_TO_CLASS: Dict[str, Type[Action]] = { ActionType.RUN: CmdRunAction, @@ -131,7 +131,7 @@ What is your next thought or action? Again, you must reply with JSON, and only w def get_hint(latest_action_id: str) -> str: - """ Returns action type hint based on given action_id """ + """Returns action type hint based on given action_id""" hints = { '': "You haven't taken any actions yet. Start by using `ls` to check out what files you're working with.", diff --git a/docs/modules/python/opendevin/controller/agent.md b/docs/modules/python/opendevin/controller/agent.md new file mode 100644 index 0000000000..f67d815f65 --- /dev/null +++ b/docs/modules/python/opendevin/controller/agent.md @@ -0,0 +1,121 @@ +--- +sidebar_label: agent +title: opendevin.controller.agent +--- + +## Agent Objects + +```python +class Agent(ABC) +``` + +This abstract base class is an general interface for an agent dedicated to +executing a specific instruction and allowing human interaction with the +agent during execution. +It tracks the execution status and maintains a history of interactions. + +#### complete + +```python +@property +def complete() -> bool +``` + +Indicates whether the current instruction execution is complete. + +**Returns**: + + - complete (bool): True if execution is complete; False otherwise. + +#### step + +```python +@abstractmethod +def step(state: 'State') -> 'Action' +``` + +Starts the execution of the assigned instruction. This method should +be implemented by subclasses to define the specific execution logic. + +#### search\_memory + +```python +@abstractmethod +def search_memory(query: str) -> List[str] +``` + +Searches the agent's memory for information relevant to the given query. + +**Arguments**: + + - query (str): The query to search for in the agent's memory. + + +**Returns**: + + - response (str): The response to the query. + +#### reset + +```python +def reset() -> None +``` + +Resets the agent's execution status and clears the history. This method can be used +to prepare the agent for restarting the instruction or cleaning up before destruction. + +#### register + +```python +@classmethod +def register(cls, name: str, agent_cls: Type['Agent']) +``` + +Registers an agent class in the registry. + +**Arguments**: + + - name (str): The name to register the class under. + - agent_cls (Type['Agent']): The class to register. + + +**Raises**: + + - AgentAlreadyRegisteredError: If name already registered + +#### get\_cls + +```python +@classmethod +def get_cls(cls, name: str) -> Type['Agent'] +``` + +Retrieves an agent class from the registry. + +**Arguments**: + + - name (str): The name of the class to retrieve + + +**Returns**: + + - agent_cls (Type['Agent']): The class registered under the specified name. + + +**Raises**: + + - AgentNotRegisteredError: If name not registered + +#### list\_agents + +```python +@classmethod +def list_agents(cls) -> list[str] +``` + +Retrieves the list of all agent names from the registry. + +**Raises**: + + - AgentNotRegisteredError: If no agent is registered + diff --git a/docs/modules/python/opendevin/controller/state/plan.md b/docs/modules/python/opendevin/controller/state/plan.md new file mode 100644 index 0000000000..ff57d8edc6 --- /dev/null +++ b/docs/modules/python/opendevin/controller/state/plan.md @@ -0,0 +1,182 @@ +--- +sidebar_label: plan +title: opendevin.controller.state.plan +--- + +## Task Objects + +```python +class Task() +``` + +#### \_\_init\_\_ + +```python +def __init__(parent: 'Task | None', + goal: str, + state: str = OPEN_STATE, + subtasks: List = []) +``` + +Initializes a new instance of the Task class. + +**Arguments**: + +- `parent` - The parent task, or None if it is the root task. +- `goal` - The goal of the task. +- `state` - The initial state of the task. +- `subtasks` - A list of subtasks associated with this task. + +#### to\_string + +```python +def to_string(indent='') +``` + +Returns a string representation of the task and its subtasks. + +**Arguments**: + +- `indent` - The indentation string for formatting the output. + + +**Returns**: + + A string representation of the task and its subtasks. + +#### to\_dict + +```python +def to_dict() +``` + +Returns a dictionary representation of the task. + +**Returns**: + + A dictionary containing the task's attributes. + +#### set\_state + +```python +def set_state(state) +``` + +Sets the state of the task and its subtasks. + +Args: state: The new state of the task. + +**Raises**: + +- `PlanInvalidStateError` - If the provided state is invalid. + +#### get\_current\_task + +```python +def get_current_task() -> 'Task | None' +``` + +Retrieves the current task in progress. + +**Returns**: + + The current task in progress, or None if no task is in progress. + +## Plan Objects + +```python +class Plan() +``` + +Represents a plan consisting of tasks. + +**Attributes**: + +- `main_goal` - The main goal of the plan. +- `task` - The root task of the plan. + +#### \_\_init\_\_ + +```python +def __init__(task: str) +``` + +Initializes a new instance of the Plan class. + +**Arguments**: + +- `task` - The main goal of the plan. + +#### \_\_str\_\_ + +```python +def __str__() +``` + +Returns a string representation of the plan. + +**Returns**: + + A string representation of the plan. + +#### get\_task\_by\_id + +```python +def get_task_by_id(id: str) -> Task +``` + +Retrieves a task by its ID. + +**Arguments**: + +- `id` - The ID of the task. + + +**Returns**: + + The task with the specified ID. + + +**Raises**: + +- `ValueError` - If the provided task ID is invalid or does not exist. + +#### add\_subtask + +```python +def add_subtask(parent_id: str, goal: str, subtasks: List = []) +``` + +Adds a subtask to a parent task. + +**Arguments**: + +- `parent_id` - The ID of the parent task. +- `goal` - The goal of the subtask. +- `subtasks` - A list of subtasks associated with the new subtask. + +#### set\_subtask\_state + +```python +def set_subtask_state(id: str, state: str) +``` + +Sets the state of a subtask. + +**Arguments**: + +- `id` - The ID of the subtask. +- `state` - The new state of the subtask. + +#### get\_current\_task + +```python +def get_current_task() +``` + +Retrieves the current task in progress. + +**Returns**: + + The current task in progress, or None if no task is in progress. + diff --git a/docs/modules/python/opendevin/core/config.md b/docs/modules/python/opendevin/core/config.md new file mode 100644 index 0000000000..0c8632a5f1 --- /dev/null +++ b/docs/modules/python/opendevin/core/config.md @@ -0,0 +1,13 @@ +--- +sidebar_label: config +title: opendevin.core.config +--- + +#### get + +```python +def get(key: ConfigType, required: bool = False) +``` + +Get a key from the environment variables or config.toml or default configs. + diff --git a/docs/modules/python/opendevin/core/logger.md b/docs/modules/python/opendevin/core/logger.md new file mode 100644 index 0000000000..77d744306d --- /dev/null +++ b/docs/modules/python/opendevin/core/logger.md @@ -0,0 +1,92 @@ +--- +sidebar_label: logger +title: opendevin.core.logger +--- + +#### get\_console\_handler + +```python +def get_console_handler() +``` + +Returns a console handler for logging. + +#### get\_file\_handler + +```python +def get_file_handler() +``` + +Returns a file handler for logging. + +#### log\_uncaught\_exceptions + +```python +def log_uncaught_exceptions(ex_cls, ex, tb) +``` + +Logs uncaught exceptions along with the traceback. + +**Arguments**: + +- `ex_cls` _type_ - The type of the exception. +- `ex` _Exception_ - The exception instance. +- `tb` _traceback_ - The traceback object. + + +**Returns**: + + None + +## LlmFileHandler Objects + +```python +class LlmFileHandler(logging.FileHandler) +``` + +__LLM prompt and response logging__ + + +#### \_\_init\_\_ + +```python +def __init__(filename, mode='a', encoding='utf-8', delay=False) +``` + +Initializes an instance of LlmFileHandler. + +**Arguments**: + +- `filename` _str_ - The name of the log file. +- `mode` _str, optional_ - The file mode. Defaults to 'a'. +- `encoding` _str, optional_ - The file encoding. Defaults to None. +- `delay` _bool, optional_ - Whether to delay file opening. Defaults to False. + +#### emit + +```python +def emit(record) +``` + +Emits a log record. + +**Arguments**: + +- `record` _logging.LogRecord_ - The log record to emit. + +#### get\_llm\_prompt\_file\_handler + +```python +def get_llm_prompt_file_handler() +``` + +Returns a file handler for LLM prompt logging. + +#### get\_llm\_response\_file\_handler + +```python +def get_llm_response_file_handler() +``` + +Returns a file handler for LLM response logging. + diff --git a/docs/modules/python/opendevin/core/main.md b/docs/modules/python/opendevin/core/main.md new file mode 100644 index 0000000000..09c41ef837 --- /dev/null +++ b/docs/modules/python/opendevin/core/main.md @@ -0,0 +1,29 @@ +--- +sidebar_label: main +title: opendevin.core.main +--- + +#### read\_task\_from\_file + +```python +def read_task_from_file(file_path: str) -> str +``` + +Read task from the specified file. + +#### read\_task\_from\_stdin + +```python +def read_task_from_stdin() -> str +``` + +Read task from stdin. + +#### main + +```python +async def main(task_str: str = '') +``` + +Main coroutine to run the agent controller with task input flexibility. + diff --git a/docs/modules/python/opendevin/core/schema/action.md b/docs/modules/python/opendevin/core/schema/action.md new file mode 100644 index 0000000000..6ed7057623 --- /dev/null +++ b/docs/modules/python/opendevin/core/schema/action.md @@ -0,0 +1,88 @@ +--- +sidebar_label: action +title: opendevin.core.schema.action +--- + +## ActionTypeSchema Objects + +```python +class ActionTypeSchema(BaseModel) +``` + +#### INIT + +Initializes the agent. Only sent by client. + +#### USER\_MESSAGE + +Sends a message from the user. Only sent by the client. + +#### START + +Starts a new development task OR send chat from the user. Only sent by the client. + +#### READ + +Reads the content of a file. + +#### WRITE + +Writes the content to a file. + +#### RUN + +Runs a command. + +#### RUN\_IPYTHON + +Runs a IPython cell. + +#### KILL + +Kills a background command. + +#### BROWSE + +Opens a web page. + +#### RECALL + +Searches long-term memory + +#### THINK + +Allows the agent to make a plan, set a goal, or record thoughts + +#### TALK + +Allows the agent to respond to the user. + +#### DELEGATE + +Delegates a task to another agent. + +#### FINISH + +If you're absolutely certain that you've completed your task and have tested your work, +use the finish action to stop working. + +#### PAUSE + +Pauses the task. + +#### RESUME + +Resumes the task. + +#### STOP + +Stops the task. Must send a start action to restart a new task. + +#### PUSH + +Push a branch to github. + +#### SEND\_PR + +Send a PR to github. + diff --git a/docs/modules/python/opendevin/core/schema/observation.md b/docs/modules/python/opendevin/core/schema/observation.md new file mode 100644 index 0000000000..af75bb0cae --- /dev/null +++ b/docs/modules/python/opendevin/core/schema/observation.md @@ -0,0 +1,39 @@ +--- +sidebar_label: observation +title: opendevin.core.schema.observation +--- + +## ObservationTypeSchema Objects + +```python +class ObservationTypeSchema(BaseModel) +``` + +#### READ + +The content of a file + +#### BROWSE + +The HTML content of a URL + +#### RUN + +The output of a command + +#### RUN\_IPYTHON + +Runs a IPython cell. + +#### RECALL + +The result of a search + +#### CHAT + +A message from the user + +#### DELEGATE + +The result of a task delegated to another agent + diff --git a/docs/modules/python/opendevin/core/schema/task.md b/docs/modules/python/opendevin/core/schema/task.md new file mode 100644 index 0000000000..bbeed28241 --- /dev/null +++ b/docs/modules/python/opendevin/core/schema/task.md @@ -0,0 +1,61 @@ +--- +sidebar_label: task +title: opendevin.core.schema.task +--- + +## TaskState Objects + +```python +class TaskState(str, Enum) +``` + +#### INIT + +Initial state of the task. + +#### RUNNING + +The task is running. + +#### AWAITING\_USER\_INPUT + +The task is awaiting user input. + +#### PAUSED + +The task is paused. + +#### STOPPED + +The task is stopped. + +#### FINISHED + +The task is finished. + +#### ERROR + +An error occurred during the task. + +## TaskStateAction Objects + +```python +class TaskStateAction(str, Enum) +``` + +#### START + +Starts the task. + +#### PAUSE + +Pauses the task. + +#### RESUME + +Resumes the task. + +#### STOP + +Stops the task. + diff --git a/docs/modules/python/opendevin/runtime/browser/browser_env.md b/docs/modules/python/opendevin/runtime/browser/browser_env.md new file mode 100644 index 0000000000..6b69aba0fa --- /dev/null +++ b/docs/modules/python/opendevin/runtime/browser/browser_env.md @@ -0,0 +1,20 @@ +--- +sidebar_label: browser_env +title: opendevin.runtime.browser.browser_env +--- + +## BrowserEnv Objects + +```python +class BrowserEnv() +``` + +#### image\_to\_png\_base64\_url + +```python +@staticmethod +def image_to_png_base64_url(image: np.ndarray | Image.Image) +``` + +Convert a numpy array to a base64 encoded png image url. + diff --git a/docs/modules/python/opendevin/runtime/docker/process.md b/docs/modules/python/opendevin/runtime/docker/process.md new file mode 100644 index 0000000000..42c6642ede Binary files /dev/null and b/docs/modules/python/opendevin/runtime/docker/process.md differ diff --git a/docs/modules/python/opendevin/runtime/e2b/sandbox.md b/docs/modules/python/opendevin/runtime/e2b/sandbox.md new file mode 100644 index 0000000000..c863cb5a98 --- /dev/null +++ b/docs/modules/python/opendevin/runtime/e2b/sandbox.md @@ -0,0 +1,19 @@ +--- +sidebar_label: sandbox +title: opendevin.runtime.e2b.sandbox +--- + +## E2BBox Objects + +```python +class E2BBox(Sandbox) +``` + +#### copy\_to + +```python +def copy_to(host_src: str, sandbox_dest: str, recursive: bool = False) +``` + +Copies a local file or directory to the sandbox. + diff --git a/docs/modules/python/opendevin/runtime/files.md b/docs/modules/python/opendevin/runtime/files.md new file mode 100644 index 0000000000..b3519d4c68 --- /dev/null +++ b/docs/modules/python/opendevin/runtime/files.md @@ -0,0 +1,40 @@ +--- +sidebar_label: files +title: opendevin.runtime.files +--- + +## WorkspaceFile Objects + +```python +class WorkspaceFile() +``` + +#### to\_dict + +```python +def to_dict() -> Dict[str, Any] +``` + +Converts the File object to a dictionary. + +**Returns**: + + The dictionary representation of the File object. + +#### get\_folder\_structure + +```python +def get_folder_structure(workdir: Path) -> WorkspaceFile +``` + +Gets the folder structure of a directory. + +**Arguments**: + +- `workdir` - The directory path. + + +**Returns**: + + The folder structure. + diff --git a/docs/modules/python/opendevin/runtime/plugins/jupyter/__init__.md b/docs/modules/python/opendevin/runtime/plugins/jupyter/__init__.md new file mode 100644 index 0000000000..4a76364180 --- /dev/null +++ b/docs/modules/python/opendevin/runtime/plugins/jupyter/__init__.md @@ -0,0 +1,16 @@ +--- +sidebar_label: jupyter +title: opendevin.runtime.plugins.jupyter +--- + +## JupyterRequirement Objects + +```python +@dataclass +class JupyterRequirement(PluginRequirement) +``` + +#### host\_src + +The directory of this file (sandbox/plugins/jupyter) + diff --git a/docs/modules/python/opendevin/runtime/plugins/mixin.md b/docs/modules/python/opendevin/runtime/plugins/mixin.md new file mode 100644 index 0000000000..0b968647cc --- /dev/null +++ b/docs/modules/python/opendevin/runtime/plugins/mixin.md @@ -0,0 +1,21 @@ +--- +sidebar_label: mixin +title: opendevin.runtime.plugins.mixin +--- + +## PluginMixin Objects + +```python +class PluginMixin() +``` + +Mixin for Sandbox to support plugins. + +#### init\_plugins + +```python +def init_plugins(requirements: List[PluginRequirement]) +``` + +Load a plugin into the sandbox. + diff --git a/docs/modules/python/opendevin/runtime/plugins/requirement.md b/docs/modules/python/opendevin/runtime/plugins/requirement.md new file mode 100644 index 0000000000..97471e5b6d --- /dev/null +++ b/docs/modules/python/opendevin/runtime/plugins/requirement.md @@ -0,0 +1,14 @@ +--- +sidebar_label: requirement +title: opendevin.runtime.plugins.requirement +--- + +## PluginRequirement Objects + +```python +@dataclass +class PluginRequirement() +``` + +Requirement for a plugin. + diff --git a/docs/modules/python/opendevin/runtime/utils/system.md b/docs/modules/python/opendevin/runtime/utils/system.md new file mode 100644 index 0000000000..02011d988a --- /dev/null +++ b/docs/modules/python/opendevin/runtime/utils/system.md @@ -0,0 +1,13 @@ +--- +sidebar_label: system +title: opendevin.runtime.utils.system +--- + +#### find\_available\_tcp\_port + +```python +def find_available_tcp_port() -> int +``` + +Find an available TCP port, return -1 if none available. + diff --git a/docs/modules/python/sidebar.json b/docs/modules/python/sidebar.json index d50b87455f..a8da03680f 100644 --- a/docs/modules/python/sidebar.json +++ b/docs/modules/python/sidebar.json @@ -71,29 +71,70 @@ "items": [ { "items": [ - "python/opendevin/action/__init__", - "python/opendevin/action/base", - "python/opendevin/action/fileop", - "python/opendevin/action/github", - "python/opendevin/action/tasks" - ], - "label": "opendevin.action", - "type": "category" - }, - { - "items": [ - "python/opendevin/browser/browser_env" - ], - "label": "opendevin.browser", - "type": "category" - }, - { - "items": [ + { + "items": [ + "python/opendevin/controller/state/plan" + ], + "label": "opendevin.controller.state", + "type": "category" + }, + "python/opendevin/controller/agent", "python/opendevin/controller/agent_controller" ], "label": "opendevin.controller", "type": "category" }, + { + "items": [ + { + "items": [ + "python/opendevin/core/schema/action", + "python/opendevin/core/schema/observation", + "python/opendevin/core/schema/task" + ], + "label": "opendevin.core.schema", + "type": "category" + }, + "python/opendevin/core/config", + "python/opendevin/core/logger", + "python/opendevin/core/main" + ], + "label": "opendevin.core", + "type": "category" + }, + { + "items": [ + { + "items": [ + "python/opendevin/events/action/__init__", + "python/opendevin/events/action/empty", + "python/opendevin/events/action/files", + "python/opendevin/events/action/github", + "python/opendevin/events/action/tasks" + ], + "label": "opendevin.events.action", + "type": "category" + }, + { + "items": [ + "python/opendevin/events/observation/__init__", + "python/opendevin/events/observation/browse", + "python/opendevin/events/observation/commands", + "python/opendevin/events/observation/delegate", + "python/opendevin/events/observation/empty", + "python/opendevin/events/observation/error", + "python/opendevin/events/observation/files", + "python/opendevin/events/observation/message", + "python/opendevin/events/observation/observation", + "python/opendevin/events/observation/recall" + ], + "label": "opendevin.events.observation", + "type": "category" + } + ], + "label": "opendevin.events", + "type": "category" + }, { "items": [ "python/opendevin/llm/llm" @@ -101,63 +142,54 @@ "label": "opendevin.llm", "type": "category" }, - { - "items": [ - "python/opendevin/observation/__init__", - "python/opendevin/observation/base", - "python/opendevin/observation/browse", - "python/opendevin/observation/delegate", - "python/opendevin/observation/error", - "python/opendevin/observation/files", - "python/opendevin/observation/message", - "python/opendevin/observation/recall", - "python/opendevin/observation/run" - ], - "label": "opendevin.observation", - "type": "category" - }, { "items": [ { "items": [ - "python/opendevin/sandbox/docker/process" + "python/opendevin/runtime/browser/browser_env" ], - "label": "opendevin.sandbox.docker", + "label": "opendevin.runtime.browser", "type": "category" }, { "items": [ - "python/opendevin/sandbox/e2b/sandbox" + "python/opendevin/runtime/docker/process" ], - "label": "opendevin.sandbox.e2b", + "label": "opendevin.runtime.docker", + "type": "category" + }, + { + "items": [ + "python/opendevin/runtime/e2b/sandbox" + ], + "label": "opendevin.runtime.e2b", "type": "category" }, { "items": [ { "items": [ - "python/opendevin/sandbox/plugins/jupyter/__init__" + "python/opendevin/runtime/plugins/jupyter/__init__" ], - "label": "opendevin.sandbox.plugins.jupyter", + "label": "opendevin.runtime.plugins.jupyter", "type": "category" }, - "python/opendevin/sandbox/plugins/mixin", - "python/opendevin/sandbox/plugins/requirement" + "python/opendevin/runtime/plugins/mixin", + "python/opendevin/runtime/plugins/requirement" ], - "label": "opendevin.sandbox.plugins", + "label": "opendevin.runtime.plugins", "type": "category" - } + }, + { + "items": [ + "python/opendevin/runtime/utils/system" + ], + "label": "opendevin.runtime.utils", + "type": "category" + }, + "python/opendevin/runtime/files" ], - "label": "opendevin.sandbox", - "type": "category" - }, - { - "items": [ - "python/opendevin/schema/action", - "python/opendevin/schema/observation", - "python/opendevin/schema/task" - ], - "label": "opendevin.schema", + "label": "opendevin.runtime", "type": "category" }, { @@ -190,20 +222,7 @@ ], "label": "opendevin.server", "type": "category" - }, - { - "items": [ - "python/opendevin/utils/system" - ], - "label": "opendevin.utils", - "type": "category" - }, - "python/opendevin/agent", - "python/opendevin/config", - "python/opendevin/files", - "python/opendevin/logger", - "python/opendevin/main", - "python/opendevin/plan" + } ], "label": "opendevin", "type": "category" diff --git a/opendevin/controller/action_manager.py b/opendevin/controller/action_manager.py index e831c80ebe..4b79dd9e18 100644 --- a/opendevin/controller/action_manager.py +++ b/opendevin/controller/action_manager.py @@ -1,6 +1,7 @@ from typing import List -from opendevin import config +from opendevin.core import config +from opendevin.core.schema import ConfigType from opendevin.events.action import ( Action, ) @@ -9,9 +10,14 @@ from opendevin.events.observation import ( CmdOutputObservation, Observation, ) -from opendevin.sandbox import DockerExecBox, DockerSSHBox, E2BBox, LocalBox, Sandbox -from opendevin.sandbox.plugins import PluginRequirement -from opendevin.schema import ConfigType +from opendevin.runtime import ( + DockerExecBox, + DockerSSHBox, + E2BBox, + LocalBox, + Sandbox, +) +from opendevin.runtime.plugins import PluginRequirement class ActionManager: diff --git a/opendevin/agent.py b/opendevin/controller/agent.py similarity index 92% rename from opendevin/agent.py rename to opendevin/controller/agent.py index 1a75767562..da57337b58 100644 --- a/opendevin/agent.py +++ b/opendevin/controller/agent.py @@ -2,11 +2,14 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Dict, List, Type if TYPE_CHECKING: + from opendevin.controller.state.state import State from opendevin.events.action import Action - from opendevin.state import State -from opendevin.exceptions import AgentAlreadyRegisteredError, AgentNotRegisteredError +from opendevin.core.exceptions import ( + AgentAlreadyRegisteredError, + AgentNotRegisteredError, +) from opendevin.llm.llm import LLM -from opendevin.sandbox.plugins import PluginRequirement +from opendevin.runtime.plugins import PluginRequirement class Agent(ABC): @@ -21,8 +24,8 @@ class Agent(ABC): sandbox_plugins: List[PluginRequirement] = [] def __init__( - self, - llm: LLM, + self, + llm: LLM, ): self.llm = llm self._complete = False diff --git a/opendevin/controller/agent_controller.py b/opendevin/controller/agent_controller.py index 13398d4c7e..2229a59a7a 100644 --- a/opendevin/controller/agent_controller.py +++ b/opendevin/controller/agent_controller.py @@ -2,9 +2,20 @@ import asyncio from typing import Callable, List, Type from agenthub.codeact_agent.codeact_agent import CodeActAgent -from opendevin import config -from opendevin.agent import Agent from opendevin.controller.action_manager import ActionManager +from opendevin.controller.agent import Agent +from opendevin.controller.state.plan import Plan +from opendevin.controller.state.state import State +from opendevin.core import config +from opendevin.core.exceptions import ( + AgentMalformedActionError, + AgentNoActionError, + LLMOutputError, + MaxCharsExceedError, +) +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema import TaskState +from opendevin.core.schema.config import ConfigType from opendevin.events.action import ( Action, AgentDelegateAction, @@ -20,21 +31,8 @@ from opendevin.events.observation import ( Observation, UserMessageObservation, ) -from opendevin.agent import Agent -from opendevin.browser.browser_env import BrowserEnv -from opendevin.controller.action_manager import ActionManager -from opendevin.exceptions import ( - AgentMalformedActionError, - AgentNoActionError, - LLMOutputError, - MaxCharsExceedError, -) -from opendevin.logger import opendevin_logger as logger -from opendevin.plan import Plan -from opendevin.sandbox import DockerSSHBox -from opendevin.schema import TaskState -from opendevin.schema.config import ConfigType -from opendevin.state import State +from opendevin.runtime import DockerSSHBox +from opendevin.runtime.browser.browser_env import BrowserEnv MAX_ITERATIONS = config.get(ConfigType.MAX_ITERATIONS) MAX_CHARS = config.get(ConfigType.MAX_CHARS) @@ -82,7 +80,6 @@ class AgentController: # Initialize browser environment self.browser = BrowserEnv() - if isinstance(agent, CodeActAgent) and not isinstance( self.action_manager.sandbox, DockerSSHBox ): diff --git a/opendevin/plan.py b/opendevin/controller/state/plan.py similarity index 90% rename from opendevin/plan.py rename to opendevin/controller/state/plan.py index 7c722a8acd..369a2b0884 100644 --- a/opendevin/plan.py +++ b/opendevin/controller/state/plan.py @@ -1,15 +1,20 @@ from typing import List -from opendevin.exceptions import PlanInvalidStateError -from opendevin.logger import opendevin_logger as logger +from opendevin.core.exceptions import PlanInvalidStateError +from opendevin.core.logger import opendevin_logger as logger OPEN_STATE = 'open' COMPLETED_STATE = 'completed' ABANDONED_STATE = 'abandoned' IN_PROGRESS_STATE = 'in_progress' VERIFIED_STATE = 'verified' -STATES = [OPEN_STATE, COMPLETED_STATE, - ABANDONED_STATE, IN_PROGRESS_STATE, VERIFIED_STATE] +STATES = [ + OPEN_STATE, + COMPLETED_STATE, + ABANDONED_STATE, + IN_PROGRESS_STATE, + VERIFIED_STATE, +] class Task: @@ -18,7 +23,13 @@ class Task: parent: 'Task | None' subtasks: List['Task'] - def __init__(self, parent: 'Task | None', goal: str, state: str = OPEN_STATE, subtasks: List = []): + def __init__( + self, + parent: 'Task | None', + goal: str, + state: str = OPEN_STATE, + subtasks: List = [], + ): """Initializes a new instance of the Task class. Args: @@ -34,7 +45,7 @@ class Task: self.parent = parent self.goal = goal self.subtasks = [] - for subtask in (subtasks or []): + for subtask in subtasks or []: if isinstance(subtask, Task): self.subtasks.append(subtask) else: @@ -80,7 +91,7 @@ class Task: 'id': self.id, 'goal': self.goal, 'state': self.state, - 'subtasks': [t.to_dict() for t in self.subtasks] + 'subtasks': [t.to_dict() for t in self.subtasks], } def set_state(self, state): @@ -95,7 +106,11 @@ class Task: logger.error('Invalid state: %s', state) raise PlanInvalidStateError(state) self.state = state - if state == COMPLETED_STATE or state == ABANDONED_STATE or state == VERIFIED_STATE: + if ( + state == COMPLETED_STATE + or state == ABANDONED_STATE + or state == VERIFIED_STATE + ): for subtask in self.subtasks: if subtask.state != ABANDONED_STATE: subtask.set_state(state) @@ -124,6 +139,7 @@ class Plan: main_goal: The main goal of the plan. task: The root task of the plan. """ + main_goal: str task: Task diff --git a/opendevin/state.py b/opendevin/controller/state/state.py similarity index 79% rename from opendevin/state.py rename to opendevin/controller/state/state.py index 7fc44b870b..4888602386 100644 --- a/opendevin/state.py +++ b/opendevin/controller/state/state.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from typing import Dict, List, Tuple +from opendevin.controller.state.plan import Plan from opendevin.events.action import ( Action, ) @@ -8,7 +9,6 @@ from opendevin.events.observation import ( CmdOutputObservation, Observation, ) -from opendevin.plan import Plan @dataclass @@ -17,10 +17,8 @@ class State: iteration: int = 0 # number of characters we have sent to and received from LLM so far for current task num_of_chars: int = 0 - background_commands_obs: List[CmdOutputObservation] = field( - default_factory=list) + background_commands_obs: List[CmdOutputObservation] = field(default_factory=list) history: List[Tuple[Action, Observation]] = field(default_factory=list) - updated_info: List[Tuple[Action, Observation] - ] = field(default_factory=list) + updated_info: List[Tuple[Action, Observation]] = field(default_factory=list) inputs: Dict = field(default_factory=dict) outputs: Dict = field(default_factory=dict) diff --git a/opendevin/config.py b/opendevin/core/config.py similarity index 91% rename from opendevin/config.py rename to opendevin/core/config.py index 1cb3733d90..90b58bc5d7 100644 --- a/opendevin/config.py +++ b/opendevin/core/config.py @@ -7,7 +7,7 @@ import platform import toml from dotenv import load_dotenv -from opendevin.schema import ConfigType +from opendevin.core.schema import ConfigType logger = logging.getLogger(__name__) @@ -55,7 +55,7 @@ DEFAULT_CONFIG: dict = { ConfigType.SANDBOX_USER_ID: os.getuid() if hasattr(os, 'getuid') else None, ConfigType.SANDBOX_TIMEOUT: 120, ConfigType.GITHUB_TOKEN: None, - ConfigType.SANDBOX_USER_ID: None + ConfigType.SANDBOX_USER_ID: None, } config_str = '' @@ -69,7 +69,9 @@ def int_value(value, default, config_key): try: return int(value) except ValueError: - logger.warning(f'Invalid value for {config_key}: {value} not applied. Using default value {default}') + logger.warning( + f'Invalid value for {config_key}: {value} not applied. Using default value {default}' + ) return default @@ -80,16 +82,22 @@ for k, v in config.items(): config[k] = os.environ[k] elif k in tomlConfig: config[k] = tomlConfig[k] - if k in [ConfigType.LLM_NUM_RETRIES, ConfigType.LLM_RETRY_MIN_WAIT, ConfigType.LLM_RETRY_MAX_WAIT]: + if k in [ + ConfigType.LLM_NUM_RETRIES, + ConfigType.LLM_RETRY_MIN_WAIT, + ConfigType.LLM_RETRY_MAX_WAIT, + ]: config[k] = int_value(config[k], v, config_key=k) # In local there is no sandbox, the workspace will have the same pwd as the host if config[ConfigType.SANDBOX_TYPE] == 'local': - config[ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX] = config[ConfigType.WORKSPACE_MOUNT_PATH] + config[ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX] = config[ + ConfigType.WORKSPACE_MOUNT_PATH + ] + def get_parser(): - parser = argparse.ArgumentParser( - description='Run an agent with a specific task') + parser = argparse.ArgumentParser(description='Run an agent with a specific task') parser.add_argument( '-d', '--directory', @@ -149,13 +157,17 @@ args = parse_arguments() def finalize_config(): - if config.get(ConfigType.WORKSPACE_MOUNT_REWRITE) and not config.get(ConfigType.WORKSPACE_MOUNT_PATH): + if config.get(ConfigType.WORKSPACE_MOUNT_REWRITE) and not config.get( + ConfigType.WORKSPACE_MOUNT_PATH + ): base = config.get(ConfigType.WORKSPACE_BASE) or os.getcwd() parts = config[ConfigType.WORKSPACE_MOUNT_REWRITE].split(':') config[ConfigType.WORKSPACE_MOUNT_PATH] = base.replace(parts[0], parts[1]) if config.get(ConfigType.WORKSPACE_MOUNT_PATH) is None: - config[ConfigType.WORKSPACE_MOUNT_PATH] = os.path.abspath(config[ConfigType.WORKSPACE_BASE]) + config[ConfigType.WORKSPACE_MOUNT_PATH] = os.path.abspath( + config[ConfigType.WORKSPACE_BASE] + ) if config.get(ConfigType.LLM_EMBEDDING_BASE_URL) is None: config[ConfigType.LLM_EMBEDDING_BASE_URL] = config.get(ConfigType.LLM_BASE_URL) @@ -171,6 +183,7 @@ def finalize_config(): if config.get(ConfigType.WORKSPACE_MOUNT_PATH) is None: config[ConfigType.WORKSPACE_MOUNT_PATH] = config.get(ConfigType.WORKSPACE_BASE) + finalize_config() diff --git a/opendevin/download.py b/opendevin/core/download.py similarity index 100% rename from opendevin/download.py rename to opendevin/core/download.py diff --git a/opendevin/exceptions.py b/opendevin/core/exceptions.py similarity index 100% rename from opendevin/exceptions.py rename to opendevin/core/exceptions.py diff --git a/opendevin/logger.py b/opendevin/core/logger.py similarity index 93% rename from opendevin/logger.py rename to opendevin/core/logger.py index 709dcd2dc0..4329ed781a 100644 --- a/opendevin/logger.py +++ b/opendevin/core/logger.py @@ -7,12 +7,10 @@ from typing import Literal, Mapping from termcolor import colored -from opendevin import config -from opendevin.schema.config import ConfigType +from opendevin.core import config +from opendevin.core.schema.config import ConfigType -DISABLE_COLOR_PRINTING = ( - config.get(ConfigType.DISABLE_COLOR).lower() == 'true' -) +DISABLE_COLOR_PRINTING = config.get(ConfigType.DISABLE_COLOR).lower() == 'true' ColorType = Literal[ 'red', @@ -48,7 +46,9 @@ class ColoredFormatter(logging.Formatter): if msg_type in LOG_COLORS and not DISABLE_COLOR_PRINTING: msg_type_color = colored(msg_type, LOG_COLORS[msg_type]) msg = colored(record.msg, LOG_COLORS[msg_type]) - time_str = colored(self.formatTime(record, self.datefmt), LOG_COLORS[msg_type]) + time_str = colored( + self.formatTime(record, self.datefmt), LOG_COLORS[msg_type] + ) name_str = colored(record.name, 'cyan') level_str = colored(record.levelname, 'yellow') if msg_type in ['ERROR', 'INFO']: @@ -69,9 +69,7 @@ file_formatter = logging.Formatter( '%(asctime)s - %(name)s:%(levelname)s: %(filename)s:%(lineno)s - %(message)s', datefmt='%H:%M:%S', ) -llm_formatter = logging.Formatter( - '%(message)s' -) +llm_formatter = logging.Formatter('%(message)s') def get_console_handler(): @@ -126,8 +124,9 @@ opendevin_logger.addHandler(get_file_handler()) opendevin_logger.addHandler(get_console_handler()) opendevin_logger.propagate = False opendevin_logger.debug('Logging initialized') -opendevin_logger.debug('Logging to %s', os.path.join( - os.getcwd(), 'logs', 'opendevin.log')) +opendevin_logger.debug( + 'Logging to %s', os.path.join(os.getcwd(), 'logs', 'opendevin.log') +) # Exclude LiteLLM from logging output logging.getLogger('LiteLLM').disabled = True diff --git a/opendevin/main.py b/opendevin/core/main.py similarity index 88% rename from opendevin/main.py rename to opendevin/core/main.py index a97ab14477..4be632c68f 100644 --- a/opendevin/main.py +++ b/opendevin/core/main.py @@ -3,9 +3,9 @@ import sys from typing import Type import agenthub # noqa F401 (we import this to get the agents registered) -from opendevin.agent import Agent -from opendevin.config import args from opendevin.controller import AgentController +from opendevin.controller.agent import Agent +from opendevin.core.config import args from opendevin.llm.llm import LLM @@ -33,8 +33,7 @@ async def main(task_str: str = ''): elif not sys.stdin.isatty(): task = read_task_from_stdin() else: - raise ValueError( - 'No task provided. Please specify a task through -t, -f.') + raise ValueError('No task provided. Please specify a task through -t, -f.') print( f'Running agent {args.agent_cls} (model: {args.model_name}) with task: "{task}"' diff --git a/opendevin/schema/__init__.py b/opendevin/core/schema/__init__.py similarity index 56% rename from opendevin/schema/__init__.py rename to opendevin/core/schema/__init__.py index 73010c3772..d52fae4584 100644 --- a/opendevin/schema/__init__.py +++ b/opendevin/core/schema/__init__.py @@ -3,4 +3,10 @@ from .config import ConfigType from .observation import ObservationType from .task import TaskState, TaskStateAction -__all__ = ['ActionType', 'ObservationType', 'ConfigType', 'TaskState', 'TaskStateAction'] +__all__ = [ + 'ActionType', + 'ObservationType', + 'ConfigType', + 'TaskState', + 'TaskStateAction', +] diff --git a/opendevin/schema/action.py b/opendevin/core/schema/action.py similarity index 98% rename from opendevin/schema/action.py rename to opendevin/core/schema/action.py index 51d21d4c83..9b39666df9 100644 --- a/opendevin/schema/action.py +++ b/opendevin/core/schema/action.py @@ -1,8 +1,6 @@ from pydantic import BaseModel, Field -__all__ = [ - 'ActionType' -] +__all__ = ['ActionType'] class ActionTypeSchema(BaseModel): diff --git a/opendevin/schema/config.py b/opendevin/core/schema/config.py similarity index 100% rename from opendevin/schema/config.py rename to opendevin/core/schema/config.py diff --git a/opendevin/schema/observation.py b/opendevin/core/schema/observation.py similarity index 96% rename from opendevin/schema/observation.py rename to opendevin/core/schema/observation.py index e800e3eb86..93ba2b1365 100644 --- a/opendevin/schema/observation.py +++ b/opendevin/core/schema/observation.py @@ -1,8 +1,6 @@ from pydantic import BaseModel, Field -__all__ = [ - 'ObservationType' -] +__all__ = ['ObservationType'] class ObservationTypeSchema(BaseModel): diff --git a/opendevin/schema/task.py b/opendevin/core/schema/task.py similarity index 100% rename from opendevin/schema/task.py rename to opendevin/core/schema/task.py diff --git a/opendevin/events/action/__init__.py b/opendevin/events/action/__init__.py index c9218d70ac..a86eb5146e 100644 --- a/opendevin/events/action/__init__.py +++ b/opendevin/events/action/__init__.py @@ -1,5 +1,4 @@ - -from opendevin.exceptions import AgentMalformedActionError +from opendevin.core.exceptions import AgentMalformedActionError from .action import Action from .agent import ( @@ -76,5 +75,5 @@ __all__ = [ 'AddTaskAction', 'ModifyTaskAction', 'TaskStateChangedAction', - 'IPythonRunCellAction' + 'IPythonRunCellAction', ] diff --git a/opendevin/events/action/agent.py b/opendevin/events/action/agent.py index 22b8fad4d8..6f8864eb36 100644 --- a/opendevin/events/action/agent.py +++ b/opendevin/events/action/agent.py @@ -1,13 +1,13 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Dict +from opendevin.core.schema import ActionType from opendevin.events.observation import ( AgentMessageObservation, AgentRecallObservation, NullObservation, Observation, ) -from opendevin.schema import ActionType from .action import Action diff --git a/opendevin/events/action/browse.py b/opendevin/events/action/browse.py index e7a4aec0cb..ebe567d6e3 100644 --- a/opendevin/events/action/browse.py +++ b/opendevin/events/action/browse.py @@ -2,9 +2,8 @@ import os from dataclasses import dataclass from typing import TYPE_CHECKING - +from opendevin.core.schema import ActionType from opendevin.events.observation import BrowserOutputObservation -from opendevin.schema import ActionType from .action import Action @@ -33,7 +32,9 @@ class BrowseURLAction(Action): active_page_index=obs['active_page_index'], # index of the active page dom_object=obs['dom_object'], # DOM object axtree_object=obs['axtree_object'], # accessibility tree object - last_browser_action=obs['last_action'], # last browser env action performed + last_browser_action=obs[ + 'last_action' + ], # last browser env action performed focused_element_bid=obs['focused_element_bid'], # focused element bid screenshot=obs['screenshot'], # base64-encoded screenshot, png url=asked_url, diff --git a/opendevin/events/action/commands.py b/opendevin/events/action/commands.py index a0977eee8f..e3d1d5cc37 100644 --- a/opendevin/events/action/commands.py +++ b/opendevin/events/action/commands.py @@ -3,8 +3,8 @@ import pathlib from dataclasses import dataclass from typing import TYPE_CHECKING -from opendevin import config -from opendevin.schema import ActionType, ConfigType +from opendevin.core import config +from opendevin.core.schema import ActionType, ConfigType from .action import Action @@ -64,8 +64,7 @@ class IPythonRunCellAction(Action): # echo "import math" | execute_cli # write code to a temporary file and pass it to `execute_cli` via stdin tmp_filepath = os.path.join( - config.get(ConfigType.WORKSPACE_BASE), - '.tmp', '.ipython_execution_tmp.py' + config.get(ConfigType.WORKSPACE_BASE), '.tmp', '.ipython_execution_tmp.py' ) pathlib.Path(os.path.dirname(tmp_filepath)).mkdir(parents=True, exist_ok=True) with open(tmp_filepath, 'w') as tmp_file: @@ -73,16 +72,13 @@ class IPythonRunCellAction(Action): tmp_filepath_inside_sandbox = os.path.join( config.get(ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX), - '.tmp', '.ipython_execution_tmp.py' + '.tmp', + '.ipython_execution_tmp.py', ) obs = controller.action_manager.run_command( - f'execute_cli < {tmp_filepath_inside_sandbox}', - background=False - ) - return IPythonRunCellObservation( - content=obs.content, - code=self.code + f'execute_cli < {tmp_filepath_inside_sandbox}', background=False ) + return IPythonRunCellObservation(content=obs.content, code=self.code) def __str__(self) -> str: ret = '**IPythonRunCellAction**\n' diff --git a/opendevin/events/action/empty.py b/opendevin/events/action/empty.py index 49a92e9ce8..9d50e24db9 100644 --- a/opendevin/events/action/empty.py +++ b/opendevin/events/action/empty.py @@ -1,14 +1,14 @@ from dataclasses import dataclass -from opendevin.schema import ActionType +from opendevin.core.schema import ActionType from .action import Action @dataclass class NullAction(Action): - """An action that does nothing. - """ + """An action that does nothing.""" + action: str = ActionType.NULL @property diff --git a/opendevin/events/action/files.py b/opendevin/events/action/files.py index ee19ee8494..b93563d4e7 100644 --- a/opendevin/events/action/files.py +++ b/opendevin/events/action/files.py @@ -2,16 +2,16 @@ import os from dataclasses import dataclass from pathlib import Path -from opendevin import config +from opendevin.core import config +from opendevin.core.schema import ActionType +from opendevin.core.schema.config import ConfigType from opendevin.events.observation import ( AgentErrorObservation, FileReadObservation, FileWriteObservation, Observation, ) -from opendevin.sandbox import E2BBox -from opendevin.schema import ActionType -from opendevin.schema.config import ConfigType +from opendevin.runtime import E2BBox from .action import Action @@ -28,14 +28,20 @@ def resolve_path(file_path, working_directory): abs_path_in_sandbox = path_in_sandbox.resolve() # If the path is outside the workspace, deny it - if not abs_path_in_sandbox.is_relative_to(config.get(ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX)): + if not abs_path_in_sandbox.is_relative_to( + config.get(ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX) + ): raise PermissionError(f'File access not permitted: {file_path}') # Get path relative to the root of the workspace inside the sandbox - path_in_workspace = abs_path_in_sandbox.relative_to(Path(config.get(ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX))) + path_in_workspace = abs_path_in_sandbox.relative_to( + Path(config.get(ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX)) + ) # Get path relative to host - path_in_host_workspace = Path(config.get(ConfigType.WORKSPACE_BASE)) / path_in_workspace + path_in_host_workspace = ( + Path(config.get(ConfigType.WORKSPACE_BASE)) / path_in_workspace + ) return path_in_host_workspace @@ -47,6 +53,7 @@ class FileReadAction(Action): Can be set to read specific lines using start and end Default lines 0:-1 (whole file) """ + path: str start: int = 0 end: int = -1 @@ -58,7 +65,7 @@ class FileReadAction(Action): if self.start == 0: return all_lines else: - return all_lines[self.start:] + return all_lines[self.start :] else: num_lines = len(all_lines) begin = max(0, min(self.start, num_lines - 2)) @@ -67,13 +74,14 @@ class FileReadAction(Action): async def run(self, controller) -> Observation: if isinstance(controller.action_manager.sandbox, E2BBox): - content = controller.action_manager.sandbox.filesystem.read( - self.path) + content = controller.action_manager.sandbox.filesystem.read(self.path) read_lines = self._read_lines(content.split('\n')) code_view = ''.join(read_lines) else: try: - whole_path = resolve_path(self.path, controller.action_manager.sandbox.get_working_directory()) + whole_path = resolve_path( + self.path, controller.action_manager.sandbox.get_working_directory() + ) self.start = max(self.start, 0) try: with open(whole_path, 'r', encoding='utf-8') as file: @@ -82,11 +90,17 @@ class FileReadAction(Action): except FileNotFoundError: return AgentErrorObservation(f'File not found: {self.path}') except UnicodeDecodeError: - return AgentErrorObservation(f'File could not be decoded as utf-8: {self.path}') + return AgentErrorObservation( + f'File could not be decoded as utf-8: {self.path}' + ) except IsADirectoryError: - return AgentErrorObservation(f'Path is a directory: {self.path}. You can only read files') + return AgentErrorObservation( + f'Path is a directory: {self.path}. You can only read files' + ) except PermissionError: - return AgentErrorObservation(f'Malformed paths not permitted: {self.path}') + return AgentErrorObservation( + f'Malformed paths not permitted: {self.path}' + ) return FileReadObservation(path=self.path, content=code_view) @property @@ -107,9 +121,9 @@ class FileWriteAction(Action): """ Insert the new content to the original content based on self.start and self.end """ - new_lines = [''] if self.start == 0 else original[:self.start] + new_lines = [''] if self.start == 0 else original[: self.start] new_lines += [i + '\n' for i in to_insert] - new_lines += [''] if self.end == -1 else original[self.end:] + new_lines += [''] if self.end == -1 else original[self.end :] return new_lines async def run(self, controller) -> Observation: @@ -120,12 +134,16 @@ class FileWriteAction(Action): if self.path in files: all_lines = controller.action_manager.sandbox.filesystem.read(self.path) new_file = self._insert_lines(self.content.split('\n'), all_lines) - controller.action_manager.sandbox.filesystem.write(self.path, ''.join(new_file)) + controller.action_manager.sandbox.filesystem.write( + self.path, ''.join(new_file) + ) else: return AgentErrorObservation(f'File not found: {self.path}') else: try: - whole_path = resolve_path(self.path, controller.action_manager.sandbox.get_working_directory()) + whole_path = resolve_path( + self.path, controller.action_manager.sandbox.get_working_directory() + ) mode = 'w' if not os.path.exists(whole_path) else 'r+' try: with open(whole_path, mode, encoding='utf-8') as file: @@ -141,11 +159,17 @@ class FileWriteAction(Action): except FileNotFoundError: return AgentErrorObservation(f'File not found: {self.path}') except IsADirectoryError: - return AgentErrorObservation(f'Path is a directory: {self.path}. You can only write to files') + return AgentErrorObservation( + f'Path is a directory: {self.path}. You can only write to files' + ) except UnicodeDecodeError: - return AgentErrorObservation(f'File could not be decoded as utf-8: {self.path}') + return AgentErrorObservation( + f'File could not be decoded as utf-8: {self.path}' + ) except PermissionError: - return AgentErrorObservation(f'Malformed paths not permitted: {self.path}') + return AgentErrorObservation( + f'Malformed paths not permitted: {self.path}' + ) return FileWriteObservation(content='', path=self.path) @property diff --git a/opendevin/events/action/github.py b/opendevin/events/action/github.py index d42c906290..1cf482a9f4 100644 --- a/opendevin/events/action/github.py +++ b/opendevin/events/action/github.py @@ -5,15 +5,15 @@ from typing import TYPE_CHECKING import requests -from opendevin import config +from opendevin.core import config +from opendevin.core.schema import ActionType +from opendevin.core.schema.config import ConfigType from opendevin.events.observation import ( AgentErrorObservation, AgentMessageObservation, CmdOutputObservation, Observation, ) -from opendevin.schema import ActionType -from opendevin.schema.config import ConfigType from .action import Action @@ -44,9 +44,7 @@ class GitHubPushAction(Action): async def run(self, controller: 'AgentController') -> Observation: github_token = config.get(ConfigType.GITHUB_TOKEN) if not github_token: - return AgentErrorObservation( - 'GITHUB_TOKEN is not set' - ) + return AgentErrorObservation('GITHUB_TOKEN is not set') # Create a random short string to use as a temporary remote random_remote = ''.join( @@ -115,9 +113,7 @@ class GitHubSendPRAction(Action): async def run(self, controller: 'AgentController') -> Observation: github_token = config.get(ConfigType.GITHUB_TOKEN) if not github_token: - return AgentErrorObservation( - 'GITHUB_TOKEN is not set' - ) + return AgentErrorObservation('GITHUB_TOKEN is not set') # API URL to create the pull request url = f'https://api.github.com/repos/{self.owner}/{self.repo}/pulls' diff --git a/opendevin/events/action/tasks.py b/opendevin/events/action/tasks.py index 61d49a095d..5c74f6cf9b 100644 --- a/opendevin/events/action/tasks.py +++ b/opendevin/events/action/tasks.py @@ -1,8 +1,8 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING +from opendevin.core.schema import ActionType from opendevin.events.observation import NullObservation -from opendevin.schema import ActionType from .action import Action @@ -48,6 +48,7 @@ class ModifyTaskAction(Action): @dataclass class TaskStateChangedAction(Action): """Fake action, just to notify the client that a task state has changed.""" + task_state: str thought: str = '' action: str = ActionType.CHANGE_TASK_STATE diff --git a/opendevin/events/observation/browse.py b/opendevin/events/observation/browse.py index 9ed2cf082b..a9f6d8d121 100644 --- a/opendevin/events/observation/browse.py +++ b/opendevin/events/observation/browse.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from opendevin.schema import ObservationType +from opendevin.core.schema import ObservationType from .observation import Observation diff --git a/opendevin/events/observation/commands.py b/opendevin/events/observation/commands.py index 973715343a..5ed6f92773 100644 --- a/opendevin/events/observation/commands.py +++ b/opendevin/events/observation/commands.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from opendevin.schema import ObservationType +from opendevin.core.schema import ObservationType from .observation import Observation diff --git a/opendevin/events/observation/delegate.py b/opendevin/events/observation/delegate.py index 2fd504d9bd..6cded9a7d8 100644 --- a/opendevin/events/observation/delegate.py +++ b/opendevin/events/observation/delegate.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from opendevin.schema import ObservationType +from opendevin.core.schema import ObservationType from .observation import Observation diff --git a/opendevin/events/observation/empty.py b/opendevin/events/observation/empty.py index 700b4ed21f..ae1c646681 100644 --- a/opendevin/events/observation/empty.py +++ b/opendevin/events/observation/empty.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from opendevin.schema import ObservationType +from opendevin.core.schema import ObservationType from .observation import Observation diff --git a/opendevin/events/observation/error.py b/opendevin/events/observation/error.py index 7751e8a557..121fbfe66c 100644 --- a/opendevin/events/observation/error.py +++ b/opendevin/events/observation/error.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from opendevin.schema import ObservationType +from opendevin.core.schema import ObservationType from .observation import Observation diff --git a/opendevin/events/observation/files.py b/opendevin/events/observation/files.py index fe4462901f..612d9ef788 100644 --- a/opendevin/events/observation/files.py +++ b/opendevin/events/observation/files.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from opendevin.schema import ObservationType +from opendevin.core.schema import ObservationType from .observation import Observation diff --git a/opendevin/events/observation/message.py b/opendevin/events/observation/message.py index 8c4ce34a22..e6532acd29 100644 --- a/opendevin/events/observation/message.py +++ b/opendevin/events/observation/message.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from opendevin.schema import ObservationType +from opendevin.core.schema import ObservationType from .observation import Observation diff --git a/opendevin/events/observation/recall.py b/opendevin/events/observation/recall.py index 8783d48499..df1a9209e9 100644 --- a/opendevin/events/observation/recall.py +++ b/opendevin/events/observation/recall.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import List -from opendevin.schema import ObservationType +from opendevin.core.schema import ObservationType from .observation import Observation diff --git a/opendevin/llm/llm.py b/opendevin/llm/llm.py index bc575d2e33..3f75ecde3e 100644 --- a/opendevin/llm/llm.py +++ b/opendevin/llm/llm.py @@ -13,10 +13,10 @@ from tenacity import ( wait_random_exponential, ) -from opendevin import config -from opendevin.logger import llm_prompt_logger, llm_response_logger -from opendevin.logger import opendevin_logger as logger -from opendevin.schema import ConfigType +from opendevin.core import config +from opendevin.core.logger import llm_prompt_logger, llm_response_logger +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema import ConfigType DEFAULT_API_KEY = config.get(ConfigType.LLM_API_KEY) DEFAULT_BASE_URL = config.get(ConfigType.LLM_BASE_URL) diff --git a/opendevin/sandbox/__init__.py b/opendevin/runtime/__init__.py similarity index 65% rename from opendevin/sandbox/__init__.py rename to opendevin/runtime/__init__.py index feedc6436d..a77d416084 100644 --- a/opendevin/sandbox/__init__.py +++ b/opendevin/runtime/__init__.py @@ -4,10 +4,4 @@ from .docker.ssh_box import DockerSSHBox from .e2b.sandbox import E2BBox from .sandbox import Sandbox -__all__ = [ - 'Sandbox', - 'DockerSSHBox', - 'DockerExecBox', - 'E2BBox', - 'LocalBox' -] +__all__ = ['Sandbox', 'DockerSSHBox', 'DockerExecBox', 'E2BBox', 'LocalBox'] diff --git a/opendevin/browser/__init__.py b/opendevin/runtime/browser/__init__.py similarity index 100% rename from opendevin/browser/__init__.py rename to opendevin/runtime/browser/__init__.py diff --git a/opendevin/browser/browser_env.py b/opendevin/runtime/browser/browser_env.py similarity index 94% rename from opendevin/browser/browser_env.py rename to opendevin/runtime/browser/browser_env.py index ab711fe74b..8389163747 100644 --- a/opendevin/browser/browser_env.py +++ b/opendevin/runtime/browser/browser_env.py @@ -12,14 +12,14 @@ import numpy as np from browsergym.utils.obs import flatten_dom_to_str from PIL import Image -from opendevin.logger import opendevin_logger as logger +from opendevin.core.logger import opendevin_logger as logger class BrowserException(Exception): pass -class BrowserEnv: +class BrowserEnv: def __init__(self): self.html_text_converter = html2text.HTML2Text() # ignore links and images @@ -32,7 +32,9 @@ class BrowserEnv: # Initialize browser environment process multiprocessing.set_start_method('spawn', force=True) self.browser_side, self.agent_side = multiprocessing.Pipe() - self.process = multiprocessing.Process(target=self.browser_process,) + self.process = multiprocessing.Process( + target=self.browser_process, + ) logger.info('Starting browser env...') self.process.start() atexit.register(self.close) @@ -50,7 +52,7 @@ class BrowserEnv: while True: try: if self.browser_side.poll(timeout=0.01): - unique_request_id , action_data = self.browser_side.recv() + unique_request_id, action_data = self.browser_side.recv() # shutdown the browser environment if unique_request_id == 'SHUTDOWN': env.close() diff --git a/opendevin/runtime/docker/__init__.py b/opendevin/runtime/docker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/opendevin/sandbox/docker/exec_box.py b/opendevin/runtime/docker/exec_box.py similarity index 88% rename from opendevin/sandbox/docker/exec_box.py rename to opendevin/runtime/docker/exec_box.py index 13dea8b29f..9f7a5d69b6 100644 --- a/opendevin/sandbox/docker/exec_box.py +++ b/opendevin/runtime/docker/exec_box.py @@ -11,13 +11,12 @@ from typing import Dict, List, Tuple import docker -from opendevin import config -from opendevin.exceptions import SandboxInvalidBackgroundCommandError -from opendevin.logger import opendevin_logger as logger -from opendevin.sandbox.docker.process import DockerProcess -from opendevin.sandbox.process import Process -from opendevin.sandbox.sandbox import Sandbox -from opendevin.schema import ConfigType +from opendevin.core import config +from opendevin.core.exceptions import SandboxInvalidBackgroundCommandError +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema import ConfigType +from opendevin.runtime.docker.process import DockerProcess, Process +from opendevin.runtime.sandbox import Sandbox InputType = namedtuple('InputType', ['content']) OutputType = namedtuple('OutputType', ['content']) @@ -47,17 +46,19 @@ class DockerExecBox(Sandbox): background_commands: Dict[int, Process] = {} def __init__( - self, - container_image: str | None = None, - timeout: int = 120, - sid: str | None = None, + self, + container_image: str | None = None, + timeout: int = 120, + sid: str | None = None, ): # Initialize docker client. Throws an exception if Docker is not reachable. try: self.docker_client = docker.from_env() except Exception as ex: logger.exception( - 'Error creating controller. Please check Docker is running and visit `https://opendevin.github.io/OpenDevin/modules/usage/troubleshooting` for more debugging information.', exc_info=False) + 'Error creating controller. Please check Docker is running and visit `https://opendevin.github.io/OpenDevin/modules/usage/troubleshooting` for more debugging information.', + exc_info=False, + ) raise ex self.instance_id = sid if sid is not None else str(uuid.uuid4()) @@ -67,7 +68,9 @@ class DockerExecBox(Sandbox): # command to finish (e.g. apt-get update) # if it is too long, the user may have to wait for a unnecessary long time self.timeout = timeout - self.container_image = CONTAINER_IMAGE if container_image is None else container_image + self.container_image = ( + CONTAINER_IMAGE if container_image is None else container_image + ) self.container_name = self.container_name_prefix + self.instance_id # always restart the container, cuz the initial be regarded as a new session @@ -116,11 +119,13 @@ class DockerExecBox(Sandbox): exit_code, logs = future.result(timeout=self.timeout) except concurrent.futures.TimeoutError: logger.exception( - 'Command timed out, killing process...', exc_info=False) + 'Command timed out, killing process...', exc_info=False + ) pid = self.get_pid(cmd) if pid is not None: self.container.exec_run( - f'kill -9 {pid}', workdir=SANDBOX_WORKSPACE_DIR) + f'kill -9 {pid}', workdir=SANDBOX_WORKSPACE_DIR + ) return -1, f'Command: "{cmd}" timed out' logs_out = logs.decode('utf-8') if logs_out.endswith('\n'): @@ -135,18 +140,25 @@ class DockerExecBox(Sandbox): ) if exit_code != 0: raise Exception( - f'Failed to create directory {sandbox_dest} in sandbox: {logs}') + f'Failed to create directory {sandbox_dest} in sandbox: {logs}' + ) if recursive: - assert os.path.isdir(host_src), 'Source must be a directory when recursive is True' + assert os.path.isdir( + host_src + ), 'Source must be a directory when recursive is True' files = glob(host_src + '/**/*', recursive=True) srcname = os.path.basename(host_src) tar_filename = os.path.join(os.path.dirname(host_src), srcname + '.tar') with tarfile.open(tar_filename, mode='w') as tar: for file in files: - tar.add(file, arcname=os.path.relpath(file, os.path.dirname(host_src))) + tar.add( + file, arcname=os.path.relpath(file, os.path.dirname(host_src)) + ) else: - assert os.path.isfile(host_src), 'Source must be a file when recursive is False' + assert os.path.isfile( + host_src + ), 'Source must be a file when recursive is False' srcname = os.path.basename(host_src) tar_filename = os.path.join(os.path.dirname(host_src), srcname + '.tar') with tarfile.open(tar_filename, mode='w') as tar: @@ -186,7 +198,8 @@ class DockerExecBox(Sandbox): bg_cmd = self.background_commands[id] if bg_cmd.pid is not None: self.container.exec_run( - f'kill -9 {bg_cmd.pid}', workdir=SANDBOX_WORKSPACE_DIR) + f'kill -9 {bg_cmd.pid}', workdir=SANDBOX_WORKSPACE_DIR + ) assert isinstance(bg_cmd, DockerProcess) bg_cmd.result.output.close() self.background_commands.pop(id) @@ -203,8 +216,7 @@ class DockerExecBox(Sandbox): elapsed += 1 if elapsed > self.timeout: break - container = self.docker_client.containers.get( - self.container_name) + container = self.docker_client.containers.get(self.container_name) except docker.errors.NotFound: pass @@ -236,8 +248,7 @@ class DockerExecBox(Sandbox): working_dir=SANDBOX_WORKSPACE_DIR, name=self.container_name, detach=True, - volumes={mount_dir: { - 'bind': SANDBOX_WORKSPACE_DIR, 'mode': 'rw'}}, + volumes={mount_dir: {'bind': SANDBOX_WORKSPACE_DIR, 'mode': 'rw'}}, ) logger.info('Container started') except Exception as ex: @@ -254,8 +265,7 @@ class DockerExecBox(Sandbox): break time.sleep(1) elapsed += 1 - self.container = self.docker_client.containers.get( - self.container_name) + self.container = self.docker_client.containers.get(self.container_name) if elapsed > self.timeout: break if self.container.status != 'running': @@ -283,7 +293,8 @@ if __name__ == '__main__': sys.exit(1) logger.info( - "Interactive Docker container started. Type 'exit' or use Ctrl+C to exit.") + "Interactive Docker container started. Type 'exit' or use Ctrl+C to exit." + ) bg_cmd = exec_box.execute_in_background( "while true; do echo -n '.' && sleep 1; done" diff --git a/opendevin/sandbox/docker/local_box.py b/opendevin/runtime/docker/local_box.py similarity index 74% rename from opendevin/sandbox/docker/local_box.py rename to opendevin/runtime/docker/local_box.py index 2dcf1e923c..5a89c37fce 100644 --- a/opendevin/sandbox/docker/local_box.py +++ b/opendevin/runtime/docker/local_box.py @@ -4,12 +4,11 @@ import subprocess import sys from typing import Dict, Tuple -from opendevin import config -from opendevin.logger import opendevin_logger as logger -from opendevin.sandbox.docker.process import DockerProcess -from opendevin.sandbox.process import Process -from opendevin.sandbox.sandbox import Sandbox -from opendevin.schema.config import ConfigType +from opendevin.core import config +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema.config import ConfigType +from opendevin.runtime.docker.process import DockerProcess, Process +from opendevin.runtime.sandbox import Sandbox # =============================================================================== # ** WARNING ** @@ -38,8 +37,12 @@ class LocalBox(Sandbox): def execute(self, cmd: str) -> Tuple[int, str]: try: completed_process = subprocess.run( - cmd, shell=True, text=True, capture_output=True, - timeout=self.timeout, cwd=config.get(ConfigType.WORKSPACE_BASE) + cmd, + shell=True, + text=True, + capture_output=True, + timeout=self.timeout, + cwd=config.get(ConfigType.WORKSPACE_BASE), ) return completed_process.returncode, completed_process.stdout.strip() except subprocess.TimeoutExpired: @@ -47,27 +50,46 @@ class LocalBox(Sandbox): def copy_to(self, host_src: str, sandbox_dest: str, recursive: bool = False): # mkdir -p sandbox_dest if it doesn't exist - res = subprocess.run(f'mkdir -p {sandbox_dest}', shell=True, text=True, cwd=config.get(ConfigType.WORKSPACE_BASE)) + res = subprocess.run( + f'mkdir -p {sandbox_dest}', + shell=True, + text=True, + cwd=config.get(ConfigType.WORKSPACE_BASE), + ) if res.returncode != 0: raise RuntimeError(f'Failed to create directory {sandbox_dest} in sandbox') if recursive: res = subprocess.run( - f'cp -r {host_src} {sandbox_dest}', shell=True, text=True, cwd=config.get(ConfigType.WORKSPACE_BASE) + f'cp -r {host_src} {sandbox_dest}', + shell=True, + text=True, + cwd=config.get(ConfigType.WORKSPACE_BASE), ) if res.returncode != 0: - raise RuntimeError(f'Failed to copy {host_src} to {sandbox_dest} in sandbox') + raise RuntimeError( + f'Failed to copy {host_src} to {sandbox_dest} in sandbox' + ) else: res = subprocess.run( - f'cp {host_src} {sandbox_dest}', shell=True, text=True, cwd=config.get(ConfigType.WORKSPACE_BASE) + f'cp {host_src} {sandbox_dest}', + shell=True, + text=True, + cwd=config.get(ConfigType.WORKSPACE_BASE), ) if res.returncode != 0: - raise RuntimeError(f'Failed to copy {host_src} to {sandbox_dest} in sandbox') + raise RuntimeError( + f'Failed to copy {host_src} to {sandbox_dest} in sandbox' + ) def execute_in_background(self, cmd: str) -> Process: process = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, cwd=config.get(ConfigType.WORKSPACE_BASE) + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + cwd=config.get(ConfigType.WORKSPACE_BASE), ) bg_cmd = DockerProcess( id=self.cur_background_id, command=cmd, result=process, pid=process.pid @@ -105,7 +127,6 @@ class LocalBox(Sandbox): if __name__ == '__main__': - local_box = LocalBox() bg_cmd = local_box.execute_in_background( "while true; do echo 'dot ' && sleep 10; done" diff --git a/opendevin/sandbox/docker/process.py b/opendevin/runtime/docker/process.py similarity index 92% rename from opendevin/sandbox/docker/process.py rename to opendevin/runtime/docker/process.py index 60b406fcf0..11b37da22c 100644 --- a/opendevin/sandbox/docker/process.py +++ b/opendevin/runtime/docker/process.py @@ -2,7 +2,7 @@ import select import sys from typing import Tuple -from opendevin.sandbox.process import Process +from opendevin.runtime.process import Process class DockerProcess(Process): @@ -68,7 +68,7 @@ class DockerProcess(Process): i = 0 byte_order = sys.byteorder while i < len(logs): - prefix = logs[i: i + 8] + prefix = logs[i : i + 8] if len(prefix) < 8: msg_type = prefix[0:1] if msg_type in [b'\x00', b'\x01', b'\x02', b'\x03']: @@ -82,10 +82,10 @@ class DockerProcess(Process): and padding == b'\x00\x00\x00' ): msg_length = int.from_bytes(prefix[4:8], byteorder=byte_order) - res += logs[i + 8: i + 8 + msg_length] + res += logs[i + 8 : i + 8 + msg_length] i += 8 + msg_length else: - res += logs[i: i + 1] + res += logs[i : i + 1] i += 1 return res, tail @@ -118,14 +118,12 @@ class DockerProcess(Process): logs = b'' last_remains = b'' while True: - ready_to_read, _, _ = select.select( - [self.result.output], [], [], 0.1) # type: ignore[has-type] + ready_to_read, _, _ = select.select([self.result.output], [], [], 0.1) # type: ignore[has-type] if ready_to_read: data = self.result.output.read(4096) # type: ignore[has-type] if not data: break - chunk, last_remains = self.parse_docker_exec_output( - last_remains + data) + chunk, last_remains = self.parse_docker_exec_output(last_remains + data) logs += chunk else: break diff --git a/opendevin/sandbox/docker/ssh_box.py b/opendevin/runtime/docker/ssh_box.py similarity index 81% rename from opendevin/sandbox/docker/ssh_box.py rename to opendevin/runtime/docker/ssh_box.py index 4fde3e7214..6df0367bc4 100644 --- a/opendevin/sandbox/docker/ssh_box.py +++ b/opendevin/runtime/docker/ssh_box.py @@ -11,15 +11,17 @@ from typing import Dict, List, Tuple, Union import docker from pexpect import pxssh -from opendevin import config -from opendevin.exceptions import SandboxInvalidBackgroundCommandError -from opendevin.logger import opendevin_logger as logger -from opendevin.sandbox.docker.process import DockerProcess -from opendevin.sandbox.plugins import JupyterRequirement, SWEAgentCommandsRequirement -from opendevin.sandbox.process import Process -from opendevin.sandbox.sandbox import Sandbox -from opendevin.schema import ConfigType -from opendevin.utils import find_available_tcp_port +from opendevin.core import config +from opendevin.core.exceptions import SandboxInvalidBackgroundCommandError +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema import ConfigType +from opendevin.runtime.docker.process import DockerProcess, Process +from opendevin.runtime.plugins import ( + JupyterRequirement, + SWEAgentCommandsRequirement, +) +from opendevin.runtime.sandbox import Sandbox +from opendevin.runtime.utils import find_available_tcp_port InputType = namedtuple('InputType', ['content']) OutputType = namedtuple('OutputType', ['content']) @@ -41,6 +43,7 @@ if SANDBOX_USER_ID := config.get(ConfigType.SANDBOX_USER_ID): elif hasattr(os, 'getuid'): USER_ID = os.getuid() + class DockerSSHBox(Sandbox): instance_id: str container_image: str @@ -61,13 +64,17 @@ class DockerSSHBox(Sandbox): timeout: int = 120, sid: str | None = None, ): - logger.info(f'SSHBox is running as {"opendevin" if RUN_AS_DEVIN else "root"} user with USER_ID={USER_ID} in the sandbox') + logger.info( + f'SSHBox is running as {"opendevin" if RUN_AS_DEVIN else "root"} user with USER_ID={USER_ID} in the sandbox' + ) # Initialize docker client. Throws an exception if Docker is not reachable. try: self.docker_client = docker.from_env() except Exception as ex: logger.exception( - 'Error creating controller. Please check Docker is running and visit `https://opendevin.github.io/OpenDevin/modules/usage/troubleshooting` for more debugging information.', exc_info=False) + 'Error creating controller. Please check Docker is running and visit `https://opendevin.github.io/OpenDevin/modules/usage/troubleshooting` for more debugging information.', + exc_info=False, + ) raise ex self.instance_id = sid if sid is not None else str(uuid.uuid4()) @@ -77,7 +84,9 @@ class DockerSSHBox(Sandbox): # command to finish (e.g. apt-get update) # if it is too long, the user may have to wait for a unnecessary long time self.timeout = timeout - self.container_image = CONTAINER_IMAGE if container_image is None else container_image + self.container_image = ( + CONTAINER_IMAGE if container_image is None else container_image + ) self.container_name = self.container_name_prefix + self.instance_id # set up random user password @@ -92,17 +101,16 @@ class DockerSSHBox(Sandbox): atexit.register(self.close) def setup_user(self): - # Make users sudoers passwordless # TODO(sandbox): add this line in the Dockerfile for next minor version of docker image exit_code, logs = self.container.exec_run( - ['/bin/bash', '-c', - r"echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers"], + ['/bin/bash', '-c', r"echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers"], workdir=SANDBOX_WORKSPACE_DIR, ) if exit_code != 0: raise Exception( - f'Failed to make all users passwordless sudoers in sandbox: {logs}') + f'Failed to make all users passwordless sudoers in sandbox: {logs}' + ) # Check if the opendevin user exists exit_code, logs = self.container.exec_run( @@ -116,22 +124,26 @@ class DockerSSHBox(Sandbox): workdir=SANDBOX_WORKSPACE_DIR, ) if exit_code != 0: - raise Exception( - f'Failed to remove opendevin user in sandbox: {logs}') + raise Exception(f'Failed to remove opendevin user in sandbox: {logs}') if RUN_AS_DEVIN: # Create the opendevin user exit_code, logs = self.container.exec_run( - ['/bin/bash', '-c', - f'useradd -rm -d /home/opendevin -s /bin/bash -g root -G sudo -u {USER_ID} opendevin'], + [ + '/bin/bash', + '-c', + f'useradd -rm -d /home/opendevin -s /bin/bash -g root -G sudo -u {USER_ID} opendevin', + ], workdir=SANDBOX_WORKSPACE_DIR, ) if exit_code != 0: - raise Exception( - f'Failed to create opendevin user in sandbox: {logs}') + raise Exception(f'Failed to create opendevin user in sandbox: {logs}') exit_code, logs = self.container.exec_run( - ['/bin/bash', '-c', - f"echo 'opendevin:{self._ssh_password}' | chpasswd"], + [ + '/bin/bash', + '-c', + f"echo 'opendevin:{self._ssh_password}' | chpasswd", + ], workdir=SANDBOX_WORKSPACE_DIR, ) if exit_code != 0: @@ -144,7 +156,8 @@ class DockerSSHBox(Sandbox): ) if exit_code != 0: raise Exception( - f'Failed to chown home directory for opendevin in sandbox: {logs}') + f'Failed to chown home directory for opendevin in sandbox: {logs}' + ) exit_code, logs = self.container.exec_run( ['/bin/bash', '-c', f'chown opendevin:root {SANDBOX_WORKSPACE_DIR}'], workdir=SANDBOX_WORKSPACE_DIR, @@ -157,13 +170,11 @@ class DockerSSHBox(Sandbox): else: exit_code, logs = self.container.exec_run( # change password for root - ['/bin/bash', '-c', - f"echo 'root:{self._ssh_password}' | chpasswd"], + ['/bin/bash', '-c', f"echo 'root:{self._ssh_password}' | chpasswd"], workdir=SANDBOX_WORKSPACE_DIR, ) if exit_code != 0: - raise Exception( - f'Failed to set password for root in sandbox: {logs}') + raise Exception(f'Failed to set password for root in sandbox: {logs}') exit_code, logs = self.container.exec_run( ['/bin/bash', '-c', "echo 'opendevin-sandbox' > /etc/hostname"], workdir=SANDBOX_WORKSPACE_DIR, @@ -178,12 +189,11 @@ class DockerSSHBox(Sandbox): else: username = 'root' logger.info( - f"Connecting to {username}@{hostname} via ssh. " + f'Connecting to {username}@{hostname} via ssh. ' f"If you encounter any issues, you can try `ssh -v -p {self._ssh_port} {username}@{hostname}` with the password '{self._ssh_password}' and report the issue on GitHub. " f"If you started OpenDevin with `docker run`, you should try `ssh -v -p {self._ssh_port} {username}@localhost` with the password '{self._ssh_password} on the host machine (where you started the container)." ) - self.ssh.login(hostname, username, self._ssh_password, - port=self._ssh_port) + self.ssh.login(hostname, username, self._ssh_password, port=self._ssh_port) # Fix: https://github.com/pexpect/pexpect/issues/669 self.ssh.sendline("bind 'set enable-bracketed-paste off'") @@ -210,14 +220,15 @@ class DockerSSHBox(Sandbox): self.ssh.sendline(cmd) success = self.ssh.prompt(timeout=self.timeout) if not success: - logger.exception( - 'Command timed out, killing process...', exc_info=False) + logger.exception('Command timed out, killing process...', exc_info=False) # send a SIGINT to the process self.ssh.sendintr() self.ssh.prompt() - command_output = self.ssh.before.decode( - 'utf-8').lstrip(cmd).strip() - return -1, f'Command: "{cmd}" timed out. Sending SIGINT to the process: {command_output}' + command_output = self.ssh.before.decode('utf-8').lstrip(cmd).strip() + return ( + -1, + f'Command: "{cmd}" timed out. Sending SIGINT to the process: {command_output}', + ) command_output = self.ssh.before.decode('utf-8').strip() # once out, make sure that we have *every* output, we while loop until we get an empty output @@ -230,7 +241,9 @@ class DockerSSHBox(Sandbox): break logger.debug('WAITING FOR .before') output = self.ssh.before.decode('utf-8').strip() - logger.debug(f'WAITING FOR END OF command output ({bool(output)}): {output}') + logger.debug( + f'WAITING FOR END OF command output ({bool(output)}): {output}' + ) if output == '': break command_output += output @@ -255,18 +268,25 @@ class DockerSSHBox(Sandbox): ) if exit_code != 0: raise Exception( - f'Failed to create directory {sandbox_dest} in sandbox: {logs}') + f'Failed to create directory {sandbox_dest} in sandbox: {logs}' + ) if recursive: - assert os.path.isdir(host_src), 'Source must be a directory when recursive is True' + assert os.path.isdir( + host_src + ), 'Source must be a directory when recursive is True' files = glob(host_src + '/**/*', recursive=True) srcname = os.path.basename(host_src) tar_filename = os.path.join(os.path.dirname(host_src), srcname + '.tar') with tarfile.open(tar_filename, mode='w') as tar: for file in files: - tar.add(file, arcname=os.path.relpath(file, os.path.dirname(host_src))) + tar.add( + file, arcname=os.path.relpath(file, os.path.dirname(host_src)) + ) else: - assert os.path.isfile(host_src), 'Source must be a file when recursive is False' + assert os.path.isfile( + host_src + ), 'Source must be a file when recursive is False' srcname = os.path.basename(host_src) tar_filename = os.path.join(os.path.dirname(host_src), srcname + '.tar') with tarfile.open(tar_filename, mode='w') as tar: @@ -306,7 +326,8 @@ class DockerSSHBox(Sandbox): bg_cmd = self.background_commands[id] if bg_cmd.pid is not None: self.container.exec_run( - f'kill -9 {bg_cmd.pid}', workdir=SANDBOX_WORKSPACE_DIR) + f'kill -9 {bg_cmd.pid}', workdir=SANDBOX_WORKSPACE_DIR + ) assert isinstance(bg_cmd, DockerProcess) bg_cmd.result.output.close() self.background_commands.pop(id) @@ -323,8 +344,7 @@ class DockerSSHBox(Sandbox): elapsed += 1 if elapsed > self.timeout: break - container = self.docker_client.containers.get( - self.container_name) + container = self.docker_client.containers.get(self.container_name) except docker.errors.NotFound: pass @@ -360,10 +380,11 @@ class DockerSSHBox(Sandbox): # FIXME: This is a temporary workaround for Mac OS network_kwargs['ports'] = {f'{self._ssh_port}/tcp': self._ssh_port} logger.warning( - ('Using port forwarding for Mac OS. ' - 'Server started by OpenDevin will not be accessible from the host machine at the moment. ' - 'See https://github.com/OpenDevin/OpenDevin/issues/897 for more information.' - ) + ( + 'Using port forwarding for Mac OS. ' + 'Server started by OpenDevin will not be accessible from the host machine at the moment. ' + 'See https://github.com/OpenDevin/OpenDevin/issues/897 for more information.' + ) ) mount_dir = config.get(ConfigType.WORKSPACE_MOUNT_PATH) @@ -378,14 +399,13 @@ class DockerSSHBox(Sandbox): name=self.container_name, detach=True, volumes={ - mount_dir: { - 'bind': SANDBOX_WORKSPACE_DIR, - 'mode': 'rw' - }, + mount_dir: {'bind': SANDBOX_WORKSPACE_DIR, 'mode': 'rw'}, # mount cache directory to /home/opendevin/.cache for pip cache reuse config.get(ConfigType.CACHE_DIR): { - 'bind': '/home/opendevin/.cache' if RUN_AS_DEVIN else '/root/.cache', - 'mode': 'rw' + 'bind': '/home/opendevin/.cache' + if RUN_AS_DEVIN + else '/root/.cache', + 'mode': 'rw', }, }, ) @@ -404,10 +424,10 @@ class DockerSSHBox(Sandbox): break time.sleep(1) elapsed += 1 - self.container = self.docker_client.containers.get( - self.container_name) + self.container = self.docker_client.containers.get(self.container_name) logger.info( - f'waiting for container to start: {elapsed}, container status: {self.container.status}') + f'waiting for container to start: {elapsed}, container status: {self.container.status}' + ) if elapsed > self.timeout: break if self.container.status != 'running': @@ -425,7 +445,6 @@ class DockerSSHBox(Sandbox): if __name__ == '__main__': - try: ssh_box = DockerSSHBox() except Exception as e: @@ -433,7 +452,8 @@ if __name__ == '__main__': sys.exit(1) logger.info( - "Interactive Docker container started. Type 'exit' or use Ctrl+C to exit.") + "Interactive Docker container started. Type 'exit' or use Ctrl+C to exit." + ) # Initialize required plugins ssh_box.init_plugins([JupyterRequirement(), SWEAgentCommandsRequirement()]) diff --git a/opendevin/sandbox/e2b/README.md b/opendevin/runtime/e2b/README.md similarity index 100% rename from opendevin/sandbox/e2b/README.md rename to opendevin/runtime/e2b/README.md diff --git a/opendevin/sandbox/e2b/process.py b/opendevin/runtime/e2b/process.py similarity index 91% rename from opendevin/sandbox/e2b/process.py rename to opendevin/runtime/e2b/process.py index 3ec999fefa..d26355c173 100644 --- a/opendevin/sandbox/e2b/process.py +++ b/opendevin/runtime/e2b/process.py @@ -1,6 +1,6 @@ from e2b import Process as E2BSandboxProcess -from opendevin.sandbox.process import Process +from opendevin.runtime.docker.process import Process class E2BProcess(Process): diff --git a/opendevin/sandbox/e2b/sandbox.py b/opendevin/runtime/e2b/sandbox.py similarity index 81% rename from opendevin/sandbox/e2b/sandbox.py rename to opendevin/runtime/e2b/sandbox.py index d95e1f6548..f865ee995f 100644 --- a/opendevin/sandbox/e2b/sandbox.py +++ b/opendevin/runtime/e2b/sandbox.py @@ -8,12 +8,12 @@ from e2b.sandbox.exception import ( TimeoutException, ) -from opendevin import config -from opendevin.logger import opendevin_logger as logger -from opendevin.sandbox.e2b.process import E2BProcess -from opendevin.sandbox.process import Process -from opendevin.sandbox.sandbox import Sandbox -from opendevin.schema.config import ConfigType +from opendevin.core import config +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema.config import ConfigType +from opendevin.runtime.e2b.process import E2BProcess +from opendevin.runtime.process import Process +from opendevin.runtime.sandbox import Sandbox class E2BBox(Sandbox): @@ -44,15 +44,21 @@ class E2BBox(Sandbox): def _archive(self, host_src: str, recursive: bool = False): if recursive: - assert os.path.isdir(host_src), 'Source must be a directory when recursive is True' + assert os.path.isdir( + host_src + ), 'Source must be a directory when recursive is True' files = glob(host_src + '/**/*', recursive=True) srcname = os.path.basename(host_src) tar_filename = os.path.join(os.path.dirname(host_src), srcname + '.tar') with tarfile.open(tar_filename, mode='w') as tar: for file in files: - tar.add(file, arcname=os.path.relpath(file, os.path.dirname(host_src))) + tar.add( + file, arcname=os.path.relpath(file, os.path.dirname(host_src)) + ) else: - assert os.path.isfile(host_src), 'Source must be a file when recursive is False' + assert os.path.isfile( + host_src + ), 'Source must be a file when recursive is False' srcname = os.path.basename(host_src) tar_filename = os.path.join(os.path.dirname(host_src), srcname + '.tar') with tarfile.open(tar_filename, mode='w') as tar: @@ -101,9 +107,13 @@ class E2BBox(Sandbox): self.sandbox.filesystem.make_dir(sandbox_dest) # Extract the archive into the destination and delete the archive - process = self.sandbox.process.start_and_wait(f'sudo tar -xf {uploaded_path} -C {sandbox_dest} && sudo rm {uploaded_path}') + process = self.sandbox.process.start_and_wait( + f'sudo tar -xf {uploaded_path} -C {sandbox_dest} && sudo rm {uploaded_path}' + ) if process.exit_code != 0: - raise Exception(f'Failed to extract {uploaded_path} to {sandbox_dest}: {process.stderr}') + raise Exception( + f'Failed to extract {uploaded_path} to {sandbox_dest}: {process.stderr}' + ) # Delete the local archive os.remove(tar_filename) diff --git a/opendevin/files.py b/opendevin/runtime/files.py similarity index 100% rename from opendevin/files.py rename to opendevin/runtime/files.py diff --git a/opendevin/sandbox/plugins/__init__.py b/opendevin/runtime/plugins/__init__.py similarity index 61% rename from opendevin/sandbox/plugins/__init__.py rename to opendevin/runtime/plugins/__init__.py index 5ea90bd9ff..10b479fd2b 100644 --- a/opendevin/sandbox/plugins/__init__.py +++ b/opendevin/runtime/plugins/__init__.py @@ -4,4 +4,9 @@ from .mixin import PluginMixin from .requirement import PluginRequirement from .swe_agent_commands import SWEAgentCommandsRequirement -__all__ = ['PluginMixin', 'PluginRequirement', 'JupyterRequirement', 'SWEAgentCommandsRequirement'] +__all__ = [ + 'PluginMixin', + 'PluginRequirement', + 'JupyterRequirement', + 'SWEAgentCommandsRequirement', +] diff --git a/opendevin/sandbox/plugins/jupyter/__init__.py b/opendevin/runtime/plugins/jupyter/__init__.py similarity index 53% rename from opendevin/sandbox/plugins/jupyter/__init__.py rename to opendevin/runtime/plugins/jupyter/__init__.py index abedf467fe..0d00ba775a 100644 --- a/opendevin/sandbox/plugins/jupyter/__init__.py +++ b/opendevin/runtime/plugins/jupyter/__init__.py @@ -1,12 +1,14 @@ import os from dataclasses import dataclass -from opendevin.sandbox.plugins.requirement import PluginRequirement +from opendevin.runtime.plugins.requirement import PluginRequirement @dataclass class JupyterRequirement(PluginRequirement): name: str = 'jupyter' - host_src: str = os.path.dirname(os.path.abspath(__file__)) # The directory of this file (sandbox/plugins/jupyter) + host_src: str = os.path.dirname( + os.path.abspath(__file__) + ) # The directory of this file (sandbox/plugins/jupyter) sandbox_dest: str = '/opendevin/plugins/jupyter' bash_script_path: str = 'setup.sh' diff --git a/opendevin/sandbox/plugins/jupyter/execute_cli b/opendevin/runtime/plugins/jupyter/execute_cli old mode 100755 new mode 100644 similarity index 100% rename from opendevin/sandbox/plugins/jupyter/execute_cli rename to opendevin/runtime/plugins/jupyter/execute_cli diff --git a/opendevin/sandbox/plugins/jupyter/execute_server b/opendevin/runtime/plugins/jupyter/execute_server old mode 100755 new mode 100644 similarity index 88% rename from opendevin/sandbox/plugins/jupyter/execute_server rename to opendevin/runtime/plugins/jupyter/execute_server index c7212e907a..134ea58f3e --- a/opendevin/sandbox/plugins/jupyter/execute_server +++ b/opendevin/runtime/plugins/jupyter/execute_server @@ -51,19 +51,16 @@ def strip_ansi(o: str) -> str: class JupyterKernel: - def __init__( - self, - url_suffix, - convid, - lang='python' - ): + def __init__(self, url_suffix, convid, lang='python'): self.base_url = f'http://{url_suffix}' self.base_ws_url = f'ws://{url_suffix}' self.lang = lang self.kernel_id = None self.ws = None self.convid = convid - logging.info(f'Jupyter kernel created for conversation {convid} at {url_suffix}') + logging.info( + f'Jupyter kernel created for conversation {convid} at {url_suffix}' + ) self.heartbeat_interval = 10000 # 10 seconds self.heartbeat_callback = None @@ -89,7 +86,9 @@ class JupyterKernel: try: await self._connect() except ConnectionRefusedError: - logging.info('ConnectionRefusedError: Failed to reconnect to kernel websocket - Is the kernel still running?') + logging.info( + 'ConnectionRefusedError: Failed to reconnect to kernel websocket - Is the kernel still running?' + ) async def _connect(self): if self.ws: @@ -128,7 +127,9 @@ class JupyterKernel: # Setup heartbeat if self.heartbeat_callback: self.heartbeat_callback.stop() - self.heartbeat_callback = PeriodicCallback(self._send_heartbeat, self.heartbeat_interval) + self.heartbeat_callback = PeriodicCallback( + self._send_heartbeat, self.heartbeat_interval + ) self.heartbeat_callback.start() async def execute(self, code, timeout=60): @@ -175,7 +176,9 @@ class JupyterKernel: continue if os.environ.get('DEBUG', False): - logging.info(f"MSG TYPE: {msg_type.upper()} DONE:{execution_done}\nCONTENT: {msg['content']}") + logging.info( + f"MSG TYPE: {msg_type.upper()} DONE:{execution_done}\nCONTENT: {msg['content']}" + ) if msg_type == 'error': traceback = '\n'.join(msg['content']['traceback']) @@ -187,7 +190,9 @@ class JupyterKernel: outputs.append(msg['content']['data']['text/plain']) if 'image/png' in msg['content']['data']: # use markdone to display image (in case of large image) - outputs.append(f"\n![image](data:image/png;base64,{msg['content']['data']['image/png']})\n") + outputs.append( + f"\n![image](data:image/png;base64,{msg['content']['data']['image/png']})\n" + ) elif msg_type == 'execute_reply': execution_done = True @@ -254,13 +259,15 @@ class ExecuteHandler(tornado.web.RequestHandler): def make_app(): jupyter_kernel = JupyterKernel( f"localhost:{os.environ.get('JUPYTER_GATEWAY_PORT')}", - os.environ.get('JUPYTER_GATEWAY_KERNEL_ID') + os.environ.get('JUPYTER_GATEWAY_KERNEL_ID'), ) asyncio.get_event_loop().run_until_complete(jupyter_kernel.initialize()) - return tornado.web.Application([ - (r'/execute', ExecuteHandler, {'jupyter_kernel': jupyter_kernel}), - ]) + return tornado.web.Application( + [ + (r'/execute', ExecuteHandler, {'jupyter_kernel': jupyter_kernel}), + ] + ) if __name__ == '__main__': diff --git a/opendevin/sandbox/plugins/jupyter/setup.sh b/opendevin/runtime/plugins/jupyter/setup.sh old mode 100755 new mode 100644 similarity index 100% rename from opendevin/sandbox/plugins/jupyter/setup.sh rename to opendevin/runtime/plugins/jupyter/setup.sh diff --git a/opendevin/runtime/plugins/mixin.py b/opendevin/runtime/plugins/mixin.py new file mode 100644 index 0000000000..9823adf486 --- /dev/null +++ b/opendevin/runtime/plugins/mixin.py @@ -0,0 +1,50 @@ +import os +from typing import List, Protocol, Tuple + +from opendevin.core.logger import opendevin_logger as logger +from opendevin.runtime.plugins.requirement import PluginRequirement + + +class SandboxProtocol(Protocol): + # https://stackoverflow.com/questions/51930339/how-do-i-correctly-add-type-hints-to-mixin-classes + + def execute(self, cmd: str) -> Tuple[int, str]: ... + + def copy_to(self, host_src: str, sandbox_dest: str, recursive: bool = False): ... + + +class PluginMixin: + """Mixin for Sandbox to support plugins.""" + + def init_plugins(self: SandboxProtocol, requirements: List[PluginRequirement]): + """Load a plugin into the sandbox.""" + for requirement in requirements: + # copy over the files + self.copy_to(requirement.host_src, requirement.sandbox_dest, recursive=True) + logger.info( + f'Copied files from [{requirement.host_src}] to [{requirement.sandbox_dest}] inside sandbox.' + ) + + # Execute the bash script + abs_path_to_bash_script = os.path.join( + requirement.sandbox_dest, requirement.bash_script_path + ) + logger.info( + f'Initalizing plugin [{requirement.name}] by executing [{abs_path_to_bash_script}] in the sandbox.' + ) + exit_code, output = self.execute(abs_path_to_bash_script) + if exit_code != 0: + raise RuntimeError( + f'Failed to initialize plugin {requirement.name} with exit code {exit_code} and output {output}' + ) + logger.info( + f'Plugin {requirement.name} initialized successfully\n:{output}' + ) + + if len(requirements) > 0: + exit_code, output = self.execute('source ~/.bashrc') + if exit_code != 0: + raise RuntimeError( + f'Failed to source ~/.bashrc with exit code {exit_code} and output {output}' + ) + logger.info('Sourced ~/.bashrc successfully') diff --git a/opendevin/sandbox/plugins/requirement.py b/opendevin/runtime/plugins/requirement.py similarity index 100% rename from opendevin/sandbox/plugins/requirement.py rename to opendevin/runtime/plugins/requirement.py diff --git a/opendevin/sandbox/plugins/swe_agent_commands/__init__.py b/opendevin/runtime/plugins/swe_agent_commands/__init__.py similarity index 63% rename from opendevin/sandbox/plugins/swe_agent_commands/__init__.py rename to opendevin/runtime/plugins/swe_agent_commands/__init__.py index 465edf636a..f7763f31fc 100644 --- a/opendevin/sandbox/plugins/swe_agent_commands/__init__.py +++ b/opendevin/runtime/plugins/swe_agent_commands/__init__.py @@ -2,17 +2,14 @@ import os from dataclasses import dataclass, field from typing import List -from opendevin.sandbox.plugins.requirement import PluginRequirement -from opendevin.sandbox.plugins.swe_agent_commands.parse_commands import ( +from opendevin.runtime.plugins.requirement import PluginRequirement +from opendevin.runtime.plugins.swe_agent_commands.parse_commands import ( parse_command_file, ) def _resolve_to_cur_dir(filename): - return os.path.join( - os.path.dirname(os.path.abspath(__file__)), - filename - ) + return os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) def check_and_parse_command_file(filepath) -> str: @@ -26,11 +23,13 @@ DEFAULT_SCRIPT_FILEPATHS = [ _resolve_to_cur_dir('search.sh'), _resolve_to_cur_dir('edit_linting.sh'), ] -DEFAULT_DOCUMENTATION = ''.join([ - check_and_parse_command_file(filepath) - for filepath in DEFAULT_SCRIPT_FILEPATHS - if filepath is not None -]) +DEFAULT_DOCUMENTATION = ''.join( + [ + check_and_parse_command_file(filepath) + for filepath in DEFAULT_SCRIPT_FILEPATHS + if filepath is not None + ] +) @dataclass @@ -40,7 +39,9 @@ class SWEAgentCommandsRequirement(PluginRequirement): sandbox_dest: str = '/opendevin/plugins/swe_agent_commands' bash_script_path: str = 'setup_default.sh' - scripts_filepaths: List[str | None] = field(default_factory=lambda: DEFAULT_SCRIPT_FILEPATHS) + scripts_filepaths: List[str | None] = field( + default_factory=lambda: DEFAULT_SCRIPT_FILEPATHS + ) documentation: str = DEFAULT_DOCUMENTATION @@ -49,11 +50,13 @@ CURSOR_SCRIPT_FILEPATHS = [ _resolve_to_cur_dir('cursors_edit_linting.sh'), _resolve_to_cur_dir('search.sh'), ] -CURSOR_DOCUMENTATION = ''.join([ - check_and_parse_command_file(filepath) - for filepath in CURSOR_SCRIPT_FILEPATHS - if filepath is not None -]) +CURSOR_DOCUMENTATION = ''.join( + [ + check_and_parse_command_file(filepath) + for filepath in CURSOR_SCRIPT_FILEPATHS + if filepath is not None + ] +) @dataclass @@ -63,5 +66,7 @@ class SWEAgentCursorCommandsRequirement(PluginRequirement): sandbox_dest: str = '/opendevin/plugins/swe_agent_commands' bash_script_path: str = 'setup_cursor_mode.sh' - scripts_filepaths: List[str | None] = field(default_factory=lambda: CURSOR_SCRIPT_FILEPATHS) + scripts_filepaths: List[str | None] = field( + default_factory=lambda: CURSOR_SCRIPT_FILEPATHS + ) documentation: str = CURSOR_DOCUMENTATION diff --git a/opendevin/sandbox/plugins/swe_agent_commands/_setup_cursor_mode_env.sh b/opendevin/runtime/plugins/swe_agent_commands/_setup_cursor_mode_env.sh old mode 100755 new mode 100644 similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/_setup_cursor_mode_env.sh rename to opendevin/runtime/plugins/swe_agent_commands/_setup_cursor_mode_env.sh diff --git a/opendevin/sandbox/plugins/swe_agent_commands/_setup_default_env.sh b/opendevin/runtime/plugins/swe_agent_commands/_setup_default_env.sh old mode 100755 new mode 100644 similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/_setup_default_env.sh rename to opendevin/runtime/plugins/swe_agent_commands/_setup_default_env.sh diff --git a/opendevin/sandbox/plugins/swe_agent_commands/_split_string b/opendevin/runtime/plugins/swe_agent_commands/_split_string old mode 100755 new mode 100644 similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/_split_string rename to opendevin/runtime/plugins/swe_agent_commands/_split_string diff --git a/opendevin/sandbox/plugins/swe_agent_commands/cursors_defaults.sh b/opendevin/runtime/plugins/swe_agent_commands/cursors_defaults.sh similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/cursors_defaults.sh rename to opendevin/runtime/plugins/swe_agent_commands/cursors_defaults.sh diff --git a/opendevin/sandbox/plugins/swe_agent_commands/cursors_edit_linting.sh b/opendevin/runtime/plugins/swe_agent_commands/cursors_edit_linting.sh similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/cursors_edit_linting.sh rename to opendevin/runtime/plugins/swe_agent_commands/cursors_edit_linting.sh diff --git a/opendevin/sandbox/plugins/swe_agent_commands/defaults.sh b/opendevin/runtime/plugins/swe_agent_commands/defaults.sh similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/defaults.sh rename to opendevin/runtime/plugins/swe_agent_commands/defaults.sh diff --git a/opendevin/sandbox/plugins/swe_agent_commands/edit_linting.sh b/opendevin/runtime/plugins/swe_agent_commands/edit_linting.sh similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/edit_linting.sh rename to opendevin/runtime/plugins/swe_agent_commands/edit_linting.sh diff --git a/opendevin/sandbox/plugins/swe_agent_commands/parse_commands.py b/opendevin/runtime/plugins/swe_agent_commands/parse_commands.py similarity index 99% rename from opendevin/sandbox/plugins/swe_agent_commands/parse_commands.py rename to opendevin/runtime/plugins/swe_agent_commands/parse_commands.py index 75aa330d16..49aa948964 100644 --- a/opendevin/sandbox/plugins/swe_agent_commands/parse_commands.py +++ b/opendevin/runtime/plugins/swe_agent_commands/parse_commands.py @@ -51,6 +51,7 @@ def parse_command_file(filepath: str) -> str: if __name__ == '__main__': import sys + if len(sys.argv) < 2: print('Usage: python parse_commands.py ') sys.exit(1) diff --git a/opendevin/sandbox/plugins/swe_agent_commands/search.sh b/opendevin/runtime/plugins/swe_agent_commands/search.sh similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/search.sh rename to opendevin/runtime/plugins/swe_agent_commands/search.sh diff --git a/opendevin/sandbox/plugins/swe_agent_commands/setup_cursor_mode.sh b/opendevin/runtime/plugins/swe_agent_commands/setup_cursor_mode.sh old mode 100755 new mode 100644 similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/setup_cursor_mode.sh rename to opendevin/runtime/plugins/swe_agent_commands/setup_cursor_mode.sh diff --git a/opendevin/sandbox/plugins/swe_agent_commands/setup_default.sh b/opendevin/runtime/plugins/swe_agent_commands/setup_default.sh old mode 100755 new mode 100644 similarity index 100% rename from opendevin/sandbox/plugins/swe_agent_commands/setup_default.sh rename to opendevin/runtime/plugins/swe_agent_commands/setup_default.sh diff --git a/opendevin/sandbox/process.py b/opendevin/runtime/process.py similarity index 100% rename from opendevin/sandbox/process.py rename to opendevin/runtime/process.py diff --git a/opendevin/sandbox/sandbox.py b/opendevin/runtime/sandbox.py similarity index 87% rename from opendevin/sandbox/sandbox.py rename to opendevin/runtime/sandbox.py index cf5e56432a..b485a2bd24 100644 --- a/opendevin/sandbox/sandbox.py +++ b/opendevin/runtime/sandbox.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod from typing import Dict, Tuple -from opendevin.sandbox.plugins.mixin import PluginMixin -from opendevin.sandbox.process import Process +from opendevin.runtime.docker.process import Process +from opendevin.runtime.plugins.mixin import PluginMixin class Sandbox(ABC, PluginMixin): diff --git a/opendevin/utils/__init__.py b/opendevin/runtime/utils/__init__.py similarity index 100% rename from opendevin/utils/__init__.py rename to opendevin/runtime/utils/__init__.py diff --git a/opendevin/utils/system.py b/opendevin/runtime/utils/system.py similarity index 80% rename from opendevin/utils/system.py rename to opendevin/runtime/utils/system.py index 47b33a31e5..e13f8405c0 100644 --- a/opendevin/utils/system.py +++ b/opendevin/runtime/utils/system.py @@ -2,8 +2,7 @@ import socket def find_available_tcp_port() -> int: - """Find an available TCP port, return -1 if none available. - """ + """Find an available TCP port, return -1 if none available.""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.bind(('localhost', 0)) diff --git a/opendevin/sandbox/plugins/mixin.py b/opendevin/sandbox/plugins/mixin.py deleted file mode 100644 index 9ca0b467df..0000000000 --- a/opendevin/sandbox/plugins/mixin.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -from typing import List, Protocol, Tuple - -from opendevin.logger import opendevin_logger as logger -from opendevin.sandbox.plugins.requirement import PluginRequirement - - -class SandboxProtocol(Protocol): - # https://stackoverflow.com/questions/51930339/how-do-i-correctly-add-type-hints-to-mixin-classes - - def execute(self, cmd: str) -> Tuple[int, str]: - ... - - def copy_to(self, host_src: str, sandbox_dest: str, recursive: bool = False): - ... - - -class PluginMixin: - """Mixin for Sandbox to support plugins.""" - - def init_plugins(self: SandboxProtocol, requirements: List[PluginRequirement]): - """Load a plugin into the sandbox.""" - for requirement in requirements: - # copy over the files - self.copy_to(requirement.host_src, requirement.sandbox_dest, recursive=True) - logger.info(f'Copied files from [{requirement.host_src}] to [{requirement.sandbox_dest}] inside sandbox.') - - # Execute the bash script - abs_path_to_bash_script = os.path.join(requirement.sandbox_dest, requirement.bash_script_path) - logger.info(f'Initalizing plugin [{requirement.name}] by executing [{abs_path_to_bash_script}] in the sandbox.') - exit_code, output = self.execute(abs_path_to_bash_script) - if exit_code != 0: - raise RuntimeError(f'Failed to initialize plugin {requirement.name} with exit code {exit_code} and output {output}') - logger.info(f'Plugin {requirement.name} initialized successfully\n:{output}') - - if len(requirements) > 0: - exit_code, output = self.execute('source ~/.bashrc') - if exit_code != 0: - raise RuntimeError(f'Failed to source ~/.bashrc with exit code {exit_code} and output {output}') - logger.info('Sourced ~/.bashrc successfully') diff --git a/opendevin/server/agent/agent.py b/opendevin/server/agent/agent.py index a834160582..d24de297a8 100644 --- a/opendevin/server/agent/agent.py +++ b/opendevin/server/agent/agent.py @@ -1,9 +1,11 @@ import asyncio from typing import Dict, List, Optional -from opendevin import config -from opendevin.agent import Agent from opendevin.controller import AgentController +from opendevin.controller.agent import Agent +from opendevin.core import config +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema import ActionType, ConfigType, TaskState, TaskStateAction from opendevin.events.action import ( Action, NullAction, @@ -14,8 +16,6 @@ from opendevin.events.observation import ( UserMessageObservation, ) from opendevin.llm.llm import LLM -from opendevin.logger import opendevin_logger as logger -from opendevin.schema import ActionType, ConfigType, TaskState, TaskStateAction from opendevin.server.session import session_manager # new task state to valid old task states diff --git a/opendevin/server/agent/manager.py b/opendevin/server/agent/manager.py index 5ae0fd6e21..436c9bae89 100644 --- a/opendevin/server/agent/manager.py +++ b/opendevin/server/agent/manager.py @@ -1,6 +1,6 @@ import atexit -from opendevin.logger import opendevin_logger as logger +from opendevin.core.logger import opendevin_logger as logger from opendevin.server.session import session_manager from .agent import AgentUnit diff --git a/opendevin/server/auth/auth.py b/opendevin/server/auth/auth.py index 6429743be7..e90a40d6e6 100644 --- a/opendevin/server/auth/auth.py +++ b/opendevin/server/auth/auth.py @@ -4,7 +4,7 @@ from typing import Dict import jwt from jwt.exceptions import InvalidTokenError -from opendevin.logger import opendevin_logger as logger +from opendevin.core.logger import opendevin_logger as logger JWT_SECRET = os.getenv('JWT_SECRET', '5ecRe7') diff --git a/opendevin/server/listen.py b/opendevin/server/listen.py index d6b9a87278..a1a8cbb076 100644 --- a/opendevin/server/listen.py +++ b/opendevin/server/listen.py @@ -11,10 +11,11 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from fastapi.staticfiles import StaticFiles import agenthub # noqa F401 (we import this to get the agents registered) -from opendevin import config, files -from opendevin.agent import Agent -from opendevin.logger import opendevin_logger as logger -from opendevin.schema.config import ConfigType +from opendevin.controller.agent import Agent +from opendevin.core import config +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema.config import ConfigType +from opendevin.runtime import files from opendevin.server.agent import agent_manager from opendevin.server.auth import get_sid_from_token, sign_token from opendevin.server.session import message_stack, session_manager @@ -115,7 +116,9 @@ async def del_messages( @app.get('/api/refresh-files') def refresh_files(): - structure = files.get_folder_structure(Path(str(config.get(ConfigType.WORKSPACE_BASE)))) + structure = files.get_folder_structure( + Path(str(config.get(ConfigType.WORKSPACE_BASE))) + ) return structure.to_dict() @@ -151,7 +154,7 @@ async def upload_file(file: UploadFile): logger.error(f'Error saving file {file.filename}: {e}', exc_info=True) return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content={'error': f'Error saving file: {e}'} + content={'error': f'Error saving file: {e}'}, ) return {'filename': file.filename, 'location': str(file_path)} diff --git a/opendevin/mock/README.md b/opendevin/server/mock/README.md similarity index 100% rename from opendevin/mock/README.md rename to opendevin/server/mock/README.md diff --git a/opendevin/mock/listen.py b/opendevin/server/mock/listen.py similarity index 97% rename from opendevin/mock/listen.py rename to opendevin/server/mock/listen.py index 3e0bb213cd..eb3752b0a4 100644 --- a/opendevin/mock/listen.py +++ b/opendevin/server/mock/listen.py @@ -1,7 +1,7 @@ import uvicorn from fastapi import FastAPI, WebSocket -from opendevin.schema import ActionType +from opendevin.core.schema import ActionType app = FastAPI() diff --git a/opendevin/server/session/manager.py b/opendevin/server/session/manager.py index 09525edbd4..f8910f012b 100644 --- a/opendevin/server/session/manager.py +++ b/opendevin/server/session/manager.py @@ -5,7 +5,7 @@ from typing import Callable, Dict from fastapi import WebSocket -from opendevin.logger import opendevin_logger as logger +from opendevin.core.logger import opendevin_logger as logger from .msg_stack import message_stack from .session import Session diff --git a/opendevin/server/session/msg_stack.py b/opendevin/server/session/msg_stack.py index d362022a5e..8c61e5fdaa 100644 --- a/opendevin/server/session/msg_stack.py +++ b/opendevin/server/session/msg_stack.py @@ -4,8 +4,8 @@ import os import uuid from typing import Dict, List -from opendevin.logger import opendevin_logger as logger -from opendevin.schema.action import ActionType +from opendevin.core.logger import opendevin_logger as logger +from opendevin.core.schema.action import ActionType CACHE_DIR = os.getenv('CACHE_DIR', 'cache') MSG_CACHE_FILE = os.path.join(CACHE_DIR, 'messages.json') @@ -62,7 +62,10 @@ class MessageStack: cnt = 0 for msg in self._messages[sid]: # Ignore assistant init message for now. - if 'action' in msg.payload and msg.payload['action'] in [ActionType.INIT, ActionType.CHANGE_TASK_STATE]: + if 'action' in msg.payload and msg.payload['action'] in [ + ActionType.INIT, + ActionType.CHANGE_TASK_STATE, + ]: continue cnt += 1 return cnt @@ -82,8 +85,7 @@ class MessageStack: with open(MSG_CACHE_FILE, 'r') as file: data = json.load(file) for sid, msgs in data.items(): - self._messages[sid] = [ - Message.from_dict(msg) for msg in msgs] + self._messages[sid] = [Message.from_dict(msg) for msg in msgs] except FileNotFoundError: pass except json.decoder.JSONDecodeError: diff --git a/opendevin/server/session/session.py b/opendevin/server/session/session.py index 19130e7815..a53914019c 100644 --- a/opendevin/server/session/session.py +++ b/opendevin/server/session/session.py @@ -3,7 +3,7 @@ from typing import Callable, Dict from fastapi import WebSocket, WebSocketDisconnect -from opendevin.logger import opendevin_logger as logger +from opendevin.core.logger import opendevin_logger as logger from .msg_stack import message_stack diff --git a/tests/integration/README.md b/tests/integration/README.md index 21befd6925..359c76b356 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -86,7 +86,7 @@ rm -rf logs rm -rf workspace mkdir workspace # Depending on the complexity of the task you want to test, you can change the number of iterations limit. Change agent accordingly. If you are adding a new test, try generating mock responses for every agent. -poetry run python ./opendevin/main.py -i 10 -t "Write a shell script 'hello.sh' that prints 'hello'." -c "MonologueAgent" -d "./workspace" +poetry run python ./opendevin/core/main.py -i 10 -t "Write a shell script 'hello.sh' that prints 'hello'." -c "MonologueAgent" -d "./workspace" ``` **NOTE**: If your agent decide to support user-agent interaction via natural language (e.g., you will prompted to enter user resposes when running the above `main.py` command), you should create a file named `tests/integration/mock///user_responses.log` containing all the responses in order you provided to the agent, delimited by newline ('\n'). This will be used to mock the STDIN during testing. diff --git a/tests/integration/test_agent.py b/tests/integration/test_agent.py index 2a3522d658..5eea4ac5c0 100644 --- a/tests/integration/test_agent.py +++ b/tests/integration/test_agent.py @@ -4,13 +4,14 @@ import subprocess import pytest -from opendevin.main import main +from opendevin.core.main import main # skip if @pytest.mark.skipif( - os.getenv('AGENT') == 'CodeActAgent' and os.getenv('SANDBOX_TYPE').lower() == 'exec', - reason='CodeActAgent does not support exec sandbox since exec sandbox is NOT stateful' + os.getenv('AGENT') == 'CodeActAgent' + and os.getenv('SANDBOX_TYPE').lower() == 'exec', + reason='CodeActAgent does not support exec sandbox since exec sandbox is NOT stateful', ) def test_write_simple_script(): task = "Write a shell script 'hello.sh' that prints 'hello'." @@ -24,4 +25,6 @@ def test_write_simple_script(): result = subprocess.run(['bash', script_path], capture_output=True, text=True) # Verify the output from the script - assert result.stdout.strip() == 'hello', f'Expected output "hello", but got "{result.stdout.strip()}"' + assert ( + result.stdout.strip() == 'hello' + ), f'Expected output "hello", but got "{result.stdout.strip()}"' diff --git a/tests/unit/test_action_github.py b/tests/unit/test_action_github.py index 1e3cc8d8da..a55a0643c9 100644 --- a/tests/unit/test_action_github.py +++ b/tests/unit/test_action_github.py @@ -1,17 +1,16 @@ - from unittest.mock import MagicMock, call, patch import pytest from agenthub.dummy_agent.agent import DummyAgent -from opendevin import config from opendevin.controller.agent_controller import AgentController +from opendevin.core import config +from opendevin.core.schema.config import ConfigType from opendevin.events.action.github import GitHubPushAction, GitHubSendPRAction from opendevin.events.observation.commands import CmdOutputObservation from opendevin.events.observation.error import AgentErrorObservation from opendevin.events.observation.message import AgentMessageObservation from opendevin.llm.llm import LLM -from opendevin.schema.config import ConfigType @pytest.fixture @@ -28,12 +27,16 @@ def agent_controller(): @patch.dict(config.config, {'GITHUB_TOKEN': 'fake_token'}, clear=True) @patch('random.choices') @patch('opendevin.controller.action_manager.ActionManager.run_command') -async def test_run_push_successful(mock_run_command, mock_random_choices, agent_controller): +async def test_run_push_successful( + mock_run_command, mock_random_choices, agent_controller +): # Setup mock for random.choices mock_random_choices.return_value = ['a', 'b', 'c', 'd', 'e'] # Create a CmdOutputObservation instance for successful command execution - successful_output = CmdOutputObservation(content='', command_id=1, command='', exit_code=0) + successful_output = CmdOutputObservation( + content='', command_id=1, command='', exit_code=0 + ) # Setup the mock for run_command to return successful output mock_run_command.return_value = successful_output @@ -64,17 +67,13 @@ async def test_run_push_successful(mock_run_command, mock_random_choices, agent_ async def test_run_push_error_missing_token( mock_run_command, mock_random_choices, agent_controller ): - # Run the method push_action = GitHubPushAction(owner='owner', repo='repo', branch='branch') result = await push_action.run(agent_controller) # Verify the result is an error due to missing token assert isinstance(result, AgentErrorObservation) - assert ( - result.message - == 'Oops. Something went wrong: GITHUB_TOKEN is not set' - ) + assert result.message == 'Oops. Something went wrong: GITHUB_TOKEN is not set' @pytest.mark.asyncio @@ -88,7 +87,15 @@ async def test_run_pull_request_created_successfully(mock_post, agent_controller mock_post.return_value = mock_response # Run the method - pr_action = GitHubSendPRAction(owner='owner', repo='repo', title='title', head='head', head_repo='head_repo', base='base', body='body') + pr_action = GitHubSendPRAction( + owner='owner', + repo='repo', + title='title', + head='head', + head_repo='head_repo', + base='base', + body='body', + ) result = await pr_action.run(agent_controller) # Verify the result is a success observation @@ -96,6 +103,7 @@ async def test_run_pull_request_created_successfully(mock_post, agent_controller assert 'Pull request created successfully' in result.content assert 'https://github.com/example/pull/1' in result.content + @pytest.mark.asyncio @patch('requests.post') @patch.dict(config.config, {'GITHUB_TOKEN': 'fake_token'}, clear=True) @@ -107,7 +115,15 @@ async def test_run_pull_request_creation_failed(mock_post, agent_controller): mock_post.return_value = mock_response # Run the method - pr_action = GitHubSendPRAction(owner='owner', repo='repo', title='title', head='head', head_repo='head_repo', base='base', body='body') + pr_action = GitHubSendPRAction( + owner='owner', + repo='repo', + title='title', + head='head', + head_repo='head_repo', + base='base', + body='body', + ) result = await pr_action.run(agent_controller) # Verify the result is an error observation @@ -116,11 +132,19 @@ async def test_run_pull_request_creation_failed(mock_post, agent_controller): assert 'Status code: 400' in result.content assert 'Bad Request' in result.content + @pytest.mark.asyncio async def test_run_error_missing_token(agent_controller): - # Run the method - pr_action = GitHubSendPRAction(owner='owner', repo='repo', title='title', head='head', head_repo='head_repo', base='base', body='body') + pr_action = GitHubSendPRAction( + owner='owner', + repo='repo', + title='title', + head='head', + head_repo='head_repo', + base='base', + body='body', + ) result = await pr_action.run(agent_controller) # Verify the result is an error due to missing token diff --git a/tests/unit/test_arg_parser.py b/tests/unit/test_arg_parser.py index 73e8e811b6..1b332c8669 100644 --- a/tests/unit/test_arg_parser.py +++ b/tests/unit/test_arg_parser.py @@ -1,6 +1,6 @@ import pytest -from opendevin.config import get_parser +from opendevin.core.config import get_parser def test_help_message(capsys): @@ -35,8 +35,12 @@ options: expected_lines = expected_help_message.strip().split('\n') # Ensure both outputs have the same number of lines - assert len(actual_lines) == len(expected_lines), 'The number of lines in the help message does not match.' + assert len(actual_lines) == len( + expected_lines + ), 'The number of lines in the help message does not match.' # Compare each line for actual, expected in zip(actual_lines, expected_lines): - assert actual.strip() == expected.strip(), f"Expected '{expected}', got '{actual}'" + assert ( + actual.strip() == expected.strip() + ), f"Expected '{expected}', got '{actual}'" diff --git a/tests/unit/test_micro_agents.py b/tests/unit/test_micro_agents.py index d08dfbc850..8533bc036c 100644 --- a/tests/unit/test_micro_agents.py +++ b/tests/unit/test_micro_agents.py @@ -5,9 +5,9 @@ from unittest.mock import MagicMock import yaml from agenthub.micro.registry import all_microagents -from opendevin.agent import Agent -from opendevin.plan import Plan -from opendevin.state import State +from opendevin.controller.agent import Agent +from opendevin.controller.state.plan import Plan +from opendevin.controller.state.state import State def test_all_agents_are_loaded(): @@ -29,9 +29,7 @@ def test_coder_agent_with_summary(): """ mock_llm = MagicMock() content = json.dumps({'action': 'finish', 'args': {}}) - mock_llm.completion.return_value = { - 'choices': [{'message': {'content': content}}] - } + mock_llm.completion.return_value = {'choices': [{'message': {'content': content}}]} coder_agent = Agent.get_cls('CoderAgent')(llm=mock_llm) assert coder_agent is not None @@ -56,9 +54,7 @@ def test_coder_agent_without_summary(): """ mock_llm = MagicMock() content = json.dumps({'action': 'finish', 'args': {}}) - mock_llm.completion.return_value = { - 'choices': [{'message': {'content': content}}] - } + mock_llm.completion.return_value = {'choices': [{'message': {'content': content}}]} coder_agent = Agent.get_cls('CoderAgent')(llm=mock_llm) assert coder_agent is not None diff --git a/tests/unit/test_sandbox.py b/tests/unit/test_sandbox.py index e815033c9a..4ab00b9454 100644 --- a/tests/unit/test_sandbox.py +++ b/tests/unit/test_sandbox.py @@ -4,8 +4,8 @@ from unittest.mock import patch import pytest -from opendevin import config -from opendevin.sandbox.docker.ssh_box import DockerSSHBox +from opendevin.core import config +from opendevin.runtime.docker.ssh_box import DockerSSHBox @pytest.fixture @@ -25,7 +25,7 @@ def test_ssh_box_run_as_devin(temp_dir): config.ConfigType.RUN_AS_DEVIN: 'true', config.ConfigType.SANDBOX_TYPE: 'ssh', }, - clear=True + clear=True, ): ssh_box = DockerSSHBox() @@ -61,7 +61,7 @@ def test_ssh_box_multi_line_cmd_run_as_devin(temp_dir): config.ConfigType.RUN_AS_DEVIN: 'true', config.ConfigType.SANDBOX_TYPE: 'ssh', }, - clear=True + clear=True, ): ssh_box = DockerSSHBox() @@ -71,6 +71,7 @@ def test_ssh_box_multi_line_cmd_run_as_devin(temp_dir): expected_lines = ['/workspacels -l', 'total 0'] assert output.strip().splitlines() == expected_lines + def test_ssh_box_stateful_cmd_run_as_devin(temp_dir): # get a temporary directory with patch.dict( @@ -80,7 +81,7 @@ def test_ssh_box_stateful_cmd_run_as_devin(temp_dir): config.ConfigType.RUN_AS_DEVIN: 'true', config.ConfigType.SANDBOX_TYPE: 'ssh', }, - clear=True + clear=True, ): ssh_box = DockerSSHBox() @@ -97,6 +98,7 @@ def test_ssh_box_stateful_cmd_run_as_devin(temp_dir): assert exit_code == 0, 'The exit code should be 0.' assert output.strip() == '/workspace/test' + def test_ssh_box_failed_cmd_run_as_devin(temp_dir): # get a temporary directory with patch.dict( @@ -106,7 +108,7 @@ def test_ssh_box_failed_cmd_run_as_devin(temp_dir): config.ConfigType.RUN_AS_DEVIN: 'true', config.ConfigType.SANDBOX_TYPE: 'ssh', }, - clear=True + clear=True, ): ssh_box = DockerSSHBox()