github/OpenHands
1
0
Fork 0
mirror of https://github.com/OpenHands/OpenHands.git synced 2025-12-26 05:48:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

doc - Added code documentation for clarity (#434)

Browse Source 
* doc - Added code documentaion to 'plan.py'

* doc - Added code documentation to 'session.py'

* doc - added code documentation for clarity

* doc - added documentation to 'conftest.py'

* doc - added code documentation to 'run_tests.pt'

* Update evaluation/regression/conftest.py

---------

Co-authored-by: Robert Brennan <accounts@rbren.io>
This commit is contained in:
Tess 2024-04-01 14:22:09 +00:00 committed by GitHub
parent 1ae3f1bf6a
commit 8796a690d5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 195 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
evaluation/regression/conftest.py
View File

@ -10,6 +10,11 @@ CASES_DIR = os.path.join(SCRIPT_DIR, 'cases')
AGENTHUB_DIR = os.path.join(SCRIPT_DIR, '../../', 'agenthub')
def agents():
"""Retrieves a list of available agents.
Returns:
A list of agent names.
"""
agents = []
for agent in os.listdir(AGENTHUB_DIR):
if os.path.isdir(os.path.join(AGENTHUB_DIR, agent)) and agent.endswith('_agent'):
@ -18,27 +23,82 @@ def agents():
@pytest.fixture(scope="session")
def test_cases_dir():
"""Fixture that provides the directory path for test cases.
Returns:
The directory path for test cases.
"""
return CASES_DIR
@pytest.fixture
def task_file(test_cases_dir, request):
"""Fixture that provides the path to the task file for a test case.
Args:
test_cases_dir: The directory path for test cases.
request: The pytest request object.
Returns:
The path to the task file for the test case.
"""
test_case_dir = os.path.dirname(request.module.__file__)
task_file_path = os.path.join(test_case_dir, 'task.txt')
return task_file_path
@pytest.fixture
def workspace_dir(test_cases_dir, request):
"""Fixture that provides the workspace directory for a test case.
Args:
test_cases_dir: The directory path for test cases.
request: The pytest request object.
Returns:
The workspace directory for the test case.
"""
test_case_dir = os.path.dirname(request.module.__file__)
workspace_dir = os.path.join(test_case_dir, 'workspace')
return workspace_dir
@pytest.fixture
def model(request):
"""Fixture that provides the model name.
Args:
request: The pytest request object.
Returns:
The model name, defaulting to "gpt-4-0125-preview".
"""
return request.config.getoption("model", default="gpt-4-0125-preview")
@pytest.fixture
def run_test_case(test_cases_dir, workspace_dir, request):
"""Fixture that provides a function to run a test case.
Args:
test_cases_dir: The directory path for test cases.
workspace_dir: The workspace directory for the test case.
request: The pytest request object.
Returns:
A function that runs a test case for a given agent and case.
"""
def _run_test_case(agent, case):
"""Runs a test case for a given agent.
Args:
agent: The name of the agent to run the test case for.
case: The name of the test case to run.
Returns:
The path to the workspace directory for the agent and test case.
Raises:
AssertionError: If the test case execution fails (non-zero return code).
Steps:
"""
case_dir = os.path.join(test_cases_dir, case)
task = open(os.path.join(case_dir, 'task.txt'), 'r').read().strip()
outputs_dir = os.path.join(case_dir, 'outputs')
@ -67,6 +127,11 @@ def run_test_case(test_cases_dir, workspace_dir, request):
return _run_test_case
def pytest_configure(config):
"""Configuration hook for pytest.
Args:
config: The pytest configuration object.
"""
now = datetime.datetime.now()
logging.basicConfig(
level=logging.INFO,

8
evaluation/regression/run_tests.py
View File

@ -4,6 +4,14 @@ import pytest
from opendevin import config
if __name__ == '__main__':
"""Main entry point of the script.
This script runs pytest with specific arguments and configuration.
Usage:
python script_name.py [--OPENAI_API_KEY=<api_key>] [--model=<model_name>]
"""
args = ['-v', 'evaluation/regression/cases']
for arg in sys.argv[1:]:
if arg.startswith('--OPENAI_API_KEY='):

78
opendevin/plan.py
View File

@ -14,6 +14,14 @@ class Task:
subtasks: List["Task"]
def __init__(self, parent: "Task | None", goal: str, state: str=OPEN_STATE, subtasks: List = []):
"""Initializes a new instance of the Task class.
Args:
parent: The parent task, or None if it is the root task.
goal: The goal of the task.
state: The initial state of the task.
subtasks: A list of subtasks associated with this task.
"""
if parent is None:
self.id = '0'
else:
@ -33,6 +41,14 @@ class Task:
self.state = OPEN_STATE
def to_string(self, indent=""):
"""Returns a string representation of the task and its subtasks.
Args:
indent: The indentation string for formatting the output.
Returns:
A string representation of the task and its subtasks.
"""
emoji = ''
if self.state == VERIFIED_STATE:
emoji = '✅'
@ -50,6 +66,11 @@ class Task:
return result
def to_dict(self):
"""Returns a dictionary representation of the task.
Returns:
A dictionary containing the task's attributes.
"""
return {
'id': self.id,
'goal': self.goal,
@ -58,6 +79,13 @@ class Task:
}
def set_state(self, state):
"""Sets the state of the task and its subtasks.
Args: state: The new state of the task.
Raises:
ValueError: If the provided state is invalid.
"""
if state not in STATES:
raise ValueError('Invalid state:' + state)
self.state = state
@ -70,6 +98,11 @@ class Task:
self.parent.set_state(state)
def get_current_task(self) -> "Task | None":
"""Retrieves the current task in progress.
Returns:
The current task in progress, or None if no task is in progress.
"""
for subtask in self.subtasks:
if subtask.state == IN_PROGRESS_STATE:
return subtask.get_current_task()
@ -78,17 +111,44 @@ class Task:
return None
class Plan:
"""Represents a plan consisting of tasks.
Attributes:
main_goal: The main goal of the plan.
task: The root task of the plan.
"""
main_goal: str
task: Task
def __init__(self, task: str):
"""Initializes a new instance of the Plan class.
Args:
task: The main goal of the plan.
"""
self.main_goal = task
self.task = Task(parent=None, goal=task, subtasks=[])
def __str__(self):
"""Returns a string representation of the plan.
Returns:
A string representation of the plan.
"""
return self.task.to_string()
def get_task_by_id(self, id: str) -> Task:
"""Retrieves a task by its ID.
Args:
id: The ID of the task.
Returns:
The task with the specified ID.
Raises:
ValueError: If the provided task ID is invalid or does not exist.
"""
try:
parts = [int(p) for p in id.split('.')]
except ValueError:
@ -104,14 +164,32 @@ class Plan:
return task
def add_subtask(self, parent_id: str, goal: str, subtasks: List = []):
"""Adds a subtask to a parent task.
Args:
parent_id: The ID of the parent task.
goal: The goal of the subtask.
subtasks: A list of subtasks associated with the new subtask.
"""
parent = self.get_task_by_id(parent_id)
child = Task(parent=parent, goal=goal, subtasks=subtasks)
parent.subtasks.append(child)
def set_subtask_state(self, id: str, state: str):
"""Sets the state of a subtask.
Args:
id: The ID of the subtask.
state: The new state of the subtask.
"""
task = self.get_task_by_id(id)
task.set_state(state)
def get_current_task(self):
"""Retrieves the current task in progress.
Returns:
The current task in progress, or None if no task is in progress.
"""
return self.task.get_current_task()

44
opendevin/server/session.py
View File

@ -22,7 +22,20 @@ LLM_MODEL = config.get_or_default("LLM_MODEL", "gpt-4-0125-preview")
CONTAINER_IMAGE = config.get_or_default("SANDBOX_CONTAINER_IMAGE", "ghcr.io/opendevin/sandbox")
class Session:
"""Represents a session with an agent.
Attributes:
websocket: The WebSocket connection associated with the session.
controller: The AgentController instance for controlling the agent.
agent: The Agent instance representing the agent.
agent_task: The task representing the agent's execution.
"""
def __init__(self, websocket):
"""Initializes a new instance of the Session class.
Args:
websocket: The WebSocket connection associated with the session.
"""
self.websocket = websocket
self.controller: Optional[AgentController] = None
self.agent: Optional[Agent] = None
@ -30,12 +43,27 @@ class Session:
asyncio.create_task(self.create_controller(), name="create controller") # FIXME: starting the docker container synchronously causes a websocket error...
async def send_error(self, message):
"""Sends an error message to the client.
Args:
message: The error message to send.
"""
await self.send({"error": True, "message": message})
async def send_message(self, message):
"""Sends a message to the client.
Args:
message: The message to send.
"""
await self.send({"message": message})
async def send(self, data):
"""Sends data to the client.
Args:
data: The data to send.
"""
if self.websocket is None:
return
try:
@ -44,6 +72,7 @@ class Session:
print("Error sending data to client", e)
async def start_listening(self):
"""Starts listening for messages from the client."""
try:
while True:
try:
@ -75,6 +104,11 @@ class Session:
print("Client websocket disconnected", e)
async def create_controller(self, start_event=None):
"""Creates an AgentController instance.
Args:
start_event: The start event data (optional).
"""
directory = DEFAULT_WORKSPACE_DIR
if start_event and "directory" in start_event["args"]:
directory = start_event["args"]["directory"]
@ -109,6 +143,11 @@ class Session:
await self.send({"action": "initialize", "message": "Control loop started."})
async def start_task(self, start_event):
"""Starts a task for the agent.
Args:
start_event: The start event data.
"""
if "task" not in start_event["args"]:
await self.send_error("No task specified")
return
@ -120,6 +159,11 @@ class Session:
self.agent_task = asyncio.create_task(self.controller.start_loop(task), name="agent loop")
def on_agent_event(self, event: Observation | Action):
"""Callback function for agent events.
Args:
event: The agent event (Observation or Action).
"""
if isinstance(event, NullAction):
return
if isinstance(event, NullObservation):