fix up serialization and deserialization of events (#1850)

* fix up serialization and deserialization of events

* fix tests

* remove prints

* fix test

* regenerate tests

* add try blocks

opendevin/controller/state/state.py

@@ -1,6 +1,9 @@
+import base64
+import pickle
 from dataclasses import dataclass, field
 
 from opendevin.controller.state.task import RootTask
+from opendevin.core.logger import opendevin_logger as logger
 from opendevin.core.schema import AgentState
 from opendevin.events.action import (
     Action,
@@ -10,6 +13,7 @@ from opendevin.events.observation import (
     CmdOutputObservation,
     Observation,
 )
+from opendevin.storage import get_file_store
 
 
 @dataclass
@@ -27,6 +31,28 @@ class State:
     error: str | None = None
     agent_state: AgentState = AgentState.LOADING
 
+    def save_to_session(self, sid: str):
+        fs = get_file_store()
+        pickled = pickle.dumps(self)
+        encoded = base64.b64encode(pickled).decode('utf-8')
+        try:
+            fs.write(f'sessions/{sid}/agent_state.pkl', encoded)
+        except Exception as e:
+            logger.error(f'Failed to save state to session: {e}')
+            raise e
+
+    @staticmethod
+    def restore_from_session(sid: str) -> 'State':
+        fs = get_file_store()
+        try:
+            encoded = fs.read(f'sessions/{sid}/agent_state.pkl')
+            pickled = base64.b64decode(encoded)
+            state = pickle.loads(pickled)
+        except Exception as e:
+            logger.error(f'Failed to restore state from session: {e}')
+            raise e
+        return state
+
     def get_current_user_intent(self):
         # TODO: this is used to understand the user's main goal, but it's possible
         # the latest message is an interruption. We should look for a space where

opendevin/events/event.py

@@ -1,12 +1,9 @@
 import datetime
 from dataclasses import dataclass
-from enum import Enum
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
-
-class EventSource(str, Enum):
-    AGENT = 'agent'
-    USER = 'user'
+if TYPE_CHECKING:
+    from opendevin.events.serialization.event import EventSource
 
 
 @dataclass
@@ -30,7 +27,7 @@ class Event:
         return None
 
     @property
-    def source(self) -> Optional[EventSource]:
+    def source(self) -> Optional['EventSource']:
         if hasattr(self, '_source'):
             return self._source  # type: ignore [attr-defined]
         return None

opendevin/events/serialization/__init__.py

@@ -2,6 +2,7 @@ from .action import (
     action_from_dict,
 )
 from .event import (
+    EventSource,
     event_from_dict,
     event_to_dict,
     event_to_memory,
@@ -16,4 +17,5 @@ __all__ = [
     'event_to_dict',
     'event_to_memory',
     'observation_from_dict',
+    'EventSource',
 ]

opendevin/events/serialization/action.py

@@ -13,11 +13,13 @@ from opendevin.events.action.commands import (
     CmdRunAction,
     IPythonRunCellAction,
 )
+from opendevin.events.action.empty import NullAction
 from opendevin.events.action.files import FileReadAction, FileWriteAction
 from opendevin.events.action.message import MessageAction
 from opendevin.events.action.tasks import AddTaskAction, ModifyTaskAction
 
 actions = (
+    NullAction,
     CmdKillAction,
     CmdRunAction,
     IPythonRunCellAction,

opendevin/events/serialization/event.py

@@ -1,5 +1,6 @@
 from dataclasses import asdict
 from datetime import datetime
+from enum import Enum
 from typing import TYPE_CHECKING
 
 from .action import action_from_dict
@@ -9,8 +10,15 @@ from .utils import remove_fields
 if TYPE_CHECKING:
     from opendevin.events.event import Event
 
+
+class EventSource(str, Enum):
+    AGENT = 'agent'
+    USER = 'user'
+
+
 # TODO: move `content` into `extras`
 TOP_KEYS = ['id', 'timestamp', 'source', 'message', 'cause', 'action', 'observation']
+UNDERSCORE_KEYS = ['id', 'timestamp', 'source', 'cause']
 
 DELETE_FROM_MEMORY_EXTRAS = {
     'screenshot',
@@ -23,21 +31,23 @@ DELETE_FROM_MEMORY_EXTRAS = {
 }
 
 
-def json_serial(obj):
-    if isinstance(obj, datetime):
-        return obj.isoformat()
-    if obj is None:
-        return None
-    return str(obj)
-
-
 def event_from_dict(data) -> 'Event':
+    evt: Event
     if 'action' in data:
-        return action_from_dict(data)
+        evt = action_from_dict(data)
     elif 'observation' in data:
-        return observation_from_dict(data)
+        evt = observation_from_dict(data)
     else:
         raise ValueError('Unknown event type: ' + data)
+    for key in UNDERSCORE_KEYS:
+        if key in data:
+            value = data[key]
+            if key == 'timestamp':
+                value = datetime.fromisoformat(value)
+            if key == 'source':
+                value = EventSource(value)
+            setattr(evt, '_' + key, value)
+    return evt
 
 
 def event_to_dict(event: 'Event') -> dict:
@@ -51,7 +61,9 @@ def event_to_dict(event: 'Event') -> dict:
         if key == 'id' and d.get('id') == -1:
             d.pop('id', None)
         if key == 'timestamp' and 'timestamp' in d:
-            d['timestamp'] = json_serial(d['timestamp'])
+            d['timestamp'] = d['timestamp'].isoformat()
+        if key == 'source' and 'source' in d:
+            d['source'] = d['source'].value
         props.pop(key, None)
     if 'action' in d:
         d['args'] = props

opendevin/events/stream.py

@@ -2,13 +2,14 @@ import asyncio
 import json
 from datetime import datetime
 from enum import Enum
-from typing import Callable
+from typing import Callable, Iterable
 
 from opendevin.core.logger import opendevin_logger as logger
 from opendevin.events.serialization.event import event_from_dict, event_to_dict
 from opendevin.storage import FileStore, get_file_store
 
-from .event import Event, EventSource
+from .event import Event
+from .serialization.event import EventSource
 
 
 class EventStreamSubscriber(str, Enum):
@@ -22,7 +23,7 @@ class EventStreamSubscriber(str, Enum):
 class EventStream:
     sid: str
     _subscribers: dict[str, Callable]
-    _events: list[Event]
+    _cur_id: int
     _lock: asyncio.Lock
     _file_store: FileStore
 
@@ -30,22 +31,36 @@ class EventStream:
         self.sid = sid
         self._file_store = get_file_store()
         self._subscribers = {}
-        self._events = []
+        self._cur_id = 0
         self._lock = asyncio.Lock()
+        self._reinitialize_from_file_store()
 
-    def _get_filename_for_event(self, event: Event):
-        # TODO: change to .id once that prop is in
-        return f'sessions/{self.sid}/events/{event._id}.json'  # type: ignore [attr-defined]
+    def _reinitialize_from_file_store(self):
+        events = self._file_store.list(f'sessions/{self.sid}/events')
+        for event_str in events:
+            id = self._get_id_from_filename(event_str)
+            if id >= self._cur_id:
+                self._cur_id = id + 1
 
-    async def _rehydrate(self):
-        async with self._lock:
-            self._events = []
-            events = self._file_store.list(f'sessions/{self.sid}/events')
-            for event_str in events:
-                content = self._file_store.read(event_str)
-                data = json.loads(content)
-                event = event_from_dict(data)
-                self._events.append(event)
+    def _get_filename_for_id(self, id: int) -> str:
+        return f'sessions/{self.sid}/events/{id}.json'
+
+    def _get_id_from_filename(self, filename: str) -> int:
+        return int(filename.split('/')[-1].split('.')[0])
+
+    def get_events(self, start_id=0, end_id=None) -> Iterable[Event]:
+        events = self._file_store.list(f'sessions/{self.sid}/events')
+        for event_str in events:
+            id = self._get_id_from_filename(event_str)
+            if start_id <= id and (end_id is None or id <= end_id):
+                event = self.get_event(id)
+                yield event
+
+    def get_event(self, id: int) -> Event:
+        filename = self._get_filename_for_id(id)
+        content = self._file_store.read(filename)
+        data = json.loads(content)
+        return event_from_dict(data)
 
     def subscribe(self, id: EventStreamSubscriber, callback: Callable):
         if id in self._subscribers:
@@ -62,12 +77,11 @@ class EventStream:
     # TODO: make this not async
     async def add_event(self, event: Event, source: EventSource):
         async with self._lock:
-            event._id = len(self._events)  # type: ignore [attr-defined]
-            event._timestamp = datetime.now()  # type: ignore [attr-defined]
-            event._source = source  # type: ignore [attr-defined]
-            self._file_store.write(
-                self._get_filename_for_event(event), json.dumps(event_to_dict(event))
-            )
-            self._events.append(event)
+            event._id = self._cur_id  # type: ignore [attr-defined]
+            self._cur_id += 1
+        event._timestamp = datetime.now()  # type: ignore [attr-defined]
+        event._source = source  # type: ignore [attr-defined]
+        data = event_to_dict(event)
+        self._file_store.write(self._get_filename_for_id(event.id), json.dumps(data))
         for key, fn in self._subscribers.items():
             await fn(event)

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_001.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.USER
+source: user
 action: message
 args:
   wait_for_response: False
@@ -72,11 +64,7 @@ content:
 extras:
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_002.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.USER
+source: user
 action: message
 args:
   wait_for_response: False
@@ -73,7 +65,7 @@ extras:
 
 Memory 1:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -85,18 +77,14 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
   path: hello.sh
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_003.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.USER
+source: user
 action: message
 args:
   wait_for_response: False
@@ -73,7 +65,7 @@ extras:
 
 Memory 1:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -85,7 +77,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -93,7 +85,7 @@ extras:
 
 Memory 2:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -105,18 +97,14 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
   path: hello.sh
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_004.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.USER
+source: user
 action: message
 args:
   wait_for_response: False
@@ -73,7 +65,7 @@ extras:
 
 Memory 1:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -85,7 +77,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -93,7 +85,7 @@ extras:
 
 Memory 2:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -105,7 +97,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -113,7 +105,7 @@ extras:
 
 Memory 3:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -125,18 +117,14 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
   path: hello.sh
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_005.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -73,7 +65,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -81,7 +73,7 @@ extras:
 
 Memory 1:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -93,7 +85,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -101,7 +93,7 @@ extras:
 
 Memory 2:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -113,7 +105,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -121,7 +113,7 @@ extras:
 
 Memory 3:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -133,18 +125,14 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
   path: hello.sh
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_006.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -73,7 +65,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -81,7 +73,7 @@ extras:
 
 Memory 1:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -93,7 +85,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -101,7 +93,7 @@ extras:
 
 Memory 2:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -113,7 +105,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -121,7 +113,7 @@ extras:
 
 Memory 3:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -133,18 +125,14 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
   path: hello.sh
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_007.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -73,7 +65,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -81,7 +73,7 @@ extras:
 
 Memory 1:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -93,7 +85,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -101,7 +93,7 @@ extras:
 
 Memory 2:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -113,7 +105,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -121,7 +113,7 @@ extras:
 
 Memory 3:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: run
 args:
   command: chmod +x hello.sh
@@ -129,7 +121,7 @@ args:
   thought:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: run
 content:
 extras:
@@ -138,11 +130,7 @@ extras:
   exit_code: 0
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_008.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -73,7 +65,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -81,7 +73,7 @@ extras:
 
 Memory 1:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -93,7 +85,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -101,7 +93,7 @@ extras:
 
 Memory 2:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: run
 args:
   command: chmod +x hello.sh
@@ -109,7 +101,7 @@ args:
   thought:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: run
 content:
 extras:
@@ -119,7 +111,7 @@ extras:
 
 Memory 3:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: run
 args:
   command: bash hello.sh
@@ -127,7 +119,7 @@ args:
   thought:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: run
 content: hello
 extras:
@@ -136,11 +128,7 @@ extras:
   exit_code: 0
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/integration/mock/SWEAgent/test_write_simple_script/prompt_009.log

@@ -1,7 +1,3 @@
-
-
-----------
-
 SYSTEM INFO:
 You am an autonomous coding agent, here to provide solutions for coding issues.
 You have been designed to assist you with a wide range of programming tasks, from code editing and debugging to testing and deployment.
@@ -52,16 +48,12 @@ None
 
 The following commands require an open file to be used: edit, scroll_up, scroll_down, goto
 To modify the current file use 'edit'. To move through the current file use 'goto' or 'scroll_up'/'scroll_down'
-when using write and edit do not surround the code with any "" just write the code.
-
-----------
-
-These are your past 4 actions:
+when using write and edit do not surround the code with any "" just write the code.These are your past 4 actions:
 ===== Previous Actions =====
 
 Memory 0:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: write
 args:
   path: hello.sh
@@ -73,7 +65,7 @@ args:
   Action:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: write
 content:
 extras:
@@ -81,7 +73,7 @@ extras:
 
 Memory 1:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: run
 args:
   command: chmod +x hello.sh
@@ -89,7 +81,7 @@ args:
   thought:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: run
 content:
 extras:
@@ -99,7 +91,7 @@ extras:
 
 Memory 2:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: run
 args:
   command: bash hello.sh
@@ -107,7 +99,7 @@ args:
   thought:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: run
 content: hello
 extras:
@@ -117,7 +109,7 @@ extras:
 
 Memory 3:
 Previous Action:
-source: EventSource.AGENT
+source: agent
 action: run
 args:
   command: bash hello.sh
@@ -125,7 +117,7 @@ args:
   thought:
 
 Output from Action:
-source: EventSource.AGENT
+source: agent
 observation: run
 content: hello
 extras:
@@ -134,11 +126,7 @@ extras:
   exit_code: 0
 ======= End Actions =======
 Use these memories to provide additional context to the problem you are solving.
-Remember that you have already completed these steps so you do not need to perform them again.
-
-----------
-
-RESPONSE FORMAT:
+Remember that you have already completed these steps so you do not need to perform them again.RESPONSE FORMAT:
 This is the format of the response you will make in order to solve the current issue.
 You will be given multiple iterations to complete this task so break it into steps and solve them one by one.
 

tests/unit/test_action_serialization.py

@@ -14,14 +14,14 @@ from opendevin.events.action import (
     ModifyTaskAction,
 )
 from opendevin.events.serialization import (
-    action_from_dict,
+    event_from_dict,
     event_to_dict,
     event_to_memory,
 )
 
 
 def serialization_deserialization(original_action_dict, cls):
-    action_instance = action_from_dict(original_action_dict)
+    action_instance = event_from_dict(original_action_dict)
     assert isinstance(
         action_instance, Action
     ), 'The action instance should be an instance of Action.'
@@ -34,11 +34,28 @@ def serialization_deserialization(original_action_dict, cls):
     assert (
         serialized_action_dict == original_action_dict
     ), 'The serialized action should match the original action dict.'
+    original_memory_dict = original_action_dict.copy()
+    original_memory_dict.pop('id', None)
+    original_memory_dict.pop('timestamp', None)
     assert (
-        serialized_action_memory == original_action_dict
+        serialized_action_memory == original_memory_dict
     ), 'The serialized action in memory should match the original action dict.'
 
 
+def test_event_props_serialization_deserialization():
+    original_action_dict = {
+        'id': 42,
+        'source': 'agent',
+        'timestamp': '2021-08-01T12:00:00',
+        'action': 'message',
+        'args': {
+            'content': 'This is a test.',
+            'wait_for_response': False,
+        },
+    }
+    serialization_deserialization(original_action_dict, MessageAction)
+
+
 def test_message_action_serialization_deserialization():
     original_action_dict = {
         'action': 'message',

tests/unit/test_event_stream.py

@@ -7,18 +7,22 @@ from opendevin.events.observation import NullObservation
 from opendevin.events.stream import EventSource, EventStream
 
 
+def collect_events(stream):
+    return [event for event in stream.get_events()]
+
+
 @pytest.mark.asyncio
 async def test_basic_flow():
     stream = EventStream('abc')
     await stream.add_event(NullAction(), EventSource.AGENT)
-    assert len(stream._events) == 1
+    assert len(collect_events(stream)) == 1
 
 
 @pytest.mark.asyncio
 async def test_stream_storage():
     stream = EventStream('def')
     await stream.add_event(NullObservation(''), EventSource.AGENT)
-    assert len(stream._events) == 1
+    assert len(collect_events(stream)) == 1
     content = stream._file_store.read('sessions/def/events/0.json')
     assert content is not None
     data = json.loads(content)
@@ -39,16 +43,13 @@ async def test_rehydration():
     stream1 = EventStream('es1')
     await stream1.add_event(NullObservation('obs1'), EventSource.AGENT)
     await stream1.add_event(NullObservation('obs2'), EventSource.AGENT)
-    assert len(stream1._events) == 2
+    assert len(collect_events(stream1)) == 2
 
     stream2 = EventStream('es2')
-    assert len(stream2._events) == 0
-    await stream2._rehydrate()
-    assert len(stream2._events) == 0
+    assert len(collect_events(stream2)) == 0
 
     stream1rehydrated = EventStream('es1')
-    assert len(stream1rehydrated._events) == 0
-    await stream1rehydrated._rehydrate()
-    assert len(stream1rehydrated._events) == 2
-    assert stream1rehydrated._events[0].content == 'obs1'
-    assert stream1rehydrated._events[1].content == 'obs2'
+    events = collect_events(stream1rehydrated)
+    assert len(events) == 2
+    assert events[0].content == 'obs1'
+    assert events[1].content == 'obs2'

tests/unit/test_micro_agents.py

@@ -9,6 +9,7 @@ from opendevin.controller.agent import Agent
 from opendevin.controller.state.state import State
 from opendevin.events.action import MessageAction
 from opendevin.events.observation import NullObservation
+from opendevin.events.serialization import EventSource
 
 
 def test_all_agents_are_loaded():
@@ -37,7 +38,7 @@ def test_coder_agent_with_summary():
 
     task = 'This is a dummy task'
     history = [(MessageAction(content=task), NullObservation(''))]
-    history[0][0]._source = 'user'
+    history[0][0]._source = EventSource.USER
     summary = 'This is a dummy summary about this repo'
     state = State(history=history, inputs={'summary': summary})
     coder_agent.step(state)
@@ -64,7 +65,7 @@ def test_coder_agent_without_summary():
 
     task = 'This is a dummy task'
     history = [(MessageAction(content=task), NullObservation(''))]
-    history[0][0]._source = 'user'
+    history[0][0]._source = EventSource.USER
     state = State(history=history)
     coder_agent.step(state)
 

tests/unit/test_observation_serialization.py

@@ -3,35 +3,52 @@ from opendevin.events.observation import (
     Observation,
 )
 from opendevin.events.serialization import (
+    event_from_dict,
     event_to_dict,
     event_to_memory,
-    observation_from_dict,
 )
 
 
-def test_observation_serialization_deserialization():
-    original_observation_dict = {
-        'observation': 'run',
-        'extras': {'exit_code': 0, 'command': 'ls -l', 'command_id': 3},
-        'message': 'Command `ls -l` executed with exit code 0.',
-        'content': 'foo.txt',
-    }
-    observation_instance = observation_from_dict(original_observation_dict)
+def serialization_deserialization(original_observation_dict, cls):
+    observation_instance = event_from_dict(original_observation_dict)
     assert isinstance(
         observation_instance, Observation
     ), 'The observation instance should be an instance of Action.'
     assert isinstance(
-        observation_instance, CmdOutputObservation
+        observation_instance, cls
     ), 'The observation instance should be an instance of CmdOutputObservation.'
     serialized_observation_dict = event_to_dict(observation_instance)
     serialized_observation_memory = event_to_memory(observation_instance)
     assert (
         serialized_observation_dict == original_observation_dict
     ), 'The serialized observation should match the original observation dict.'
-    original_observation_dict.pop('message')
+    original_observation_dict.pop('message', None)
+    original_observation_dict.pop('id', None)
+    original_observation_dict.pop('timestamp', None)
     assert (
         serialized_observation_memory == original_observation_dict
-    ), 'The serialized observation in memory should match the original observation dict.'
+    ), 'The serialized observation memory should match the original observation dict.'
 
 
 # Additional tests for various observation subclasses can be included here
+def test_observation_event_props_serialization_deserialization():
+    original_observation_dict = {
+        'id': 42,
+        'source': 'agent',
+        'timestamp': '2021-08-01T12:00:00',
+        'observation': 'run',
+        'message': 'Command `ls -l` executed with exit code 0.',
+        'extras': {'exit_code': 0, 'command': 'ls -l', 'command_id': 3},
+        'content': 'foo.txt',
+    }
+    serialization_deserialization(original_observation_dict, CmdOutputObservation)
+
+
+def test_command_output_observation_serialization_deserialization():
+    original_observation_dict = {
+        'observation': 'run',
+        'extras': {'exit_code': 0, 'command': 'ls -l', 'command_id': 3},
+        'message': 'Command `ls -l` executed with exit code 0.',
+        'content': 'foo.txt',
+    }
+    serialization_deserialization(original_observation_dict, CmdOutputObservation)