diff --git a/frontend/src/api/open-hands.ts b/frontend/src/api/open-hands.ts index b6d6550998..084c3a3a1c 100644 --- a/frontend/src/api/open-hands.ts +++ b/frontend/src/api/open-hands.ts @@ -226,12 +226,14 @@ class OpenHands { selectedRepository?: string, initialUserMsg?: string, imageUrls?: string[], + replayJson?: string, ): Promise { const body = { selected_repository: selectedRepository, selected_branch: undefined, initial_user_msg: initialUserMsg, image_urls: imageUrls, + replay_json: replayJson, }; const { data } = await openHands.post( diff --git a/frontend/src/components/features/chat/chat-interface.tsx b/frontend/src/components/features/chat/chat-interface.tsx index 37a1603120..05c983c72f 100644 --- a/frontend/src/components/features/chat/chat-interface.tsx +++ b/frontend/src/components/features/chat/chat-interface.tsx @@ -24,8 +24,12 @@ import { useGetTrajectory } from "#/hooks/mutation/use-get-trajectory"; import { downloadTrajectory } from "#/utils/download-trajectory"; import { displayErrorToast } from "#/utils/custom-toast-handlers"; -function getEntryPoint(hasRepository: boolean | null): string { +function getEntryPoint( + hasRepository: boolean | null, + hasReplayJson: boolean | null, +): string { if (hasRepository) return "github"; + if (hasReplayJson) return "replay"; return "direct"; } @@ -44,7 +48,7 @@ export function ChatInterface() { >("positive"); const [feedbackModalIsOpen, setFeedbackModalIsOpen] = React.useState(false); const [messageToSend, setMessageToSend] = React.useState(null); - const { selectedRepository } = useSelector( + const { selectedRepository, replayJson } = useSelector( (state: RootState) => state.initialQuery, ); const params = useParams(); @@ -53,8 +57,12 @@ export function ChatInterface() { const handleSendMessage = async (content: string, files: File[]) => { if (messages.length === 0) { posthog.capture("initial_query_submitted", { - entry_point: getEntryPoint(selectedRepository !== null), + entry_point: getEntryPoint( + selectedRepository !== null, + replayJson !== null, + ), query_character_length: content.length, + replay_json_size: replayJson?.length, }); } else { posthog.capture("user_message_sent", { diff --git a/frontend/src/components/features/suggestions/replay-suggestion-box.tsx b/frontend/src/components/features/suggestions/replay-suggestion-box.tsx new file mode 100644 index 0000000000..b9c177fcec --- /dev/null +++ b/frontend/src/components/features/suggestions/replay-suggestion-box.tsx @@ -0,0 +1,34 @@ +import { useTranslation } from "react-i18next"; +import { I18nKey } from "#/i18n/declaration"; +import { SuggestionBox } from "./suggestion-box"; + +interface ReplaySuggestionBoxProps { + onChange: (event: React.ChangeEvent) => void; +} + +export function ReplaySuggestionBox({ onChange }: ReplaySuggestionBoxProps) { + const { t } = useTranslation(); + return ( + + + {t(I18nKey.LANDING$UPLOAD_TRAJECTORY)} + + + + } + /> + ); +} diff --git a/frontend/src/hooks/mutation/use-create-conversation.ts b/frontend/src/hooks/mutation/use-create-conversation.ts index 75b9314bdc..52fe804950 100644 --- a/frontend/src/hooks/mutation/use-create-conversation.ts +++ b/frontend/src/hooks/mutation/use-create-conversation.ts @@ -11,18 +11,28 @@ export const useCreateConversation = () => { const dispatch = useDispatch(); const queryClient = useQueryClient(); - const { selectedRepository, files } = useSelector( + const { selectedRepository, files, replayJson } = useSelector( (state: RootState) => state.initialQuery, ); return useMutation({ mutationFn: async (variables: { q?: string }) => { + if ( + !variables.q?.trim() && + !selectedRepository && + files.length === 0 && + !replayJson + ) { + throw new Error("No query provided"); + } + if (variables.q) dispatch(setInitialPrompt(variables.q)); return OpenHands.createConversation( selectedRepository || undefined, variables.q, files, + replayJson || undefined, ); }, onSuccess: async ({ conversation_id: conversationId }, { q }) => { @@ -31,6 +41,7 @@ export const useCreateConversation = () => { query_character_length: q?.length, has_repository: !!selectedRepository, has_files: files.length > 0, + has_replay_json: !!replayJson, }); await queryClient.invalidateQueries({ queryKey: ["user", "conversations"], diff --git a/frontend/src/i18n/declaration.ts b/frontend/src/i18n/declaration.ts index ace7cfd4df..3c83f1fa85 100644 --- a/frontend/src/i18n/declaration.ts +++ b/frontend/src/i18n/declaration.ts @@ -172,8 +172,8 @@ export enum I18nKey { BUTTON$STOP = "BUTTON$STOP", LANDING$ATTACH_IMAGES = "LANDING$ATTACH_IMAGES", LANDING$OPEN_REPO = "LANDING$OPEN_REPO", - LANDING$IMPORT_PROJECT = "LANDING$IMPORT_PROJECT", - LANDING$UPLOAD_ZIP = "LANDING$UPLOAD_ZIP", + LANDING$REPLAY = "LANDING$REPLAY", + LANDING$UPLOAD_TRAJECTORY = "LANDING$UPLOAD_TRAJECTORY", LANDING$RECENT_CONVERSATION = "LANDING$RECENT_CONVERSATION", LANDING$OR = "LANDING$OR", SUGGESTIONS$TEST_COVERAGE = "SUGGESTIONS$TEST_COVERAGE", diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.json index dce5472842..26b3822222 100644 --- a/frontend/src/i18n/translation.json +++ b/frontend/src/i18n/translation.json @@ -2558,35 +2558,34 @@ "no": "Åpne et repo", "tr": "Depo aç" }, - "LANDING$IMPORT_PROJECT": { - "en": "+ Import Project", - "ja": "+ プロジェクトをインポート", - "zh-CN": "+ 导入项目", - "zh-TW": "+ 匯入專案", - "ko-KR": "+ 프로젝트 가져오기", - "fr": "+ Importer un projet", - "es": "+ Importar proyecto", - "de": "+ Projekt importieren", - "it": "+ Importa progetto", - "pt": "+ Importar projeto", - "ar": "+ استيراد مشروع", - "no": "+ Importer prosjekt", - "tr": "Proje içe aktar" + "LANDING$REPLAY": { + "en": "+ Replay Trajectory", + "ja": "+ Replay Trajectory", + "zh-CN": "+ Replay Trajectory", + "zh-TW": "+ Replay Trajectory", + "ko-KR": "+ Replay Trajectory", + "fr": "+ Replay Trajectory", + "es": "+ Replay Trajectory", + "de": "+ Replay Trajectory", + "it": "+ Replay Trajectory", + "pt": "+ Replay Trajectory", + "ar": "+ Replay Trajectory", + "no": "+ Replay Trajectory", + "tr": "+ Replay Trajectory" }, - "LANDING$UPLOAD_ZIP": { - "en": "Upload a .zip", - "ja": ".zipファイルをアップロード", - "zh-CN": "上传.zip文件", - "zh-TW": "上傳 .zip 檔案", - "ko-KR": ".zip 파일 업로드", - "fr": "Télécharger un .zip", - "es": "Subir un .zip", - "de": "Eine .zip-Datei hochladen", - "it": "Carica un .zip", - "pt": "Fazer upload de um .zip", - "ar": "تحميل ملف .zip", - "no": "Last opp en .zip", - "tr": "ZIP yükle" + "LANDING$UPLOAD_TRAJECTORY": { + "en": "Upload a .json", + "zh-CN": "Upload a .json", + "zh-TW": "Upload a .json", + "ko-KR": "Upload a .json", + "fr": "Upload a .json", + "es": "Upload a .json", + "de": "Upload a .json", + "it": "Upload a .json", + "pt": "Upload a .json", + "ar": "Upload a .json", + "no": "Upload a .json", + "tr": "Upload a .json" }, "LANDING$RECENT_CONVERSATION": { "en": "jump back to your most recent conversation", diff --git a/frontend/src/routes/_oh._index/route.tsx b/frontend/src/routes/_oh._index/route.tsx index ac0736bc71..ff33476bc0 100644 --- a/frontend/src/routes/_oh._index/route.tsx +++ b/frontend/src/routes/_oh._index/route.tsx @@ -1,13 +1,20 @@ import React from "react"; +import { useDispatch } from "react-redux"; +import posthog from "posthog-js"; +import { setReplayJson } from "#/state/initial-query-slice"; import { useGitHubUser } from "#/hooks/query/use-github-user"; import { useGitHubAuthUrl } from "#/hooks/use-github-auth-url"; import { useConfig } from "#/hooks/query/use-config"; +import { ReplaySuggestionBox } from "../../components/features/suggestions/replay-suggestion-box"; import { GitHubRepositoriesSuggestionBox } from "#/components/features/github/github-repositories-suggestion-box"; import { CodeNotInGitHubLink } from "#/components/features/github/code-not-in-github-link"; import { HeroHeading } from "#/components/shared/hero-heading"; import { TaskForm } from "#/components/shared/task-form"; +import { convertFileToText } from "#/utils/convert-file-to-text"; +import { ENABLE_TRAJECTORY_REPLAY } from "#/utils/feature-flags"; function Home() { + const dispatch = useDispatch(); const formRef = React.useRef(null); const { data: config } = useConfig(); @@ -35,6 +42,20 @@ function Home() { gitHubAuthUrl={gitHubAuthUrl} user={user || null} /> + {ENABLE_TRAJECTORY_REPLAY() && ( + { + if (event.target.files) { + const json = event.target.files[0]; + dispatch(setReplayJson(await convertFileToText(json))); + posthog.capture("json_file_uploaded"); + formRef.current?.requestSubmit(); + } else { + // TODO: handle error + } + }} + /> + )}
diff --git a/frontend/src/state/initial-query-slice.ts b/frontend/src/state/initial-query-slice.ts index 5cbacb6304..00a628d43a 100644 --- a/frontend/src/state/initial-query-slice.ts +++ b/frontend/src/state/initial-query-slice.ts @@ -4,12 +4,14 @@ type SliceState = { files: string[]; // base64 encoded images initialPrompt: string | null; selectedRepository: string | null; + replayJson: string | null; }; const initialState: SliceState = { files: [], initialPrompt: null, selectedRepository: null, + replayJson: null, }; export const selectedFilesSlice = createSlice({ @@ -37,6 +39,9 @@ export const selectedFilesSlice = createSlice({ clearSelectedRepository(state) { state.selectedRepository = null; }, + setReplayJson(state, action: PayloadAction) { + state.replayJson = action.payload; + }, }, }); @@ -48,5 +53,6 @@ export const { clearInitialPrompt, setSelectedRepository, clearSelectedRepository, + setReplayJson, } = selectedFilesSlice.actions; export default selectedFilesSlice.reducer; diff --git a/frontend/src/utils/convert-file-to-text.ts b/frontend/src/utils/convert-file-to-text.ts new file mode 100644 index 0000000000..a8cf56e19a --- /dev/null +++ b/frontend/src/utils/convert-file-to-text.ts @@ -0,0 +1,10 @@ +export const convertFileToText = async (file: File) => { + const reader = new FileReader(); + + return new Promise((resolve) => { + reader.onload = () => { + resolve(reader.result as string); + }; + reader.readAsText(file); + }); +}; diff --git a/frontend/src/utils/feature-flags.ts b/frontend/src/utils/feature-flags.ts index 7508417bdc..6976dc91bb 100644 --- a/frontend/src/utils/feature-flags.ts +++ b/frontend/src/utils/feature-flags.ts @@ -11,3 +11,8 @@ export function loadFeatureFlag( return defaultValue; } } + +export const BILLING_SETTINGS = () => loadFeatureFlag("BILLING_SETTINGS"); +export const HIDE_LLM_SETTINGS = () => loadFeatureFlag("HIDE_LLM_SETTINGS"); +export const ENABLE_TRAJECTORY_REPLAY = () => + loadFeatureFlag("TRAJECTORY_REPLAY"); diff --git a/openhands/controller/replay.py b/openhands/controller/replay.py index bae54b10e7..81b17ee542 100644 --- a/openhands/controller/replay.py +++ b/openhands/controller/replay.py @@ -3,6 +3,7 @@ from openhands.events.action.action import Action from openhands.events.action.message import MessageAction from openhands.events.event import Event, EventSource from openhands.events.observation.empty import NullObservation +from openhands.events.serialization.event import event_from_dict class ReplayManager: @@ -76,3 +77,21 @@ class ReplayManager: assert isinstance(event, Action) self.replay_index += 1 return event + + @staticmethod + def get_replay_events(trajectory) -> list[Event]: + if not isinstance(trajectory, list): + raise ValueError( + f'Expected a list in {trajectory}, got {type(trajectory).__name__}' + ) + replay_events = [] + for item in trajectory: + event = event_from_dict(item) + if event.source == EventSource.ENVIRONMENT: + # ignore ENVIRONMENT events as they are not issued by + # the user or agent, and should not be replayed + continue + # cannot add an event with _id to event stream + event._id = None # type: ignore[attr-defined] + replay_events.append(event) + return replay_events diff --git a/openhands/core/main.py b/openhands/core/main.py index a99fddbd5e..cda5941c70 100644 --- a/openhands/core/main.py +++ b/openhands/core/main.py @@ -6,6 +6,7 @@ from typing import Callable, Protocol import openhands.agenthub # noqa F401 (we import this to get the agents registered) from openhands.controller.agent import Agent +from openhands.controller.replay import ReplayManager from openhands.controller.state.state import State from openhands.core.config import ( AppConfig, @@ -28,7 +29,6 @@ from openhands.events.action import MessageAction, NullAction from openhands.events.action.action import Action from openhands.events.event import Event from openhands.events.observation import AgentStateChangedObservation -from openhands.events.serialization import event_from_dict from openhands.io import read_input, read_task from openhands.memory.memory import Memory from openhands.runtime.base import Runtime @@ -250,21 +250,7 @@ def load_replay_log(trajectory_path: str) -> tuple[list[Event] | None, Action]: raise ValueError(f'Trajectory path is a directory, not a file: {path}') with open(path, 'r', encoding='utf-8') as file: - data = json.load(file) - if not isinstance(data, list): - raise ValueError( - f'Expected a list in {path}, got {type(data).__name__}' - ) - events = [] - for item in data: - event = event_from_dict(item) - if event.source == EventSource.ENVIRONMENT: - # ignore ENVIRONMENT events as they are not issued by - # the user or agent, and should not be replayed - continue - # cannot add an event with _id to event stream - event._id = None # type: ignore[attr-defined] - events.append(event) + events = ReplayManager.get_replay_events(json.load(file)) assert isinstance(events[0], MessageAction) return events[1:], events[0] except json.JSONDecodeError as e: diff --git a/openhands/server/conversation_manager/conversation_manager.py b/openhands/server/conversation_manager/conversation_manager.py index 86bae39970..32601d152d 100644 --- a/openhands/server/conversation_manager/conversation_manager.py +++ b/openhands/server/conversation_manager/conversation_manager.py @@ -81,6 +81,7 @@ class ConversationManager(ABC): settings: Settings, user_id: str | None, initial_user_msg: MessageAction | None = None, + replay_json: str | None = None, github_user_id: str | None = None, ) -> EventStream: """Start an event loop if one is not already running""" diff --git a/openhands/server/conversation_manager/standalone_conversation_manager.py b/openhands/server/conversation_manager/standalone_conversation_manager.py index 7ca97fc742..08076727ff 100644 --- a/openhands/server/conversation_manager/standalone_conversation_manager.py +++ b/openhands/server/conversation_manager/standalone_conversation_manager.py @@ -252,6 +252,7 @@ class StandaloneConversationManager(ConversationManager): settings: Settings, user_id: str | None, initial_user_msg: MessageAction | None = None, + replay_json: str | None = None, github_user_id: str | None = None, ) -> EventStream: logger.info(f'maybe_start_agent_loop:{sid}', extra={'session_id': sid}) @@ -284,7 +285,9 @@ class StandaloneConversationManager(ConversationManager): user_id=user_id, ) self._local_agent_loops_by_sid[sid] = session - asyncio.create_task(session.initialize_agent(settings, initial_user_msg)) + asyncio.create_task( + session.initialize_agent(settings, initial_user_msg, replay_json) + ) # This does not get added when resuming an existing conversation try: session.agent_session.event_stream.subscribe( diff --git a/openhands/server/routes/manage_conversations.py b/openhands/server/routes/manage_conversations.py index 0f8772995d..453d686594 100644 --- a/openhands/server/routes/manage_conversations.py +++ b/openhands/server/routes/manage_conversations.py @@ -43,6 +43,7 @@ class InitSessionRequest(BaseModel): selected_branch: str | None = None initial_user_msg: str | None = None image_urls: list[str] | None = None + replay_json: str | None = None async def _create_new_conversation( @@ -52,6 +53,7 @@ async def _create_new_conversation( selected_branch: str | None, initial_user_msg: str | None, image_urls: list[str] | None, + replay_json: str | None, attach_convo_id: bool = False, ): logger.info( @@ -132,6 +134,7 @@ async def _create_new_conversation( conversation_init_data, user_id, initial_user_msg=initial_message_action, + replay_json=replay_json, ) logger.info(f'Finished initializing conversation {conversation_id}') @@ -151,6 +154,7 @@ async def new_conversation(request: Request, data: InitSessionRequest): selected_branch = data.selected_branch initial_user_msg = data.initial_user_msg image_urls = data.image_urls or [] + replay_json = data.replay_json try: # Create conversation with initial message @@ -161,6 +165,7 @@ async def new_conversation(request: Request, data: InitSessionRequest): selected_branch, initial_user_msg, image_urls, + replay_json, ) return JSONResponse( diff --git a/openhands/server/session/agent_session.py b/openhands/server/session/agent_session.py index 32a6259168..f2a22235c6 100644 --- a/openhands/server/session/agent_session.py +++ b/openhands/server/session/agent_session.py @@ -1,4 +1,5 @@ import asyncio +import json import time from logging import LoggerAdapter from types import MappingProxyType @@ -6,13 +7,14 @@ from typing import Callable, cast from openhands.controller import AgentController from openhands.controller.agent import Agent +from openhands.controller.replay import ReplayManager from openhands.controller.state.state import State from openhands.core.config import AgentConfig, AppConfig, LLMConfig from openhands.core.exceptions import AgentRuntimeUnavailableError from openhands.core.logger import OpenHandsLoggerAdapter from openhands.core.schema.agent import AgentState from openhands.events.action import ChangeAgentStateAction, MessageAction -from openhands.events.event import EventSource +from openhands.events.event import Event, EventSource from openhands.events.stream import EventStream from openhands.integrations.provider import PROVIDER_TOKEN_TYPE, ProviderHandler from openhands.memory.memory import Memory @@ -85,6 +87,7 @@ class AgentSession: selected_repository: str | None = None, selected_branch: str | None = None, initial_message: MessageAction | None = None, + replay_json: str | None = None, ): """Starts the Agent session Parameters: @@ -120,14 +123,26 @@ class AgentSession: selected_branch=selected_branch, ) - self.controller = self._create_controller( - agent, - config.security.confirmation_mode, - max_iterations, - max_budget_per_task=max_budget_per_task, - agent_to_llm_config=agent_to_llm_config, - agent_configs=agent_configs, - ) + if replay_json: + initial_message = self._run_replay( + initial_message, + replay_json, + agent, + config, + max_iterations, + max_budget_per_task, + agent_to_llm_config, + agent_configs, + ) + else: + self.controller = self._create_controller( + agent, + config.security.confirmation_mode, + max_iterations, + max_budget_per_task=max_budget_per_task, + agent_to_llm_config=agent_to_llm_config, + agent_configs=agent_configs, + ) repo_directory = None if self.runtime and runtime_connected and selected_repository: @@ -192,6 +207,37 @@ class AgentSession: if self.security_analyzer is not None: await self.security_analyzer.close() + def _run_replay( + self, + initial_message: MessageAction | None, + replay_json: str, + agent: Agent, + config: AppConfig, + max_iterations: int, + max_budget_per_task: float | None, + agent_to_llm_config: dict[str, LLMConfig] | None, + agent_configs: dict[str, AgentConfig] | None, + ) -> MessageAction: + """ + Replays a trajectory from a JSON file. Note that once the replay session + finishes, the controller will continue to run with further user instructions, + so we still need to pass llm configs, budget, etc., even though the replay + itself does not call LLM or cost money. + """ + assert initial_message is None + replay_events = ReplayManager.get_replay_events(json.loads(replay_json)) + self.controller = self._create_controller( + agent, + config.security.confirmation_mode, + max_iterations, + max_budget_per_task=max_budget_per_task, + agent_to_llm_config=agent_to_llm_config, + agent_configs=agent_configs, + replay_events=replay_events[1:], + ) + assert isinstance(replay_events[0], MessageAction) + return replay_events[0] + def _create_security_analyzer(self, security_analyzer: str | None): """Creates a SecurityAnalyzer instance that will be used to analyze the agent actions @@ -298,6 +344,7 @@ class AgentSession: max_budget_per_task: float | None = None, agent_to_llm_config: dict[str, LLMConfig] | None = None, agent_configs: dict[str, AgentConfig] | None = None, + replay_events: list[Event] | None = None, ) -> AgentController: """Creates an AgentController instance @@ -343,6 +390,7 @@ class AgentSession: headless_mode=False, status_callback=self._status_callback, initial_state=self._maybe_restore_state(), + replay_events=replay_events, ) return controller diff --git a/openhands/server/session/conversation_init_data.py b/openhands/server/session/conversation_init_data.py index 397d119c50..6f825db0d4 100644 --- a/openhands/server/session/conversation_init_data.py +++ b/openhands/server/session/conversation_init_data.py @@ -11,6 +11,7 @@ class ConversationInitData(Settings): git_provider_tokens: PROVIDER_TOKEN_TYPE | None = Field(default=None, frozen=True) selected_repository: str | None = Field(default=None) + replay_json: str | None = Field(default=None) selected_branch: str | None = Field(default=None) model_config = { diff --git a/openhands/server/session/session.py b/openhands/server/session/session.py index f1d74799e0..751adc90cd 100644 --- a/openhands/server/session/session.py +++ b/openhands/server/session/session.py @@ -84,7 +84,10 @@ class Session: await self.agent_session.close() async def initialize_agent( - self, settings: Settings, initial_message: MessageAction | None + self, + settings: Settings, + initial_message: MessageAction | None, + replay_json: str | None, ): self.agent_session.event_stream.add_event( AgentStateChangedObservation('', AgentState.LOADING), @@ -154,6 +157,7 @@ class Session: selected_repository=selected_repository, selected_branch=selected_branch, initial_message=initial_message, + replay_json=replay_json, ) except Exception as e: self.logger.exception(f'Error creating agent_session: {e}')