github/OpenHands
1
0
Fork 0
mirror of https://github.com/OpenHands/OpenHands.git synced 2025-12-26 05:48:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Trajectory replay on web app (under feature flag) (#6348)

Browse Source
This commit is contained in:
Boxuan Li 2025-03-24 14:40:07 -07:00 committed by GitHub
parent de05ea898e
commit f7d3516dec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 224 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
frontend/src/api/open-hands.ts
View File

@ -226,12 +226,14 @@ class OpenHands {
selectedRepository?: string,
initialUserMsg?: string,
imageUrls?: string[],
replayJson?: string,
): Promise<Conversation> {
const body = {
selected_repository: selectedRepository,
selected_branch: undefined,
initial_user_msg: initialUserMsg,
image_urls: imageUrls,
replay_json: replayJson,
};
const { data } = await openHands.post<Conversation>(

14
frontend/src/components/features/chat/chat-interface.tsx
View File

@ -24,8 +24,12 @@ import { useGetTrajectory } from "#/hooks/mutation/use-get-trajectory";
import { downloadTrajectory } from "#/utils/download-trajectory";
import { displayErrorToast } from "#/utils/custom-toast-handlers";
function getEntryPoint(hasRepository: boolean | null): string {
function getEntryPoint(
hasRepository: boolean | null,
hasReplayJson: boolean | null,
): string {
if (hasRepository) return "github";
if (hasReplayJson) return "replay";
return "direct";
}
@ -44,7 +48,7 @@ export function ChatInterface() {
>("positive");
const [feedbackModalIsOpen, setFeedbackModalIsOpen] = React.useState(false);
const [messageToSend, setMessageToSend] = React.useState<string | null>(null);
const { selectedRepository } = useSelector(
const { selectedRepository, replayJson } = useSelector(
(state: RootState) => state.initialQuery,
);
const params = useParams();
@ -53,8 +57,12 @@ export function ChatInterface() {
const handleSendMessage = async (content: string, files: File[]) => {
if (messages.length === 0) {
posthog.capture("initial_query_submitted", {
entry_point: getEntryPoint(selectedRepository !== null),
entry_point: getEntryPoint(
selectedRepository !== null,
replayJson !== null,
),
query_character_length: content.length,
replay_json_size: replayJson?.length,
});
} else {
posthog.capture("user_message_sent", {

34
frontend/src/components/features/suggestions/replay-suggestion-box.tsx Normal file
View File

@ -0,0 +1,34 @@
import { useTranslation } from "react-i18next";
import { I18nKey } from "#/i18n/declaration";
import { SuggestionBox } from "./suggestion-box";
interface ReplaySuggestionBoxProps {
onChange: (event: React.ChangeEvent<HTMLInputElement>) => void;
}
export function ReplaySuggestionBox({ onChange }: ReplaySuggestionBoxProps) {
const { t } = useTranslation();
return (
<SuggestionBox
title={t(I18nKey.LANDING$REPLAY)}
content={
<label
htmlFor="import-trajectory"
className="w-full flex justify-center"
>
<span className="border-2 border-dashed border-neutral-600 rounded px-2 py-1 cursor-pointer">
{t(I18nKey.LANDING$UPLOAD_TRAJECTORY)}
</span>
<input
hidden
type="file"
accept="application/json"
id="import-trajectory"
multiple={false}
onChange={onChange}
/>
</label>
}
/>
);
}

13
frontend/src/hooks/mutation/use-create-conversation.ts
View File

@ -11,18 +11,28 @@ export const useCreateConversation = () => {
const dispatch = useDispatch();
const queryClient = useQueryClient();
const { selectedRepository, files } = useSelector(
const { selectedRepository, files, replayJson } = useSelector(
(state: RootState) => state.initialQuery,
);
return useMutation({
mutationFn: async (variables: { q?: string }) => {
if (
!variables.q?.trim() &&
!selectedRepository &&
files.length === 0 &&
!replayJson
) {
throw new Error("No query provided");
}
if (variables.q) dispatch(setInitialPrompt(variables.q));
return OpenHands.createConversation(
selectedRepository || undefined,
variables.q,
files,
replayJson || undefined,
);
},
onSuccess: async ({ conversation_id: conversationId }, { q }) => {
@ -31,6 +41,7 @@ export const useCreateConversation = () => {
query_character_length: q?.length,
has_repository: !!selectedRepository,
has_files: files.length > 0,
has_replay_json: !!replayJson,
});
await queryClient.invalidateQueries({
queryKey: ["user", "conversations"],

4
frontend/src/i18n/declaration.ts
View File

@ -172,8 +172,8 @@ export enum I18nKey {
BUTTON$STOP = "BUTTON$STOP",
LANDING$ATTACH_IMAGES = "LANDING$ATTACH_IMAGES",
LANDING$OPEN_REPO = "LANDING$OPEN_REPO",
LANDING$IMPORT_PROJECT = "LANDING$IMPORT_PROJECT",
LANDING$UPLOAD_ZIP = "LANDING$UPLOAD_ZIP",
LANDING$REPLAY = "LANDING$REPLAY",
LANDING$UPLOAD_TRAJECTORY = "LANDING$UPLOAD_TRAJECTORY",
LANDING$RECENT_CONVERSATION = "LANDING$RECENT_CONVERSATION",
LANDING$OR = "LANDING$OR",
SUGGESTIONS$TEST_COVERAGE = "SUGGESTIONS$TEST_COVERAGE",

55
frontend/src/i18n/translation.json
View File

@ -2558,35 +2558,34 @@
"no": "Åpne et repo",
"tr": "Depo aç"
},
"LANDING$IMPORT_PROJECT": {
"en": "+ Import Project",
"ja": "+ プロジェクトをインポート",
"zh-CN": "+ 导入项目",
"zh-TW": "+ 匯入專案",
"ko-KR": "+ 프로젝트 가져오기",
"fr": "+ Importer un projet",
"es": "+ Importar proyecto",
"de": "+ Projekt importieren",
"it": "+ Importa progetto",
"pt": "+ Importar projeto",
"ar": "+ استيراد مشروع",
"no": "+ Importer prosjekt",
"tr": "Proje içe aktar"
"LANDING$REPLAY": {
"en": "+ Replay Trajectory",
"ja": "+ Replay Trajectory",
"zh-CN": "+ Replay Trajectory",
"zh-TW": "+ Replay Trajectory",
"ko-KR": "+ Replay Trajectory",
"fr": "+ Replay Trajectory",
"es": "+ Replay Trajectory",
"de": "+ Replay Trajectory",
"it": "+ Replay Trajectory",
"pt": "+ Replay Trajectory",
"ar": "+ Replay Trajectory",
"no": "+ Replay Trajectory",
"tr": "+ Replay Trajectory"
},
"LANDING$UPLOAD_ZIP": {
"en": "Upload a .zip",
"ja": ".zipファイルをアップロード",
"zh-CN": "上传.zip文件",
"zh-TW": "上傳 .zip 檔案",
"ko-KR": ".zip 파일 업로드",
"fr": "Télécharger un .zip",
"es": "Subir un .zip",
"de": "Eine .zip-Datei hochladen",
"it": "Carica un .zip",
"pt": "Fazer upload de um .zip",
"ar": "تحميل ملف .zip",
"no": "Last opp en .zip",
"tr": "ZIP yükle"
"LANDING$UPLOAD_TRAJECTORY": {
"en": "Upload a .json",
"zh-CN": "Upload a .json",
"zh-TW": "Upload a .json",
"ko-KR": "Upload a .json",
"fr": "Upload a .json",
"es": "Upload a .json",
"de": "Upload a .json",
"it": "Upload a .json",
"pt": "Upload a .json",
"ar": "Upload a .json",
"no": "Upload a .json",
"tr": "Upload a .json"
},
"LANDING$RECENT_CONVERSATION": {
"en": "jump back to your most recent conversation",

21
frontend/src/routes/_oh._index/route.tsx
View File

@ -1,13 +1,20 @@
import React from "react";
import { useDispatch } from "react-redux";
import posthog from "posthog-js";
import { setReplayJson } from "#/state/initial-query-slice";
import { useGitHubUser } from "#/hooks/query/use-github-user";
import { useGitHubAuthUrl } from "#/hooks/use-github-auth-url";
import { useConfig } from "#/hooks/query/use-config";
import { ReplaySuggestionBox } from "../../components/features/suggestions/replay-suggestion-box";
import { GitHubRepositoriesSuggestionBox } from "#/components/features/github/github-repositories-suggestion-box";
import { CodeNotInGitHubLink } from "#/components/features/github/code-not-in-github-link";
import { HeroHeading } from "#/components/shared/hero-heading";
import { TaskForm } from "#/components/shared/task-form";
import { convertFileToText } from "#/utils/convert-file-to-text";
import { ENABLE_TRAJECTORY_REPLAY } from "#/utils/feature-flags";
function Home() {
const dispatch = useDispatch();
const formRef = React.useRef<HTMLFormElement>(null);
const { data: config } = useConfig();
@ -35,6 +42,20 @@ function Home() {
gitHubAuthUrl={gitHubAuthUrl}
user={user || null}
/>
{ENABLE_TRAJECTORY_REPLAY() && (
<ReplaySuggestionBox
onChange={async (event) => {
if (event.target.files) {
const json = event.target.files[0];
dispatch(setReplayJson(await convertFileToText(json)));
posthog.capture("json_file_uploaded");
formRef.current?.requestSubmit();
} else {
// TODO: handle error
}
}}
/>
)}
</div>
<div className="w-full flex justify-start mt-2 ml-2">
<CodeNotInGitHubLink />

6
frontend/src/state/initial-query-slice.ts
View File

@ -4,12 +4,14 @@ type SliceState = {
files: string[]; // base64 encoded images
initialPrompt: string | null;
selectedRepository: string | null;
replayJson: string | null;
};
const initialState: SliceState = {
files: [],
initialPrompt: null,
selectedRepository: null,
replayJson: null,
};
export const selectedFilesSlice = createSlice({
@ -37,6 +39,9 @@ export const selectedFilesSlice = createSlice({
clearSelectedRepository(state) {
state.selectedRepository = null;
},
setReplayJson(state, action: PayloadAction<string | null>) {
state.replayJson = action.payload;
},
},
});
@ -48,5 +53,6 @@ export const {
clearInitialPrompt,
setSelectedRepository,
clearSelectedRepository,
setReplayJson,
} = selectedFilesSlice.actions;
export default selectedFilesSlice.reducer;

10
frontend/src/utils/convert-file-to-text.ts Normal file
View File

@ -0,0 +1,10 @@
export const convertFileToText = async (file: File) => {
const reader = new FileReader();
return new Promise<string>((resolve) => {
reader.onload = () => {
resolve(reader.result as string);
};
reader.readAsText(file);
});
};

5
frontend/src/utils/feature-flags.ts
View File

@ -11,3 +11,8 @@ export function loadFeatureFlag(
return defaultValue;
}
}
export const BILLING_SETTINGS = () => loadFeatureFlag("BILLING_SETTINGS");
export const HIDE_LLM_SETTINGS = () => loadFeatureFlag("HIDE_LLM_SETTINGS");
export const ENABLE_TRAJECTORY_REPLAY = () =>
loadFeatureFlag("TRAJECTORY_REPLAY");

19
openhands/controller/replay.py
View File

@ -3,6 +3,7 @@ from openhands.events.action.action import Action
from openhands.events.action.message import MessageAction
from openhands.events.event import Event, EventSource
from openhands.events.observation.empty import NullObservation
from openhands.events.serialization.event import event_from_dict
class ReplayManager:
@ -76,3 +77,21 @@ class ReplayManager:
assert isinstance(event, Action)
self.replay_index += 1
return event
@staticmethod
def get_replay_events(trajectory) -> list[Event]:
if not isinstance(trajectory, list):
raise ValueError(
f'Expected a list in {trajectory}, got {type(trajectory).__name__}'
)
replay_events = []
for item in trajectory:
event = event_from_dict(item)
if event.source == EventSource.ENVIRONMENT:
# ignore ENVIRONMENT events as they are not issued by
# the user or agent, and should not be replayed
continue
# cannot add an event with _id to event stream
event._id = None # type: ignore[attr-defined]
replay_events.append(event)
return replay_events

18
openhands/core/main.py
View File

@ -6,6 +6,7 @@ from typing import Callable, Protocol
import openhands.agenthub # noqa F401 (we import this to get the agents registered)
from openhands.controller.agent import Agent
from openhands.controller.replay import ReplayManager
from openhands.controller.state.state import State
from openhands.core.config import (
AppConfig,
@ -28,7 +29,6 @@ from openhands.events.action import MessageAction, NullAction
from openhands.events.action.action import Action
from openhands.events.event import Event
from openhands.events.observation import AgentStateChangedObservation
from openhands.events.serialization import event_from_dict
from openhands.io import read_input, read_task
from openhands.memory.memory import Memory
from openhands.runtime.base import Runtime
@ -250,21 +250,7 @@ def load_replay_log(trajectory_path: str) -> tuple[list[Event] | None, Action]:
raise ValueError(f'Trajectory path is a directory, not a file: {path}')
with open(path, 'r', encoding='utf-8') as file:
data = json.load(file)
if not isinstance(data, list):
raise ValueError(
f'Expected a list in {path}, got {type(data).__name__}'
)
events = []
for item in data:
event = event_from_dict(item)
if event.source == EventSource.ENVIRONMENT:
# ignore ENVIRONMENT events as they are not issued by
# the user or agent, and should not be replayed
continue
# cannot add an event with _id to event stream
event._id = None # type: ignore[attr-defined]
events.append(event)
events = ReplayManager.get_replay_events(json.load(file))
assert isinstance(events[0], MessageAction)
return events[1:], events[0]
except json.JSONDecodeError as e:

1
openhands/server/conversation_manager/conversation_manager.py
View File

@ -81,6 +81,7 @@ class ConversationManager(ABC):
settings: Settings,
user_id: str | None,
initial_user_msg: MessageAction | None = None,
replay_json: str | None = None,
github_user_id: str | None = None,
) -> EventStream:
"""Start an event loop if one is not already running"""

5
openhands/server/conversation_manager/standalone_conversation_manager.py
View File

@ -252,6 +252,7 @@ class StandaloneConversationManager(ConversationManager):
settings: Settings,
user_id: str | None,
initial_user_msg: MessageAction | None = None,
replay_json: str | None = None,
github_user_id: str | None = None,
) -> EventStream:
logger.info(f'maybe_start_agent_loop:{sid}', extra={'session_id': sid})
@ -284,7 +285,9 @@ class StandaloneConversationManager(ConversationManager):
user_id=user_id,
)
self._local_agent_loops_by_sid[sid] = session
asyncio.create_task(session.initialize_agent(settings, initial_user_msg))
asyncio.create_task(
session.initialize_agent(settings, initial_user_msg, replay_json)
)
# This does not get added when resuming an existing conversation
try:
session.agent_session.event_stream.subscribe(

5
openhands/server/routes/manage_conversations.py
View File

@ -43,6 +43,7 @@ class InitSessionRequest(BaseModel):
selected_branch: str | None = None
initial_user_msg: str | None = None
image_urls: list[str] | None = None
replay_json: str | None = None
async def _create_new_conversation(
@ -52,6 +53,7 @@ async def _create_new_conversation(
selected_branch: str | None,
initial_user_msg: str | None,
image_urls: list[str] | None,
replay_json: str | None,
attach_convo_id: bool = False,
):
logger.info(
@ -132,6 +134,7 @@ async def _create_new_conversation(
conversation_init_data,
user_id,
initial_user_msg=initial_message_action,
replay_json=replay_json,
)
logger.info(f'Finished initializing conversation {conversation_id}')
@ -151,6 +154,7 @@ async def new_conversation(request: Request, data: InitSessionRequest):
selected_branch = data.selected_branch
initial_user_msg = data.initial_user_msg
image_urls = data.image_urls or []
replay_json = data.replay_json
try:
# Create conversation with initial message
@ -161,6 +165,7 @@ async def new_conversation(request: Request, data: InitSessionRequest):
selected_branch,
initial_user_msg,
image_urls,
replay_json,
)
return JSONResponse(

66
openhands/server/session/agent_session.py
View File

@ -1,4 +1,5 @@
import asyncio
import json
import time
from logging import LoggerAdapter
from types import MappingProxyType
@ -6,13 +7,14 @@ from typing import Callable, cast
from openhands.controller import AgentController
from openhands.controller.agent import Agent
from openhands.controller.replay import ReplayManager
from openhands.controller.state.state import State
from openhands.core.config import AgentConfig, AppConfig, LLMConfig
from openhands.core.exceptions import AgentRuntimeUnavailableError
from openhands.core.logger import OpenHandsLoggerAdapter
from openhands.core.schema.agent import AgentState
from openhands.events.action import ChangeAgentStateAction, MessageAction
from openhands.events.event import EventSource
from openhands.events.event import Event, EventSource
from openhands.events.stream import EventStream
from openhands.integrations.provider import PROVIDER_TOKEN_TYPE, ProviderHandler
from openhands.memory.memory import Memory
@ -85,6 +87,7 @@ class AgentSession:
selected_repository: str | None = None,
selected_branch: str | None = None,
initial_message: MessageAction | None = None,
replay_json: str | None = None,
):
"""Starts the Agent session
Parameters:
@ -120,14 +123,26 @@ class AgentSession:
selected_branch=selected_branch,
)
self.controller = self._create_controller(
agent,
config.security.confirmation_mode,
max_iterations,
max_budget_per_task=max_budget_per_task,
agent_to_llm_config=agent_to_llm_config,
agent_configs=agent_configs,
)
if replay_json:
initial_message = self._run_replay(
initial_message,
replay_json,
agent,
config,
max_iterations,
max_budget_per_task,
agent_to_llm_config,
agent_configs,
)
else:
self.controller = self._create_controller(
agent,
config.security.confirmation_mode,
max_iterations,
max_budget_per_task=max_budget_per_task,
agent_to_llm_config=agent_to_llm_config,
agent_configs=agent_configs,
)
repo_directory = None
if self.runtime and runtime_connected and selected_repository:
@ -192,6 +207,37 @@ class AgentSession:
if self.security_analyzer is not None:
await self.security_analyzer.close()
def _run_replay(
self,
initial_message: MessageAction | None,
replay_json: str,
agent: Agent,
config: AppConfig,
max_iterations: int,
max_budget_per_task: float | None,
agent_to_llm_config: dict[str, LLMConfig] | None,
agent_configs: dict[str, AgentConfig] | None,
) -> MessageAction:
"""
Replays a trajectory from a JSON file. Note that once the replay session
finishes, the controller will continue to run with further user instructions,
so we still need to pass llm configs, budget, etc., even though the replay
itself does not call LLM or cost money.
"""
assert initial_message is None
replay_events = ReplayManager.get_replay_events(json.loads(replay_json))
self.controller = self._create_controller(
agent,
config.security.confirmation_mode,
max_iterations,
max_budget_per_task=max_budget_per_task,
agent_to_llm_config=agent_to_llm_config,
agent_configs=agent_configs,
replay_events=replay_events[1:],
)
assert isinstance(replay_events[0], MessageAction)
return replay_events[0]
def _create_security_analyzer(self, security_analyzer: str | None):
"""Creates a SecurityAnalyzer instance that will be used to analyze the agent actions
@ -298,6 +344,7 @@ class AgentSession:
max_budget_per_task: float | None = None,
agent_to_llm_config: dict[str, LLMConfig] | None = None,
agent_configs: dict[str, AgentConfig] | None = None,
replay_events: list[Event] | None = None,
) -> AgentController:
"""Creates an AgentController instance
@ -343,6 +390,7 @@ class AgentSession:
headless_mode=False,
status_callback=self._status_callback,
initial_state=self._maybe_restore_state(),
replay_events=replay_events,
)
return controller

1
openhands/server/session/conversation_init_data.py
View File

@ -11,6 +11,7 @@ class ConversationInitData(Settings):
git_provider_tokens: PROVIDER_TOKEN_TYPE | None = Field(default=None, frozen=True)
selected_repository: str | None = Field(default=None)
replay_json: str | None = Field(default=None)
selected_branch: str | None = Field(default=None)
model_config = {

6
openhands/server/session/session.py
View File

@ -84,7 +84,10 @@ class Session:
await self.agent_session.close()
async def initialize_agent(
self, settings: Settings, initial_message: MessageAction | None
self,
settings: Settings,
initial_message: MessageAction | None,
replay_json: str | None,
):
self.agent_session.event_stream.add_event(
AgentStateChangedObservation('', AgentState.LOADING),
@ -154,6 +157,7 @@ class Session:
selected_repository=selected_repository,
selected_branch=selected_branch,
initial_message=initial_message,
replay_json=replay_json,
)
except Exception as e:
self.logger.exception(f'Error creating agent_session: {e}')