diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts b/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts index fe6bf842c3..702c308733 100644 --- a/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts +++ b/frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts @@ -4,6 +4,7 @@ import i18n from "#/i18n"; import { SecurityRisk } from "#/types/v1/core/base/common"; import { ExecuteBashAction, + TerminalAction, FileEditorAction, StrReplaceEditorAction, MCPToolAction, @@ -58,7 +59,7 @@ const getFileEditorActionContent = ( // Command Actions const getExecuteBashActionContent = ( - event: ActionEvent, + event: ActionEvent, ): string => { let content = `Command:\n\`${event.action.command}\``; @@ -164,8 +165,9 @@ export const getActionContent = (event: ActionEvent): string => { return getFileEditorActionContent(action); case "ExecuteBashAction": + case "TerminalAction": return getExecuteBashActionContent( - event as ActionEvent, + event as ActionEvent, ); case "MCPToolAction": diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx b/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx index 5c93b11be5..f6c67d9fcc 100644 --- a/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx +++ b/frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx @@ -50,6 +50,7 @@ const getActionEventTitle = (event: OpenHandsEvent): React.ReactNode => { switch (actionType) { case "ExecuteBashAction": + case "TerminalAction": actionKey = "ACTION_MESSAGE$RUN"; actionValues = { command: trimText(event.action.command, 80), @@ -111,6 +112,7 @@ const getObservationEventTitle = (event: OpenHandsEvent): React.ReactNode => { switch (observationType) { case "ExecuteBashObservation": + case "TerminalObservation": observationKey = "OBSERVATION_MESSAGE$RUN"; observationValues = { command: event.observation.command diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts b/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts index 6bfc3734e5..e043142e90 100644 --- a/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts +++ b/frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts @@ -8,6 +8,7 @@ import { ThinkObservation, BrowserObservation, ExecuteBashObservation, + TerminalObservation, FileEditorObservation, StrReplaceEditorObservation, TaskTrackerObservation, @@ -42,8 +43,8 @@ const getFileEditorObservationContent = ( }; // Command Observations -const getExecuteBashObservationContent = ( - event: ObservationEvent, +const getTerminalObservationContent = ( + event: ObservationEvent, ): string => { const { observation } = event; @@ -179,8 +180,9 @@ export const getObservationContent = (event: ObservationEvent): string => { ); case "ExecuteBashObservation": - return getExecuteBashObservationContent( - event as ObservationEvent, + case "TerminalObservation": + return getTerminalObservationContent( + event as ObservationEvent, ); case "BrowserObservation": diff --git a/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts b/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts index e5a52bfe95..790ecb00cf 100644 --- a/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts +++ b/frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts @@ -17,6 +17,15 @@ export const getObservationResult = ( if (exitCode === 0 || metadata.exit_code === 0) return "success"; // Command executed successfully return "error"; // Command failed } + case "TerminalObservation": { + const exitCode = + observation.exit_code ?? observation.metadata.exit_code ?? null; + + if (observation.timeout || exitCode === -1) return "timeout"; + if (exitCode === 0) return "success"; + if (observation.is_error) return "error"; + return "success"; + } case "FileEditorObservation": case "StrReplaceEditorObservation": // Check if there's an error diff --git a/frontend/src/types/v1/core/base/action.ts b/frontend/src/types/v1/core/base/action.ts index ce08d5a1b9..77a76df848 100644 --- a/frontend/src/types/v1/core/base/action.ts +++ b/frontend/src/types/v1/core/base/action.ts @@ -41,6 +41,25 @@ export interface ExecuteBashAction extends ActionBase<"ExecuteBashAction"> { reset: boolean; } +export interface TerminalAction extends ActionBase<"TerminalAction"> { + /** + * The terminal command to execute. + */ + command: string; + /** + * If True, the command is an input to the running process. If False, the command is executed directly. + */ + is_input: boolean; + /** + * Optional max time limit (seconds) for the command. + */ + timeout: number | null; + /** + * If True, reset the terminal session before running the command. + */ + reset: boolean; +} + export interface FileEditorAction extends ActionBase<"FileEditorAction"> { /** * The commands to run. Allowed options are: `view`, `create`, `str_replace`, `insert`, `undo_edit`. @@ -206,6 +225,7 @@ export type Action = | FinishAction | ThinkAction | ExecuteBashAction + | TerminalAction | FileEditorAction | StrReplaceEditorAction | TaskTrackerAction diff --git a/frontend/src/types/v1/core/base/base.ts b/frontend/src/types/v1/core/base/base.ts index 5925e8599d..30c39cec89 100644 --- a/frontend/src/types/v1/core/base/base.ts +++ b/frontend/src/types/v1/core/base/base.ts @@ -3,6 +3,7 @@ type EventType = | "Finish" | "Think" | "ExecuteBash" + | "Terminal" | "FileEditor" | "StrReplaceEditor" | "TaskTracker"; @@ -24,7 +25,8 @@ type ObservationOnlyType = "Browser"; type ActionEventType = `${ActionOnlyType}Action` | `${EventType}Action`; type ObservationEventType = | `${ObservationOnlyType}Observation` - | `${EventType}Observation`; + | `${EventType}Observation` + | "TerminalObservation"; export interface ActionBase { kind: T; diff --git a/frontend/src/types/v1/core/base/observation.ts b/frontend/src/types/v1/core/base/observation.ts index e406e30593..0625c9d336 100644 --- a/frontend/src/types/v1/core/base/observation.ts +++ b/frontend/src/types/v1/core/base/observation.ts @@ -81,6 +81,34 @@ export interface ExecuteBashObservation metadata: CmdOutputMetadata; } +export interface TerminalObservation + extends ObservationBase<"TerminalObservation"> { + /** + * Content returned from the terminal as a list of TextContent/ImageContent objects. + */ + content: Array; + /** + * The bash command that was executed. + */ + command: string | null; + /** + * The exit code of the command if it has finished. + */ + exit_code: number | null; + /** + * Whether the command execution produced an error. + */ + is_error: boolean; + /** + * Whether the command execution timed out. + */ + timeout: boolean; + /** + * Additional metadata captured from the shell after command execution. + */ + metadata: CmdOutputMetadata; +} + export interface FileEditorObservation extends ObservationBase<"FileEditorObservation"> { /** @@ -168,6 +196,7 @@ export type Observation = | ThinkObservation | BrowserObservation | ExecuteBashObservation + | TerminalObservation | FileEditorObservation | StrReplaceEditorObservation | TaskTrackerObservation; diff --git a/frontend/src/types/v1/type-guards.ts b/frontend/src/types/v1/type-guards.ts index b479e4697b..e88946e4bc 100644 --- a/frontend/src/types/v1/type-guards.ts +++ b/frontend/src/types/v1/type-guards.ts @@ -3,7 +3,9 @@ import { ObservationEvent, BaseEvent, ExecuteBashAction, + TerminalAction, ExecuteBashObservation, + TerminalObservation, } from "./core"; import { AgentErrorEvent } from "./core/events/observation-event"; import { MessageEvent } from "./core/events/message-event"; @@ -98,17 +100,20 @@ export const isActionEvent = (event: OpenHandsEvent): event is ActionEvent => */ export const isExecuteBashActionEvent = ( event: OpenHandsEvent, -): event is ActionEvent => - isActionEvent(event) && event.action.kind === "ExecuteBashAction"; +): event is ActionEvent => + isActionEvent(event) && + (event.action.kind === "ExecuteBashAction" || + event.action.kind === "TerminalAction"); /** - * Type guard function to check if an observation event is an ExecuteBashObservation + * Type guard function to check if an observation event contains terminal output */ export const isExecuteBashObservationEvent = ( event: OpenHandsEvent, -): event is ObservationEvent => +): event is ObservationEvent => isObservationEvent(event) && - event.observation.kind === "ExecuteBashObservation"; + (event.observation.kind === "ExecuteBashObservation" || + event.observation.kind === "TerminalObservation"); /** * Type guard function to check if an event is a system prompt event