refactor(frontend): support TerminalObservation event (#11819)

frontend/src/components/v1/chat/event-content-helpers/get-action-content.ts

@@ -4,6 +4,7 @@ import i18n from "#/i18n";
 import { SecurityRisk } from "#/types/v1/core/base/common";
 import {
   ExecuteBashAction,
+  TerminalAction,
   FileEditorAction,
   StrReplaceEditorAction,
   MCPToolAction,
@@ -58,7 +59,7 @@ const getFileEditorActionContent = (
 
 // Command Actions
 const getExecuteBashActionContent = (
-  event: ActionEvent<ExecuteBashAction>,
+  event: ActionEvent<ExecuteBashAction | TerminalAction>,
 ): string => {
   let content = `Command:\n\`${event.action.command}\``;
 
@@ -164,8 +165,9 @@ export const getActionContent = (event: ActionEvent): string => {
       return getFileEditorActionContent(action);
 
     case "ExecuteBashAction":
+    case "TerminalAction":
       return getExecuteBashActionContent(
-        event as ActionEvent<ExecuteBashAction>,
+        event as ActionEvent<ExecuteBashAction | TerminalAction>,
       );
 
     case "MCPToolAction":

frontend/src/components/v1/chat/event-content-helpers/get-event-content.tsx

@@ -50,6 +50,7 @@ const getActionEventTitle = (event: OpenHandsEvent): React.ReactNode => {
 
   switch (actionType) {
     case "ExecuteBashAction":
+    case "TerminalAction":
       actionKey = "ACTION_MESSAGE$RUN";
       actionValues = {
         command: trimText(event.action.command, 80),
@@ -111,6 +112,7 @@ const getObservationEventTitle = (event: OpenHandsEvent): React.ReactNode => {
 
   switch (observationType) {
     case "ExecuteBashObservation":
+    case "TerminalObservation":
       observationKey = "OBSERVATION_MESSAGE$RUN";
       observationValues = {
         command: event.observation.command

frontend/src/components/v1/chat/event-content-helpers/get-observation-content.ts

@@ -8,6 +8,7 @@ import {
   ThinkObservation,
   BrowserObservation,
   ExecuteBashObservation,
+  TerminalObservation,
   FileEditorObservation,
   StrReplaceEditorObservation,
   TaskTrackerObservation,
@@ -42,8 +43,8 @@ const getFileEditorObservationContent = (
 };
 
 // Command Observations
-const getExecuteBashObservationContent = (
-  event: ObservationEvent<ExecuteBashObservation>,
+const getTerminalObservationContent = (
+  event: ObservationEvent<ExecuteBashObservation | TerminalObservation>,
 ): string => {
   const { observation } = event;
 
@@ -179,8 +180,9 @@ export const getObservationContent = (event: ObservationEvent): string => {
       );
 
     case "ExecuteBashObservation":
-      return getExecuteBashObservationContent(
-        event as ObservationEvent<ExecuteBashObservation>,
+    case "TerminalObservation":
+      return getTerminalObservationContent(
+        event as ObservationEvent<ExecuteBashObservation | TerminalObservation>,
       );
 
     case "BrowserObservation":

frontend/src/components/v1/chat/event-content-helpers/get-observation-result.ts

@@ -17,6 +17,15 @@ export const getObservationResult = (
       if (exitCode === 0 || metadata.exit_code === 0) return "success"; // Command executed successfully
       return "error"; // Command failed
     }
+    case "TerminalObservation": {
+      const exitCode =
+        observation.exit_code ?? observation.metadata.exit_code ?? null;
+
+      if (observation.timeout || exitCode === -1) return "timeout";
+      if (exitCode === 0) return "success";
+      if (observation.is_error) return "error";
+      return "success";
+    }
     case "FileEditorObservation":
     case "StrReplaceEditorObservation":
       // Check if there's an error

frontend/src/types/v1/core/base/action.ts

@@ -41,6 +41,25 @@ export interface ExecuteBashAction extends ActionBase<"ExecuteBashAction"> {
   reset: boolean;
 }
 
+export interface TerminalAction extends ActionBase<"TerminalAction"> {
+  /**
+   * The terminal command to execute.
+   */
+  command: string;
+  /**
+   * If True, the command is an input to the running process. If False, the command is executed directly.
+   */
+  is_input: boolean;
+  /**
+   * Optional max time limit (seconds) for the command.
+   */
+  timeout: number | null;
+  /**
+   * If True, reset the terminal session before running the command.
+   */
+  reset: boolean;
+}
+
 export interface FileEditorAction extends ActionBase<"FileEditorAction"> {
   /**
    * The commands to run. Allowed options are: `view`, `create`, `str_replace`, `insert`, `undo_edit`.
@@ -206,6 +225,7 @@ export type Action =
   | FinishAction
   | ThinkAction
   | ExecuteBashAction
+  | TerminalAction
   | FileEditorAction
   | StrReplaceEditorAction
   | TaskTrackerAction

frontend/src/types/v1/core/base/base.ts

@@ -3,6 +3,7 @@ type EventType =
   | "Finish"
   | "Think"
   | "ExecuteBash"
+  | "Terminal"
   | "FileEditor"
   | "StrReplaceEditor"
   | "TaskTracker";
@@ -24,7 +25,8 @@ type ObservationOnlyType = "Browser";
 type ActionEventType = `${ActionOnlyType}Action` | `${EventType}Action`;
 type ObservationEventType =
   | `${ObservationOnlyType}Observation`
-  | `${EventType}Observation`;
+  | `${EventType}Observation`
+  | "TerminalObservation";
 
 export interface ActionBase<T extends ActionEventType = ActionEventType> {
   kind: T;

frontend/src/types/v1/core/base/observation.ts

@@ -81,6 +81,34 @@ export interface ExecuteBashObservation
   metadata: CmdOutputMetadata;
 }
 
+export interface TerminalObservation
+  extends ObservationBase<"TerminalObservation"> {
+  /**
+   * Content returned from the terminal as a list of TextContent/ImageContent objects.
+   */
+  content: Array<TextContent | ImageContent>;
+  /**
+   * The bash command that was executed.
+   */
+  command: string | null;
+  /**
+   * The exit code of the command if it has finished.
+   */
+  exit_code: number | null;
+  /**
+   * Whether the command execution produced an error.
+   */
+  is_error: boolean;
+  /**
+   * Whether the command execution timed out.
+   */
+  timeout: boolean;
+  /**
+   * Additional metadata captured from the shell after command execution.
+   */
+  metadata: CmdOutputMetadata;
+}
+
 export interface FileEditorObservation
   extends ObservationBase<"FileEditorObservation"> {
   /**
@@ -168,6 +196,7 @@ export type Observation =
   | ThinkObservation
   | BrowserObservation
   | ExecuteBashObservation
+  | TerminalObservation
   | FileEditorObservation
   | StrReplaceEditorObservation
   | TaskTrackerObservation;

frontend/src/types/v1/type-guards.ts

@@ -3,7 +3,9 @@ import {
   ObservationEvent,
   BaseEvent,
   ExecuteBashAction,
+  TerminalAction,
   ExecuteBashObservation,
+  TerminalObservation,
 } from "./core";
 import { AgentErrorEvent } from "./core/events/observation-event";
 import { MessageEvent } from "./core/events/message-event";
@@ -98,17 +100,20 @@ export const isActionEvent = (event: OpenHandsEvent): event is ActionEvent =>
  */
 export const isExecuteBashActionEvent = (
   event: OpenHandsEvent,
-): event is ActionEvent<ExecuteBashAction> =>
-  isActionEvent(event) && event.action.kind === "ExecuteBashAction";
+): event is ActionEvent<ExecuteBashAction | TerminalAction> =>
+  isActionEvent(event) &&
+  (event.action.kind === "ExecuteBashAction" ||
+    event.action.kind === "TerminalAction");
 
 /**
- * Type guard function to check if an observation event is an ExecuteBashObservation
+ * Type guard function to check if an observation event contains terminal output
  */
 export const isExecuteBashObservationEvent = (
   event: OpenHandsEvent,
-): event is ObservationEvent<ExecuteBashObservation> =>
+): event is ObservationEvent<ExecuteBashObservation | TerminalObservation> =>
   isObservationEvent(event) &&
-  event.observation.kind === "ExecuteBashObservation";
+  (event.observation.kind === "ExecuteBashObservation" ||
+    event.observation.kind === "TerminalObservation");
 
 /**
  * Type guard function to check if an event is a system prompt event