(frontend): Implement BrowseInteractiveAction in frontend (#7452)

Co-authored-by: openhands <openhands@all-hands.dev>

frontend/src/i18n/declaration.ts

@@ -281,6 +281,7 @@ export enum I18nKey {
   ACTION_MESSAGE$EDIT = "ACTION_MESSAGE$EDIT",
   ACTION_MESSAGE$WRITE = "ACTION_MESSAGE$WRITE",
   ACTION_MESSAGE$BROWSE = "ACTION_MESSAGE$BROWSE",
+  ACTION_MESSAGE$BROWSE_INTERACTIVE = "ACTION_MESSAGE$BROWSE_INTERACTIVE",
   ACTION_MESSAGE$THINK = "ACTION_MESSAGE$THINK",
   OBSERVATION_MESSAGE$RUN = "OBSERVATION_MESSAGE$RUN",
   OBSERVATION_MESSAGE$RUN_IPYTHON = "OBSERVATION_MESSAGE$RUN_IPYTHON",

frontend/src/i18n/translation.json

@@ -4193,6 +4193,21 @@
         "es": "Navegando en la web",
         "tr": "Web'de geziniyor"
     },
+    "ACTION_MESSAGE$BROWSE_INTERACTIVE": {
+        "en": "Interactive browsing in progress...",
+        "zh-CN": "交互式浏览进行中...",
+        "zh-TW": "互動式瀏覽進行中...",
+        "ko-KR": "인터랙티브 브라우징 진행 중...",
+        "ja": "インタラクティブブラウジング進行中...",
+        "no": "Interaktiv surfing pågår...",
+        "ar": "التصفح التفاعلي قيد التقدم...",
+        "de": "Interaktives Browsen läuft...",
+        "fr": "Navigation interactive en cours...",
+        "it": "Navigazione interattiva in corso...",
+        "pt": "Navegação interativa em andamento...",
+        "es": "Navegación interactiva en progreso...",
+        "tr": "Etkileşimli tarama devam ediyor..."
+    },
     "ACTION_MESSAGE$THINK": {
         "en": "Thinking",
         "zh-CN": "思考",

frontend/src/services/observations.ts

@@ -30,6 +30,7 @@ export function handleObservationMessage(message: ObservationMessage) {
       store.dispatch(appendJupyterOutput(message.content));
       break;
     case ObservationType.BROWSE:
+    case ObservationType.BROWSE_INTERACTIVE:
       if (message.extras?.screenshot) {
         store.dispatch(setScreenshotSrc(message.extras?.screenshot));
       }
@@ -178,6 +179,46 @@ export function handleObservationMessage(message: ObservationMessage) {
           }),
         );
         break;
+      case "browse_interactive":
+        store.dispatch(
+          addAssistantObservation({
+            ...baseObservation,
+            observation: "browse_interactive" as const,
+            extras: {
+              url: String(message.extras.url || ""),
+              screenshot: String(message.extras.screenshot || ""),
+              error: Boolean(message.extras.error),
+              open_page_urls: Array.isArray(message.extras.open_page_urls)
+                ? message.extras.open_page_urls
+                : [],
+              active_page_index: Number(message.extras.active_page_index || 0),
+              dom_object:
+                typeof message.extras.dom_object === "object"
+                  ? (message.extras.dom_object as Record<string, unknown>)
+                  : {},
+              axtree_object:
+                typeof message.extras.axtree_object === "object"
+                  ? (message.extras.axtree_object as Record<string, unknown>)
+                  : {},
+              extra_element_properties:
+                typeof message.extras.extra_element_properties === "object"
+                  ? (message.extras.extra_element_properties as Record<
+                      string,
+                      unknown
+                    >)
+                  : {},
+              last_browser_action: String(
+                message.extras.last_browser_action || "",
+              ),
+              last_browser_action_error:
+                message.extras.last_browser_action_error,
+              focused_element_bid: String(
+                message.extras.focused_element_bid || "",
+              ),
+            },
+          }),
+        );
+        break;
       case "error":
         store.dispatch(
           addAssistantObservation({

frontend/src/state/chat-slice.ts

@@ -20,6 +20,7 @@ const HANDLED_ACTIONS: OpenHandsEventType[] = [
   "write",
   "read",
   "browse",
+  "browse_interactive",
   "edit",
 ];
 
@@ -108,6 +109,9 @@ export const chatSlice = createSlice({
         text = `${action.payload.args.path}\n${content}`;
       } else if (actionID === "browse") {
         text = `Browsing ${action.payload.args.url}`;
+      } else if (actionID === "browse_interactive") {
+        // Include the browser_actions in the content
+        text = `**Action:**\n\n\`\`\`python\n${action.payload.args.browser_actions}\n\`\`\``;
       }
       if (actionID === "run" || actionID === "run_ipython") {
         if (
@@ -127,6 +131,7 @@ export const chatSlice = createSlice({
         imageUrls: [],
         timestamp: new Date().toISOString(),
       };
+
       state.messages.push(message);
     },
 
@@ -191,11 +196,11 @@ export const chatSlice = createSlice({
       } else if (observationID === "browse") {
         let content = `**URL:** ${observation.payload.extras.url}\n`;
         if (observation.payload.extras.error) {
-          content += `**Error:**\n${observation.payload.extras.error}\n`;
+          content += `\n\n**Error:**\n${observation.payload.extras.error}\n`;
         }
-        content += `**Output:**\n${observation.payload.content}`;
+        content += `\n\n**Output:**\n${observation.payload.content}`;
         if (content.length > MAX_CONTENT_LENGTH) {
-          content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`;
+          content = `${content.slice(0, MAX_CONTENT_LENGTH)}...(truncated)`;
         }
         causeMessage.content = content;
       }

frontend/src/types/core/observations.ts

@@ -51,6 +51,24 @@ export interface BrowseObservation extends OpenHandsObservationEvent<"browse"> {
   };
 }
 
+export interface BrowseInteractiveObservation
+  extends OpenHandsObservationEvent<"browse_interactive"> {
+  source: "agent";
+  extras: {
+    url: string;
+    screenshot: string;
+    error: boolean;
+    open_page_urls: string[];
+    active_page_index: number;
+    dom_object: Record<string, unknown>;
+    axtree_object: Record<string, unknown>;
+    extra_element_properties: Record<string, unknown>;
+    last_browser_action: string;
+    last_browser_action_error: unknown;
+    focused_element_bid: string;
+  };
+}
+
 export interface WriteObservation extends OpenHandsObservationEvent<"write"> {
   source: "agent";
   extras: {
@@ -98,6 +116,7 @@ export type OpenHandsObservation =
   | IPythonObservation
   | DelegateObservation
   | BrowseObservation
+  | BrowseInteractiveObservation
   | WriteObservation
   | ReadObservation
   | EditObservation

frontend/src/types/observation-type.tsx

@@ -8,6 +8,9 @@ enum ObservationType {
   // The HTML contents of a URL
   BROWSE = "browse",
 
+  // Interactive browsing
+  BROWSE_INTERACTIVE = "browse_interactive",
+
   // The output of a command
   RUN = "run",