From 5e43dbadcbcb02a827e903ff8c16590b2627d449 Mon Sep 17 00:00:00 2001 From: Xingyao Wang Date: Sat, 24 May 2025 01:21:22 +0800 Subject: [PATCH] feat(frontend): Display MCP tool name and arguments as JSON in MCPObservation when visualized in frontend (#8644) Co-authored-by: openhands --- frontend/public/mockServiceWorker.js | 2 +- .../get-observation-content.ts | 10 --- .../features/chat/event-message.tsx | 15 ++++ .../features/chat/generic-event-message.tsx | 29 ++++---- .../features/chat/mcp-observation-content.tsx | 73 +++++++++++++++++++ .../system-message-modal.tsx | 24 +----- frontend/src/i18n/declaration.ts | 2 + frontend/src/i18n/translation.json | 32 ++++++++ frontend/src/types/core/guards.ts | 6 ++ frontend/src/types/core/observations.ts | 1 + frontend/src/utils/constants.ts | 19 +++++ openhands/events/observation/mcp.py | 4 +- openhands/mcp/utils.py | 4 +- tests/unit/test_mcp_action_observation.py | 37 ++++++++++ 14 files changed, 211 insertions(+), 47 deletions(-) create mode 100644 frontend/src/components/features/chat/mcp-observation-content.tsx diff --git a/frontend/public/mockServiceWorker.js b/frontend/public/mockServiceWorker.js index 58eb75a35a..0c8e0df6af 100644 --- a/frontend/public/mockServiceWorker.js +++ b/frontend/public/mockServiceWorker.js @@ -8,7 +8,7 @@ * - Please do NOT serve this file on production. */ -const PACKAGE_VERSION = '2.7.6' +const PACKAGE_VERSION = '2.8.4' const INTEGRITY_CHECKSUM = '00729d72e3b82faf54ca8b9621dbb96f' const IS_MOCKED_RESPONSE = Symbol('isMockedResponse') const activeClientIds = new Set() diff --git a/frontend/src/components/features/chat/event-content-helpers/get-observation-content.ts b/frontend/src/components/features/chat/event-content-helpers/get-observation-content.ts index 567d6a6f3e..1481f71b58 100644 --- a/frontend/src/components/features/chat/event-content-helpers/get-observation-content.ts +++ b/frontend/src/components/features/chat/event-content-helpers/get-observation-content.ts @@ -46,14 +46,6 @@ const getBrowseObservationContent = (event: BrowseObservation) => { return contentDetails; }; -const getMcpObservationContent = (event: OpenHandsObservation): string => { - let { content } = event; - if (content.length > MAX_CONTENT_LENGTH) { - content = `${content.slice(0, MAX_CONTENT_LENGTH)}...`; - } - return `**Output:**\n\`\`\`\n${content.trim() || i18n.t("OBSERVATION$MCP_NO_OUTPUT")}\n\`\`\``; -}; - const getRecallObservationContent = (event: RecallObservation): string => { let content = ""; @@ -124,8 +116,6 @@ export const getObservationContent = (event: OpenHandsObservation): string => { return getCommandObservationContent(event); case "browse": return getBrowseObservationContent(event); - case "mcp": - return getMcpObservationContent(event); case "recall": return getRecallObservationContent(event); default: diff --git a/frontend/src/components/features/chat/event-message.tsx b/frontend/src/components/features/chat/event-message.tsx index 1d4a3ac51c..5d50861c97 100644 --- a/frontend/src/components/features/chat/event-message.tsx +++ b/frontend/src/components/features/chat/event-message.tsx @@ -8,11 +8,13 @@ import { isOpenHandsObservation, isFinishAction, isRejectObservation, + isMcpObservation, } from "#/types/core/guards"; import { OpenHandsObservation } from "#/types/core/observations"; import { ImageCarousel } from "../images/image-carousel"; import { ChatMessage } from "./chat-message"; import { ErrorMessage } from "./error-message"; +import { MCPObservationContent } from "./mcp-observation-content"; import { getObservationResult } from "./event-content-helpers/get-observation-result"; import { getEventContent } from "./event-content-helpers/get-event-content"; import { GenericEventMessage } from "./generic-event-message"; @@ -78,6 +80,19 @@ export function EventMessage({ return ; } + if (isMcpObservation(event)) { + return ( +
+ } + success={getObservationResult(event)} + /> + {shouldShowConfirmationButtons && } +
+ ); + } + return (
{isOpenHandsAction(event) && hasThoughtProperty(event.args) && ( diff --git a/frontend/src/components/features/chat/generic-event-message.tsx b/frontend/src/components/features/chat/generic-event-message.tsx index 161e6bcce6..0588a1026b 100644 --- a/frontend/src/components/features/chat/generic-event-message.tsx +++ b/frontend/src/components/features/chat/generic-event-message.tsx @@ -10,7 +10,7 @@ import { ObservationResultStatus } from "./event-content-helpers/get-observation interface GenericEventMessageProps { title: React.ReactNode; - details: string; + details: string | React.ReactNode; success?: ObservationResultStatus; } @@ -44,18 +44,21 @@ export function GenericEventMessage({ {success && }
- {showDetails && ( - - {details} - - )} + {showDetails && + (typeof details === "string" ? ( + + {details} + + ) : ( + details + ))} ); } diff --git a/frontend/src/components/features/chat/mcp-observation-content.tsx b/frontend/src/components/features/chat/mcp-observation-content.tsx new file mode 100644 index 0000000000..0eef3b2132 --- /dev/null +++ b/frontend/src/components/features/chat/mcp-observation-content.tsx @@ -0,0 +1,73 @@ +import React from "react"; +import ReactJsonView from "@microlink/react-json-view"; +import { useTranslation } from "react-i18next"; +import { MCPObservation } from "#/types/core/observations"; +import { JSON_VIEW_THEME } from "#/utils/constants"; + +interface MCPObservationContentProps { + event: MCPObservation; +} + +export function MCPObservationContent({ event }: MCPObservationContentProps) { + const { t } = useTranslation(); + + // Parse the content as JSON if possible + let outputData: unknown; + try { + outputData = JSON.parse(event.content); + } catch (e) { + // If parsing fails, use the raw content + outputData = event.content; + } + + const hasArguments = + event.extras.arguments && Object.keys(event.extras.arguments).length > 0; + + return ( +
+ {/* Arguments section */} + {hasArguments && ( +
+
+

+ {t("MCP_OBSERVATION$ARGUMENTS")} +

+
+
+ +
+
+ )} + + {/* Output section */} +
+
+

+ {t("MCP_OBSERVATION$OUTPUT")} +

+
+
+ {typeof outputData === "object" && outputData !== null ? ( + + ) : ( + 
+              {event.content.trim() || t("OBSERVATION$MCP_NO_OUTPUT")}
+
+ )} +
+
+
+ ); +} diff --git a/frontend/src/components/features/conversation-panel/system-message-modal.tsx b/frontend/src/components/features/conversation-panel/system-message-modal.tsx index 414c1d9080..c71535a69b 100644 --- a/frontend/src/components/features/conversation-panel/system-message-modal.tsx +++ b/frontend/src/components/features/conversation-panel/system-message-modal.tsx @@ -6,26 +6,7 @@ import { BaseModalTitle } from "#/components/shared/modals/confirmation-modals/b import { ModalBackdrop } from "#/components/shared/modals/modal-backdrop"; import { ModalBody } from "#/components/shared/modals/modal-body"; import { cn } from "#/utils/utils"; - -// Custom JSON viewer theme that matches our application theme -const jsonViewTheme = { - base00: "transparent", // background - base01: "#2d2d2d", // lighter background - base02: "#4e4e4e", // selection background - base03: "#6c6c6c", // comments, invisibles - base04: "#969896", // dark foreground - base05: "#d9d9d9", // default foreground - base06: "#e8e8e8", // light foreground - base07: "#ffffff", // light background - base08: "#ff5370", // variables, red - base09: "#f78c6c", // integers, orange - base0A: "#ffcb6b", // booleans, yellow - base0B: "#c3e88d", // strings, green - base0C: "#89ddff", // support, cyan - base0D: "#82aaff", // functions, blue - base0E: "#c792ea", // keywords, purple - base0F: "#ff5370", // deprecated, red -}; +import { JSON_VIEW_THEME } from "#/utils/constants"; interface SystemMessageModalProps { isOpen: boolean; @@ -207,8 +188,9 @@ export function SystemMessageModal({
diff --git a/frontend/src/i18n/declaration.ts b/frontend/src/i18n/declaration.ts index cb3d638c1d..fec19d7480 100644 --- a/frontend/src/i18n/declaration.ts +++ b/frontend/src/i18n/declaration.ts @@ -11,6 +11,8 @@ export enum I18nKey { EVENT$UNKNOWN_EVENT = "EVENT$UNKNOWN_EVENT", OBSERVATION$COMMAND_NO_OUTPUT = "OBSERVATION$COMMAND_NO_OUTPUT", OBSERVATION$MCP_NO_OUTPUT = "OBSERVATION$MCP_NO_OUTPUT", + MCP_OBSERVATION$ARGUMENTS = "MCP_OBSERVATION$ARGUMENTS", + MCP_OBSERVATION$OUTPUT = "MCP_OBSERVATION$OUTPUT", OBSERVATION$ERROR_PREFIX = "OBSERVATION$ERROR_PREFIX", TASK$ADDRESSING_TASK = "TASK$ADDRESSING_TASK", SECRETS$SECRET_VALUE_REQUIRED = "SECRETS$SECRET_VALUE_REQUIRED", diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.json index c3a861f900..66f4bb0f6e 100644 --- a/frontend/src/i18n/translation.json +++ b/frontend/src/i18n/translation.json @@ -175,6 +175,38 @@ "de": "[MCP-Tool wurde ohne Ausgabe ausgeführt]", "uk": "[Інструмент MCP завершив виконання без виводу]" }, + "MCP_OBSERVATION$ARGUMENTS": { + "en": "Arguments", + "ja": "引数", + "zh-CN": "参数", + "zh-TW": "參數", + "ko-KR": "인수", + "no": "Argumenter", + "it": "Argomenti", + "pt": "Argumentos", + "es": "Argumentos", + "ar": "المعاملات", + "fr": "Arguments", + "tr": "Argümanlar", + "de": "Argumente", + "uk": "Аргументи" + }, + "MCP_OBSERVATION$OUTPUT": { + "en": "Output", + "ja": "出力", + "zh-CN": "输出", + "zh-TW": "輸出", + "ko-KR": "출력", + "no": "Utdata", + "it": "Output", + "pt": "Saída", + "es": "Salida", + "ar": "المخرجات", + "fr": "Sortie", + "tr": "Çıktı", + "de": "Ausgabe", + "uk": "Вивід" + }, "OBSERVATION$ERROR_PREFIX": { "en": "error:", "ja": "エラー:", diff --git a/frontend/src/types/core/guards.ts b/frontend/src/types/core/guards.ts index 70dc5c6aa1..d6d6003392 100644 --- a/frontend/src/types/core/guards.ts +++ b/frontend/src/types/core/guards.ts @@ -8,6 +8,7 @@ import { import { CommandObservation, ErrorObservation, + MCPObservation, OpenHandsObservation, } from "./observations"; @@ -57,3 +58,8 @@ export const isRejectObservation = ( event: OpenHandsParsedEvent, ): event is OpenHandsObservation => isOpenHandsObservation(event) && event.observation === "user_rejected"; + +export const isMcpObservation = ( + event: OpenHandsParsedEvent, +): event is MCPObservation => + isOpenHandsObservation(event) && event.observation === "mcp"; diff --git a/frontend/src/types/core/observations.ts b/frontend/src/types/core/observations.ts index 5e16155c62..eb7b0de5e5 100644 --- a/frontend/src/types/core/observations.ts +++ b/frontend/src/types/core/observations.ts @@ -135,6 +135,7 @@ export interface MCPObservation extends OpenHandsObservationEvent<"mcp"> { source: "agent"; extras: { name: string; + arguments: Record; }; } diff --git a/frontend/src/utils/constants.ts b/frontend/src/utils/constants.ts index b4b512172a..718be6a0c8 100644 --- a/frontend/src/utils/constants.ts +++ b/frontend/src/utils/constants.ts @@ -9,3 +9,22 @@ export const ASSET_FILE_TYPES = [ ".webm", ".ogg", ]; + +export const JSON_VIEW_THEME = { + base00: "transparent", // background + base01: "#2d2d2d", // lighter background + base02: "#4e4e4e", // selection background + base03: "#6c6c6c", // comments, invisibles + base04: "#969896", // dark foreground + base05: "#d9d9d9", // default foreground + base06: "#e8e8e8", // light foreground + base07: "#ffffff", // light background + base08: "#ff5370", // variables, red + base09: "#f78c6c", // integers, orange + base0A: "#ffcb6b", // booleans, yellow + base0B: "#c3e88d", // strings, green + base0C: "#89ddff", // support, cyan + base0D: "#82aaff", // functions, blue + base0E: "#c792ea", // keywords, purple + base0F: "#ff5370", // deprecated, red +}; diff --git a/openhands/events/observation/mcp.py b/openhands/events/observation/mcp.py index 532dfa8340..ca4aa5623e 100644 --- a/openhands/events/observation/mcp.py +++ b/openhands/events/observation/mcp.py @@ -1,4 +1,5 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Any from openhands.core.schema import ObservationType from openhands.events.observation.observation import Observation @@ -10,6 +11,7 @@ class MCPObservation(Observation): observation: str = ObservationType.MCP name: str = '' # The name of the MCP tool that was called + arguments: dict[str, Any] = field(default_factory=dict) # The arguments passed to the MCP tool @property def message(self) -> str: diff --git a/openhands/mcp/utils.py b/openhands/mcp/utils.py index f81d7371b8..212ccaef8e 100644 --- a/openhands/mcp/utils.py +++ b/openhands/mcp/utils.py @@ -155,7 +155,9 @@ async def call_tool_mcp(mcp_clients: list[MCPClient], action: MCPAction) -> Obse logger.debug(f'MCP response: {response}') return MCPObservation( - content=json.dumps(response.model_dump(mode='json')), name=action.name + content=json.dumps(response.model_dump(mode='json')), + name=action.name, + arguments=action.arguments, ) diff --git a/tests/unit/test_mcp_action_observation.py b/tests/unit/test_mcp_action_observation.py index 2dd8796a0d..918057ee8d 100644 --- a/tests/unit/test_mcp_action_observation.py +++ b/tests/unit/test_mcp_action_observation.py @@ -106,3 +106,40 @@ def test_mcp_action_with_complex_arguments(): assert 'nested_arg' in message assert 'inner_key' in message assert 'inner_value' in message + + +def test_mcp_observation_with_arguments(): + """Test MCPObservation with arguments.""" + complex_args = { + 'simple_arg': 'value', + 'number_arg': 42, + 'boolean_arg': True, + 'nested_arg': {'inner_key': 'inner_value', 'inner_list': [1, 2, 3]}, + 'list_arg': ['a', 'b', 'c'], + } + + observation = MCPObservation( + content=json.dumps({'result': 'success', 'data': 'test data'}), + name='test_tool', + arguments=complex_args, + ) + + assert observation.content == json.dumps({'result': 'success', 'data': 'test data'}) + assert observation.observation == ObservationType.MCP + assert observation.name == 'test_tool' + assert observation.arguments == complex_args + assert observation.arguments['nested_arg']['inner_key'] == 'inner_value' + assert observation.arguments['list_arg'] == ['a', 'b', 'c'] + + # Test serialization + from openhands.events.serialization import event_to_dict + + serialized = event_to_dict(observation) + + assert serialized['observation'] == ObservationType.MCP + assert serialized['content'] == json.dumps( + {'result': 'success', 'data': 'test data'} + ) + assert serialized['extras']['name'] == 'test_tool' + assert serialized['extras']['arguments'] == complex_args + assert serialized['extras']['arguments']['nested_arg']['inner_key'] == 'inner_value'