From 2b7ceea667b490af73bee48e3755557e30683976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=90=BE=20smolpaws?= Date: Wed, 15 Jul 2026 11:40:57 +0200 Subject: [PATCH] refactor: define canvas UI as an SDK client tool (#1797) * refactor: define canvas UI as an SDK client tool Send a JSON-defined canvas_ui_client tool on new, profile-based, and resumed conversation requests while retaining the legacy Python registration for persisted conversations. Normalize the new SDK event kinds to the existing Canvas UI rendering. Co-authored-by: smolpaws Co-authored-by: openhands * fix: omit canvas client tool from ACP launches * refactor: rename canvas client tool Use the semantic canvas_ui_control name and contain the SDK-generated action discriminator behind exported constants. Co-authored-by: Engel Nyst --------- Co-authored-by: Engel Nyst Co-authored-by: openhands Co-authored-by: Debug Agent <157206163+simonrosenberg@users.noreply.github.com> --- __tests__/api/agent-server-adapter.test.ts | 106 +++++++++++++----- .../get-event-content.test.tsx | 68 ++++++++++- .../mutation/use-create-conversation.test.tsx | 5 +- __tests__/services/canvas-ui.test.ts | 23 +++- __tests__/tools/canvas-ui-tool.test.ts | 40 +++++-- docker/Dockerfile | 5 +- docker/entrypoint.sh | 5 +- docs/ACP_AGENTS.md | 7 +- examples/acp-docker/docker-compose.yml | 11 +- scripts/dev-safe.mjs | 11 +- src/api/agent-server-adapter.test.ts | 5 + src/api/agent-server-adapter.ts | 61 +++++----- src/api/canvas-ui-client-tool.ts | 91 +++++++++++++++ .../agent-server-conversation-service.api.ts | 4 +- .../get-event-content.tsx | 31 +++++ .../get-observation-content.ts | 8 ++ src/constants/canvas-ui.ts | 10 ++ .../conversation-websocket-context.tsx | 7 +- src/hooks/mutation/use-create-conversation.ts | 31 ++--- src/types/agent-server/core/base/action.ts | 11 +- src/types/agent-server/core/base/base.ts | 15 ++- .../agent-server/core/base/observation.ts | 4 +- src/types/agent-server/type-guards.ts | 17 ++- tests/e2e/live-acp/README.md | 6 +- tests/e2e/live-acp/acp-docker-e2e.mts | 10 +- .../live/utils/agent-server-conversation.ts | 6 +- tools/canvas_ui_tool.py | 22 ++-- 27 files changed, 466 insertions(+), 154 deletions(-) create mode 100644 src/api/canvas-ui-client-tool.ts create mode 100644 src/constants/canvas-ui.ts diff --git a/__tests__/api/agent-server-adapter.test.ts b/__tests__/api/agent-server-adapter.test.ts index 17f699d4f1..fa3d767dc6 100644 --- a/__tests__/api/agent-server-adapter.test.ts +++ b/__tests__/api/agent-server-adapter.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CANVAS_UI_CLIENT_TOOL_NAME } from "#/constants/canvas-ui"; import { ACP_SERVER_TAG_KEY, @@ -117,7 +118,6 @@ describe("buildStartConversationRequest", () => { { name: "terminal", params: {} }, { name: "file_editor", params: {} }, { name: "task_tracker", params: {} }, - { name: "canvas_ui", params: {} }, { name: "browser_tool_set", params: {} }, { name: "task_tool_set", params: {} }, ]); @@ -412,11 +412,7 @@ describe("buildStartConversationRequest", () => { }) as Record; expect(payload.hook_config).toEqual({ on_start: [] }); - // Canvas-UI tool is auto-injected; user-supplied entries are merged in - // alongside it. The dedicated canvas_ui describe block below pins the - // exact merge semantics. expect(payload.tool_module_qualnames).toEqual({ - canvas_ui: "canvas_ui_tool", demo_tool: "pkg.tools.demo", }); expect(payload.agent_definitions).toEqual([ @@ -595,40 +591,88 @@ describe("buildStartConversationRequest", () => { }); }); - describe("canvas_ui tool injection", () => { - it("registers canvas_ui_tool in tool_module_qualnames when the backend advertises canvas_ui", () => { + describe("canvas_ui client tool injection", () => { + it("sends canvas_ui as a client-defined JSON tool", () => { const payload = buildStartConversationRequest({ settings: DEFAULT_SETTINGS, - }) as { tool_module_qualnames: Record }; - - expect(payload.tool_module_qualnames).toMatchObject({ - canvas_ui: "canvas_ui_tool", }); - }); - - it("omits canvas_ui and its module qualname when the backend does not advertise canvas_ui", () => { - mockIsAgentServerToolAvailable.mockImplementation( - (toolName: string) => toolName !== "canvas_ui", - ); - - const payload = buildStartConversationRequest({ - settings: DEFAULT_SETTINGS, - }) as { - agent_settings: { tools: Array<{ name: string }> }; - tool_module_qualnames?: Record; - }; + expect(payload.client_tools).toHaveLength(1); + expect(payload.client_tools[0]).toMatchObject({ + name: CANVAS_UI_CLIENT_TOOL_NAME, + parameters: { + type: "object", + properties: { + command: { + enum: ["navigate_to_file", "open_tab", "show_preview"], + }, + }, + required: ["command"], + }, + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }); expect( - payload.agent_settings.tools.map((tool) => tool.name), + payload.agent_settings?.tools?.map((tool) => tool.name) ?? [], ).not.toContain("canvas_ui"); expect(payload.tool_module_qualnames).toBeUndefined(); }); - it("drops a user-supplied canvas_ui module qualname when the backend does not advertise canvas_ui", () => { - mockIsAgentServerToolAvailable.mockImplementation( - (toolName: string) => toolName !== "canvas_ui", - ); + it("omits the client tool for an inline ACP agent", () => { + const payload = buildStartConversationRequest({ + settings: { + ...DEFAULT_SETTINGS, + agent_settings: { + ...DEFAULT_SETTINGS.agent_settings, + agent_kind: "acp", + acp_server: "custom", + acp_command: ["custom-acp"], + }, + }, + }); + expect(payload.client_tools).toEqual([]); + }); + + it("omits the client tool for an ACP profile when global settings are stale", () => { + const payload = buildStartConversationRequest({ + settings: DEFAULT_SETTINGS, + agentProfileId: "profile-acp", + agentProfileKind: "acp", + }); + + expect(payload.client_tools).toEqual([]); + }); + + it("sends the client tool for an OpenHands profile", () => { + const payload = buildStartConversationRequest({ + settings: DEFAULT_SETTINGS, + agentProfileId: "profile-openhands", + agentProfileKind: "openhands", + }); + + expect(payload.client_tools.map((tool) => tool.name)).toEqual([ + CANVAS_UI_CLIENT_TOOL_NAME, + ]); + }); + + it("sends the client tool when resuming a conversation", () => { + const payload = buildStartConversationRequest({ + settings: DEFAULT_SETTINGS, + conversationId: "legacy-conversation-id", + }) as { conversation_id: string; client_tools: Array<{ name: string }> }; + + expect(payload.conversation_id).toBe("legacy-conversation-id"); + expect(payload.client_tools.map((tool) => tool.name)).toEqual([ + CANVAS_UI_CLIENT_TOOL_NAME, + ]); + }); + + it("drops conflicting user-supplied Canvas module qualnames", () => { const payload = buildStartConversationRequest({ settings: { ...DEFAULT_SETTINGS, @@ -636,6 +680,7 @@ describe("buildStartConversationRequest", () => { ...DEFAULT_SETTINGS.conversation_settings, tool_module_qualnames: { canvas_ui: "custom_canvas_ui_tool", + [CANVAS_UI_CLIENT_TOOL_NAME]: "custom_canvas_ui_control_tool", my_tool: "my_package.my_tool", }, }, @@ -647,7 +692,7 @@ describe("buildStartConversationRequest", () => { }); }); - it("merges user-supplied tool_module_qualnames alongside canvas_ui_tool without dropping either side", () => { + it("preserves non-Canvas custom tool modules", () => { const payload = buildStartConversationRequest({ settings: { ...DEFAULT_SETTINGS, @@ -659,7 +704,6 @@ describe("buildStartConversationRequest", () => { }) as { tool_module_qualnames: Record }; expect(payload.tool_module_qualnames).toEqual({ - canvas_ui: "canvas_ui_tool", my_tool: "my_package.my_tool", }); }); diff --git a/__tests__/components/conversation-events/get-event-content.test.tsx b/__tests__/components/conversation-events/get-event-content.test.tsx index f2b9c9472b..44eefa10ae 100644 --- a/__tests__/components/conversation-events/get-event-content.test.tsx +++ b/__tests__/components/conversation-events/get-event-content.test.tsx @@ -1,7 +1,15 @@ import { render, screen } from "@testing-library/react"; import { describe, expect, it } from "vitest"; +import { + CANVAS_UI_CLIENT_ACTION_KIND, + CANVAS_UI_CLIENT_TOOL_NAME, +} from "#/constants/canvas-ui"; import { getEventContent } from "#/components/conversation-events/chat"; -import { ActionEvent, ObservationEvent, SecurityRisk } from "#/types/agent-server/core"; +import { + ActionEvent, + ObservationEvent, + SecurityRisk, +} from "#/types/agent-server/core"; const terminalActionEvent: ActionEvent = { id: "action-1", @@ -326,4 +334,62 @@ describe("getEventContent", () => { "UI command 'open_tab' dispatched to the Agent Canvas frontend.", ); }); + + it("renders client-defined Canvas UI events like legacy events", () => { + const canvasUIAction: ActionEvent = { + id: "action-canvas-client", + timestamp: new Date().toISOString(), + source: "agent", + thought: [], + thinking_blocks: [], + action: { + kind: CANVAS_UI_CLIENT_ACTION_KIND, + command: "open_tab", + path: null, + tab: "files", + }, + tool_name: CANVAS_UI_CLIENT_TOOL_NAME, + tool_call_id: "tool-canvas-client", + tool_call: { + id: "tool-canvas-client", + type: "function", + function: { + name: CANVAS_UI_CLIENT_TOOL_NAME, + arguments: '{"command":"open_tab","tab":"files"}', + }, + }, + llm_response_id: "response-canvas-client", + security_risk: SecurityRisk.LOW, + summary: "", + }; + const canvasUIObservation: ObservationEvent = { + id: "obs-canvas-client", + timestamp: new Date().toISOString(), + source: "environment", + tool_name: CANVAS_UI_CLIENT_TOOL_NAME, + tool_call_id: "tool-canvas-client", + action_id: "action-canvas-client", + observation: { + kind: "ClientToolObservation", + content: [{ type: "text", text: "Tool call dispatched to client." }], + is_error: false, + }, + }; + + const actionContent = getEventContent(canvasUIAction); + render({actionContent.title}); + expect(screen.getByText("CANVASUI")).toBeInTheDocument(); + + const { title, details } = getEventContent( + canvasUIObservation, + canvasUIAction, + ); + render({title}); + expect( + screen.getByText("OBSERVATION_MESSAGE$CANVAS_UI"), + ).toBeInTheDocument(); + expect(details).toBe( + "UI command 'open_tab' dispatched to the Agent Canvas frontend.", + ); + }); }); diff --git a/__tests__/hooks/mutation/use-create-conversation.test.tsx b/__tests__/hooks/mutation/use-create-conversation.test.tsx index 47db81bb56..0b4e2cb9a4 100644 --- a/__tests__/hooks/mutation/use-create-conversation.test.tsx +++ b/__tests__/hooks/mutation/use-create-conversation.test.tsx @@ -382,8 +382,8 @@ describe("useCreateConversation", () => { // The active profile IS the well-known default → it's the enriched baseline // (mirrors agent_settings), not a deliberate profile pick, so the launch // stays on the agent_settings path (no profile tail) even though its - // llm_profile_ref resolves. Keeps /canvas_ui/project - // skills, which the profile-resolution path drops. + // llm_profile_ref resolves. Keeps and project skills, + // which the profile-resolution path drops. listAgentProfilesMock.mockResolvedValue({ profiles: [ { @@ -461,6 +461,7 @@ describe("useCreateConversation", () => { const call = createConversationSpy.mock.lastCall; expect(call?.[9]).toBe("profile-acp-default"); + expect(call?.[10]).toBe("acp"); }); it("launches the seeded `default` profile from its resolved id on cloud (no agent_settings fallback exists there) (#1571)", async () => { diff --git a/__tests__/services/canvas-ui.test.ts b/__tests__/services/canvas-ui.test.ts index 71b084fe64..9eb1193fb6 100644 --- a/__tests__/services/canvas-ui.test.ts +++ b/__tests__/services/canvas-ui.test.ts @@ -1,5 +1,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + CANVAS_UI_CLIENT_ACTION_KIND, + CANVAS_UI_CLIENT_TOOL_NAME, + LEGACY_CANVAS_UI_TOOL_NAME, +} from "#/constants/canvas-ui"; import { handleCanvasUIAction } from "#/services/canvas-ui"; import { useConversationStore } from "#/stores/conversation-store"; import { useFilesTabStore } from "#/stores/files-tab-store"; @@ -98,14 +103,24 @@ describe("isCanvasUIActionEvent", () => { timestamp: "2026-05-13T00:00:00Z", source: "agent", action: { kind: "CanvasUIAction" }, - tool_name: "canvas_ui", + tool_name: LEGACY_CANVAS_UI_TOOL_NAME, tool_call_id: "call-1", ...overrides, }; } - it("returns true for an ActionEvent whose tool_name is canvas_ui", () => { - expect(isCanvasUIActionEvent(makeActionEvent() as never)).toBe(true); + it.each([ + ["CanvasUIAction", LEGACY_CANVAS_UI_TOOL_NAME], + [CANVAS_UI_CLIENT_ACTION_KIND, CANVAS_UI_CLIENT_TOOL_NAME], + ])("returns true for a %s ActionEvent from %s", (kind, toolName) => { + expect( + isCanvasUIActionEvent( + makeActionEvent({ + action: { kind, command: "open_tab" }, + tool_name: toolName, + }) as never, + ), + ).toBe(true); }); it("returns false when tool_name belongs to a different tool", () => { @@ -122,7 +137,7 @@ describe("isCanvasUIActionEvent", () => { timestamp: "2026-05-13T00:00:00Z", source: "environment", observation: { kind: "ExecuteBashObservation" }, - tool_name: "canvas_ui", + tool_name: LEGACY_CANVAS_UI_TOOL_NAME, tool_call_id: "call-1", }; diff --git a/__tests__/tools/canvas-ui-tool.test.ts b/__tests__/tools/canvas-ui-tool.test.ts index e4c9e146a5..acae157204 100644 --- a/__tests__/tools/canvas-ui-tool.test.ts +++ b/__tests__/tools/canvas-ui-tool.test.ts @@ -3,18 +3,44 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; -const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."); -const toolSource = readFileSync(resolve(repoRoot, "tools/canvas_ui_tool.py"), "utf8"); +import { + CANVAS_UI_CLIENT_ACTION_KIND, + CANVAS_UI_CLIENT_TOOL, + CANVAS_UI_CLIENT_TOOL_NAME, + LEGACY_CANVAS_UI_TOOL_NAME, +} from "#/api/canvas-ui-client-tool"; -describe("canvas_ui browser guidance", () => { +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const legacyToolSource = readFileSync( + resolve(repoRoot, "tools/canvas_ui_tool.py"), + "utf8", +); + +describe("canvas_ui client tool", () => { it("tells the agent to capture a browser screenshot before opening the browser tab", () => { const captureInstruction = "browser_get_state(include_screenshot=true)"; const openBrowserInstruction = 'command="open_tab", tab="browser"'; - expect(toolSource).toContain(captureInstruction); - expect(toolSource).toContain(openBrowserInstruction); - expect(toolSource.indexOf(captureInstruction)).toBeLessThan( - toolSource.indexOf(openBrowserInstruction), + expect(CANVAS_UI_CLIENT_TOOL.description).toContain(captureInstruction); + expect(CANVAS_UI_CLIENT_TOOL.description).toContain(openBrowserInstruction); + expect( + CANVAS_UI_CLIENT_TOOL.description.indexOf(captureInstruction), + ).toBeLessThan( + CANVAS_UI_CLIENT_TOOL.description.indexOf(openBrowserInstruction), + ); + }); + + it("exports the semantic tool name and generated action kind", () => { + expect(CANVAS_UI_CLIENT_TOOL_NAME).toBe("canvas_ui_control"); + expect(CANVAS_UI_CLIENT_ACTION_KIND).toBe("ClientAction_canvas_ui_control"); + expect(CANVAS_UI_CLIENT_TOOL.name).toBe(CANVAS_UI_CLIENT_TOOL_NAME); + expect(CANVAS_UI_CLIENT_TOOL.name).not.toBe(LEGACY_CANVAS_UI_TOOL_NAME); + }); + + it("retains the Python registration shim for persisted conversations", () => { + expect(legacyToolSource).toContain("Legacy conversation compatibility"); + expect(legacyToolSource).toContain( + 'register_tool("canvas_ui", CanvasUITool)', ); }); }); diff --git a/docker/Dockerfile b/docker/Dockerfile index bcfbed22b2..4535f23d0c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -126,8 +126,9 @@ COPY --from=frontend-build /build/node_modules/totalist /opt/agent-canvas/node_m # emit the agent's block; same builder the dev stack uses). COPY scripts/runtime-services-info.mjs /opt/agent-canvas/runtime-services-info.mjs -# Copy custom tools (e.g. canvas_ui_tool.py) so the agent-server can import -# them via tool_module_qualnames. OH_EXTRA_PYTHON_PATH is set in entrypoint.sh. +# Persisted conversations created before the client_tools migration still +# import canvas_ui_tool by qualname. Keep the compatibility module available; +# new conversations define canvas_ui_control through a JSON client tool. COPY tools/ /opt/agent-canvas/tools/ # Copy generated defaults.env (from config/defaults.json via config-gen stage) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 27f9b22ad6..1e9d303807 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -124,9 +124,8 @@ export AGENT_SERVER_URL="${AGENT_SERVER_URL:-http://127.0.0.1:${AGENT_SERVER_POR # for locally-generated session keys. export AUTOMATION_AGENT_SERVER_URL="${AUTOMATION_AGENT_SERVER_URL:-http://127.0.0.1:${AGENT_SERVER_PORT}}" -# Make custom tools (e.g. canvas_ui_tool.py) importable by the agent-server -# via tool_module_qualnames. Matches what scripts/dev-safe.mjs does with -# OH_EXTRA_PYTHON_PATH: config.canvasToolsDir. +# Keep the legacy canvas_ui_tool module importable when the agent-server restores +# conversations whose persisted metadata still references its module qualname. export OH_EXTRA_PYTHON_PATH="${OH_EXTRA_PYTHON_PATH:-/opt/agent-canvas/tools}" # Track child PIDs so we can clean up on exit. diff --git a/docs/ACP_AGENTS.md b/docs/ACP_AGENTS.md index ab2be745e6..2e61a1ed52 100644 --- a/docs/ACP_AGENTS.md +++ b/docs/ACP_AGENTS.md @@ -144,9 +144,10 @@ point Canvas at it). In short: ```bash # 1. Agent Server in a container (CORS allows localhost, so the browser talks -# to it directly). The image pre-installs the ACP CLI wrappers. The -# canvas_ui tool is mounted so the agent-server can import the module Canvas -# references in every start request. +# to it directly). The image pre-installs the ACP CLI wrappers. New +# canvas_ui_control calls use client_tools; the Python mount keeps +# pre-migration conversations loadable when persisted metadata imports +# canvas_ui_tool. # Minimum image: 1.28.0-python (first compatible ACP provider/model protocol # surface for current Canvas). Override SHA with a newer build. docker run -d --name oh-acp -p 8010:8000 -v acp-data:/workspace \ diff --git a/examples/acp-docker/docker-compose.yml b/examples/acp-docker/docker-compose.yml index 8e14a7d2b6..1a653c1c5f 100644 --- a/examples/acp-docker/docker-compose.yml +++ b/examples/acp-docker/docker-compose.yml @@ -30,10 +30,9 @@ services: # host:container — Canvas points VITE_BACKEND_BASE_URL at http://localhost:8010. - "8010:8000" environment: - # Canvas's start request always references the bundled ``canvas_ui`` tool - # via ``tool_module_qualnames``; the agent-server imports that module from - # OH_EXTRA_PYTHON_PATH at conversation creation. Without the mount below + - # this var, conversation creation fails to import ``canvas_ui_tool``. + # New conversations define canvas_ui_control through client_tools. Keep the + # old Python module importable so conversations persisted before that migration + # can restore CanvasUIAction / CanvasUIObservation events after a restart. - OH_EXTRA_PYTHON_PATH=/canvas-tools # Optional cipher key. ACP conversations work without it (Canvas sends ACP # provider credentials as loopback LookupSecrets resolved from the @@ -66,8 +65,8 @@ services: # Persist conversations AND the credential files the SDK materialises # (Codex auth.json under CODEX_HOME, Gemini ADC/SA JSON) across restarts. - acp-data:/workspace - # Canvas-specific Python tools (the canvas_ui tool) the agent-server loads - # via OH_EXTRA_PYTHON_PATH. Path is relative to this compose file. + # Legacy canvas_ui module used only when restoring pre-client_tools state. + # Path is relative to this compose file. - ../../tools:/canvas-tools:ro restart: unless-stopped diff --git a/scripts/dev-safe.mjs b/scripts/dev-safe.mjs index 71378eb82e..59247d86e8 100644 --- a/scripts/dev-safe.mjs +++ b/scripts/dev-safe.mjs @@ -636,10 +636,9 @@ function buildConfigFromPorts(ports, cwd, env) { env.LOCAL_BACKEND_API_KEY || getOrCreatePersistedApiKeyFile(persistedKeyPath); - // Host directory containing Agent-Canvas-specific Python tools (e.g. the - // canvas_ui tool). Added to OH_EXTRA_PYTHON_PATH below so the agent-server - // can import the modules listed in `tool_module_qualnames`. Lives at - // /tools relative to this script. + // Host directory containing the legacy canvas_ui Python module. Persisted + // conversations created before the client_tools migration still reference + // its module qualname, so the agent-server can import it when resuming them. const canvasToolsDir = fileURLToPath(new URL("../tools", import.meta.url)); return { @@ -719,8 +718,8 @@ export function buildAgentServerEnv(config) { // a follow-up change to the automation preset reads // OH_SESSION_API_KEYS_0 directly (which is already in env). AGENT_SERVER_URL: config.backendBaseUrl, - // Make the host tools/ directory importable so the agent-server can - // resolve modules listed in tool_module_qualnames (e.g. canvas_ui_tool). + // Let the agent-server resolve canvas_ui_tool when old persisted metadata + // requests that compatibility module during startup. OH_EXTRA_PYTHON_PATH: config.canvasToolsDir, }; } diff --git a/src/api/agent-server-adapter.test.ts b/src/api/agent-server-adapter.test.ts index 7248b9058d..05a549b3db 100644 --- a/src/api/agent-server-adapter.test.ts +++ b/src/api/agent-server-adapter.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { CANVAS_UI_CLIENT_TOOL_NAME } from "#/constants/canvas-ui"; import { DEFAULT_SETTINGS } from "#/services/settings"; import type { Settings } from "#/types/settings"; import { buildStartConversationRequest } from "./agent-server-adapter"; @@ -120,10 +121,14 @@ describe("buildStartConversationRequest — agentProfileId path", () => { const payload = buildStartConversationRequest({ settings, agentProfileId: "profile-xyz", + agentProfileKind: "openhands", }); expect(payload.agent_profile_id).toBe("profile-xyz"); expect(payload.agent_settings).toBeUndefined(); + expect(payload.client_tools.map((tool) => tool.name)).toEqual([ + CANVAS_UI_CLIENT_TOOL_NAME, + ]); }); it("suppresses the ACP server tag when launching from a profile", () => { diff --git a/src/api/agent-server-adapter.ts b/src/api/agent-server-adapter.ts index ed07c28a56..38cb1e15d8 100644 --- a/src/api/agent-server-adapter.ts +++ b/src/api/agent-server-adapter.ts @@ -2,7 +2,7 @@ import { ACP_SETTINGS_KEYS } from "@openhands/typescript-client"; import { SKILLS_CATALOG } from "@openhands/extensions/skills"; import { DEFAULT_SETTINGS } from "#/services/settings"; import { ExecutionStatus } from "#/types/agent-server/core"; -import { Settings, SettingsValue } from "#/types/settings"; +import { AgentKind, Settings, SettingsValue } from "#/types/settings"; import { getAcpPreferredDefaultModel, getAcpProvider, @@ -28,6 +28,12 @@ import { OPENAI_SUBSCRIPTION_VENDOR, isSubscriptionLlmConfig, } from "#/constants/llm-subscription"; +import { + CANVAS_UI_CLIENT_TOOL, + CANVAS_UI_CLIENT_TOOL_NAME, + LEGACY_CANVAS_UI_TOOL_NAME, + type ClientToolSpec, +} from "./canvas-ui-client-tool"; export interface DirectConversationInfo { id: string; @@ -88,18 +94,7 @@ export interface DirectConversationInfo { } | null; } -// Module qualname for the Canvas-UI tool. The agent-server imports this via -// tool_module_qualnames; the host directory is exposed via OH_EXTRA_PYTHON_PATH -// (see scripts/dev-safe.mjs). -const CANVAS_UI_TOOL_NAME = "canvas_ui"; -const CANVAS_UI_TOOL_MODULE = "canvas_ui_tool"; - -const DEFAULT_TOOL_NAMES = [ - "terminal", - "file_editor", - "task_tracker", - CANVAS_UI_TOOL_NAME, -]; +const DEFAULT_TOOL_NAMES = ["terminal", "file_editor", "task_tracker"]; const BROWSER_TOOL_SET_NAME = "browser_tool_set"; const TASK_TOOL_SET_NAME = "task_tool_set"; @@ -518,10 +513,6 @@ function isToolRecord( } function shouldIncludeTool(name: string, agentSettings: SettingsRecord) { - if (name === CANVAS_UI_TOOL_NAME) { - return isAgentServerToolAvailable(name); - } - if (name === BROWSER_TOOL_SET_NAME) { return browserToolsEnabled() && isAgentServerToolAvailable(name); } @@ -889,6 +880,7 @@ type StartConversationPayload = Record & { conversation_id?: string; secrets?: Record; tags?: Record; + client_tools: ClientToolSpec[]; tool_module_qualnames?: Record; }; @@ -907,6 +899,7 @@ export interface StartConversationOptions { // When set, the conversation launches from this AgentProfile (resolved // server-side) instead of an inline ``agent_settings`` dump (#3727). agentProfileId?: string; + agentProfileKind?: AgentKind; } export function buildStartConversationRequest( @@ -917,6 +910,11 @@ export function buildStartConversationRequest( : options.settings; const acpMode = isAcpAgent(sourceAgentSettings); + const launchAgentKind = options.agentProfileId + ? options.agentProfileKind + : acpMode + ? "acp" + : "openhands"; const agentSettings = buildConfiguredAgentSettings(sourceAgentSettings); const acpServerTag = acpMode ? getAcpServerTag(sourceAgentSettings) @@ -947,14 +945,15 @@ export function buildStartConversationRequest( // server/SDK's responsibility to restore on the profile path — tracked in // software-agent-sdk#3967 (profile resolution must attach the default // toolset + public skills, else a profile-launched OpenHands agent has only - // Finish/Think). The two genuinely canvas-only enrichments — the - // ``canvas_ui`` tool and the dev ``RUNTIME_SERVICES`` system-message-suffix - // (``buildAgentContext``) — have no server-side representation and are - // intentionally not carried on the profile path. + // Finish/Think). The dev ``RUNTIME_SERVICES`` system-message suffix remains + // agent-settings-only; the Canvas UI tool is a top-level client tool and + // therefore works on both inline-agent and profile launch paths. ...(options.agentProfileId ? { agent_profile_id: options.agentProfileId } : { agent_settings: agentSettings }), workspace: conversationSettings.workspace, + client_tools: + launchAgentKind === "openhands" ? [CANVAS_UI_CLIENT_TOOL] : [], confirmation_policy: getConversationConfirmationPolicy(conversationSettings), max_iterations: @@ -1008,20 +1007,13 @@ export function buildStartConversationRequest( payload.hook_config = conversationSettings.hook_config; } - const toolModuleQualnames: Record = {}; - const canvasUiAvailable = isAgentServerToolAvailable(CANVAS_UI_TOOL_NAME); - if (canvasUiAvailable) { - toolModuleQualnames[CANVAS_UI_TOOL_NAME] = CANVAS_UI_TOOL_MODULE; - } - Object.assign( - toolModuleQualnames, - (conversationSettings.tool_module_qualnames as + const toolModuleQualnames = { + ...((conversationSettings.tool_module_qualnames as | Record - | undefined) ?? {}, - ); - if (!canvasUiAvailable) { - delete toolModuleQualnames[CANVAS_UI_TOOL_NAME]; - } + | undefined) ?? {}), + }; + delete toolModuleQualnames[LEGACY_CANVAS_UI_TOOL_NAME]; + delete toolModuleQualnames[CANVAS_UI_CLIENT_TOOL_NAME]; if (Object.keys(toolModuleQualnames).length > 0) { payload.tool_module_qualnames = toolModuleQualnames; } @@ -1090,6 +1082,7 @@ export async function buildStartConversationRequestWithEncryptedSettings(options workingDir?: string; worktree?: boolean; agentProfileId?: string; + agentProfileKind?: AgentKind; }): Promise> { const { SecretsService } = await import("./secrets-service"); diff --git a/src/api/canvas-ui-client-tool.ts b/src/api/canvas-ui-client-tool.ts new file mode 100644 index 0000000000..168c83eab4 --- /dev/null +++ b/src/api/canvas-ui-client-tool.ts @@ -0,0 +1,91 @@ +import { CANVAS_UI_CLIENT_TOOL_NAME } from "#/constants/canvas-ui"; + +export { + CANVAS_UI_CLIENT_ACTION_KIND, + CANVAS_UI_CLIENT_TOOL_NAME, + LEGACY_CANVAS_UI_TOOL_NAME, +} from "#/constants/canvas-ui"; + +export interface ClientToolSpec { + name: string; + description: string; + parameters: Record; + annotations?: { + title?: string | null; + readOnlyHint: boolean; + destructiveHint: boolean; + idempotentHint: boolean; + openWorldHint: boolean; + }; +} + +const CANVAS_UI_DESCRIPTION = `The user is interacting with you inside Agent Canvas — a web UI with a chat panel on the left and a tabbed right-side panel (files, terminal, browser, vscode, planner, tasklist). This tool lets you drive that right-side panel so the user sees what you just produced. + +They will NOT see the files you wrote, the terminal output, or the browser +unless you call this tool to switch the right-side panel to the relevant +tab. Call this every time you finish work that produces something the user +should look at — don't rely on them noticing on their own. + +When to call (pick the most specific option that matches your last action): + +* You wrote or modified a single file (ANY language, ANY size — including + small scripts like a hello-world bash file) → + command="navigate_to_file", path= + +* You generated an HTML page, image, SVG, PDF, markdown report, or other + previewable artifact → + command="show_preview", path= + +* You finished editing multiple files in one logical step → + command="open_tab", tab="files" + (The Files tab automatically renders a diff view when the workspace has + uncommitted git changes, which covers the "highlight changes" case.) + +* You ran a long-running terminal command, or one whose output the user + should inspect → + command="open_tab", tab="terminal" + +* You browsed to a URL the user should see → + First call browser_get_state(include_screenshot=true) after your final + browser interaction so Agent Canvas has a screenshot to display, then call + command="open_tab", tab="browser" + (browser_navigate alone only updates the URL; without browser_get_state, + the Browser tab will open without a screenshot.) + +Call this BEFORE writing your chat-message summary of the change, so the +artifact is visible while the user reads what you did. One canvas_ui_control +call per logical step is enough — don't repeat it for the same file or tab in +the same turn.`; + +export const CANVAS_UI_CLIENT_TOOL: ClientToolSpec = { + name: CANVAS_UI_CLIENT_TOOL_NAME, + description: CANVAS_UI_DESCRIPTION, + parameters: { + type: "object", + additionalProperties: false, + properties: { + command: { + type: "string", + enum: ["navigate_to_file", "open_tab", "show_preview"], + description: "UI command to dispatch.", + }, + path: { + type: "string", + description: + "Workspace-relative file path. Required for navigate_to_file and show_preview; ignored otherwise.", + }, + tab: { + type: "string", + enum: ["files", "browser", "vscode", "terminal", "planner", "tasklist"], + description: "Tab to open. Required for open_tab; ignored otherwise.", + }, + }, + required: ["command"], + }, + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, +}; diff --git a/src/api/conversation-service/agent-server-conversation-service.api.ts b/src/api/conversation-service/agent-server-conversation-service.api.ts index f330a8bcff..fe7d6c991a 100644 --- a/src/api/conversation-service/agent-server-conversation-service.api.ts +++ b/src/api/conversation-service/agent-server-conversation-service.api.ts @@ -10,7 +10,7 @@ import { VSCodeClient, } from "@openhands/typescript-client/clients"; import { v4 as uuidv4 } from "uuid"; -import { Provider } from "#/types/settings"; +import { AgentKind, Provider } from "#/types/settings"; import type { ConversationRuntimeContext } from "#/api/conversation-file-upload.api"; import { buildHttpBaseUrl } from "#/utils/websocket-url"; import { @@ -376,6 +376,7 @@ class AgentServerConversationService { // cloud app-server (OpenHands #15060): local threads it through the // encrypted-settings builder; cloud sends it as a flat request field. agentProfileId?: string, + agentProfileKind?: AgentKind, ): Promise { if (getActiveBackend().backend.kind === "cloud") { // Cloud path mirrors OpenHands' frontend: build a flat @@ -429,6 +430,7 @@ class AgentServerConversationService { workingDir, worktree: resolvedWorkspaceMode === "new_worktree", agentProfileId, + agentProfileKind, }); const data = await new ConversationClient( diff --git a/src/components/conversation-events/chat/event-content-helpers/get-event-content.tsx b/src/components/conversation-events/chat/event-content-helpers/get-event-content.tsx index abc66215bf..e5f96f2c1d 100644 --- a/src/components/conversation-events/chat/event-content-helpers/get-event-content.tsx +++ b/src/components/conversation-events/chat/event-content-helpers/get-event-content.tsx @@ -1,5 +1,9 @@ import { Trans } from "react-i18next"; import React from "react"; +import { + CANVAS_UI_CLIENT_ACTION_KIND, + CANVAS_UI_CLIENT_TOOL_NAME, +} from "#/constants/canvas-ui"; import { OpenHandsEvent, ObservationEvent, @@ -9,6 +13,7 @@ import { isActionEvent, isObservationEvent, isACPToolCallEvent, + isCanvasUIActionEvent, } from "#/types/agent-server/type-guards"; import { MonoComponent } from "../../../features/chat/mono-component"; import { PathComponent } from "../../../features/chat/path-component"; @@ -236,6 +241,9 @@ const getActionEventTitle = (event: OpenHandsEvent): React.ReactNode => { case "BrowserCloseTabAction": actionKey = "ACTION_MESSAGE$BROWSE"; break; + case "CanvasUIAction": + case CANVAS_UI_CLIENT_ACTION_KIND: + return "CANVASUI"; default: // For unknown actions, use the type name return String(actionType).replace("Action", "").toUpperCase(); @@ -324,6 +332,12 @@ const getObservationEventTitle = ( case "CanvasUIObservation": observationKey = "OBSERVATION_MESSAGE$CANVAS_UI"; break; + case "ClientToolObservation": + if (event.tool_name === CANVAS_UI_CLIENT_TOOL_NAME) { + observationKey = "OBSERVATION_MESSAGE$CANVAS_UI"; + break; + } + return observationType.replace("Observation", "").toUpperCase(); case "SwitchLLMObservation": observationKey = event.observation.is_error ? "MODEL$SWITCH_FAILED" @@ -376,6 +390,22 @@ const getObservationEventTitle = ( return observationType; }; +const getCanvasUIClientObservationContent = ( + event: ObservationEvent, + correspondingAction?: ActionEvent, +): string | null => { + if ( + event.observation.kind !== "ClientToolObservation" || + event.tool_name !== CANVAS_UI_CLIENT_TOOL_NAME || + !correspondingAction || + !isCanvasUIActionEvent(correspondingAction) + ) { + return null; + } + + return `UI command '${correspondingAction.action.command}' dispatched to the Agent Canvas frontend.`; +}; + export const getEventContent = ( event: OpenHandsEvent | SkillReadyEvent, correspondingAction?: ActionEvent, @@ -409,6 +439,7 @@ export const getEventContent = ( ); } else { details = + getCanvasUIClientObservationContent(event, correspondingAction) ?? resolveVisualizerBody(event, correspondingAction) ?? getObservationContent(event); } diff --git a/src/components/conversation-events/chat/event-content-helpers/get-observation-content.ts b/src/components/conversation-events/chat/event-content-helpers/get-observation-content.ts index e209bb6642..45ec656e43 100644 --- a/src/components/conversation-events/chat/event-content-helpers/get-observation-content.ts +++ b/src/components/conversation-events/chat/event-content-helpers/get-observation-content.ts @@ -1,3 +1,4 @@ +import { CANVAS_UI_CLIENT_TOOL_NAME } from "#/constants/canvas-ui"; import { ObservationEvent } from "#/types/agent-server/core"; import { getObservationResult } from "./get-observation-result"; import { getDefaultEventContent, MAX_CONTENT_LENGTH } from "./shared"; @@ -417,6 +418,13 @@ export const getObservationContent = (event: ObservationEvent): string => { event as ObservationEvent, ); + case "ClientToolObservation": + return event.tool_name === CANVAS_UI_CLIENT_TOOL_NAME + ? getCanvasUIObservationContent( + event as ObservationEvent, + ) + : getDefaultEventContent(event); + case "SwitchLLMObservation": return getSwitchLLMObservationContent( event as ObservationEvent, diff --git a/src/constants/canvas-ui.ts b/src/constants/canvas-ui.ts new file mode 100644 index 0000000000..4981cef02a --- /dev/null +++ b/src/constants/canvas-ui.ts @@ -0,0 +1,10 @@ +export const LEGACY_CANVAS_UI_TOOL_NAME = "canvas_ui"; +export const CANVAS_UI_CLIENT_TOOL_NAME = "canvas_ui_control"; + +/** + * Action discriminator generated by the SDK for the client-defined Canvas UI + * tool (`ClientAction_`). Keep the SDK naming convention contained + * here instead of repeating its generated class name throughout the frontend. + */ +export const CANVAS_UI_CLIENT_ACTION_KIND = + `ClientAction_${CANVAS_UI_CLIENT_TOOL_NAME}` as const; diff --git a/src/contexts/conversation-websocket-context.tsx b/src/contexts/conversation-websocket-context.tsx index 44da40624f..14940e9354 100644 --- a/src/contexts/conversation-websocket-context.tsx +++ b/src/contexts/conversation-websocket-context.tsx @@ -637,10 +637,9 @@ export function ConversationWebSocketProvider({ invalidateConversationQueries(queryClient, conversationId); } - // Handle canvas_ui custom-tool ActionEvents - drive the frontend - // (navigate to a file, switch tabs, show a preview). The tool - // executes server-side as a no-op; the actual UI change happens - // here on the client. + // Handle canvas_ui ActionEvents from both the legacy Python tool and + // the client-defined JSON tool. The server acknowledges immediately; + // the actual UI change happens here on the client. if (isCanvasUIActionEvent(event)) { handleCanvasUIAction(event.action, conversationId ?? null); } diff --git a/src/hooks/mutation/use-create-conversation.ts b/src/hooks/mutation/use-create-conversation.ts index 7632e4d37b..5296818d25 100644 --- a/src/hooks/mutation/use-create-conversation.ts +++ b/src/hooks/mutation/use-create-conversation.ts @@ -2,7 +2,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import AgentServerConversationService from "#/api/conversation-service/agent-server-conversation-service.api"; import { PluginSpec } from "#/api/conversation-service/agent-server-conversation-service.types"; import { SuggestedTask } from "#/utils/types"; -import { Provider } from "#/types/settings"; +import { AgentKind, Provider } from "#/types/settings"; import { useTracking } from "#/hooks/use-tracking"; import { useLlmProfiles } from "#/hooks/query/use-llm-profiles"; import { useAgentProfiles } from "#/hooks/query/use-agent-profiles"; @@ -149,17 +149,17 @@ export const useCreateConversation = () => { // deliberate profile pick — it mirrors global agent_settings. Launch it // via agent_settings so the canvas-only enrichments the profile-resolution // path drops survive for the common home-launch: the - // system-message suffix, the canvas_ui tool, and project-skill loading - // (buildAgentContext). Named profiles are deliberate custom configs and - // still use the profile path (accepting that enrichment boundary). + // system-message suffix and project-skill loading (buildAgentContext). + // Named profiles are deliberate custom configs and still use the profile + // path (accepting that enrichment boundary). // Trade-off: per-profile fields set on `default` itself don't apply on // home-launch — custom per-profile config belongs in a named profile. // // Scoped to OpenHands: an ACP `default` must keep the profile path. // Activation is pointer-only, so global agent_settings is stale (often // still OpenHands) when an ACP profile is active — launching it via - // agent_settings would start the wrong agent. ACP also carries no - // /canvas_ui enrichment, so there's nothing to preserve. + // agent_settings would start the wrong agent. ACP carries no + // enrichment, so there's nothing to preserve. // // Scoped to local: cloud never writes agent_settings, so it always // resolves `default` server-side via agent_profile_id (validated below). @@ -198,15 +198,20 @@ export const useCreateConversation = () => { } } - // Only extend the call with the [sandboxId, agentProfileId] tail when - // launching from a profile, so a plain create stays byte-identical to - // the legacy agent_settings path (#3727). sandboxId is unused here. - // TODO(#1587): createConversation has grown to 10 positional params; + // Only extend the call with the profile tail when launching from a + // profile, so a plain create stays byte-identical to the legacy + // agent_settings path (#3727). sandboxId is unused here. + // TODO(#1587): createConversation has grown to 11 positional params; // refactor it to an options object so this position-skipping tail isn't // needed. - const profileArgs: [undefined, string] | [] = effectiveAgentProfileId - ? [undefined, effectiveAgentProfileId] - : []; + const profileArgs: [undefined, string, AgentKind | undefined] | [] = + effectiveAgentProfileId + ? [ + undefined, + effectiveAgentProfileId, + resolvedAgentProfile?.agent_kind, + ] + : []; const conversation = await AgentServerConversationService.createConversation( diff --git a/src/types/agent-server/core/base/action.ts b/src/types/agent-server/core/base/action.ts index 2595c1cb64..a316bf584c 100644 --- a/src/types/agent-server/core/base/action.ts +++ b/src/types/agent-server/core/base/action.ts @@ -1,3 +1,4 @@ +import { CANVAS_UI_CLIENT_ACTION_KIND } from "#/constants/canvas-ui"; import { ActionBase } from "./base"; import { TaskItem } from "./common"; @@ -308,11 +309,13 @@ export interface SwitchLLMAction extends ActionBase<"SwitchLLMAction"> { } /** - * Frontend-injected custom tool. Emitted over the existing WebSocket as a - * regular ActionEvent; intercepted client-side by handleCanvasUIAction. - * The Python definition lives in tools/canvas_ui_tool.py. + * Canvas UI action emitted over the existing WebSocket and intercepted by + * handleCanvasUIAction. Legacy conversations use CanvasUIAction; new + * conversations use the SDK-generated client action kind. */ -export interface CanvasUIAction extends ActionBase<"CanvasUIAction"> { +export interface CanvasUIAction extends ActionBase< + "CanvasUIAction" | typeof CANVAS_UI_CLIENT_ACTION_KIND +> { command: "navigate_to_file" | "open_tab" | "show_preview"; path?: string | null; tab?: string | null; diff --git a/src/types/agent-server/core/base/base.ts b/src/types/agent-server/core/base/base.ts index 2f1700a58e..891e105851 100644 --- a/src/types/agent-server/core/base/base.ts +++ b/src/types/agent-server/core/base/base.ts @@ -1,3 +1,5 @@ +import { CANVAS_UI_CLIENT_ACTION_KIND } from "#/constants/canvas-ui"; + type EventType = | "MCPTool" | "Finish" @@ -22,9 +24,8 @@ type ActionOnlyType = | "BrowserListTabs" | "BrowserSwitchTab" | "BrowserCloseTab" - // Frontend-injected custom tool. Not part of the upstream SDK Action - // union but emitted as a regular ActionEvent over the WebSocket. See - // tools/canvas_ui_tool.py and src/services/canvas-ui.ts. + // Legacy Python-defined Canvas tool kind. The client-tool kind is added + // separately below because the SDK generates its full discriminator. | "CanvasUI"; type ObservationOnlyType = "Browser"; @@ -35,7 +36,8 @@ type ActionEventType = | "GlobAction" | "GrepAction" // The `task` tool delegating work to a spawned subagent. - | "TaskAction"; + | "TaskAction" + | typeof CANVAS_UI_CLIENT_ACTION_KIND; type ObservationEventType = | `${ObservationOnlyType}Observation` | `${EventType}Observation` @@ -44,8 +46,9 @@ type ObservationEventType = | "GrepObservation" // Result of the `task` tool, which delegates work to a spawned subagent. | "TaskObservation" - // Acknowledgement emitted after a `canvas_ui` command is dispatched. - | "CanvasUIObservation"; + // Legacy and client-defined acknowledgements for Canvas UI dispatches. + | "CanvasUIObservation" + | "ClientToolObservation"; export interface ActionBase { kind: T; diff --git a/src/types/agent-server/core/base/observation.ts b/src/types/agent-server/core/base/observation.ts index 6fc6d5127e..9a6fd719aa 100644 --- a/src/types/agent-server/core/base/observation.ts +++ b/src/types/agent-server/core/base/observation.ts @@ -325,7 +325,9 @@ export interface TaskObservation extends ObservationBase<"TaskObservation"> { status: string; } -export interface CanvasUIObservation extends ObservationBase<"CanvasUIObservation"> { +export interface CanvasUIObservation extends ObservationBase< + "CanvasUIObservation" | "ClientToolObservation" +> { /** * Acknowledgement text returned after the canvas UI command is dispatched. */ diff --git a/src/types/agent-server/type-guards.ts b/src/types/agent-server/type-guards.ts index 485ba2358f..542a9da400 100644 --- a/src/types/agent-server/type-guards.ts +++ b/src/types/agent-server/type-guards.ts @@ -1,3 +1,8 @@ +import { + CANVAS_UI_CLIENT_TOOL_NAME, + LEGACY_CANVAS_UI_TOOL_NAME, +} from "#/constants/canvas-ui"; + import { OpenHandsEvent, ObservationEvent, @@ -167,17 +172,17 @@ export const isBrowserNavigateActionEvent = ( isActionEvent(event) && event.action.kind === "BrowserNavigateAction"; /** - * Type guard for the canvas_ui custom tool's ActionEvent. + * Type guard for Canvas UI tool ActionEvents. * - * The tool is injected via tool_module_qualnames (see canvas_ui_tool.py and - * agent-server-adapter.ts). We discriminate on tool_name (which we control - * via register_tool("canvas_ui", ...)). The predicate narrows the event so - * the call site can read `event.action.command` etc. without further casts. + * Discriminating on tool_name supports legacy CanvasUIAction events and the + * SDK-generated action kind without leaking that generated name here. */ export const isCanvasUIActionEvent = ( event: OpenHandsEvent, ): event is ActionEvent => - isActionEvent(event) && event.tool_name === "canvas_ui"; + isActionEvent(event) && + (event.tool_name === LEGACY_CANVAS_UI_TOOL_NAME || + event.tool_name === CANVAS_UI_CLIENT_TOOL_NAME); /** * Type guard function to check if an event is a system prompt event diff --git a/tests/e2e/live-acp/README.md b/tests/e2e/live-acp/README.md index 68016aeadd..f4b6e02b0a 100644 --- a/tests/e2e/live-acp/README.md +++ b/tests/e2e/live-acp/README.md @@ -18,12 +18,12 @@ and needs a running container + real host credentials). ## Run it ```bash -# 1. Agent-server container with the canvas_ui tool mounted (as the dev stack does). -# Minimum 1.25.0-python (software-agent-sdk#3510); override for a newer build. +# 1. Agent-server container. v1.28.0 adds the client_tools API used by Canvas. +# The Python mount keeps pre-migration conversation state loadable. docker run -d --name oh-acp -p 8010:8000 \ -v oh-acp-data:/workspace \ -v "$(pwd)/tools:/canvas-tools:ro" -e OH_EXTRA_PYTHON_PATH=/canvas-tools \ - ghcr.io/openhands/agent-server:1.25.0-python + ghcr.io/openhands/agent-server:1.28.0-python # 2. Run the e2e (all providers, or a subset). npx vite-node -c tests/e2e/live-acp/vite-node.config.mts \ diff --git a/tests/e2e/live-acp/acp-docker-e2e.mts b/tests/e2e/live-acp/acp-docker-e2e.mts index f2858119e0..0ed722313d 100644 --- a/tests/e2e/live-acp/acp-docker-e2e.mts +++ b/tests/e2e/live-acp/acp-docker-e2e.mts @@ -10,17 +10,17 @@ * LookupSecrets Canvas emits resolve back from the store and authenticate the * CLI end-to-end (including the SDK's acp_file_secrets materialisation). * - * Requires an agent-server with software-agent-sdk#3510 (first in v1.25.0): the - * ACP credentials ride as loopback LookupSecrets, and only #3510 resolves them - * off the event loop — an older image deadlocks ("Failed to start ACP server: - * timed out"). + * Requires agent-server v1.28.0: it includes both software-agent-sdk#3510 for + * off-loop LookupSecret resolution and the client_tools API used by + * canvas_ui_control. Older images either deadlock resolving ACP credentials or + * omit the Canvas UI tool. * * Excluded from `npm test` (lives under tests/). Run it by hand against a * running container: * * docker run -d --name oh-acp -p 8010:8000 -v oh-acp-data:/workspace \ * -v "$(pwd)/tools:/canvas-tools:ro" -e OH_EXTRA_PYTHON_PATH=/canvas-tools \ - * ghcr.io/openhands/agent-server:1.25.0-python + * ghcr.io/openhands/agent-server:1.28.0-python * npx vite-node -c tests/e2e/live-acp/vite-node.config.mts \ * tests/e2e/live-acp/acp-docker-e2e.mts -- codex claude gemini * diff --git a/tests/e2e/live/utils/agent-server-conversation.ts b/tests/e2e/live/utils/agent-server-conversation.ts index 9001abe27e..dc8d1fc662 100644 --- a/tests/e2e/live/utils/agent-server-conversation.ts +++ b/tests/e2e/live/utils/agent-server-conversation.ts @@ -5,6 +5,7 @@ import { type Locator, type Page, } from "@playwright/test"; +import { CANVAS_UI_CLIENT_TOOL } from "../../../../src/api/canvas-ui-client-tool"; export const BACKEND_URL = process.env.LIVE_E2E_BACKEND_URL ?? "http://127.0.0.1:18100"; @@ -46,7 +47,6 @@ const DEFAULT_AGENT_TOOLS = [ { name: "terminal", params: {} }, { name: "file_editor", params: {} }, { name: "task_tracker", params: {} }, - { name: "canvas_ui", params: {} }, ]; export const sessionApiKey = firstNonEmpty( process.env.LIVE_E2E_SESSION_API_KEY, @@ -249,9 +249,7 @@ export async function createLiveConversation( verification: buildLiveVerificationSettings(options), tools: DEFAULT_AGENT_TOOLS, }, - tool_module_qualnames: { - canvas_ui: "canvas_ui_tool", - }, + client_tools: [CANVAS_UI_CLIENT_TOOL], }, }); diff --git a/tools/canvas_ui_tool.py b/tools/canvas_ui_tool.py index 851997f981..1fe91b1842 100644 --- a/tools/canvas_ui_tool.py +++ b/tools/canvas_ui_tool.py @@ -1,12 +1,18 @@ -"""Canvas UI control tool. +"""Legacy conversation compatibility for the former Python Canvas UI tool. -Shipped with Agent Canvas. Mounted into the agent-server container at -``/canvas-tools`` and loaded via ``tool_module_qualnames`` so the agent can -direct the frontend (navigate to a file, switch tabs, show a preview). +New conversations define ``canvas_ui_control`` through the agent-server's +``client_tools`` JSON API. The distinct name avoids colliding with this +process-global legacy registration. This module remains importable because +persisted conversations store +``tool_module_qualnames = {"canvas_ui": "canvas_ui_tool"}`` and may contain +``CanvasUIAction`` / ``CanvasUIObservation`` events. Agent-server imports the +module before restoring those conversations so the legacy event kinds and tool +registration remain resolvable. The server-side executor is a no-op that returns an acknowledgment. The actual UI effect happens client-side: the frontend watches the WebSocket stream for -``ActionEvent``s with ``tool_name == "canvas_ui"`` and dispatches the command. +legacy ``canvas_ui`` and current ``canvas_ui_control`` ActionEvents and dispatches +the command. """ from collections.abc import Sequence @@ -127,7 +133,7 @@ class CanvasUITool(ToolDefinition[CanvasUIAction, CanvasUIObservation]): ] -# Auto-register at import time. The agent-server imports this module via -# tool_module_qualnames; this call wires the tool into the registry so it can -# be referenced by name in conversation tool lists. +# Persisted pre-client_tools conversations import this module by qualname before +# restoring their agent and events. Keep the registration until those records +# have a server-side migration path. register_tool("canvas_ui", CanvasUITool)