diff --git a/__tests__/components/browser.test.tsx b/__tests__/components/browser.test.tsx index 68c76b168d..0f66c4a705 100644 --- a/__tests__/components/browser.test.tsx +++ b/__tests__/components/browser.test.tsx @@ -1,4 +1,4 @@ -import { describe, it, expect, afterEach, vi } from "vitest"; +import { describe, it, expect, afterEach, beforeEach, vi } from "vitest"; import { screen, render } from "@testing-library/react"; import React from "react"; @@ -31,7 +31,12 @@ import { BrowserPanel } from "#/components/features/browser/browser"; import { useBrowserStore } from "#/stores/browser-store"; describe("Browser", () => { + beforeEach(() => { + useBrowserStore.getState().reset(); + }); + afterEach(() => { + useBrowserStore.getState().reset(); vi.clearAllMocks(); }); @@ -39,7 +44,6 @@ describe("Browser", () => { useBrowserStore.setState({ url: "https://example.com", screenshotSrc: "", - reset: vi.fn(), }); render(); @@ -52,7 +56,6 @@ describe("Browser", () => { url: "https://example.com", screenshotSrc: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mN0uGvyHwAFCAJS091fQwAAAABJRU5ErkJggg==", - reset: vi.fn(), }); render(); @@ -60,4 +63,20 @@ describe("Browser", () => { expect(screen.getByText("https://example.com")).toBeInTheDocument(); expect(screen.getByAltText("BROWSER$SCREENSHOT_ALT")).toBeInTheDocument(); }); + + it("does not clear a preloaded screenshot when the browser tab first mounts", () => { + const screenshotSrc = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mN0uGvyHwAFCAJS091fQwAAAABJRU5ErkJggg=="; + + useBrowserStore.setState({ + url: "https://example.com", + screenshotSrc, + }); + + render(); + + expect(useBrowserStore.getState().screenshotSrc).toBe(screenshotSrc); + expect(screen.getByAltText("BROWSER$SCREENSHOT_ALT")).toBeInTheDocument(); + expect(screen.queryByText("BROWSER$NO_PAGE_LOADED")).not.toBeInTheDocument(); + }); }); diff --git a/__tests__/contexts/conversation-websocket-context.test.tsx b/__tests__/contexts/conversation-websocket-context.test.tsx index f41996af41..9e934db027 100644 --- a/__tests__/contexts/conversation-websocket-context.test.tsx +++ b/__tests__/contexts/conversation-websocket-context.test.tsx @@ -5,6 +5,7 @@ import { createUserMessageEvent } from "test-utils"; import { ConversationWebSocketProvider } from "#/contexts/conversation-websocket-context"; import { useEventStore } from "#/stores/use-event-store"; import { useOptimisticUserMessageStore } from "#/stores/optimistic-user-message-store"; +import { useBrowserStore } from "#/stores/browser-store"; import { useUserConversation } from "#/hooks/query/use-user-conversation"; import EventService from "#/api/event-service/event-service.api"; import type { MessageEvent } from "#/types/agent-server/core"; @@ -62,6 +63,7 @@ describe("ConversationWebSocketProvider — conversation-scoped event store", () loadedConversationId: null, }); useOptimisticUserMessageStore.setState({ pendingMessages: [] }); + useBrowserStore.getState().reset(); vi.mocked(useUserConversation).mockReturnValue({ data: { conversation_url: "http://localhost/api", session_api_key: null }, @@ -102,6 +104,32 @@ describe("ConversationWebSocketProvider — conversation-scoped event store", () await waitFor(() => expect(eventIds()).toEqual(["user-msg-conv-b"])); }); + it("resets browser-panel state when switching conversations", async () => { + const { rerender } = renderProvider("conv-a"); + await waitFor(() => expect(eventIds()).toEqual(["user-msg-conv-a"])); + + useBrowserStore.setState({ + url: "https://example.com", + screenshotSrc: "data:image/png;base64,abc123", + }); + + rerender( + + +
+ + , + ); + + await waitFor(() => + expect(useBrowserStore.getState().screenshotSrc).toBe(""), + ); + expect(useBrowserStore.getState().url).toBe(""); + }); + it("keeps events that arrived after history when re-entering the same conversation", async () => { // Arrange: open conversation A, then receive an agent reply over the socket // that is not part of the cached REST history page. diff --git a/__tests__/tools/canvas-ui-tool.test.ts b/__tests__/tools/canvas-ui-tool.test.ts new file mode 100644 index 0000000000..e4c9e146a5 --- /dev/null +++ b/__tests__/tools/canvas-ui-tool.test.ts @@ -0,0 +1,20 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const toolSource = readFileSync(resolve(repoRoot, "tools/canvas_ui_tool.py"), "utf8"); + +describe("canvas_ui browser guidance", () => { + it("tells the agent to capture a browser screenshot before opening the browser tab", () => { + const captureInstruction = "browser_get_state(include_screenshot=true)"; + const openBrowserInstruction = 'command="open_tab", tab="browser"'; + + expect(toolSource).toContain(captureInstruction); + expect(toolSource).toContain(openBrowserInstruction); + expect(toolSource.indexOf(captureInstruction)).toBeLessThan( + toolSource.indexOf(openBrowserInstruction), + ); + }); +}); diff --git a/src/components/features/browser/browser.tsx b/src/components/features/browser/browser.tsx index c2636a3f35..85a502cbbe 100644 --- a/src/components/features/browser/browser.tsx +++ b/src/components/features/browser/browser.tsx @@ -1,16 +1,9 @@ -import { useEffect } from "react"; import { BrowserSnapshot } from "./browser-snapshot"; import { EmptyBrowserMessage } from "./empty-browser-message"; -import { useConversationId } from "#/hooks/use-conversation-id"; import { useBrowserStore } from "#/stores/browser-store"; export function BrowserPanel() { - const { url, screenshotSrc, reset } = useBrowserStore(); - const { conversationId } = useConversationId(); - - useEffect(() => { - reset(); - }, [conversationId, reset]); + const { url, screenshotSrc } = useBrowserStore(); const imgSrc = screenshotSrc?.startsWith("data:image/png;base64,") ? screenshotSrc diff --git a/src/contexts/conversation-websocket-context.tsx b/src/contexts/conversation-websocket-context.tsx index 3c77f719c8..aaf1459a32 100644 --- a/src/contexts/conversation-websocket-context.tsx +++ b/src/contexts/conversation-websocket-context.tsx @@ -143,6 +143,7 @@ export function ConversationWebSocketProvider({ ); const { setExecutionStatus } = useConversationStateStore(); const { appendInput, appendOutput } = useCommandStore(); + const resetBrowserStore = useBrowserStore((state) => state.reset); // History loading state. // - Main conversation history is now loaded via REST (`useConversationHistory`), @@ -241,7 +242,8 @@ export function ConversationWebSocketProvider({ // records the new loaded id in one `set`, so no subscriber can observe a // half-applied state (events gone but the old id still reported). clearEventsForConversation(nextId); - }, [conversationId, clearEventsForConversation]); + resetBrowserStore(); + }, [conversationId, clearEventsForConversation, resetBrowserStore]); useLayoutEffect(() => { if (!preloadedHistory || preloadedHistory.events.length === 0) { diff --git a/src/stores/browser-store.ts b/src/stores/browser-store.ts index a627702853..8842373fa6 100644 --- a/src/stores/browser-store.ts +++ b/src/stores/browser-store.ts @@ -1,9 +1,9 @@ import { create } from "zustand"; interface BrowserState { - // URL of browser window (placeholder for now, will be replaced with the actual URL later) + // URL of the last page the agent navigated to in the browser panel. url: string; - // Base64-encoded screenshot of browser window (placeholder for now, will be replaced with the actual screenshot later) + // Base64-encoded screenshot of the browser window, when the tool provides one. screenshotSrc: string; } @@ -14,7 +14,7 @@ interface BrowserStore extends BrowserState { } const initialState: BrowserState = { - url: "https://github.com/OpenHands/OpenHands", + url: "", screenshotSrc: "", }; diff --git a/tools/canvas_ui_tool.py b/tools/canvas_ui_tool.py index c402b45df6..851997f981 100644 --- a/tools/canvas_ui_tool.py +++ b/tools/canvas_ui_tool.py @@ -90,7 +90,11 @@ When to call (pick the most specific option that matches your last action): command="open_tab", tab="terminal" * You browsed to a URL the user should see → + First call browser_get_state(include_screenshot=true) after your final + browser interaction so Agent Canvas has a screenshot to display, then call command="open_tab", tab="browser" + (browser_navigate alone only updates the URL; without browser_get_state, + the Browser tab will open without a screenshot.) Call this BEFORE writing your chat-message summary of the change, so the artifact is visible while the user reads what you did. One canvas_ui call