mirror of
https://github.com/OpenHands/OpenHands.git
synced 2026-10-06 14:33:11 +08:00
Remove visual snapshot tests and bump Docker E2E timeout (#1332)
* Remove visual snapshot tests Co-authored-by: openhands <openhands@all-hands.dev> * Bump Docker E2E timeout Co-authored-by: openhands <openhands@all-hands.dev> * Align Docker E2E timeout caps Co-authored-by: openhands <openhands@all-hands.dev> --------- Co-authored-by: openhands <openhands@all-hands.dev>
This commit is contained in:
Rohit Malhotra
committed by
GitHub
GitHub
co-authored by
openhands
openhands
parent
7537dab99d
commit
c7c8862c11
BIN
Binary file not shown.
|
Before Width: | Height: | Size: 36 KiB |
@@ -1,376 +0,0 @@
|
||||
# Snapshot Test Coverage Plan
|
||||
|
||||
Visual regression tests that should be added to this directory. Each proposed
|
||||
spec file maps to one feature area. Tests follow the existing patterns:
|
||||
`setupMocks` + `dismissConsentModal` for static pages; `navigateToConversation`
|
||||
+ `injectEvents` / `window.__OH_EVENT_STORE__` for conversation-page scenarios;
|
||||
`animations: "disabled"` and `maxDiffPixelRatio: 0.01` on every
|
||||
`toHaveScreenshot()` call.
|
||||
|
||||
Existing coverage is marked **✅ done**. Everything else is proposed.
|
||||
|
||||
---
|
||||
|
||||
## 1. Home Screen
|
||||
|
||||
**File:** `home-screen.snapshot.spec.ts`
|
||||
**Existing:** ✅ `home-screen.png` in `settings-page.snapshot.spec.ts`
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `home-empty-no-workspace` | No workspaces stored |
|
||||
| `home-with-workspace-selected` | One workspace selected in the dropdown |
|
||||
| `home-workspace-dropdown-open` | Workspace dropdown expanded showing stored workspaces |
|
||||
| `home-folder-browser-modal` | `FolderBrowserModal` open with a mocked directory listing |
|
||||
| `home-manage-workspaces-modal` | `ManageWorkspacesModal` open with two workspaces listed |
|
||||
| `home-task-suggestions-loading` | Task suggestions skeleton (loading state) |
|
||||
| `home-task-suggestions-loaded` | Task suggestion cards visible |
|
||||
|
||||
---
|
||||
|
||||
## 2. Onboarding Modal
|
||||
|
||||
**File:** `onboarding.snapshot.spec.ts`
|
||||
|
||||
Mock `openhands-onboarded` absent from localStorage so the modal appears. Use
|
||||
`page.addInitScript` to clear the key. Step navigation is driven by clicking
|
||||
the "Next" / "Back" buttons between snapshots.
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `onboarding-step-0-choose-agent` | Step 0 – agent cards; OpenHands selected, others "coming soon" |
|
||||
| `onboarding-step-1-check-backend-connecting` | Step 1 – backend form with status banner in connecting state |
|
||||
| `onboarding-step-1-check-backend-connected` | Step 1 – status banner green (mock `/server_info` 200) |
|
||||
| `onboarding-step-1-check-backend-error` | Step 1 – status banner red (mock `/server_info` 500) |
|
||||
| `onboarding-step-2-setup-llm` | Step 2 – LLM settings form embedded (basic view) |
|
||||
| `onboarding-step-3-say-hello` | Step 3 – pre-filled message input |
|
||||
| `onboarding-progress-bar-step-2` | Progress bar at step 2 (two segments completed) |
|
||||
|
||||
---
|
||||
|
||||
## 3. Settings Pages
|
||||
|
||||
**File:** `settings.snapshot.spec.ts`
|
||||
**Existing:** ✅ `settings-page.png`, `settings-app-page.png` in `settings-page.snapshot.spec.ts`
|
||||
|
||||
### LLM Settings
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `settings-llm-basic-view` | Model picker + API key (no custom base URL) |
|
||||
| `settings-llm-basic-api-key-set` | API key field showing the "key set" icon |
|
||||
| `settings-llm-advanced-view` | "All settings" view: custom model string + base URL + API key |
|
||||
| `settings-llm-openhands-model-help` | OpenHands model selected → help link visible below API key |
|
||||
| `settings-llm-profiles-list` | LLM profiles manager showing two profile rows |
|
||||
| `settings-llm-profiles-rename-modal` | Rename profile modal open |
|
||||
| `settings-llm-profiles-delete-modal` | Delete profile confirmation modal open |
|
||||
| `settings-llm-dirty-save-button` | Save button enabled after a field change |
|
||||
|
||||
### Condenser Settings
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `settings-condenser` | Schema-driven condenser form loaded |
|
||||
|
||||
### Verification Settings
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `settings-verification-confirmation-off` | Confirmation mode toggle off (security analyzer hidden) |
|
||||
| `settings-verification-confirmation-on` | Confirmation mode toggle on → security analyzer dropdown visible |
|
||||
|
||||
### App Settings
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `settings-app-default` | ✅ Already captured as `settings-app-page.png`; keep as baseline |
|
||||
| `settings-app-dirty` | At least one field changed, Save button enabled |
|
||||
|
||||
### Secrets Settings
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `settings-secrets-empty` | No secrets, only the "Add New Secret" button |
|
||||
| `settings-secrets-list` | Three secrets listed in the table |
|
||||
| `settings-secrets-add-form` | Add-secret form open |
|
||||
| `settings-secrets-edit-form` | Edit-secret form open (secret name pre-populated) |
|
||||
| `settings-secrets-delete-confirm` | Confirmation modal open with secret name in text |
|
||||
|
||||
### Settings Navigation
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `settings-nav-llm-active` | Left nav with LLM link highlighted |
|
||||
| `settings-nav-secrets-active` | Left nav with Secrets link highlighted |
|
||||
|
||||
---
|
||||
|
||||
## 4. MCP Page
|
||||
|
||||
**File:** `mcp-page.snapshot.spec.ts`
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `mcp-page-empty` | No installed servers, full marketplace visible |
|
||||
| `mcp-page-with-installed` | Two installed servers (one SSE, one STDIO) in the installed section |
|
||||
| `mcp-marketplace-card-uninstalled` | A marketplace card in default state |
|
||||
| `mcp-marketplace-card-installed` | Same card showing "Installed" badge |
|
||||
| `mcp-search-filtered` | Search query "slack" – only matching cards visible |
|
||||
| `mcp-install-modal` | `InstallServerModal` open for a marketplace entry |
|
||||
| `mcp-custom-server-editor-sse` | Custom server editor showing SSE form |
|
||||
| `mcp-custom-server-editor-stdio` | Custom server editor showing stdio form |
|
||||
| `mcp-delete-confirm-modal` | Delete confirmation modal open |
|
||||
|
||||
---
|
||||
|
||||
## 5. Skills Page
|
||||
|
||||
**File:** `skills-page.snapshot.spec.ts`
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `skills-loading` | Skeleton pulse placeholders |
|
||||
| `skills-loaded` | 2-column grid with several skill cards (enabled + disabled) |
|
||||
| `skills-card-enabled` | Single enabled skill card (close-up via `getByTestId`) |
|
||||
| `skills-card-disabled` | Single disabled skill card |
|
||||
| `skills-search-filtered` | Search query applied, subset of cards visible |
|
||||
| `skills-no-match` | Search query returns zero results – "no match" message |
|
||||
| `skills-empty-from-server` | Server returns no skills – empty state text |
|
||||
|
||||
---
|
||||
|
||||
## 6. Automations
|
||||
|
||||
**File:** `automations.snapshot.spec.ts`
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `automations-loading` | Skeleton cards (health check in-flight) |
|
||||
| `automations-backend-not-configured` | `BackendNotConfigured` full-page state |
|
||||
| `automations-empty` | Backend healthy but zero automations |
|
||||
| `automations-list-active-inactive` | Active group (2 cards) + Inactive group (1 card) |
|
||||
| `automations-search-filtered` | Search narrows list to one card |
|
||||
| `automations-delete-modal` | Delete confirmation modal open over the list |
|
||||
| `automations-detail-active` | Detail page – active automation, all sections visible |
|
||||
| `automations-detail-activity-log` | Detail page – activity log section with run entries |
|
||||
|
||||
---
|
||||
|
||||
## 7. Backend Management
|
||||
|
||||
**File:** `backends.snapshot.spec.ts`
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `backend-selector-single` | Dropdown closed, one backend, green status dot |
|
||||
| `backend-selector-multiple` | Dropdown open, two backends with status dots |
|
||||
| `backend-selector-error-dot` | Dropdown open, one backend with red status dot |
|
||||
| `backend-add-modal` | `BackendFormModal` open in "add" mode |
|
||||
| `backend-edit-modal` | `BackendFormModal` open in "edit" mode, fields pre-filled |
|
||||
| `backend-manage-modal` | `ManageBackendsModal` listing two backends |
|
||||
| `backend-unavailable-recovery` | Full-screen recovery: "Manage Backends" modal over dark backdrop |
|
||||
| `environment-switch-overlay` | `EnvironmentSwitchOverlay` visible during backend switch |
|
||||
|
||||
---
|
||||
|
||||
## 8. Analytics Consent Modal
|
||||
|
||||
**File:** `settings-page.snapshot.spec.ts` (already exists)
|
||||
**Existing:** ✅ `analytics-consent-modal.png`
|
||||
|
||||
No new snapshots needed; keep existing baseline.
|
||||
|
||||
---
|
||||
|
||||
## 9. Conversation Page – Happy Path
|
||||
|
||||
**File:** `conversation-happy-path.snapshot.spec.ts`
|
||||
|
||||
All tests use `navigateToConversation` + `injectEvents` via
|
||||
`window.__OH_EVENT_STORE__`. Use serial mode.
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `conversation-chat-messages-skeleton` | Before events load (store empty, skeleton visible) |
|
||||
| `conversation-user-message` | Single user message injected |
|
||||
| `conversation-assistant-message` | User message + assistant text message |
|
||||
| `conversation-event-group-collapsed` | Two back-to-back action events folded into a group (collapsed) |
|
||||
| `conversation-event-group-expanded` | Same group after clicking to expand |
|
||||
| `conversation-finish-event` | `FinishAction` event rendered (green check or finish card) |
|
||||
| `conversation-typing-indicator` | `RUNNING` state injected – typing indicator visible |
|
||||
| `conversation-right-panel-closed` | Chat takes full width, right-panel toggle button visible |
|
||||
| `conversation-right-panel-files` | Right panel open on Files tab |
|
||||
| `conversation-right-panel-terminal` | Right panel open on Terminal tab |
|
||||
| `conversation-right-panel-planner` | Right panel open on Planner tab, Build button visible |
|
||||
| `conversation-tab-nav-full` | All tabs visible including Task List (inject task tracker event) |
|
||||
| `conversation-btw-message-pending` | BTW message with spinner in "pending" state |
|
||||
| `conversation-btw-message-answered` | BTW message answered, "Got it" button visible |
|
||||
|
||||
---
|
||||
|
||||
## 10. Conversation Page – Confirmation Mode
|
||||
|
||||
**File:** `conversation-confirmation-mode.snapshot.spec.ts`
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `confirmation-mode-locked-icon` | Lock icon visible in chat input row (mock `confirmation_mode: true`) |
|
||||
| `confirmation-awaiting-normal-risk` | `WAITING_FOR_CONFIRMATION` state + action event → Approve/Reject buttons visible, no risk banner |
|
||||
| `confirmation-awaiting-high-risk` | Same but action has `security_risk: "high"` → red `RiskAlert` banner above buttons |
|
||||
|
||||
---
|
||||
|
||||
## 11. Conversation Page – Collapsible Thinking
|
||||
|
||||
**File:** `collapsible-thinking.snapshot.spec.ts` (already exists)
|
||||
**Existing:** ✅ `think-action-collapsed.png`, `think-action-expanded.png`, `reasoning-content-collapsed.png`, `reasoning-content-expanded.png`
|
||||
|
||||
No new snapshots needed; keep existing baselines.
|
||||
|
||||
---
|
||||
|
||||
## 12. Conversation Creation – Error States
|
||||
|
||||
**File:** `conversation-creation-errors.snapshot.spec.ts`
|
||||
|
||||
These tests mock the relevant API endpoints to trigger each failure path.
|
||||
|
||||
| Snapshot name | What to mock | State to capture |
|
||||
|---|---|---|
|
||||
| `creation-backend-unavailable` | `GET /server_info` → network error / timeout | Full-screen backend recovery modal rendered over the home route |
|
||||
| `creation-api-failure-toast` | `POST /api/conversations` → 500 | Home screen + error toast visible in top-right corner |
|
||||
| `creation-task-polling-error` | `GET /api/conversations/start-tasks/:id` → `{ status: "ERROR", detail: "Sandbox failed to start" }` | Conversation route showing error toast; button returns to idle |
|
||||
| `creation-conversation-not-found` | `GET /api/conversations/:id` → 404 | Error toast + redirect back to home (capture just before redirect navigates away) |
|
||||
| `creation-loading-spinner` | `GET /api/conversations/start-tasks/:id` → `{ status: "PENDING" }` (never resolves) | `ConversationLoading` full-screen spinner |
|
||||
|
||||
---
|
||||
|
||||
## 13. Conversation Page – In-Progress Error States
|
||||
|
||||
**File:** `conversation-errors.snapshot.spec.ts`
|
||||
|
||||
All tests use `navigateToConversation` + inject agent state or events. Serial mode.
|
||||
|
||||
### Agent error events (inline in chat)
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `error-event-known-id` | `AgentErrorEvent` with a known i18n `errorId` – translated title shown, details collapsed |
|
||||
| `error-event-unknown-id` | `AgentErrorEvent` with unknown `errorId` – generic fallback title |
|
||||
| `error-event-expanded` | Same event after clicking the chevron – raw markdown details visible |
|
||||
| `error-event-long-message` | Error with a very long `defaultMessage` – details scroll within the card |
|
||||
|
||||
### Agent status bar error states
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `agent-status-error` | `ExecutionStatus.ERROR` injected → red `CircleErrorIcon` in status button, error status text in pill |
|
||||
| `agent-status-stuck` | `ExecutionStatus.STUCK` → same error icon path |
|
||||
| `agent-status-paused-resume` | `ExecutionStatus.PAUSED` → play/resume button visible |
|
||||
| `agent-status-running-stop` | `ExecutionStatus.RUNNING` → stop button visible, pulsing dot |
|
||||
| `agent-status-websocket-closed` | WebSocket mock closed after open → `CircleErrorIcon` |
|
||||
|
||||
### ErrorMessageBanner (scroll-up pagination / anchor error)
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `error-banner-short` | `setErrorMessage` called with a short message (< 220 chars) – no toggle button |
|
||||
| `error-banner-long-collapsed` | Long message (> 220 chars) – banner shows line-clamp with "View more" |
|
||||
| `error-banner-long-expanded` | Same banner after clicking "View more" – full text visible |
|
||||
| `error-banner-dismissed` | Banner after clicking × – banner no longer rendered |
|
||||
|
||||
### Session / WebSocket errors (toast)
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `ws-session-expired-toast` | Inject `{ error: true, error_code: 401, message: "..." }` event → "Session expired." toast |
|
||||
| `ws-generic-error-toast` | Inject `{ error: "Something went wrong", message: "..." }` event → generic error toast |
|
||||
| `ws-max-iterations-paused` | Inject `{ type: "error", message: "Agent reached maximum..." }` → agent transitions to PAUSED state |
|
||||
|
||||
### Send-message failure (optimistic pending bubble)
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `pending-message-sending` | Message sent but not echoed yet – faded "sending" bubble |
|
||||
| `pending-message-error` | Mock `send()` throws → bubble shows error treatment + Retry button |
|
||||
| `pending-message-after-retry` | After clicking Retry – bubble back to "sending" state |
|
||||
|
||||
### File upload validation
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `file-upload-too-large-toast` | Attach files exceeding 3 MB limit → error toast (no upload starts) |
|
||||
|
||||
---
|
||||
|
||||
## 14. Conversation Page – Alert Banner
|
||||
|
||||
**File:** `alert-banner.snapshot.spec.ts`
|
||||
|
||||
Mock `GET /server_info` (or the relevant endpoint) to return alert payload.
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `alert-banner-maintenance` | Maintenance start time set → banner with clock/triangle icon and formatted local time |
|
||||
| `alert-banner-faulty-models` | Faulty models list → banner listing model names |
|
||||
| `alert-banner-error-message` | Free-form error string → banner with raw text |
|
||||
| `alert-banner-dismissed` | Banner after clicking × – not rendered, page unobstructed |
|
||||
|
||||
---
|
||||
|
||||
## 15. Changes Tab – Diff Viewer Edge Cases
|
||||
|
||||
**File:** `changes-tab.snapshot.spec.ts`
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `diff-viewer-modified-file` | Normal modified file – Monaco diff editor visible |
|
||||
| `diff-viewer-deleted-file` | File with `type: "D"` selected → "file deleted" placeholder, no Monaco editor |
|
||||
| `diff-viewer-no-changes` | Empty git changes list – empty state text |
|
||||
|
||||
---
|
||||
|
||||
## 16. Sidebar
|
||||
|
||||
**File:** `sidebar.snapshot.spec.ts`
|
||||
|
||||
| Snapshot name | State to capture |
|
||||
|---|---|
|
||||
| `sidebar-with-conversations` | Conversation list populated, active conversation highlighted |
|
||||
| `sidebar-empty` | No conversations |
|
||||
| `sidebar-new-conversation-popover` | "+ New Conversation" popover open showing workspace entries |
|
||||
| `sidebar-status-dots` | Conversation list showing multiple status dots (finished, running, error, paused) |
|
||||
|
||||
---
|
||||
|
||||
## Key User Flow Sequences
|
||||
|
||||
The following multi-step flows should each be captured as a `test.step()` series
|
||||
within a single test, producing a sequence of named snapshots that together tell
|
||||
the story of the flow.
|
||||
|
||||
### Flow A – Modify LLM settings → start conversation
|
||||
**File:** `flow-llm-settings-to-conversation.snapshot.spec.ts`
|
||||
|
||||
1. `flow-llm-settings-initial` — LLM settings page, default model
|
||||
2. `flow-llm-settings-dirty` — Model changed to a new value, Save button enabled
|
||||
3. `flow-llm-settings-saved` — After save, success toast, button disabled again
|
||||
4. `flow-llm-conversation-started` — New conversation page, status indicator shows model in use
|
||||
|
||||
### Flow B – Enable confirmation mode → agent action requires approval
|
||||
**File:** `flow-confirmation-mode.snapshot.spec.ts`
|
||||
|
||||
1. `flow-confirmation-settings` — Verification settings page, confirmation mode off
|
||||
2. `flow-confirmation-settings-toggled` — Toggle turned on, security analyzer dropdown appears
|
||||
3. `flow-confirmation-settings-saved` — Saved, lock icon now shows in conversation input
|
||||
4. `flow-confirmation-action-pending` — Conversation page: agent at `WAITING_FOR_CONFIRMATION`, Approve/Reject buttons visible
|
||||
5. `flow-confirmation-high-risk` — Same with `security_risk: "high"` — red RiskAlert above buttons
|
||||
|
||||
### Flow C – Add a secret → confirm it appears in the secrets list
|
||||
**File:** `flow-secrets.snapshot.spec.ts`
|
||||
|
||||
1. `flow-secrets-empty` — Secrets settings, empty table
|
||||
2. `flow-secrets-add-form` — "Add New Secret" form open
|
||||
3. `flow-secrets-list-populated` — After save, secret appears in the table
|
||||
|
||||
### Flow D – Install MCP server → appears as installed
|
||||
**File:** `flow-mcp-install.snapshot.spec.ts`
|
||||
|
||||
1. `flow-mcp-marketplace` — Marketplace, server shows as uninstalled
|
||||
2. `flow-mcp-install-modal` — Install modal open with form
|
||||
3. `flow-mcp-installed` — Modal closed, server now in Installed section
|
||||
|
||||
### Flow E – Conversation creation failure recovery
|
||||
**File:** `flow-creation-failure-recovery.snapshot.spec.ts`
|
||||
|
||||
1. `flow-recovery-api-down` — Home screen, `/api/conversations` returning 500
|
||||
2. `flow-recovery-error-toast` — Toast visible after clicking "New Conversation"
|
||||
3. `flow-recovery-home-intact` — Toast dismissed, home screen unchanged and ready to retry
|
||||
@@ -1,123 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
import { stubWebSocket } from "./support/stub-websocket";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for archived / sandbox-error conversation states.
|
||||
*
|
||||
* Mock conversations pre-seeded in src/mocks/conversation-handlers.ts:
|
||||
* 4. "Archived Project" — sandbox_status: "MISSING"
|
||||
* 5. "Errored Project" — sandbox_status: "ERROR"
|
||||
*
|
||||
* Snapshots:
|
||||
* 1. conversation-panel-with-archived-badges — archive icon + ERROR pill for MISSING/ERROR
|
||||
* 2. conversation-view-archived — chat interface for conv 4 with the
|
||||
* read-only "Sandbox no longer available" banner (no chat input)
|
||||
* 3. conversation-view-sandbox-error — same for conv 5, "Sandbox error" variant
|
||||
*
|
||||
* NOTE: We do NOT inject events into the chat for these tests. In dev mode
|
||||
* React 18 strict mode double-fires effects in child-before-parent order, so
|
||||
* ConversationWebSocketProvider's addEvents runs before conversation.tsx's
|
||||
* clearEvents — any REST-loaded or store-injected events get wiped. The
|
||||
* tests verify the banner + hidden chat input (the actual feature), not
|
||||
* event rendering.
|
||||
*/
|
||||
|
||||
const ARCHIVED_CONVERSATION_ID = "4"; // sandbox_status: "MISSING"
|
||||
const ERROR_CONVERSATION_ID = "5"; // sandbox_status: "ERROR"
|
||||
|
||||
/** Dismisses the analytics consent modal if it appears. */
|
||||
async function dismissConsentModal(page: Page) {
|
||||
try {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 5_000 });
|
||||
await page
|
||||
.getByRole("dialog", { name: "Help improve OpenHands" })
|
||||
.waitFor({ state: "hidden", timeout: 5_000 });
|
||||
} catch {
|
||||
// Modal didn't appear — fine.
|
||||
}
|
||||
}
|
||||
|
||||
test.describe("Archived Conversation Visual Snapshots", () => {
|
||||
test.describe.configure({ mode: "serial" });
|
||||
test.setTimeout(60_000);
|
||||
|
||||
// ── 1. Sidebar panel ───────────────────────────────────────────────────
|
||||
|
||||
test("conversation panel shows archive icon and error badge for MISSING/ERROR sandboxes", async ({
|
||||
page,
|
||||
}) => {
|
||||
await seedLocalStorage(page);
|
||||
await page.goto("/conversations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const conversationPanel = page.getByTestId("conversation-panel");
|
||||
await expect(conversationPanel).toBeVisible({ timeout: 15_000 });
|
||||
// 6 mock conversations: 1–3 normal + 4 MISSING + 5 ERROR + pagination-local
|
||||
await expect(page.getByTestId("conversation-card")).toHaveCount(6, {
|
||||
timeout: 10_000,
|
||||
});
|
||||
await expect(page.getByTestId("conversation-status-archived")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await expect(page.getByTestId("error-badge")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
await expect(conversationPanel).toHaveScreenshot(
|
||||
"conversation-panel-with-archived-badges.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
// ── 2. Conversation view — MISSING sandbox (archived) ──────────────────
|
||||
|
||||
test("archived conversation view shows read-only banner and hides chat input", async ({
|
||||
page,
|
||||
}) => {
|
||||
await seedLocalStorage(page);
|
||||
await stubWebSocket(page);
|
||||
await page.goto(`/conversations/${ARCHIVED_CONVERSATION_ID}`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await dismissConsentModal(page);
|
||||
|
||||
// Wait for the archived banner — proves useActiveConversation resolved
|
||||
// with sandbox_status: "MISSING" and the component fully initialized.
|
||||
await expect(
|
||||
page.getByTestId("archived-conversation-banner"),
|
||||
).toBeVisible({ timeout: 20_000 });
|
||||
await expect(page.getByTestId("interactive-chat-box")).toHaveCount(0);
|
||||
|
||||
await expect(page.getByTestId("chat-interface")).toHaveScreenshot(
|
||||
"conversation-view-archived.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
// ── 3. Conversation view — ERROR sandbox ──────────────────────────────
|
||||
|
||||
test("error sandbox conversation view shows error banner and hides chat input", async ({
|
||||
page,
|
||||
}) => {
|
||||
await seedLocalStorage(page);
|
||||
await stubWebSocket(page);
|
||||
await page.goto(`/conversations/${ERROR_CONVERSATION_ID}`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await dismissConsentModal(page);
|
||||
|
||||
await expect(
|
||||
page.getByTestId("archived-conversation-banner"),
|
||||
).toBeVisible({ timeout: 20_000 });
|
||||
await expect(page.getByTestId("interactive-chat-box")).toHaveCount(0);
|
||||
|
||||
await expect(page.getByTestId("chat-interface")).toHaveScreenshot(
|
||||
"conversation-view-sandbox-error.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,188 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the Automations pages.
|
||||
*
|
||||
* Three states are covered:
|
||||
* 1. Backend not configured (health check returns an error)
|
||||
* 2. List view with active and inactive automation groups (MSW serves 5 automations)
|
||||
* 3. Empty list (health OK, list returns zero items)
|
||||
*
|
||||
* The test server runs with VITE_MOCK_API=true (npm run dev:mock).
|
||||
* MSW handles GET /api/automation/v1 with pre-built data from
|
||||
* src/mocks/automation-handlers.ts. The health endpoint is NOT
|
||||
* covered by MSW so page.route() owns it in all tests.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Dismiss the analytics consent modal if it appears (MSW settings return
|
||||
* user_consents_to_analytics: null so the modal can show on any page).
|
||||
*/
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire up the routes that every automations test needs:
|
||||
* - Skip onboarding so the modal never blocks the page
|
||||
* - Conversations search (empty – keeps the sidebar quiet)
|
||||
*
|
||||
* NOTE: Settings requests go to the same-origin Vite dev server where MSW
|
||||
* intercepts them before page.route(). We call dismissConsentModal() after
|
||||
* navigation to handle the MSW consent modal instead of suppressing it here.
|
||||
*
|
||||
* Automation health and list requests go cross-origin to :8000 where
|
||||
* page.route() takes precedence. Health is mocked per-test; list is mocked
|
||||
* here with INLINE_AUTOMATIONS so all tests have a consistent baseline.
|
||||
*/
|
||||
async function setupMocks(page: Page) {
|
||||
await seedLocalStorage(page);
|
||||
|
||||
await page.route("**/api/conversations/search**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ results: [] }),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe("Automations Visual Snapshots", () => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
// TODO: add automations-backend-not-configured snapshot once window.__MSW_WORKER__
|
||||
// is exposed in mock mode so tests can call worker.use() to override the health
|
||||
// handler per-test. See COVERAGE_PLAN.md §6 for the full snapshot list.
|
||||
|
||||
test("list with active and inactive groups renders correctly", async ({ page }) => {
|
||||
// MSW (automation-handlers.ts) serves health OK and the full automations list
|
||||
// (3 active, 2 inactive) so no page.route() overrides are needed here.
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/automations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// Wait for at least one automation card to confirm the list loaded
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toBeVisible({ timeout: 15_000 });
|
||||
await expect(page.getByRole("button", { name: "Automation actions" }).first()).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"automations-list-active-inactive.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
test("search with no results renders correctly", async ({ page }) => {
|
||||
// Load the full list then type a query that matches nothing.
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/automations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
await expect(page.getByRole("button", { name: "Automation actions" }).first()).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
// Type a query that matches no automation name, repository, or prompt
|
||||
const searchInput = page.getByRole("textbox");
|
||||
await searchInput.fill("zzznomatchquery");
|
||||
|
||||
// Wait for the filtered state to settle
|
||||
await page.waitForTimeout(300);
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("automations-search-no-results.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("no automations shows empty state", async ({ page }) => {
|
||||
// Each Playwright test gets a fresh browser context, so MSW starts with
|
||||
// the full MOCK_AUTOMATIONS_RESPONSE. We delete every automation via the
|
||||
// REST API (MSW handles DELETE) then reload so React Query fetches the
|
||||
// now-empty list and renders the EmptyState component.
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/automations");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
// Wait for initial list to confirm the service worker is active
|
||||
await expect(
|
||||
page.getByRole("button", { name: "Automation actions" }).first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
// Delete every automation from the MSW mutable state.
|
||||
// Important: the `automations` Map lives in PAGE-level JS (the MSW handlers
|
||||
// are compiled into the client bundle). A page.reload() would re-run the
|
||||
// module initialiser and reset the Map to 5 items, so instead we:
|
||||
// 1. Make the DELETE fetches (MSW handles them in the page context)
|
||||
// 2. Call window.__TEST_INVALIDATE_QUERIES__() to ask React Query to
|
||||
// refetch without a reload
|
||||
await page.evaluate(async () => {
|
||||
const res = await fetch("/api/automation/v1?limit=100");
|
||||
const data = (await res.json()) as { automations: { id: string }[] };
|
||||
await Promise.all(
|
||||
data.automations.map((a) =>
|
||||
fetch(`/api/automation/v1/${a.id}`, { method: "DELETE" }),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
// Trigger React Query to refetch the now-empty list
|
||||
await page.evaluate(() => {
|
||||
(
|
||||
window as Window & { __TEST_INVALIDATE_QUERIES__?: () => void }
|
||||
).__TEST_INVALIDATE_QUERIES__?.();
|
||||
});
|
||||
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
// No kebab buttons means the list is truly empty
|
||||
await expect(
|
||||
page.getByRole("button", { name: "Automation actions" }),
|
||||
).toHaveCount(0, { timeout: 10_000 });
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot("automations-no-automations.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("delete confirmation modal renders correctly", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/automations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// Wait for the first kebab button to be visible before interacting
|
||||
const kebab = page.getByRole("button", { name: "Automation actions" }).first();
|
||||
await expect(kebab).toBeVisible({ timeout: 10_000 });
|
||||
await kebab.click();
|
||||
|
||||
await page.getByRole("button", { name: "Delete" }).click();
|
||||
|
||||
// DeleteConfirmationModal is a plain div (no dialog role).
|
||||
// Wait for its title to confirm it appeared.
|
||||
await expect(page.getByText("Delete automation")).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"automations-delete-modal.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,504 +0,0 @@
|
||||
import { test, expect, type Page } from "@playwright/test";
|
||||
import type { Backend } from "../../../src/api/backend-registry/types";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Extended visual snapshot tests for the backend management UI.
|
||||
*
|
||||
* The add-backend modal is a two-column layout:
|
||||
* Left — Manual connection: name, host, API-key, Connect button.
|
||||
* Kind is inferred from the host URL (local vs cloud).
|
||||
* Right — Cloud OAuth: one-click "Login with OpenHands" device flow.
|
||||
*
|
||||
* Flow 1 — Add form validation gates (manual connection column)
|
||||
* Connect is disabled until name + valid host are filled; cloud-inferred
|
||||
* hosts additionally require an API key.
|
||||
*
|
||||
* Flow 2 — Two-column layout renders correctly
|
||||
* The add modal shows both manual-connection and cloud-login columns
|
||||
* side by side with an OR divider.
|
||||
*
|
||||
* Flow 3 — Cloud login column renders OAuth section
|
||||
* The right column shows the OpenHands logo, description, and a
|
||||
* "Login with OpenHands" device-flow button.
|
||||
*
|
||||
* Flow 4 — Remove backend with confirmation step
|
||||
* Clicking "Remove" opens a ConfirmationModal; confirming removes the
|
||||
* row; cancelling keeps it.
|
||||
*
|
||||
* Flow 5 — Edit backend pre-fills form fields
|
||||
* Opening the edit modal for an existing backend populates name, host,
|
||||
* and API-key inputs from the stored backend data.
|
||||
*
|
||||
* Flow 6 — Switch active backend via dropdown
|
||||
* Selecting a different backend fires the environment-switch overlay,
|
||||
* then updates the selector trigger label once the overlay fades.
|
||||
*
|
||||
* Flow 7 — Malformed / empty host blocks submission
|
||||
* A host with only whitespace keeps the Connect button disabled.
|
||||
* A syntactically invalid URL also keeps Connect disabled.
|
||||
*
|
||||
* Flow 8 — Close add form dismisses without saving
|
||||
* Clicking the close (✕) button closes the modal without altering
|
||||
* the backend list.
|
||||
*/
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** Two pre-seeded backends used by multi-backend tests. */
|
||||
const LOCAL_BACKEND: Backend = {
|
||||
id: "default-local",
|
||||
name: "Local",
|
||||
host: "http://localhost:3000",
|
||||
apiKey: "",
|
||||
kind: "local",
|
||||
};
|
||||
|
||||
const CLOUD_BACKEND: Backend = {
|
||||
id: "test-production",
|
||||
name: "Production",
|
||||
host: "https://app.all-hands.dev",
|
||||
apiKey: "sk-test-key",
|
||||
kind: "cloud",
|
||||
};
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Seed localStorage with one or two backends and navigate to the
|
||||
* conversations list so the BackendSelector is visible in the sidebar.
|
||||
* Routes file API and cloud-proxy requests so they don't produce
|
||||
* console errors that could affect timing.
|
||||
*/
|
||||
async function setupPage(
|
||||
page: Page,
|
||||
{
|
||||
backends = [LOCAL_BACKEND],
|
||||
activeBackendId,
|
||||
}: { backends?: Backend[]; activeBackendId?: string } = {},
|
||||
) {
|
||||
const extra: [string, string][] = [
|
||||
["openhands-backends", JSON.stringify(backends)],
|
||||
];
|
||||
if (activeBackendId) {
|
||||
extra.push([
|
||||
"openhands-active-backend",
|
||||
JSON.stringify({ backendId: activeBackendId, orgId: null }),
|
||||
]);
|
||||
}
|
||||
await seedLocalStorage(page, { extra });
|
||||
|
||||
// Prevent workspace-scan 404s in the sidebar from cluttering timing.
|
||||
await page.route("**/api/file/**", (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ path: "/home", subdirs: [] }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to the conversations list and hover the backend selector to
|
||||
* open the dropdown. Returns the root-layout locator for snapshots.
|
||||
*/
|
||||
async function openDropdown(page: Page) {
|
||||
await page.goto("/conversations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
const selector = page.getByTestId("backend-selector");
|
||||
await expect(selector).toBeVisible({ timeout: 10_000 });
|
||||
await selector.hover();
|
||||
|
||||
await expect(page.getByTestId("add-backend-menu-item")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
return rootLayout;
|
||||
}
|
||||
|
||||
/** Open the Add Backend modal via the dropdown footer. */
|
||||
async function openAddModal(page: Page) {
|
||||
const rootLayout = await openDropdown(page);
|
||||
await page.getByTestId("add-backend-menu-item").click();
|
||||
await expect(page.getByTestId("add-backend-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await expect(page.getByTestId("add-backend-name")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
return rootLayout;
|
||||
}
|
||||
|
||||
/** Open the Manage Backends modal via the dropdown footer. */
|
||||
async function openManageModal(page: Page) {
|
||||
const rootLayout = await openDropdown(page);
|
||||
await page.getByTestId("manage-backends-menu-item").click();
|
||||
await expect(page.getByTestId("manage-backends-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
return rootLayout;
|
||||
}
|
||||
|
||||
const SNAP_OPTS = { animations: "disabled" as const, maxDiffPixelRatio: 0.01 };
|
||||
|
||||
// ── Test Suite ─────────────────────────────────────────────────────────────
|
||||
|
||||
test.describe("Backend Management — Extended Flow Snapshots", () => {
|
||||
test.setTimeout(90_000);
|
||||
|
||||
// ── Flow 1: Add-form validation gates ─────────────────────────────────
|
||||
|
||||
test("Flow 1a — add form blank: Connect button disabled until required fields filled", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page);
|
||||
const rootLayout = await openAddModal(page);
|
||||
|
||||
// 1. Completely blank form — Connect must be disabled.
|
||||
await expect(page.getByTestId("add-backend-submit")).toBeDisabled();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-blank-disabled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
|
||||
// 2. Fill only the name; host still empty → Connect still disabled.
|
||||
await page.getByTestId("add-backend-name").fill("My Backend");
|
||||
await expect(page.getByTestId("add-backend-submit")).toBeDisabled();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-name-only-disabled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
test("Flow 1b — local backend becomes Connect-ready with name + host, no API key required", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page);
|
||||
const rootLayout = await openAddModal(page);
|
||||
|
||||
// A localhost URL infers local kind — no API key needed.
|
||||
await page.getByTestId("add-backend-name").fill("Dev Server");
|
||||
await page.getByTestId("add-backend-host").fill("http://localhost:8080");
|
||||
|
||||
// API key left empty — Connect must be enabled for local kind.
|
||||
await expect(page.getByTestId("add-backend-submit")).not.toBeDisabled();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-local-ready.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
test("Flow 1c — cloud-inferred backend requires API key; Connect stays disabled without it", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page);
|
||||
const rootLayout = await openAddModal(page);
|
||||
|
||||
// An all-hands.dev URL infers cloud kind — API key required.
|
||||
await page.getByTestId("add-backend-name").fill("Cloud Prod");
|
||||
await page.getByTestId("add-backend-host").fill("https://app.all-hands.dev");
|
||||
|
||||
// No API key → Connect disabled.
|
||||
await expect(page.getByTestId("add-backend-submit")).toBeDisabled();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-cloud-no-key-disabled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
|
||||
// Fill API key → Connect enabled.
|
||||
await page.getByTestId("add-backend-api-key").fill("sk-live-abc123");
|
||||
await expect(page.getByTestId("add-backend-submit")).not.toBeDisabled();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-cloud-with-key-enabled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Flow 2: Two-column layout ──────────────────────────────────────
|
||||
|
||||
test("Flow 2 — add modal shows two-column layout with manual and cloud login", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page);
|
||||
const rootLayout = await openAddModal(page);
|
||||
|
||||
// Left column: manual connection form is visible.
|
||||
await expect(page.getByTestId("add-backend-form")).toBeVisible();
|
||||
|
||||
// Right column: cloud login section with device-flow OAuth.
|
||||
await expect(page.getByTestId("add-backend-cloud-title")).toBeVisible();
|
||||
await expect(page.getByTestId("add-backend-device-flow")).toBeVisible();
|
||||
await expect(page.getByTestId("add-backend-login-button")).toBeVisible();
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-two-column-layout.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Flow 3: Cloud login column ────────────────────────────────────────
|
||||
|
||||
test("Flow 3 — cloud login column shows advanced host override", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page);
|
||||
const rootLayout = await openAddModal(page);
|
||||
|
||||
// Expand advanced settings in the cloud column.
|
||||
await page.getByTestId("add-backend-advanced-toggle").click();
|
||||
await expect(page.getByTestId("add-backend-cloud-host")).toBeVisible({
|
||||
timeout: 3_000,
|
||||
});
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-cloud-advanced-open.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Flow 4: Remove backend with confirmation ──────────────────────────
|
||||
|
||||
test("Flow 4 — removing a backend: confirmation modal then row disappears", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page, { backends: [LOCAL_BACKEND, CLOUD_BACKEND] });
|
||||
const rootLayout = await openManageModal(page);
|
||||
|
||||
// Both backend rows visible.
|
||||
await expect(
|
||||
page.getByTestId("manage-backends-row-Local"),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId("manage-backends-row-Production"),
|
||||
).toBeVisible();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-manage-two-listed.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
|
||||
// Click Remove on "Production".
|
||||
await page.getByTestId("manage-backends-remove-Production").click();
|
||||
|
||||
// ConfirmationModal should appear with the backend name in the text.
|
||||
await expect(page.getByTestId("confirmation-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-remove-confirmation.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
|
||||
// Click Cancel — Production row should still be present.
|
||||
await page.getByTestId("cancel-button").click();
|
||||
await expect(page.getByTestId("confirmation-modal")).not.toBeVisible({
|
||||
timeout: 3_000,
|
||||
});
|
||||
await expect(
|
||||
page.getByTestId("manage-backends-row-Production"),
|
||||
).toBeVisible();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-remove-cancelled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
|
||||
// Remove again and CONFIRM this time.
|
||||
await page.getByTestId("manage-backends-remove-Production").click();
|
||||
await expect(page.getByTestId("confirmation-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await page.getByTestId("confirm-button").click();
|
||||
|
||||
// Row disappears from the manage list.
|
||||
await expect(
|
||||
page.getByTestId("manage-backends-row-Production"),
|
||||
).not.toBeVisible({ timeout: 5_000 });
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-manage-after-removal.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Flow 5: Edit backend modal pre-fills form ─────────────────────────
|
||||
|
||||
test("Flow 5 — edit modal pre-populates existing backend's name, host and key", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page, { backends: [LOCAL_BACKEND, CLOUD_BACKEND] });
|
||||
const rootLayout = await openManageModal(page);
|
||||
|
||||
// Open Edit for the Production backend.
|
||||
await page.getByTestId("manage-backends-edit-Production").click();
|
||||
await expect(page.getByTestId("edit-backend-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
// Assert the pre-filled values.
|
||||
await expect(page.getByTestId("edit-backend-name")).toHaveValue(
|
||||
CLOUD_BACKEND.name,
|
||||
);
|
||||
await expect(page.getByTestId("edit-backend-host")).toHaveValue(
|
||||
CLOUD_BACKEND.host,
|
||||
);
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-edit-prefilled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Flow 6: Switch active backend ────────────────────────────────────
|
||||
|
||||
test("Flow 6 — switching backends shows environment-switch overlay then updates selector", async ({
|
||||
page,
|
||||
}) => {
|
||||
// Start with Local active; Production is a second registered backend.
|
||||
await setupPage(page, {
|
||||
backends: [LOCAL_BACKEND, CLOUD_BACKEND],
|
||||
activeBackendId: LOCAL_BACKEND.id,
|
||||
});
|
||||
const rootLayout = await openDropdown(page);
|
||||
|
||||
// Both options should be visible in the open dropdown.
|
||||
await expect(page.getByRole("option", { name: "Local" })).toBeVisible();
|
||||
await expect(
|
||||
page.getByRole("option", { name: "Production" }),
|
||||
).toBeVisible();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-dropdown-two-backends.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
|
||||
// Click Production option — triggers the environment-switch overlay.
|
||||
// The overlay is rendered via createPortal into document.body, so it
|
||||
// lives outside the root-layout subtree. Use a full-page screenshot
|
||||
// to capture it reliably.
|
||||
//
|
||||
// body[data-environment-switching="true"] is set synchronously inside
|
||||
// triggerEnvironmentSwitch before any React re-render, giving us a
|
||||
// stable early signal that the overlay is imminent even before React
|
||||
// paints the portal div.
|
||||
await page.getByRole("option", { name: "Production" }).click();
|
||||
await page.waitForSelector('body[data-environment-switching="true"]', {
|
||||
timeout: 2_000,
|
||||
});
|
||||
// Now wait for the actual portal div (React needs one render tick).
|
||||
await page.waitForSelector('[data-testid="environment-switch-overlay"]', {
|
||||
timeout: 2_000,
|
||||
});
|
||||
|
||||
// The overlay card animates from opacity:0 → 1 over 980ms. Playwright's
|
||||
// `animations: "disabled"` freezes CSS animations at frame 0, making the
|
||||
// card invisible in the screenshot. Override that so the card renders
|
||||
// fully opaque for a deterministic snapshot.
|
||||
await page.addStyleTag({
|
||||
content:
|
||||
".environment-switch-overlay > div { animation: none !important; opacity: 1 !important; transform: none !important; }",
|
||||
});
|
||||
|
||||
await expect(page).toHaveScreenshot("backend-switch-overlay.png", SNAP_OPTS);
|
||||
|
||||
// After overlay fades (980 ms), the selector should show "Production".
|
||||
await page.waitForSelector('[data-testid="environment-switch-overlay"]', {
|
||||
state: "hidden",
|
||||
timeout: 3_000,
|
||||
});
|
||||
// Re-hover to show the updated active backend in the dropdown.
|
||||
await page.getByTestId("backend-selector").hover();
|
||||
await expect(
|
||||
page.getByRole("option", { name: "Production" }),
|
||||
).toBeVisible({ timeout: 5_000 });
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-after-switch.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Flow 7: Malformed/empty host ──────────────────────────────────────
|
||||
|
||||
test("Flow 7 — empty or invalid host keeps Connect disabled", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page);
|
||||
const rootLayout = await openAddModal(page);
|
||||
|
||||
// Seed just the name, leave host blank.
|
||||
await page.getByTestId("add-backend-name").fill("Bad URL Test");
|
||||
|
||||
// Whitespace-only host → isValidHostUrl returns false → disabled.
|
||||
await page.getByTestId("add-backend-host").fill(" ");
|
||||
await expect(page.getByTestId("add-backend-submit")).toBeDisabled();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-whitespace-host-disabled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
|
||||
// A syntactically invalid URL is rejected by isValidHostUrl().
|
||||
await page.getByTestId("add-backend-host").fill("not://:::a valid url!!!");
|
||||
await expect(page.getByTestId("add-backend-submit")).toBeDisabled();
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-invalid-url-disabled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
|
||||
// ── Flow 8: Cancel add form ───────────────────────────────────────────
|
||||
|
||||
test("Flow 8 — closing the add form dismisses modal without persisting data", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupPage(page);
|
||||
const rootLayout = await openAddModal(page);
|
||||
|
||||
// Partially fill the form.
|
||||
await page.getByTestId("add-backend-name").fill("Temp Backend");
|
||||
await page.getByTestId("add-backend-host").fill("http://localhost:9999");
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-add-form-partially-filled.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
|
||||
// Click the close (✕) button.
|
||||
await page.getByTestId("add-backend-close").click();
|
||||
|
||||
// Modal is dismissed.
|
||||
await expect(page.getByTestId("add-backend-modal")).not.toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
// Open Manage Backends to confirm "Temp Backend" was NOT saved.
|
||||
await page.getByTestId("backend-selector").hover();
|
||||
await expect(page.getByTestId("manage-backends-menu-item")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await page.getByTestId("manage-backends-menu-item").click();
|
||||
await expect(page.getByTestId("manage-backends-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
// Only the original "Local" backend should be present.
|
||||
await expect(
|
||||
page.getByTestId("manage-backends-row-Local"),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.locator('[data-testid*="manage-backends-row-Temp"]'),
|
||||
).not.toBeVisible();
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"backend-cancel-nothing-saved.png",
|
||||
SNAP_OPTS,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,138 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the backend management UI.
|
||||
*
|
||||
* The BackendSelector lives in the sidebar footer and opens a dropdown on
|
||||
* hover. Its footer contains two action buttons:
|
||||
* - data-testid="add-backend-menu-item" → opens BackendFormModal (add)
|
||||
* - data-testid="manage-backends-menu-item" → opens ManageBackendsModal
|
||||
*
|
||||
* Backend state is seeded from the registry's default local backend
|
||||
* (DEFAULT_LOCAL_BACKEND_NAME = "Local") which is auto-created in
|
||||
* localStorage on first load.
|
||||
*
|
||||
* Three snapshots are captured:
|
||||
* 1. Selector dropdown open — shows the "Local" backend with status dot
|
||||
* and the Add / Manage footer actions.
|
||||
* 2. Add Backend modal — BackendFormModal in "add" mode (empty form).
|
||||
* 3. Manage Backends modal — ManageBackendsModal listing the default backend.
|
||||
*/
|
||||
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
async function setupMocks(page: Page) {
|
||||
await seedLocalStorage(page);
|
||||
|
||||
// Suppress file-API proxy errors emitted when the home page scans the
|
||||
// workspace directory (same suppression used in sidebar.snapshot.spec.ts).
|
||||
await page.route("**/api/file/**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ path: "/home", subdirs: [] }),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to the home page, wait for it to stabilise, then hover over the
|
||||
* backend selector to open the dropdown. Returns the rootLayout locator.
|
||||
*/
|
||||
async function openBackendDropdown(page: Page) {
|
||||
await page.goto("/conversations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// Wait for the sidebar to be fully rendered.
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
// The BackendSelector renders its Dropdown with openOnHover=true in the
|
||||
// expanded sidebar footer. Hovering over data-testid="backend-selector"
|
||||
// fires onMouseEnter → openMenu().
|
||||
const backendSelector = page.getByTestId("backend-selector");
|
||||
await expect(backendSelector).toBeVisible({ timeout: 10_000 });
|
||||
await backendSelector.hover();
|
||||
|
||||
// Wait for the dropdown footer actions to confirm the menu is open.
|
||||
await expect(page.getByTestId("add-backend-menu-item")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
return rootLayout;
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
test.describe("Backend Management Visual Snapshots", () => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
test("backend selector dropdown shows registered backend with status dot", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
const rootLayout = await openBackendDropdown(page);
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot("backend-selector-open.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("add backend modal opens with empty form", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
const rootLayout = await openBackendDropdown(page);
|
||||
|
||||
// Click "Add backend" in the dropdown footer.
|
||||
// onMouseDown has stopPropagation to keep the menu open; onClick opens the modal.
|
||||
await page.getByTestId("add-backend-menu-item").click();
|
||||
|
||||
// BackendFormModal (mode="add") has data-testid="add-backend-modal".
|
||||
await expect(page.getByTestId("add-backend-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
// Wait for the name input to confirm the form has rendered.
|
||||
await expect(page.getByTestId("add-backend-name")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot("backend-add-modal.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("manage backends modal lists the default local backend", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
const rootLayout = await openBackendDropdown(page);
|
||||
|
||||
// Click "Manage backends" in the dropdown footer.
|
||||
await page.getByTestId("manage-backends-menu-item").click();
|
||||
|
||||
// ManageBackendsModal has data-testid="manage-backends-modal".
|
||||
await expect(page.getByTestId("manage-backends-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
// Confirm at least one backend row is visible (the default "Local" backend).
|
||||
// Row testids follow the pattern: manage-backends-row-${backend.name}.
|
||||
await expect(page.getByTestId("manage-backends-row-Local")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot("backend-manage-modal.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,246 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
import { stubWebSocket } from "./support/stub-websocket";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the Changes (diff viewer) UI.
|
||||
*
|
||||
* The Changes view is rendered by src/routes/changes-tab.tsx inside the Files
|
||||
* tab (src/routes/files-tab.tsx) when the "Diff view" toggle is ON. We force
|
||||
* that toggle by pre-seeding the conversation's localStorage state with
|
||||
* `filesTabDiffView: true` before navigation.
|
||||
*
|
||||
* MSW pre-seeds three git changes in src/mocks/git-repository-handlers.ts:
|
||||
* - src/components/hello.tsx (M — modified)
|
||||
* - src/utils/new-helper.ts (A — added)
|
||||
* - src/old-module.py (D — deleted)
|
||||
*
|
||||
* Three snapshots are captured:
|
||||
* 1. Empty state — no files changed (window.__setMockGitChanges__([]) used
|
||||
* to clear MSW's in-memory list after boot, then a query invalidation
|
||||
* triggers a re-fetch that returns []).
|
||||
* 2. Diff viewer — modified file (hello.tsx) expanded to show Monaco.
|
||||
* 3. Deleted file placeholder — deleted file (old-module.py) shows the
|
||||
* "file deleted" message instead of a Monaco editor (the diff query is
|
||||
* disabled for type "D" per useUnifiedGitDiff).
|
||||
*
|
||||
* NOTE on MSW vs page.route():
|
||||
* MSW 2.x browser-mode handlers run in the page's main thread, not the
|
||||
* service worker. Playwright's page.route() is blocked by the service worker
|
||||
* for same-origin requests. We therefore manipulate MSW state via
|
||||
* page.evaluate() rather than page.route() (same pattern as the automations
|
||||
* empty-state test).
|
||||
*/
|
||||
|
||||
// Mock conversation IDs "1", "2", "3" are pre-defined in MSW handlers.
|
||||
const CONVERSATION_ID = "1";
|
||||
|
||||
// Pre-enable diff view for conversation 1.
|
||||
// NOTE: rightPanelShown is intentionally omitted — it is session-only state
|
||||
// stripped by sanitizeStoredState on read. The right panel is opened
|
||||
// programmatically via a right-panel-toggle click in navigateAndWaitForFilesTab.
|
||||
const CONVERSATION_STATE_KEY = `conversation-state-${CONVERSATION_ID}`;
|
||||
const CONVERSATION_STATE_VALUE = JSON.stringify({
|
||||
selectedTab: "files",
|
||||
filesTabDiffView: true,
|
||||
filesTabContentViewMode: "rich",
|
||||
unpinnedTabs: [],
|
||||
conversationMode: "code",
|
||||
subConversationTaskId: null,
|
||||
draftMessage: null,
|
||||
});
|
||||
|
||||
/**
|
||||
* Skip onboarding and pre-enable the diff view for conversation 1.
|
||||
*/
|
||||
async function setupMocks(page: Page) {
|
||||
await seedLocalStorage(page, {
|
||||
extra: [[CONVERSATION_STATE_KEY, CONVERSATION_STATE_VALUE]],
|
||||
});
|
||||
|
||||
await stubWebSocket(page);
|
||||
}
|
||||
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to the conversation and wait for the Files tab (diff view) to
|
||||
* be rendered. Returns the `data-testid="files-tab"` locator.
|
||||
*
|
||||
* `isRightPanelShown` is session-only Zustand state (always false on load;
|
||||
* `sanitizeStoredState` strips any persisted `rightPanelShown` key).
|
||||
* We open the right panel by clicking the `right-panel-toggle` button,
|
||||
* which calls `setHasRightPanelToggled(true)` → synced to
|
||||
* `setIsRightPanelShown(true)` by `use-chat-input-logic`.
|
||||
*/
|
||||
async function navigateAndWaitForFilesTab(page: Page) {
|
||||
await page.goto(`/conversations/${CONVERSATION_ID}`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await dismissConsentModal(page);
|
||||
|
||||
// Open the right panel — it always starts closed on page load.
|
||||
const toggle = page.getByTestId("right-panel-toggle");
|
||||
await expect(toggle).toBeVisible({ timeout: 15_000 });
|
||||
await toggle.click();
|
||||
|
||||
// The FilesTab is lazy-loaded inside the now-open right panel.
|
||||
const filesTab = page.getByTestId("files-tab");
|
||||
await expect(filesTab).toBeVisible({ timeout: 20_000 });
|
||||
return filesTab;
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
test.describe("Changes Tab Visual Snapshots", () => {
|
||||
// Heavier conversation-page setup — run serially to avoid flakiness.
|
||||
test.describe.configure({ mode: "serial" });
|
||||
test.setTimeout(60_000);
|
||||
|
||||
test("changes tab shows empty state when no files changed", async ({
|
||||
page,
|
||||
}) => {
|
||||
// MSW pre-seeds MOCK_GIT_CHANGES with M/A/D files. We call the exposed
|
||||
// window setter (installed by git-repository-handlers.ts) AFTER the app
|
||||
// boots to replace the list with [], then ask React Query to refetch.
|
||||
// This avoids a page.reload() which would re-seed the module state.
|
||||
await setupMocks(page);
|
||||
|
||||
await navigateAndWaitForFilesTab(page);
|
||||
|
||||
// Pin the RandomTip section to a fixed height so the flex-1 container
|
||||
// above it is deterministic across runs. RandomTip renders a randomly
|
||||
// selected tip whose text can vary in line count, causing different
|
||||
// layout heights between the baseline-generation run and verification run.
|
||||
// The class combination ".text-m.bg-tertiary.p-4" is unique to this element
|
||||
// in changes-tab.tsx (confirmed by grep). Hiding the content removes the
|
||||
// visual variable; the fixed height keeps the surrounding flex layout stable.
|
||||
await page.addStyleTag({
|
||||
content: `.text-m.bg-tertiary.p-4 {
|
||||
height: 80px !important;
|
||||
overflow: hidden !important;
|
||||
visibility: hidden !important;
|
||||
}`,
|
||||
});
|
||||
|
||||
// Wait for the initial (non-empty) render to settle before mutating state.
|
||||
await expect(
|
||||
page.locator('[data-testid="file-diff-viewer-outer"]').first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
// Clear the changes via the exposed window helper and refetch.
|
||||
await page.evaluate(() => {
|
||||
(
|
||||
window as unknown as {
|
||||
__setMockGitChanges__?: (changes: unknown[]) => void;
|
||||
}
|
||||
).__setMockGitChanges__?.([]);
|
||||
});
|
||||
await page.evaluate(() => {
|
||||
(
|
||||
window as unknown as {
|
||||
__TEST_INVALIDATE_QUERIES__?: (queryKey?: unknown[]) => void;
|
||||
}
|
||||
).__TEST_INVALIDATE_QUERIES__?.(["file_changes"]);
|
||||
});
|
||||
|
||||
// Wait for the empty-state message from EmptyChangesMessage component.
|
||||
await expect(
|
||||
page.getByText("OpenHands hasn't made any changes yet"),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
// Screenshot the diff-content div (direct child of files-tab when diff view
|
||||
// is enabled) rather than the full files-tab. This avoids capturing any
|
||||
// adjacent-panel artefacts that may bleed into the outer element's bounding
|
||||
// box during CI rendering, while still showing the full empty-state UI.
|
||||
//
|
||||
// DOM path: main[data-testid="files-tab"] > div.flex-1.min-h-0 > main
|
||||
// The inner <main> rendered by GitChanges is the safest stable target.
|
||||
const changesContent = page
|
||||
.getByTestId("files-tab")
|
||||
.locator("> div")
|
||||
.last();
|
||||
await expect(changesContent).toHaveScreenshot("changes-empty.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("changes tab shows file list and diff viewer for modified file", async ({
|
||||
page,
|
||||
}) => {
|
||||
// MOCK_GIT_CHANGES is pre-seeded; the file list renders without any override.
|
||||
await setupMocks(page);
|
||||
|
||||
const filesTab = await navigateAndWaitForFilesTab(page);
|
||||
|
||||
// Wait for at least one file row to appear.
|
||||
await expect(
|
||||
page.locator('[data-testid="file-diff-viewer-outer"]').first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
// Click the modified file (hello.tsx) header row to expand the diff editor.
|
||||
// The header row is the first child div of file-diff-viewer-outer and has
|
||||
// the cursor-pointer class; clicking the strong element (file path) is the
|
||||
// most reliable targeting.
|
||||
await page
|
||||
.locator('[data-testid="file-diff-viewer-outer"]')
|
||||
.filter({ hasText: "hello.tsx" })
|
||||
.locator("strong")
|
||||
.click();
|
||||
|
||||
// Wait for the EditorContainer (wraps the Monaco DiffEditor) to appear.
|
||||
await expect(page.getByTestId("editor-container").first()).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
// Mask the Monaco DiffEditor container. Monaco renders text content
|
||||
// progressively and uses sub-pixel font hinting that varies between OS/CI
|
||||
// environments. Masking editor-container captures the panel layout (toolbar,
|
||||
// file list, editor frame) without the volatile text-rendering pixels.
|
||||
await expect(filesTab).toHaveScreenshot("changes-diff-viewer.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
mask: [page.getByTestId("editor-container")],
|
||||
});
|
||||
});
|
||||
|
||||
test("changes tab shows deleted-file placeholder instead of diff editor", async ({
|
||||
page,
|
||||
}) => {
|
||||
// src/old-module.py has type "D" (deleted). useUnifiedGitDiff disables the
|
||||
// query for deleted files; clicking the row expands the file-deleted-message
|
||||
// placeholder instead of a Monaco editor.
|
||||
await setupMocks(page);
|
||||
|
||||
const filesTab = await navigateAndWaitForFilesTab(page);
|
||||
|
||||
// Wait for at least one file row to appear.
|
||||
await expect(
|
||||
page.locator('[data-testid="file-diff-viewer-outer"]').first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
// Click the deleted file row to expand it.
|
||||
await page
|
||||
.locator('[data-testid="file-diff-viewer-outer"]')
|
||||
.filter({ hasText: "old-module.py" })
|
||||
.locator("strong")
|
||||
.click();
|
||||
|
||||
// The deleted-file placeholder (data-testid="file-deleted-message") is
|
||||
// shown when !isCollapsed && type === "D".
|
||||
await expect(page.getByTestId("file-deleted-message")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
await expect(filesTab).toHaveScreenshot("changes-deleted-file.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,364 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
import { stubWebSocket } from "./support/stub-websocket";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the CollapsibleThinking component.
|
||||
*
|
||||
* These tests navigate to a conversation in the mock dev server, inject
|
||||
* synthetic events into the event store, and capture screenshots of the
|
||||
* collapsible thinking sections in both collapsed and expanded states.
|
||||
*
|
||||
* The mock dev server (VITE_MOCK_API=true) uses MSW service workers to
|
||||
* handle API requests. We rely on MSW for conversations, settings, etc.
|
||||
* and only inject events directly into the Zustand store.
|
||||
*
|
||||
* To update baselines after intentional UI changes:
|
||||
* npm run test:e2e:snapshots:update
|
||||
*/
|
||||
|
||||
// Use mock conversation ID "1" which is pre-defined in MSW handlers
|
||||
const CONVERSATION_ID = "1";
|
||||
|
||||
/** ThinkAction event with thinking content */
|
||||
const THINK_ACTION_EVENT = {
|
||||
id: "think-event-1",
|
||||
timestamp: "2026-01-01T00:00:01.000Z",
|
||||
source: "agent",
|
||||
thought: [
|
||||
{
|
||||
type: "text",
|
||||
text: 'think: {"thought": "I need to analyze the codebase structure first. Let me look at the directory layout and understand the project architecture before making any changes."}',
|
||||
},
|
||||
],
|
||||
reasoning_content: null,
|
||||
thinking_blocks: [],
|
||||
action: {
|
||||
kind: "ThinkAction",
|
||||
thought:
|
||||
"I need to analyze the codebase structure first. Let me look at the directory layout and understand the project architecture before making any changes.",
|
||||
},
|
||||
tool_name: "think",
|
||||
tool_call_id: "call_think_1",
|
||||
tool_call: {
|
||||
id: "call_think_1",
|
||||
type: "function",
|
||||
function: {
|
||||
name: "think",
|
||||
arguments: JSON.stringify({
|
||||
thought:
|
||||
"I need to analyze the codebase structure first. Let me look at the directory layout and understand the project architecture before making any changes.",
|
||||
}),
|
||||
},
|
||||
},
|
||||
llm_response_id: "response_1",
|
||||
security_risk: "unknown",
|
||||
};
|
||||
|
||||
/** Bash action event with reasoning_content (extended thinking) */
|
||||
const BASH_WITH_REASONING_EVENT = {
|
||||
id: "bash-event-1",
|
||||
timestamp: "2026-01-01T00:00:02.000Z",
|
||||
source: "agent",
|
||||
thought: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Let me list the project files to understand the structure.",
|
||||
},
|
||||
],
|
||||
reasoning_content:
|
||||
"The user wants to understand the project. I should start by listing the files in the root directory. This will give us a high-level overview of the codebase. Then I can dive deeper into specific areas based on what we find.",
|
||||
thinking_blocks: [],
|
||||
action: {
|
||||
kind: "ExecuteBashAction",
|
||||
command: "ls -la",
|
||||
is_input: false,
|
||||
timeout: null,
|
||||
reset: false,
|
||||
},
|
||||
tool_name: "execute_bash",
|
||||
tool_call_id: "call_bash_1",
|
||||
tool_call: {
|
||||
id: "call_bash_1",
|
||||
type: "function",
|
||||
function: {
|
||||
name: "execute_bash",
|
||||
arguments: JSON.stringify({ command: "ls -la" }),
|
||||
},
|
||||
},
|
||||
llm_response_id: "response_2",
|
||||
security_risk: "unknown",
|
||||
};
|
||||
|
||||
/** User message event */
|
||||
const USER_MESSAGE_EVENT = {
|
||||
id: "user-msg-1",
|
||||
timestamp: "2026-01-01T00:00:00.000Z",
|
||||
source: "user",
|
||||
llm_message: {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Help me understand this project" }],
|
||||
},
|
||||
activated_microagents: [],
|
||||
extended_content: [],
|
||||
};
|
||||
|
||||
/**
|
||||
* Dismisses the analytics consent modal if it appears.
|
||||
* The MSW mock settings don't include user_consents_to_analytics, so
|
||||
* the consent modal may appear on first load.
|
||||
*/
|
||||
async function dismissConsentModal(page: Page) {
|
||||
const confirmButton = page.getByRole("button", {
|
||||
name: "Confirm preferences",
|
||||
});
|
||||
try {
|
||||
await confirmButton.waitFor({ state: "visible", timeout: 5000 });
|
||||
await confirmButton.click();
|
||||
await page
|
||||
.getByRole("dialog", { name: "Help improve OpenHands" })
|
||||
.waitFor({ state: "hidden", timeout: 5000 });
|
||||
} catch {
|
||||
// Modal didn't appear — that's fine
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inject events into the event store via the exposed Zustand API.
|
||||
*/
|
||||
async function injectEvents(page: Page, events: unknown[]) {
|
||||
await page.waitForFunction(() => {
|
||||
const store = (
|
||||
window as unknown as {
|
||||
__OH_EVENT_STORE__?: {
|
||||
getState: () => { addEvents?: (e: unknown[]) => void };
|
||||
};
|
||||
}
|
||||
).__OH_EVENT_STORE__;
|
||||
return Boolean(store?.getState().addEvents);
|
||||
});
|
||||
|
||||
await expect
|
||||
.poll(
|
||||
async () =>
|
||||
page.evaluate((evts) => {
|
||||
const store = (
|
||||
window as unknown as {
|
||||
__OH_EVENT_STORE__?: {
|
||||
getState: () => {
|
||||
addEvents: (e: unknown[]) => void;
|
||||
events: unknown[];
|
||||
};
|
||||
};
|
||||
}
|
||||
).__OH_EVENT_STORE__;
|
||||
|
||||
if (!store) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const state = store.getState();
|
||||
state.addEvents(evts);
|
||||
return store.getState().events.length;
|
||||
}, events),
|
||||
{ timeout: 10000 },
|
||||
)
|
||||
.toBeGreaterThanOrEqual(events.length);
|
||||
|
||||
// Wait for React to re-render
|
||||
await page.waitForTimeout(500);
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to a conversation page and seed it with events so the chat
|
||||
* interface renders instead of the empty launch prompt.
|
||||
* Uses mock conversation "1" which exists in the MSW handlers.
|
||||
*/
|
||||
async function navigateToConversation(page: Page, events: unknown[]) {
|
||||
await seedLocalStorage(page);
|
||||
|
||||
await page.route("**/api/bash/execute_bash_command", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
command: "ls -la",
|
||||
exit_code: 0,
|
||||
output: "",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
await page.route("**/api/file/**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ path: "/home", subdirs: [] }),
|
||||
});
|
||||
});
|
||||
|
||||
// The GUI hides ChatSuggestions (and shows the "LLM not configured" banner)
|
||||
// unless an active profile with a key exists — see useLlmConfigured. Seed
|
||||
// one so the empty conversation renders normally and the banner stays out
|
||||
// of the captured screenshots (matching the baselines).
|
||||
await page.route(
|
||||
(url) => url.pathname.endsWith("/api/profiles"),
|
||||
async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
profiles: [
|
||||
{
|
||||
name: "mock",
|
||||
model: "anthropic/claude-sonnet-4-20250514",
|
||||
base_url: null,
|
||||
api_key_set: true,
|
||||
},
|
||||
],
|
||||
active_profile: "mock",
|
||||
}),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
await stubWebSocket(page);
|
||||
|
||||
await page.goto(`/conversations/${CONVERSATION_ID}`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await dismissConsentModal(page);
|
||||
|
||||
// Wait for the conversation route to mount before injecting events.
|
||||
// ConversationWebSocketContext runs a useLayoutEffect on mount that
|
||||
// calls clearEventsForConversation(<id>) — if we inject events first,
|
||||
// that effect wipes them and the UI renders the empty state. Use the
|
||||
// store's `loadedConversationId` as a route-stable readiness signal:
|
||||
// it flips from null → CONVERSATION_ID inside that layoutEffect, so
|
||||
// observing it means the clear-and-set has already happened and any
|
||||
// subsequent addEvents will survive.
|
||||
//
|
||||
// We deliberately don't wait on the home route's "Let's start
|
||||
// building!" text here — it only renders on the home route, so relying
|
||||
// on a brief home flash before the conversation hydrates is racy and
|
||||
// caused the 20s timeout flake tracked in #1200.
|
||||
await page.waitForFunction(
|
||||
(expectedId) => {
|
||||
const store = (
|
||||
window as unknown as {
|
||||
__OH_EVENT_STORE__?: {
|
||||
getState: () => { loadedConversationId: string | null };
|
||||
};
|
||||
}
|
||||
).__OH_EVENT_STORE__;
|
||||
return store?.getState().loadedConversationId === expectedId;
|
||||
},
|
||||
CONVERSATION_ID,
|
||||
{ timeout: 20000 },
|
||||
);
|
||||
|
||||
await injectEvents(page, events);
|
||||
|
||||
const chatInterface = page.getByTestId("chat-interface");
|
||||
await expect(chatInterface).toBeVisible({ timeout: 20000 });
|
||||
return chatInterface;
|
||||
}
|
||||
|
||||
test.describe("Collapsible Thinking Visual Snapshots", () => {
|
||||
// Run serially: the conversation page + WebSocket stub is heavier than
|
||||
// static pages, and parallel workers hitting the shared dev server can
|
||||
// cause intermittent load failures.
|
||||
test.describe.configure({ mode: "serial" });
|
||||
test.setTimeout(60000);
|
||||
|
||||
test("ThinkAction renders as collapsed section", async ({ page }) => {
|
||||
const chatInterface = await navigateToConversation(page, [
|
||||
USER_MESSAGE_EVENT,
|
||||
THINK_ACTION_EVENT,
|
||||
]);
|
||||
|
||||
// Verify the collapsible thinking section is rendered
|
||||
const collapsibleThinking = page.getByTestId("collapsible-thinking");
|
||||
await expect(collapsibleThinking).toBeVisible({ timeout: 5000 });
|
||||
|
||||
// The content should be collapsed (not visible)
|
||||
const content = page.getByTestId("collapsible-thinking-content");
|
||||
await expect(content).toHaveCount(0);
|
||||
|
||||
// Snapshot: collapsed state
|
||||
await expect(chatInterface).toHaveScreenshot("think-action-collapsed.png", {
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
});
|
||||
});
|
||||
|
||||
test("ThinkAction expands on click", async ({ page }) => {
|
||||
const chatInterface = await navigateToConversation(page, [
|
||||
USER_MESSAGE_EVENT,
|
||||
THINK_ACTION_EVENT,
|
||||
]);
|
||||
|
||||
const toggle = page.getByTestId("collapsible-thinking-toggle");
|
||||
await expect(toggle).toBeVisible({ timeout: 5000 });
|
||||
|
||||
// Click to expand
|
||||
await toggle.click();
|
||||
|
||||
// Content should now be visible
|
||||
const content = page.getByTestId("collapsible-thinking-content");
|
||||
await expect(content).toBeVisible({ timeout: 5000 });
|
||||
|
||||
// Snapshot: expanded state
|
||||
await expect(chatInterface).toHaveScreenshot("think-action-expanded.png", {
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
});
|
||||
});
|
||||
|
||||
test("Reasoning content renders as collapsed section with action", async ({
|
||||
page,
|
||||
}) => {
|
||||
const chatInterface = await navigateToConversation(page, [
|
||||
USER_MESSAGE_EVENT,
|
||||
BASH_WITH_REASONING_EVENT,
|
||||
]);
|
||||
|
||||
// Verify both the collapsible section and the action event are present
|
||||
const collapsibleThinking = page.getByTestId("collapsible-thinking");
|
||||
await expect(collapsibleThinking).toBeVisible({ timeout: 5000 });
|
||||
|
||||
// Snapshot: reasoning content collapsed alongside the bash action
|
||||
await expect(chatInterface).toHaveScreenshot(
|
||||
"reasoning-content-collapsed.png",
|
||||
{
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("Reasoning content expands on click", async ({ page }) => {
|
||||
const chatInterface = await navigateToConversation(page, [
|
||||
USER_MESSAGE_EVENT,
|
||||
BASH_WITH_REASONING_EVENT,
|
||||
]);
|
||||
|
||||
const toggle = page.getByTestId("collapsible-thinking-toggle");
|
||||
await expect(toggle).toBeVisible({ timeout: 5000 });
|
||||
|
||||
// Click to expand
|
||||
await toggle.click();
|
||||
|
||||
const content = page.getByTestId("collapsible-thinking-content");
|
||||
await expect(content).toBeVisible({ timeout: 5000 });
|
||||
|
||||
// Snapshot: reasoning content expanded
|
||||
await expect(chatInterface).toHaveScreenshot(
|
||||
"reasoning-content-expanded.png",
|
||||
{
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,342 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the MCP page (/mcp).
|
||||
*
|
||||
* The MCP marketplace catalog is imported from @openhands/extensions/integrations,
|
||||
* so it never requires an API call. Installed servers are read from
|
||||
* settings.agent_settings.mcp_config (SDK format: { mcpServers: { ... } }).
|
||||
*
|
||||
* Three states are covered:
|
||||
* 1. No installed servers – empty installed section, full marketplace visible
|
||||
* 2. Two installed servers (one SSE, one stdio)
|
||||
* 3. Search query "slack" filtering both sections simultaneously
|
||||
*/
|
||||
|
||||
/**
|
||||
* Dismiss the analytics consent modal if MSW shows it (settings return
|
||||
* user_consents_to_analytics: null by default in mock mode).
|
||||
*/
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire up the base routes every MCP page test needs.
|
||||
*
|
||||
* Settings are owned here rather than by the real local backend. MCP saves do a
|
||||
* pre-clear PATCH followed by the new config write, so the snapshot flow needs
|
||||
* a deterministic in-memory settings response.
|
||||
*/
|
||||
async function setupMocks(page: Page) {
|
||||
await seedLocalStorage(page);
|
||||
|
||||
let agentSettings: Record<string, unknown> = {};
|
||||
let conversationSettings: Record<string, unknown> = {};
|
||||
|
||||
await page.route("**/api/settings", async (route) => {
|
||||
const request = route.request();
|
||||
|
||||
if (request.method() === "GET") {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
agent_settings: agentSettings,
|
||||
conversation_settings: conversationSettings,
|
||||
llm_api_key_is_set: true,
|
||||
}),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method() === "PATCH") {
|
||||
const body = request.postDataJSON() as {
|
||||
agent_settings_diff?: Record<string, unknown>;
|
||||
conversation_settings_diff?: Record<string, unknown>;
|
||||
};
|
||||
agentSettings = {
|
||||
...agentSettings,
|
||||
...(body.agent_settings_diff ?? {}),
|
||||
};
|
||||
conversationSettings = {
|
||||
...conversationSettings,
|
||||
...(body.conversation_settings_diff ?? {}),
|
||||
};
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
agent_settings: agentSettings,
|
||||
conversation_settings: conversationSettings,
|
||||
llm_api_key_is_set: true,
|
||||
}),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await route.fallback();
|
||||
});
|
||||
|
||||
await page.route("**/api/mcp/test", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ ok: true, tools: ["mock_tool"] }),
|
||||
});
|
||||
});
|
||||
|
||||
await page.route("**/api/conversations/search**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ results: [] }),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe("MCP Page Visual Snapshots", () => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
test("empty installed section with marketplace renders correctly", async ({
|
||||
page,
|
||||
}) => {
|
||||
// MSW settings have no mcp_config → installed section is empty
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/mcp");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const mcpPage = page.getByTestId("mcp-page");
|
||||
await expect(mcpPage).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await expect(mcpPage).toHaveScreenshot("mcp-empty-installed.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("add custom server editor form renders correctly", async ({ page }) => {
|
||||
// The "Add custom server" modal does not depend on settings state so it
|
||||
// is reliably testable regardless of MSW's default settings response.
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/mcp");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const mcpPage = page.getByTestId("mcp-page");
|
||||
await expect(mcpPage).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
// Open the custom server editor
|
||||
await page.getByTestId("mcp-add-custom-server").click();
|
||||
|
||||
// Wait for the editor form to appear inside the modal
|
||||
const modal = page.locator(".fixed.inset-0").last();
|
||||
await expect(modal).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
await expect(page.getByTestId("root-layout")).toHaveScreenshot(
|
||||
"mcp-custom-server-editor.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
test("search query filters marketplace", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/mcp");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const mcpPage = page.getByTestId("mcp-page");
|
||||
await expect(mcpPage).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
// Type "slack" into the unified search box
|
||||
const searchInput = page.getByTestId("mcp-search-input");
|
||||
await expect(searchInput).toBeVisible({ timeout: 5_000 });
|
||||
await searchInput.fill("slack");
|
||||
|
||||
// Wait for the filtered results to stabilise
|
||||
await page.waitForTimeout(300);
|
||||
|
||||
await expect(mcpPage).toHaveScreenshot("mcp-search-filtered.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Iterative snapshot test: install the Slack server from the marketplace.
|
||||
*
|
||||
* Simulates the full user journey with one snapshot per step:
|
||||
* step 1 – marketplace view with Slack card visible
|
||||
* step 2 – Slack install modal open with empty fields
|
||||
* step 3 – Slack install modal with bot-token and team-ID filled in
|
||||
* step 4 – after clicking Install, Slack card appears in Installed section
|
||||
*
|
||||
* The MSW settings PATCH handler persists the new mcp_config so the
|
||||
* refetch shows the installed server without any page.route() trickery.
|
||||
*/
|
||||
test("install Slack from marketplace (iterative snapshots)", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/mcp");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const mcpPage = page.getByTestId("mcp-page");
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(mcpPage).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await test.step("step 1 – marketplace before install", async () => {
|
||||
await expect(page.getByTestId("mcp-marketplace-card-slack")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await expect(mcpPage).toHaveScreenshot(
|
||||
"mcp-slack-install-1-marketplace.png",
|
||||
{
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
await test.step("step 2 – Slack install modal open", async () => {
|
||||
await page.getByTestId("mcp-marketplace-card-slack").click();
|
||||
await expect(page.getByTestId("mcp-install-modal")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"mcp-slack-install-2-modal.png",
|
||||
{
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
await test.step("step 3 – fill in bot token and team ID", async () => {
|
||||
await page
|
||||
.getByTestId("mcp-install-field-SLACK_BOT_TOKEN")
|
||||
.fill("xoxb-test-bot-token-1234567890");
|
||||
await page
|
||||
.getByTestId("mcp-install-field-SLACK_TEAM_ID")
|
||||
.fill("T01ABC123");
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"mcp-slack-install-3-filled.png",
|
||||
{
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
await test.step("step 4 – submit and confirm Slack is installed", async () => {
|
||||
await page.getByTestId("mcp-install-submit").click();
|
||||
|
||||
// Modal should disappear once the mutation completes
|
||||
await expect(page.getByTestId("mcp-install-modal")).not.toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// The installed section should now contain a Slack server card
|
||||
await expect(page.getByTestId("mcp-server-item")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
// Brief wait to let the toast and any animations settle
|
||||
await page.waitForTimeout(400);
|
||||
|
||||
await expect(mcpPage).toHaveScreenshot(
|
||||
"mcp-slack-install-4-installed.png",
|
||||
{
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Iterative snapshot test: manually add a custom SSE server via the editor.
|
||||
*
|
||||
* Simulates the full user journey with one snapshot per step:
|
||||
* step 1 – custom server editor open (SSE type, empty fields)
|
||||
* step 2 – URL field filled in
|
||||
* step 3 – API key field filled in (optional field shown)
|
||||
* step 4 – after clicking Add Server, the custom server appears installed
|
||||
*
|
||||
* Uses the same MSW settings PATCH + GET refetch flow as the marketplace
|
||||
* install test so no page.route() overrides are needed.
|
||||
*/
|
||||
test("add custom SSE server via editor (iterative snapshots)", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/mcp");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const mcpPage = page.getByTestId("mcp-page");
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(mcpPage).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await test.step("step 1 – custom server editor open (empty)", async () => {
|
||||
await page.getByTestId("mcp-add-custom-server").click();
|
||||
await expect(page.getByTestId("mcp-custom-editor")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"mcp-custom-server-1-editor-open.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
await test.step("step 2 – URL filled in", async () => {
|
||||
await page
|
||||
.getByTestId("url-input")
|
||||
.fill("https://api.example-mcp.com/sse");
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"mcp-custom-server-2-url-filled.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
await test.step("step 3 – API key filled in (optional)", async () => {
|
||||
await page.getByTestId("api-key-input").fill("test-api-key-xyz-123");
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"mcp-custom-server-3-all-filled.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
await test.step("step 4 – save and confirm custom server is installed", async () => {
|
||||
await page.getByTestId("submit-button").click();
|
||||
|
||||
// Editor modal should close
|
||||
await expect(page.getByTestId("mcp-custom-editor")).not.toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// Installed section should now contain the custom SSE server
|
||||
await expect(page.getByTestId("mcp-server-item")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await page.waitForTimeout(400);
|
||||
|
||||
await expect(mcpPage).toHaveScreenshot(
|
||||
"mcp-custom-server-4-installed.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,166 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import {
|
||||
clickOnboardingStepButton,
|
||||
ONBOARDING_AGENT_STEP,
|
||||
ONBOARDING_BACKEND_STEP,
|
||||
ONBOARDING_HELLO_STEP,
|
||||
ONBOARDING_LLM_STEP,
|
||||
waitForOnboardingBackendConnected,
|
||||
waitForOnboardingLlmSettingsReady,
|
||||
waitForOnboardingStep,
|
||||
} from "../support/onboarding-helpers";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the 4-step onboarding modal.
|
||||
*
|
||||
* The modal is shown automatically on first visit when the
|
||||
* `openhands-onboarded` key is absent from localStorage.
|
||||
* It lives on the home route (`routes/home.tsx` → `OnboardingHost`).
|
||||
*
|
||||
* Steps:
|
||||
* 0. Check backend — backend form + connection status banner
|
||||
* 1. Choose agent — static agent cards; mock mode renders deterministic ACP options
|
||||
* 2. Setup LLM — LLM settings form (pre-filled with Anthropic/Claude Opus)
|
||||
* 3. Say hello — pre-filled message input to start a conversation
|
||||
*
|
||||
* All four slides are mounted at once; inactive slides are translated
|
||||
* off-screen and clipped by `overflow: clip`. We wait for the
|
||||
* `data-current-step` attribute on the slide rail instead of relying on
|
||||
* `toBeVisible()` for the step container elements, which can be unreliable
|
||||
* for absolutely-positioned off-screen slides.
|
||||
*
|
||||
* In MSW mock mode `/server_info` returns HTTP 200 so the backend health
|
||||
* probe in step 0 resolves to "connected", enabling the Next button.
|
||||
*/
|
||||
|
||||
test.describe.configure({ mode: "serial" });
|
||||
|
||||
async function setupMocks(page: Page) {
|
||||
// removeOnboarded: true ensures the onboarding modal appears.
|
||||
// Analytics consent modal is suppressed (separate concern).
|
||||
await seedLocalStorage(page, { removeOnboarded: true });
|
||||
}
|
||||
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
test.describe("Onboarding Modal Visual Snapshots", () => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
test("onboarding step 0 shows backend connection form", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
// Modal appears because openhands-onboarded is absent
|
||||
await expect(page.getByTestId("onboarding-modal")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
|
||||
|
||||
// Wait for the backend connection banner to settle.
|
||||
// In MSW mode /server_info returns 200, so the health probe should
|
||||
// quickly resolve to "connected".
|
||||
await waitForOnboardingBackendConnected(page);
|
||||
|
||||
const modal = page.getByTestId("onboarding-modal");
|
||||
await expect(modal).toHaveScreenshot(
|
||||
"onboarding-step-0-check-backend.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
test("onboarding step 1 shows agent selection cards", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
await expect(page.getByTestId("onboarding-modal")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
|
||||
|
||||
await waitForOnboardingBackendConnected(page);
|
||||
await clickOnboardingStepButton(page, "onboarding-backend-next");
|
||||
await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);
|
||||
|
||||
const modal = page.getByTestId("onboarding-modal");
|
||||
await expect(modal).toHaveScreenshot("onboarding-step-1-choose-agent.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("onboarding step 2 shows LLM settings form", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
await expect(page.getByTestId("onboarding-modal")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
|
||||
|
||||
// Step 0 → 1
|
||||
await waitForOnboardingBackendConnected(page);
|
||||
await clickOnboardingStepButton(page, "onboarding-backend-next");
|
||||
await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);
|
||||
|
||||
// Step 1 → 2
|
||||
await clickOnboardingStepButton(page, "onboarding-agent-next");
|
||||
await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);
|
||||
|
||||
await waitForOnboardingLlmSettingsReady(page);
|
||||
|
||||
const modal = page.getByTestId("onboarding-modal");
|
||||
await expect(modal).toHaveScreenshot("onboarding-step-2-setup-llm.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("onboarding step 3 shows pre-filled message input", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
await expect(page.getByTestId("onboarding-modal")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
|
||||
|
||||
// Step 0 → 1 (requires backend connected)
|
||||
await waitForOnboardingBackendConnected(page);
|
||||
await clickOnboardingStepButton(page, "onboarding-backend-next");
|
||||
await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);
|
||||
|
||||
// Step 1 → 2
|
||||
await clickOnboardingStepButton(page, "onboarding-agent-next");
|
||||
await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);
|
||||
|
||||
await waitForOnboardingLlmSettingsReady(page);
|
||||
|
||||
// Step 2 → 3:
|
||||
// If the LLM form is dirty (it is, because ONBOARDING_LLM_OVERRIDES differs
|
||||
// from the mock default model), clicking Next will trigger a PATCH settings
|
||||
// mutation. MSW handles the PATCH and resolves onSaveSuccess → onNext.
|
||||
await clickOnboardingStepButton(page, "onboarding-llm-next");
|
||||
await waitForOnboardingStep(page, ONBOARDING_HELLO_STEP);
|
||||
|
||||
// Wait for the say-hello input to be ready
|
||||
await expect(page.getByTestId("onboarding-hello-input")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
const modal = page.getByTestId("onboarding-modal");
|
||||
await expect(modal).toHaveScreenshot("onboarding-step-3-say-hello.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,57 +0,0 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
test("captures /projects workspace browser state", async ({ page }) => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
await seedLocalStorage(page, {
|
||||
extra: [
|
||||
["analytics-consent", "true"],
|
||||
["openhands-telemetry-first-use", "true"],
|
||||
],
|
||||
});
|
||||
|
||||
await page.goto("/conversations", { waitUntil: "domcontentloaded" });
|
||||
|
||||
const consentDialog = page.getByRole("dialog", {
|
||||
name: "Help improve OpenHands",
|
||||
});
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 5000 })
|
||||
.catch(() => undefined);
|
||||
await expect(consentDialog).toHaveCount(0, { timeout: 5000 });
|
||||
|
||||
// The home screen is now a chat-first launcher (#514) — the workspace
|
||||
// dropdown lives inside OpenWorkspaceDialog, opened via "Open workspace".
|
||||
const openWorkspaceButton = page.getByTestId("open-workspace-button");
|
||||
await expect(openWorkspaceButton).toBeEnabled({ timeout: 15_000 });
|
||||
await openWorkspaceButton.click();
|
||||
await expect(page.getByTestId("open-workspace-dialog-body")).toBeVisible();
|
||||
|
||||
const workspaceDropdown = page.getByTestId("workspace-dropdown");
|
||||
await expect(workspaceDropdown).toBeEnabled({ timeout: 15_000 });
|
||||
await workspaceDropdown.click();
|
||||
await page.getByTestId("add-workspaces-button").click();
|
||||
|
||||
const modal = page.getByTestId("folder-browser-modal");
|
||||
await expect(modal).toBeVisible();
|
||||
await expect(page.getByTestId("folder-browser-current-path")).toHaveText(
|
||||
"/projects",
|
||||
);
|
||||
await expect(
|
||||
page.getByTestId("folder-browser-sidebar-/projects"),
|
||||
).toBeVisible();
|
||||
await expect(page.getByTestId("folder-browser-entry-demo-app")).toBeVisible();
|
||||
|
||||
await page.mouse.move(5, 5);
|
||||
await expect(modal).toHaveScreenshot("projects-workspace-browser.png");
|
||||
|
||||
await page.getByTestId("folder-browser-entry-demo-app").click();
|
||||
await expect(page.getByTestId("folder-browser-current-path")).toHaveText(
|
||||
"/projects/demo-app",
|
||||
);
|
||||
await expect(
|
||||
page.getByTestId("folder-browser-entry-web-client"),
|
||||
).toBeVisible();
|
||||
});
|
||||
@@ -1,510 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Post a snapshot test report as a PR comment with embedded images.
|
||||
*
|
||||
* Reads environment variables set by the snapshot-tests.yml workflow:
|
||||
* GH_TOKEN — GitHub token for API calls and git push
|
||||
* PR_NUMBER — Pull request number
|
||||
* REPO — "owner/repo"
|
||||
* RUN_ID — GitHub Actions run ID
|
||||
* HEAD_REF — PR branch name (used only for the log message)
|
||||
* MAIN_BASELINES_DIR — Path to the copied main-branch baselines (e.g. /tmp/main-baselines)
|
||||
* SNAPSHOTS_APPROVED — "true" when the update-snapshots label is set
|
||||
*
|
||||
* The script:
|
||||
* 1. Scans tests/e2e/__snapshots__/ (PR's current snapshots) and MAIN_BASELINES_DIR
|
||||
* 2. Classifies each snapshot as NEW, CHANGED, or UNCHANGED
|
||||
* 3. For CHANGED: locates diff/actual/expected images in test-results/
|
||||
* 4. Creates an orphan commit on snapshot-artifacts/pr-<N> with the images and
|
||||
* pushes it there (NOT to the PR branch — avoids invalidating required checks)
|
||||
* 5. Posts or updates a PR comment with inline image tables using raw.githubusercontent.com URLs
|
||||
*/
|
||||
|
||||
import { execSync } from "node:child_process";
|
||||
import {
|
||||
appendFileSync,
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
} from "node:fs";
|
||||
import { basename, dirname, join, relative } from "node:path";
|
||||
|
||||
// ── Environment ────────────────────────────────────────────────────────────
|
||||
|
||||
const GH_TOKEN = requireEnv("GH_TOKEN");
|
||||
const PR_NUMBER = requireEnv("PR_NUMBER");
|
||||
const REPO = requireEnv("REPO");
|
||||
const RUN_ID = requireEnv("RUN_ID");
|
||||
const HEAD_REF = requireEnv("HEAD_REF");
|
||||
const MAIN_BASELINES_DIR =
|
||||
process.env.MAIN_BASELINES_DIR ?? "/tmp/main-baselines";
|
||||
const SNAPSHOTS_APPROVED = process.env.SNAPSHOTS_APPROVED === "true";
|
||||
const GENERATE_OUTCOME = process.env.GENERATE_OUTCOME ?? "success";
|
||||
const COMPARE_OUTCOME = process.env.COMPARE_OUTCOME ?? "success";
|
||||
|
||||
const SNAPSHOTS_DIR = "tests/e2e/__snapshots__";
|
||||
// The workflow saves comparison test-results to this path before the
|
||||
// --update-snapshots pass wipes test-results/. Fall back to the default
|
||||
// Playwright output directory when running outside CI.
|
||||
const TEST_RESULTS_DIR =
|
||||
process.env.COMPARISON_RESULTS_DIR ?? "test-results";
|
||||
// Images are pushed to this dedicated branch, NOT to the PR branch.
|
||||
// Pushing to the PR branch with [skip ci] was blocking required checks on the HEAD commit.
|
||||
const ARTIFACTS_BRANCH = `snapshot-artifacts/pr-${PR_NUMBER}`;
|
||||
const COMMENT_MARKER = "<!-- snapshot-test-report -->";
|
||||
const GITHUB_API = process.env.GITHUB_API_URL ?? "https://api.github.com";
|
||||
const RAW_BASE = "https://raw.githubusercontent.com";
|
||||
const [OWNER, REPO_NAME] = REPO.split("/");
|
||||
|
||||
function requireEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) {
|
||||
throw new Error(`Missing required environment variable: ${name}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
// ── File utilities ─────────────────────────────────────────────────────────
|
||||
|
||||
/** Recursively find all files with a given extension under a directory. */
|
||||
function findFiles(dir, ext) {
|
||||
if (!existsSync(dir)) return [];
|
||||
const results = [];
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
results.push(...findFiles(full, ext));
|
||||
} else if (!ext || entry.name.endsWith(ext)) {
|
||||
results.push(full);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Copy a file, creating parent directories as needed. */
|
||||
function copyFile(src, dest) {
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
copyFileSync(src, dest);
|
||||
}
|
||||
|
||||
// ── Snapshot classification ────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Classify snapshots as changed, new, or unchanged.
|
||||
*
|
||||
* "Changed" is determined by whether Playwright produced a diff file in
|
||||
* test-results/ — this respects the configured threshold/maxDiffPixels so
|
||||
* minor rendering noise below the tolerance is not flagged as a change.
|
||||
*
|
||||
* "New" means the snapshot exists in the PR but has no baseline on main.
|
||||
* "Unchanged" means all other snapshots that Playwright accepted.
|
||||
*/
|
||||
function classifySnapshots() {
|
||||
const currentFiles = findFiles(SNAPSHOTS_DIR, ".png");
|
||||
const baselineFiles = findFiles(MAIN_BASELINES_DIR, ".png");
|
||||
|
||||
// Build a set of relative paths from the baselines directory
|
||||
const baselineRelPaths = new Set(
|
||||
baselineFiles.map((f) => relative(MAIN_BASELINES_DIR, f)),
|
||||
);
|
||||
|
||||
// Index diff files from test-results by their canonical snapshot name.
|
||||
// Playwright names diff files "<snapshot-name>-<N>-diff.png" — strip the
|
||||
// Playwright-appended "-<N>" so "sidebar-filter-menu-1" → "sidebar-filter-menu".
|
||||
const allDiffFiles = findFiles(TEST_RESULTS_DIR, "-diff.png");
|
||||
const diffBySnapshotName = new Map();
|
||||
for (const diffFile of allDiffFiles) {
|
||||
const key = basename(diffFile, "-diff.png").replace(/-\d+$/, "");
|
||||
if (!diffBySnapshotName.has(key)) {
|
||||
diffBySnapshotName.set(key, diffFile);
|
||||
}
|
||||
}
|
||||
|
||||
const changed = [];
|
||||
const newSnapshots = [];
|
||||
const unchanged = [];
|
||||
|
||||
for (const currentFile of currentFiles) {
|
||||
const relPath = relative(SNAPSHOTS_DIR, currentFile);
|
||||
const snapshotName = basename(relPath, ".png");
|
||||
|
||||
if (!baselineRelPaths.has(relPath)) {
|
||||
newSnapshots.push({ relPath, currentFile });
|
||||
} else if (diffBySnapshotName.has(snapshotName)) {
|
||||
const diffFile = diffBySnapshotName.get(snapshotName);
|
||||
changed.push({
|
||||
relPath,
|
||||
currentFile,
|
||||
baselineFile: join(MAIN_BASELINES_DIR, relPath),
|
||||
diffFile: existsSync(diffFile) ? diffFile : null,
|
||||
});
|
||||
} else {
|
||||
unchanged.push({ relPath });
|
||||
}
|
||||
}
|
||||
|
||||
return { changed, newSnapshots, unchanged };
|
||||
}
|
||||
|
||||
// ── Image publishing ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Push snapshot images to a dedicated orphan branch (snapshot-artifacts/pr-<N>)
|
||||
* so they can be embedded in the PR comment via raw.githubusercontent.com URLs.
|
||||
*
|
||||
* Images are intentionally NOT pushed to the PR branch. Pushing to the PR branch
|
||||
* (even with [skip ci]) invalidates required checks on the HEAD commit and hangs
|
||||
* the PR. The orphan artifacts branch is invisible to all CI workflows.
|
||||
*
|
||||
* Returns the commit SHA on the artifacts branch, or null on failure.
|
||||
*/
|
||||
function publishImages(changed, newSnapshots) {
|
||||
const hasImages = changed.length > 0 || newSnapshots.length > 0;
|
||||
if (!hasImages) return null;
|
||||
|
||||
// Build a temp directory containing only the images, mirroring the layout
|
||||
// that buildComment expects: changed/<relPath>-{actual,expected,diff}.png
|
||||
// new/<relPath>.png
|
||||
const tmpDir = execSync("mktemp -d").toString().trim();
|
||||
try {
|
||||
for (const { relPath, currentFile, baselineFile, diffFile } of changed) {
|
||||
const dest = join(tmpDir, "changed", relPath);
|
||||
copyFile(currentFile, dest.replace(".png", "-actual.png"));
|
||||
if (baselineFile && existsSync(baselineFile)) {
|
||||
copyFile(baselineFile, dest.replace(".png", "-expected.png"));
|
||||
}
|
||||
if (diffFile) {
|
||||
copyFile(diffFile, dest.replace(".png", "-diff.png"));
|
||||
}
|
||||
}
|
||||
for (const { relPath, currentFile } of newSnapshots) {
|
||||
copyFile(currentFile, join(tmpDir, "new", relPath));
|
||||
}
|
||||
|
||||
// Create an orphan commit in tmpDir and force-push to the artifacts branch.
|
||||
// Using a fresh git repo avoids touching any tracked files in the PR checkout.
|
||||
const git = (cmd) => execSync(`git -C "${tmpDir}" ${cmd}`);
|
||||
git("init");
|
||||
git(`config user.name "github-actions[bot]"`);
|
||||
git(`config user.email "41898282+github-actions[bot]@users.noreply.github.com"`);
|
||||
git("add .");
|
||||
git(`commit -m "snapshot images for PR #${PR_NUMBER} run ${RUN_ID}"`);
|
||||
git(
|
||||
`push --force "https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" ` +
|
||||
`HEAD:refs/heads/${ARTIFACTS_BRANCH}`,
|
||||
);
|
||||
return git("rev-parse HEAD").toString().trim();
|
||||
} catch (err) {
|
||||
console.error("Warning: failed to push snapshot images:", err.message);
|
||||
return null;
|
||||
} finally {
|
||||
execSync(`rm -rf "${tmpDir}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Markdown generation ────────────────────────────────────────────────────
|
||||
|
||||
function rawUrl(commitSha, filePath) {
|
||||
return `${RAW_BASE}/${OWNER}/${REPO_NAME}/${commitSha}/${filePath}`;
|
||||
}
|
||||
|
||||
/** Extract the human-readable spec name from a relative snapshot path.
|
||||
* "snapshots/mcp-page.snapshot.spec.ts/chromium/foo.png" → "mcp-page"
|
||||
*/
|
||||
function specFromRelPath(relPath) {
|
||||
const segment = relPath.replace(/^snapshots\//, "").split("/")[0] ?? "";
|
||||
return segment.replace(".snapshot.spec.ts", "");
|
||||
}
|
||||
|
||||
/** Group an array of snapshot objects by their spec name. */
|
||||
function groupBySpec(items) {
|
||||
const groups = /** @type {Map<string, typeof items>} */ (new Map());
|
||||
for (const item of items) {
|
||||
const spec = specFromRelPath(item.relPath);
|
||||
if (!groups.has(spec)) groups.set(spec, []);
|
||||
groups.get(spec).push(item);
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
function buildComment(changed, newSnapshots, unchanged, commitSha) {
|
||||
const total = changed.length + newSnapshots.length + unchanged.length;
|
||||
const hasDifferences = changed.length > 0;
|
||||
|
||||
let statusIcon;
|
||||
let statusText;
|
||||
if (hasDifferences && SNAPSHOTS_APPROVED) {
|
||||
statusIcon = "✅";
|
||||
statusText =
|
||||
`${changed.length} snapshot${changed.length !== 1 ? "s" : ""} changed — ` +
|
||||
`acknowledged via the \`update-snapshots\` label. New baselines will be uploaded when this PR merges.`;
|
||||
} else if (hasDifferences) {
|
||||
statusIcon = "❌";
|
||||
statusText =
|
||||
`${changed.length} snapshot${changed.length !== 1 ? "s" : ""} differ from the main branch baseline${changed.length !== 1 ? "s" : ""}. ` +
|
||||
`Add the \`update-snapshots\` label to acknowledge intentional changes.`;
|
||||
} else if (unchanged.length === 0) {
|
||||
statusIcon = "✅";
|
||||
statusText =
|
||||
`No baseline found on main — all ${newSnapshots.length} snapshot${newSnapshots.length !== 1 ? "s" : ""} are new ` +
|
||||
`and will become the baseline once this PR merges.`;
|
||||
} else {
|
||||
statusIcon = "✅";
|
||||
statusText = "All snapshots match the main branch baselines.";
|
||||
}
|
||||
|
||||
const lines = [
|
||||
COMMENT_MARKER,
|
||||
`## 📸 Snapshot Test Report`,
|
||||
"",
|
||||
];
|
||||
|
||||
if (COMPARE_OUTCOME === "failure") {
|
||||
lines.push(
|
||||
`> [!WARNING]`,
|
||||
`> **Snapshot comparison step crashed** (timeout, OOM, or runner error) — diff results below may be incomplete or absent.`,
|
||||
`> Check the [CI logs](https://github.com/${REPO}/actions/runs/${RUN_ID}) for the full error output (look for the "Run snapshot comparison" step).`,
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
if (GENERATE_OUTCOME === "failure") {
|
||||
lines.push(
|
||||
`> [!WARNING]`,
|
||||
`> **One or more snapshot tests crashed during generation** — some snapshots below may be incomplete.`,
|
||||
`> Check the [CI logs](https://github.com/${REPO}/actions/runs/${RUN_ID}) for the full error output (look for the "Generate current PR snapshots" step).`,
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
`${statusIcon} ${statusText}`,
|
||||
"",
|
||||
`| Category | Count |`,
|
||||
`|---|---|`,
|
||||
`| 🔴 Changed | ${changed.length} |`,
|
||||
`| 🆕 New | ${newSnapshots.length} |`,
|
||||
`| ✅ Unchanged | ${unchanged.length} |`,
|
||||
`| **Total** | **${total}** |`,
|
||||
"",
|
||||
);
|
||||
|
||||
if (hasDifferences && !SNAPSHOTS_APPROVED) {
|
||||
lines.push(
|
||||
`> **How to resolve:**`,
|
||||
`> - **Unintentional diffs** — the baselines on \`main\` may have moved since this branch was created. Merge the latest \`main\` into this branch and re-run CI.`,
|
||||
`> - **Intentional changes** — add the \`update-snapshots\` label. CI will pass and the new screenshots become the baseline when this PR merges.`,
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
// Changed snapshots — grouped by spec file
|
||||
if (changed.length > 0) {
|
||||
lines.push(
|
||||
`<details>`,
|
||||
`<summary>🔴 Changed snapshots (${changed.length})</summary>`,
|
||||
"",
|
||||
);
|
||||
for (const [spec, items] of groupBySpec(changed)) {
|
||||
lines.push(
|
||||
`### \`${spec}\`${items.length > 1 ? ` — ${items.length} snapshots` : ""}`,
|
||||
"",
|
||||
);
|
||||
for (const { relPath, diffFile } of items) {
|
||||
const name = basename(relPath, ".png");
|
||||
lines.push(`**${name}**`, "");
|
||||
|
||||
if (commitSha) {
|
||||
const base = join("changed", relPath);
|
||||
const expectedUrl = rawUrl(commitSha, base.replace(".png", "-expected.png"));
|
||||
const actualUrl = rawUrl(commitSha, base.replace(".png", "-actual.png"));
|
||||
const diffUrl = diffFile
|
||||
? rawUrl(commitSha, base.replace(".png", "-diff.png"))
|
||||
: null;
|
||||
|
||||
lines.push(
|
||||
`| Expected (main) | Actual (PR) |${diffUrl ? " Diff |" : ""}`,
|
||||
`|---|---|${diffUrl ? "---|" : ""}`,
|
||||
`|  |  |${diffUrl ? `  |` : ""}`,
|
||||
"",
|
||||
);
|
||||
} else {
|
||||
lines.push(
|
||||
`_Images could not be embedded (fork PR or push failed). ` +
|
||||
`Download the [\`snapshot-test-results\` artifact](https://github.com/${REPO}/actions/runs/${RUN_ID}) for visual diffs._`,
|
||||
"",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
lines.push(`</details>`, "");
|
||||
}
|
||||
|
||||
// New snapshots — grouped by spec file
|
||||
if (newSnapshots.length > 0) {
|
||||
lines.push(
|
||||
`<details>`,
|
||||
`<summary>🆕 New snapshots (${newSnapshots.length})</summary>`,
|
||||
"",
|
||||
`These snapshots have no baseline on main and will become the new baseline once this PR merges.`,
|
||||
"",
|
||||
);
|
||||
if (commitSha) {
|
||||
for (const [spec, items] of groupBySpec(newSnapshots)) {
|
||||
lines.push(
|
||||
`### \`${spec}\`${items.length > 1 ? ` — ${items.length} snapshots` : ""}`,
|
||||
"",
|
||||
);
|
||||
for (const { relPath } of items) {
|
||||
const name = basename(relPath, ".png");
|
||||
const actualUrl = rawUrl(commitSha, join("new", relPath));
|
||||
lines.push(`**${name}**`, "", ``, "");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
lines.push(
|
||||
`_Images could not be embedded (fork PR or push failed). ` +
|
||||
`Download the [\`snapshot-test-results\` artifact](https://github.com/${REPO}/actions/runs/${RUN_ID}) for screenshots._`,
|
||||
"",
|
||||
);
|
||||
}
|
||||
lines.push(`</details>`, "");
|
||||
}
|
||||
|
||||
// Unchanged — compact grouped list by spec file
|
||||
if (unchanged.length > 0) {
|
||||
lines.push(
|
||||
`<details>`,
|
||||
`<summary>✅ Unchanged snapshots (${unchanged.length})</summary>`,
|
||||
"",
|
||||
);
|
||||
for (const [spec, items] of groupBySpec(unchanged)) {
|
||||
lines.push(`**\`${spec}\`**`);
|
||||
for (const { relPath } of items) {
|
||||
lines.push(`- ${basename(relPath, ".png")}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
lines.push(`</details>`, "");
|
||||
}
|
||||
|
||||
lines.push(
|
||||
`---`,
|
||||
`_Generated by the [Snapshot Tests](https://github.com/${REPO}/actions/runs/${RUN_ID}) workflow. ` +
|
||||
`This comment was created by an AI agent (OpenHands) on behalf of the repo maintainers._`,
|
||||
);
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
// ── GitHub API ─────────────────────────────────────────────────────────────
|
||||
|
||||
async function githubFetch(path, options = {}) {
|
||||
const url = path.startsWith("http") ? path : `${GITHUB_API}${path}`;
|
||||
const res = await fetch(url, {
|
||||
...options,
|
||||
headers: {
|
||||
Authorization: `Bearer ${GH_TOKEN}`,
|
||||
Accept: "application/vnd.github.v3+json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "agent-canvas-snapshot-bot",
|
||||
...options.headers,
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`GitHub API ${res.status} for ${url}: ${text}`);
|
||||
}
|
||||
// DELETE returns 204 No Content
|
||||
if (res.status === 204) return null;
|
||||
return res.headers.get("content-type")?.includes("json") ? res.json() : res.text();
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete any existing snapshot report comment and post a fresh one.
|
||||
*
|
||||
* We always delete-then-create (rather than edit in-place) so that the new
|
||||
* comment always references the current run's image URLs. Editing would
|
||||
* leave stale raw.githubusercontent.com URLs pointing at the previous run's
|
||||
* .pr/snapshots/<old_run_id>/ images.
|
||||
*/
|
||||
async function postFreshComment(body) {
|
||||
const comments = await githubFetch(
|
||||
`/repos/${OWNER}/${REPO_NAME}/issues/${PR_NUMBER}/comments`,
|
||||
);
|
||||
const existing = comments.find((c) => c.body.includes(COMMENT_MARKER));
|
||||
|
||||
if (existing) {
|
||||
await githubFetch(
|
||||
`/repos/${OWNER}/${REPO_NAME}/issues/comments/${existing.id}`,
|
||||
{ method: "DELETE" },
|
||||
);
|
||||
console.log(`Deleted stale PR comment ${existing.id}`);
|
||||
}
|
||||
|
||||
await githubFetch(
|
||||
`/repos/${OWNER}/${REPO_NAME}/issues/${PR_NUMBER}/comments`,
|
||||
{ method: "POST", body: JSON.stringify({ body }) },
|
||||
);
|
||||
console.log("Posted fresh PR comment");
|
||||
}
|
||||
|
||||
// ── Main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
console.log(`Classifying snapshots...`);
|
||||
console.log(` Current snapshots: ${SNAPSHOTS_DIR}`);
|
||||
console.log(` Main baselines: ${MAIN_BASELINES_DIR}`);
|
||||
|
||||
const { changed, newSnapshots, unchanged } = classifySnapshots();
|
||||
|
||||
console.log(
|
||||
` Changed: ${changed.length}, New: ${newSnapshots.length}, Unchanged: ${unchanged.length}`,
|
||||
);
|
||||
|
||||
let commitSha = null;
|
||||
if (changed.length > 0 || newSnapshots.length > 0) {
|
||||
console.log(`Publishing images to branch ${ARTIFACTS_BRANCH}...`);
|
||||
commitSha = publishImages(changed, newSnapshots);
|
||||
if (commitSha) {
|
||||
console.log(` Images published at ${commitSha}`);
|
||||
} else {
|
||||
console.log(` Image publishing failed — comment will link to artifact download`);
|
||||
}
|
||||
}
|
||||
|
||||
const body = buildComment(changed, newSnapshots, unchanged, commitSha);
|
||||
// Commenting requires `pull-requests: write`. GitHub silently downgrades
|
||||
// that to `read` for `pull_request` runs triggered from a fork, so the
|
||||
// POST below 403s on cross-repo PRs. Treat the failure the same way
|
||||
// publishImages already treats its push failure: log it and keep going
|
||||
// so the workflow can still write `has_changes` and let the
|
||||
// "Fail if differences" step do its job.
|
||||
try {
|
||||
await postFreshComment(body);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
"Warning: failed to post PR comment (expected on fork PRs):",
|
||||
err.message,
|
||||
);
|
||||
}
|
||||
|
||||
// Tell the workflow whether there are actual pixel-diff failures so the
|
||||
// "Fail if differences" step can distinguish changed snapshots (should
|
||||
// fail CI) from missing baselines (new tests from this PR, should pass).
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
`has_changes=${changed.length > 0}\n`,
|
||||
);
|
||||
console.log(` has_changes=${changed.length > 0} written to GITHUB_OUTPUT`);
|
||||
}
|
||||
|
||||
console.log("Done.");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("post-snapshot-comment failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,247 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for UI pages.
|
||||
*
|
||||
* These tests capture screenshots of pages and compare them against
|
||||
* baseline images to detect unintended visual regressions.
|
||||
*
|
||||
* To update baselines after intentional UI changes:
|
||||
* npm run test:e2e:snapshots:update
|
||||
*/
|
||||
|
||||
/** Mock settings response with analytics consent already given */
|
||||
const SETTINGS_WITH_CONSENT = {
|
||||
llm_model: "anthropic/claude-sonnet-4-20250514",
|
||||
llm_base_url: "",
|
||||
agent: "CodeActAgent",
|
||||
language: "en",
|
||||
llm_api_key: null,
|
||||
llm_api_key_set: true,
|
||||
search_api_key_set: false,
|
||||
confirmation_mode: false,
|
||||
security_analyzer: "llm",
|
||||
remote_runtime_resource_factor: 1,
|
||||
provider_tokens_set: { github: "" },
|
||||
enable_default_condenser: true,
|
||||
condenser_max_size: 240,
|
||||
enable_sound_notifications: false,
|
||||
// Analytics consent already given - modal won't show
|
||||
user_consents_to_analytics: false,
|
||||
enable_proactive_conversation_starters: false,
|
||||
enable_solvability_analysis: false,
|
||||
max_budget_per_task: null,
|
||||
};
|
||||
|
||||
/** Mock settings response with analytics consent pending (null = show modal) */
|
||||
const SETTINGS_WITHOUT_CONSENT = {
|
||||
...SETTINGS_WITH_CONSENT,
|
||||
// null means user hasn't made a choice yet - modal will show
|
||||
user_consents_to_analytics: null,
|
||||
};
|
||||
|
||||
/**
|
||||
* Sets up common API mocks for snapshot tests.
|
||||
* @param page - Playwright page
|
||||
* @param showConsentModal - Whether to show the analytics consent modal
|
||||
*/
|
||||
async function setupMocks(page: Page, showConsentModal = false) {
|
||||
await seedLocalStorage(page, { showConsentModal });
|
||||
|
||||
// Mock settings API - consent modal appears when user_consents_to_analytics is null
|
||||
const settingsResponse = showConsentModal
|
||||
? SETTINGS_WITHOUT_CONSENT
|
||||
: SETTINGS_WITH_CONSENT;
|
||||
|
||||
await page.route("**/api/settings", async (route) => {
|
||||
if (route.request().method() === "GET") {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify(settingsResponse),
|
||||
});
|
||||
} else {
|
||||
await route.continue();
|
||||
}
|
||||
});
|
||||
|
||||
// Mock settings schemas
|
||||
await page.route("**/api/settings/agent-schema", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({}),
|
||||
});
|
||||
});
|
||||
|
||||
await page.route("**/api/settings/conversation-schema", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({}),
|
||||
});
|
||||
});
|
||||
|
||||
// Mock conversations search for home page
|
||||
await page.route("**/api/conversations/search**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ results: [] }),
|
||||
});
|
||||
});
|
||||
|
||||
await page.route("**/api/workspaces**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ workspaces: [], workspaceParents: [] }),
|
||||
});
|
||||
});
|
||||
|
||||
await page.route("**/api/bash/execute_bash_command", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
command: "",
|
||||
exit_code: 0,
|
||||
output: "",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
// Mock file APIs to prevent proxy errors
|
||||
await page.route("**/api/file/**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ path: "/home", subdirs: [] }),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Dismisses the analytics consent modal if it appears.
|
||||
*/
|
||||
async function dismissConsentModal(page: Page) {
|
||||
const consentDialog = page.getByRole("dialog", {
|
||||
name: "Help improve OpenHands",
|
||||
});
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3000 })
|
||||
.catch(() => undefined);
|
||||
await expect(consentDialog).toHaveCount(0, { timeout: 3000 });
|
||||
}
|
||||
|
||||
test.describe("UI Visual Snapshots", () => {
|
||||
// Increase timeout for this test - modal loading can be slow
|
||||
test.setTimeout(60000);
|
||||
|
||||
test("Analytics consent modal renders correctly", async ({ page }) => {
|
||||
// Use setupMocks with showConsentModal=true to guarantee modal appears
|
||||
await setupMocks(page, true);
|
||||
|
||||
await page.goto("/conversations", { waitUntil: "networkidle" });
|
||||
|
||||
// Wait for the page to stabilize
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toBeVisible({ timeout: 15000 });
|
||||
|
||||
// Wait for the consent modal (lazy-loaded) with extended timeout
|
||||
const consentModal = page.getByRole("dialog", {
|
||||
name: "Help improve OpenHands",
|
||||
});
|
||||
await expect(consentModal).toBeVisible({ timeout: 15000 });
|
||||
|
||||
// Snapshot the full page with the consent modal
|
||||
await expect(rootLayout).toHaveScreenshot("analytics-consent-modal.png", {
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
});
|
||||
});
|
||||
|
||||
test("Home page renders correctly", async ({ page }) => {
|
||||
await setupMocks(page, false);
|
||||
await page.goto("/conversations");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
const homeScreen = page.getByTestId("home-screen");
|
||||
await expect(homeScreen).toBeVisible({ timeout: 15_000 });
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("home-screen.png", {
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
});
|
||||
});
|
||||
|
||||
test("Settings page renders correctly", async ({ page }) => {
|
||||
await setupMocks(page, false);
|
||||
await page.goto("/settings");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toBeVisible({ timeout: 15000 });
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot("settings-page.png", {
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
});
|
||||
});
|
||||
|
||||
test("Settings app page renders correctly", async ({ page }) => {
|
||||
await setupMocks(page, false);
|
||||
await page.goto("/settings/app");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toBeVisible({ timeout: 15000 });
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
await expect(rootLayout).toHaveScreenshot("settings-app-page.png", {
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
});
|
||||
});
|
||||
|
||||
test("Add backend modal renders correctly", async ({ page }) => {
|
||||
await setupMocks(page, false);
|
||||
|
||||
// Mock the server-info health-check endpoint. Without this, the
|
||||
// periodic health poll may prevent networkidle or cause hangs.
|
||||
await page.route("**/server_info", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ version: "mock" }),
|
||||
});
|
||||
});
|
||||
|
||||
await page.goto("/conversations");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
const homeScreen = page.getByTestId("home-screen");
|
||||
await expect(homeScreen).toBeVisible({ timeout: 15000 });
|
||||
|
||||
// The backend selector uses openOnHover, so hovering opens the
|
||||
// dropdown. Clicking the toggle would close it again, so we hover
|
||||
// to open and then click the menu item directly.
|
||||
const backendSelector = page.getByTestId("backend-selector");
|
||||
await expect(backendSelector).toBeVisible({ timeout: 15_000 });
|
||||
await backendSelector.hover();
|
||||
await page.getByTestId("add-backend-menu-item").click();
|
||||
|
||||
const addBackendModal = page.getByTestId("add-backend-modal");
|
||||
await expect(addBackendModal).toBeVisible({ timeout: 15000 });
|
||||
|
||||
await expect(addBackendModal).toHaveScreenshot("add-backend-modal.png", {
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: "disabled",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,161 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the Secrets Settings page (/settings/secrets).
|
||||
*
|
||||
* MSW pre-seeds two secrets in src/mocks/secrets-handlers.ts:
|
||||
* - OpenAI_API_Key
|
||||
* - Google_Maps_API_Key
|
||||
*
|
||||
* All five snapshots tell an iterative story:
|
||||
* 1. Default list (two pre-seeded rows)
|
||||
* 2. "Add New Secret" form open (empty)
|
||||
* 3. Form filled in with name + value
|
||||
* 4. After saving — list shows a third secret
|
||||
* 5. Delete confirmation modal open
|
||||
*/
|
||||
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
async function setupMocks(page: Page) {
|
||||
await seedLocalStorage(page);
|
||||
// Keep conversations sidebar quiet (page.route wins for this cross-origin path
|
||||
// only; for same-origin MSW takes precedence and we rely on MSW data).
|
||||
await page.route("**/api/conversations/search**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ items: [], next_page_id: null }),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe("Settings – Secrets Visual Snapshots", () => {
|
||||
test.setTimeout(60_000);
|
||||
// Run in serial so the MSW secrets Map stays consistent across steps
|
||||
test.describe.configure({ mode: "serial" });
|
||||
|
||||
test("secrets list shows two pre-seeded rows", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/secrets");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// Both secrets must be visible before taking the snapshot
|
||||
await expect(
|
||||
page.getByTestId("secret-item").first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
await expect(page.getByTestId("secret-item")).toHaveCount(2);
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("secrets-list.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("add-new-secret form opens empty", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/secrets");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
await expect(
|
||||
page.getByTestId("secret-item").first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
// Click the "Add New Secret" button
|
||||
await page.getByTestId("add-secret-button").click();
|
||||
|
||||
// The inline form should appear
|
||||
await expect(page.getByTestId("value-input")).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("secrets-add-form.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("add-new-secret form filled with name and value", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/secrets");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
await expect(
|
||||
page.getByTestId("secret-item").first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
await page.getByTestId("add-secret-button").click();
|
||||
await expect(page.getByTestId("value-input")).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
// Fill the secret name field
|
||||
await page.getByTestId("name-input").fill("ANTHROPIC_API_KEY");
|
||||
|
||||
// Fill the secret value field
|
||||
await page.getByTestId("value-input").fill("sk-ant-snapshot-test-key");
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("secrets-add-form-filled.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("after saving a secret the list shows three rows", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/secrets");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
await expect(
|
||||
page.getByTestId("secret-item").first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
await page.getByTestId("add-secret-button").click();
|
||||
await expect(page.getByTestId("value-input")).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
await page.getByTestId("name-input").fill("ANTHROPIC_API_KEY");
|
||||
await page.getByTestId("value-input").fill("sk-ant-snapshot-test-key");
|
||||
|
||||
// Submit the form ("Add secret" is the i18n text for the submit button in add mode)
|
||||
await page.getByTestId("submit-button").click();
|
||||
|
||||
// Wait for the form to close and the new secret to appear
|
||||
await expect(page.getByTestId("secret-item")).toHaveCount(3, {
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("secrets-after-save.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("delete confirmation modal is shown for a secret", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/secrets");
|
||||
await dismissConsentModal(page);
|
||||
|
||||
await expect(
|
||||
page.getByTestId("secret-item").first(),
|
||||
).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
// Click the delete button on the first secret row
|
||||
await page.getByTestId("delete-secret-button").first().click();
|
||||
|
||||
// The ConfirmationModal should appear
|
||||
await expect(
|
||||
page.getByTestId("confirmation-modal"),
|
||||
).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("secrets-delete-confirm.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,173 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for:
|
||||
* - /settings/verification (Confirmation Mode toggle + Security Analyzer)
|
||||
* - /settings/condenser (Schema-driven condenser form)
|
||||
*
|
||||
* The verification page is now fully schema-driven (no hand-written header
|
||||
* for the confirmation-mode toggle), and `confirmation_mode` is a
|
||||
* `prominence: "major"` field so it lives in the Advanced/All views, not
|
||||
* Basic. Each verification test therefore switches to the "All" view
|
||||
* before snapshotting so both the critic-related and confirmation-mode
|
||||
* controls are on screen.
|
||||
*
|
||||
* MSW provides the default settings (confirmation_mode: false) so the first
|
||||
* verification snapshot shows the toggle in the OFF position with a dimmed
|
||||
* Save Changes button. Toggling it ON reveals the Security Analyzer dropdown
|
||||
* and enables Save Changes — captured in the second snapshot.
|
||||
*
|
||||
* There is no separate "dirty" snapshot for confirmation mode because clicking
|
||||
* the toggle IS the dirty action — the "ON" snapshot already captures the
|
||||
* dirty/enabled-Save-Changes state.
|
||||
*/
|
||||
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
async function setupMocks(page: Page) {
|
||||
await seedLocalStorage(page);
|
||||
}
|
||||
|
||||
test.describe("Settings – Verification & Condenser Visual Snapshots", () => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
/**
|
||||
* Helper: wait for the schema-driven verification page to be ready, then
|
||||
* switch to the "All" view so `confirmation_mode` (a `major`-prominence
|
||||
* field) is rendered alongside the `critic_enabled` toggle.
|
||||
*
|
||||
* Readiness signal: the "Enable Critic" label is the first critical-
|
||||
* prominence field and is always visible once the schema is loaded,
|
||||
* regardless of view. The underlying checkbox is `hidden` in the DOM
|
||||
* (styled toggle pattern), so we assert on the label text instead.
|
||||
*/
|
||||
async function waitForVerificationPage(page: Page) {
|
||||
await expect(
|
||||
page.getByTestId("sdk-settings-verification.critic_enabled"),
|
||||
).toBeAttached({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await page.getByTestId("sdk-section-all-toggle").click();
|
||||
// The visible label is "Confirmation Mode" — the schema's raw "Confirmation
|
||||
// mode" goes through the i18n translation table (SCHEMA$CONFIRMATION_MODE$LABEL).
|
||||
await expect(page.getByText("Confirmation Mode")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
}
|
||||
|
||||
async function ensureCriticEnabled(page: Page) {
|
||||
const apiKeyInput = page.getByTestId(
|
||||
"sdk-settings-verification.critic_api_key",
|
||||
);
|
||||
if (!(await apiKeyInput.isVisible())) {
|
||||
await page
|
||||
.locator(
|
||||
`label:has([data-testid="sdk-settings-verification.critic_enabled"])`,
|
||||
)
|
||||
.click();
|
||||
}
|
||||
await expect(apiKeyInput).toBeVisible({ timeout: 5_000 });
|
||||
}
|
||||
|
||||
test("verification settings with confirmation mode OFF (default)", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/verification");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
await waitForVerificationPage(page);
|
||||
|
||||
// Security Analyzer combobox must NOT be present when confirmation_mode
|
||||
// is off (it depends_on the toggle). HeroUI Autocomplete does not
|
||||
// forward data-testid, so match by accessible role + label (case
|
||||
// insensitive since the schema label is "Security analyzer").
|
||||
await expect(
|
||||
page.getByRole("combobox", { name: /security analyzer/i }),
|
||||
).toHaveCount(0);
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("verification-settings-off.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("verification settings with critic enabled shows API key guidance", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/verification");
|
||||
await dismissConsentModal(page);
|
||||
await waitForVerificationPage(page);
|
||||
|
||||
await ensureCriticEnabled(page);
|
||||
await expect(
|
||||
page.getByText(
|
||||
/Critic API Key is the same as your OpenHands Provider LLM Key/i,
|
||||
),
|
||||
).toBeVisible();
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"verification-settings-critic-enabled.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
test("verification settings with confirmation mode ON shows security analyzer", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/verification");
|
||||
await dismissConsentModal(page);
|
||||
await waitForVerificationPage(page);
|
||||
|
||||
// The schema-rendered SettingsSwitch's underlying <input type="checkbox">
|
||||
// is `hidden`; clicking the visible label that wraps it activates the
|
||||
// form control through standard HTML label–control association. The
|
||||
// testId now comes from SchemaField's `sdk-settings-${field.key}` scheme.
|
||||
await page
|
||||
.locator(`label:has([data-testid="sdk-settings-confirmation_mode"])`)
|
||||
.click();
|
||||
|
||||
// Security Analyzer dropdown should now appear.
|
||||
await expect(
|
||||
page.getByRole("combobox", { name: /security analyzer/i }),
|
||||
).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("verification-settings-on.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("condenser settings page renders schema form", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/settings/condenser");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// The wrapper div with data-testid should be present once the form renders
|
||||
await expect(page.getByTestId("condenser-settings-screen")).toBeAttached({
|
||||
timeout: 5_000,
|
||||
});
|
||||
await expect(
|
||||
page.getByText(/Enable (default condenser|Memory Condensation)/i),
|
||||
).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("condenser-settings.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,156 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the sidebar / conversation panel.
|
||||
*
|
||||
* MSW pre-seeds three conversations in src/mocks/conversation-handlers.ts:
|
||||
* 1. "My New Project" — execution_status: "waiting_for_confirmation"
|
||||
* 2. "Repo Testing" — execution_status: "idle"
|
||||
* 3. "Another Project" — execution_status: "idle"
|
||||
*
|
||||
* Three snapshots:
|
||||
* 1. conversation-panel — the panel itself with status dots + relative timestamps
|
||||
* 2. sidebar-collapsed — after clicking the collapse toggle (thin icon rail)
|
||||
* 3. new-conversation-popover — popover open showing workspace options
|
||||
*/
|
||||
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
async function setupMocks(page: Page) {
|
||||
await seedLocalStorage(page);
|
||||
// Suppress proxy errors for file API (home page workspace scan)
|
||||
await page.route("**/api/file/**", async (route) => {
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ path: "/home", subdirs: [] }),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test.describe("Sidebar Visual Snapshots", () => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
test("conversation panel shows three conversations with status dots", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/conversations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// Wait for all five conversation cards to be present
|
||||
const conversationPanel = page.getByTestId("conversation-panel");
|
||||
await expect(conversationPanel).toBeVisible({ timeout: 15_000 });
|
||||
await expect(page.getByTestId("conversation-card")).toHaveCount(6, {
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
// Scope the screenshot to just the sidebar panel (includes status dots)
|
||||
await expect(conversationPanel).toHaveScreenshot(
|
||||
"sidebar-conversation-panel.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
test("sidebar collapses to an icon rail on toggle", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/conversations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
await expect(page.getByTestId("conversation-panel")).toBeVisible({
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
||||
// Click the collapse toggle (visible when sidebar is expanded)
|
||||
await page.getByTestId("sidebar-collapse-toggle").click();
|
||||
|
||||
// Sidebar should now be in collapsed state
|
||||
const sidebarAside = page.locator("aside[data-collapsed='true']");
|
||||
await expect(sidebarAside).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot("sidebar-collapsed.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("conversations filter menu opens from the filter toggle button", async ({
|
||||
page,
|
||||
}) => {
|
||||
await setupMocks(page);
|
||||
await page.goto("/conversations");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
await expect(page.getByTestId("conversation-panel")).toBeVisible({
|
||||
timeout: 15_000,
|
||||
});
|
||||
await expect(page.getByTestId("conversation-card")).toHaveCount(6, {
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
// Click the filter toggle in the Conversations header
|
||||
await page.getByTestId("older-conversations-filter-toggle").click();
|
||||
|
||||
// Filter menu should appear
|
||||
await expect(
|
||||
page.getByTestId("older-conversations-filter-menu"),
|
||||
).toBeVisible({ timeout: 5_000 });
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"sidebar-filter-menu.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
|
||||
test("new conversation popover opens with no-workspace entry", async ({
|
||||
page,
|
||||
}) => {
|
||||
// The `NewConversationButton` component (and its `new-conversation-button`
|
||||
// testid) is temporarily removed from the sidebar. The component lives at
|
||||
// src/components/features/conversation-panel/new-conversation-button-local.tsx
|
||||
// but is commented out in sidebar.tsx (lines 241-244). This test documents
|
||||
// the intended snapshot once the button is re-wired. Mark as fixme so CI
|
||||
// stays green while the implementation is pending.
|
||||
// eslint-disable-next-line playwright/no-skipped-test
|
||||
test.fixme(
|
||||
true,
|
||||
"NewConversationButton is temporarily hidden from the sidebar (sidebar.tsx:241-244)",
|
||||
);
|
||||
|
||||
await setupMocks(page);
|
||||
await page.goto("/");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
// Find and click the new-conversation trigger in the sidebar
|
||||
const newConvButton = page.getByTestId("new-conversation-button");
|
||||
await expect(newConvButton).toBeVisible({ timeout: 10_000 });
|
||||
await newConvButton.click();
|
||||
|
||||
// Popover should appear
|
||||
await expect(page.getByTestId("new-conversation-popover")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
// "No workspace" entry is always present (no workspaces stored in
|
||||
// localStorage in this mock setup)
|
||||
await expect(page.getByTestId("launch-no-workspace")).toBeVisible();
|
||||
|
||||
const rootLayout = page.getByTestId("root-layout");
|
||||
await expect(rootLayout).toHaveScreenshot(
|
||||
"sidebar-new-conversation-popover.png",
|
||||
{ animations: "disabled", maxDiffPixelRatio: 0.01 },
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,237 +0,0 @@
|
||||
import { test, expect, Page } from "@playwright/test";
|
||||
import { seedLocalStorage } from "./support/seed-local-storage";
|
||||
|
||||
/**
|
||||
* Visual snapshot tests for the Skills page (/skills).
|
||||
*
|
||||
* SkillsService.getSkills() issues POST /api/skills. In mock-API mode
|
||||
* MSW returns { skills: [] }, so the empty state is free. For the loaded
|
||||
* and search/filter states we seed the React Query cache directly via
|
||||
* window.__OH_QUERY_CLIENT__ (exposed in dev/mock mode) to bypass MSW's
|
||||
* same-origin intercept.
|
||||
*
|
||||
* Four states are covered:
|
||||
* 1. Empty – server returns no skills (MSW default)
|
||||
* 2. Loaded – four skill cards visible
|
||||
* 3. Search – filtered to one card after typing "docker"
|
||||
* 4. No match – empty message after typing an unrecognised term
|
||||
* 5. Type filter – only agentskills cards visible after clicking filter
|
||||
*/
|
||||
|
||||
const MOCK_SKILLS = [
|
||||
{
|
||||
name: "code-review",
|
||||
type: "agentskills" as const,
|
||||
source: "github:OpenHands/extensions/skills/codereview",
|
||||
description:
|
||||
"Rigorous code review focusing on data structures, simplicity, " +
|
||||
"security, pragmatism, and risk/safety evaluation.",
|
||||
triggers: ["/review"],
|
||||
version: "1.2.0",
|
||||
license: "MIT",
|
||||
allowed_tools: ["terminal", "file_editor"],
|
||||
},
|
||||
{
|
||||
name: "docker",
|
||||
type: "agentskills" as const,
|
||||
source: "github:OpenHands/extensions/skills/docker",
|
||||
description:
|
||||
"Run Docker commands within a container environment, including " +
|
||||
"starting the Docker daemon and managing containers.",
|
||||
triggers: ["/docker"],
|
||||
version: "0.9.1",
|
||||
license: "Apache-2.0",
|
||||
allowed_tools: ["terminal"],
|
||||
},
|
||||
{
|
||||
name: "prd",
|
||||
type: "knowledge" as const,
|
||||
source: "github:OpenHands/extensions/skills/prd",
|
||||
description: "Generate a Product Requirements Document for a new feature.",
|
||||
triggers: ["/prd"],
|
||||
version: "1.0.0",
|
||||
license: null,
|
||||
allowed_tools: null,
|
||||
},
|
||||
{
|
||||
name: "repo-rules",
|
||||
type: "repo" as const,
|
||||
source: null,
|
||||
description: "Project-specific rules for this repository.",
|
||||
triggers: [],
|
||||
version: undefined,
|
||||
license: null,
|
||||
allowed_tools: ["file_editor", "terminal"],
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Dismiss the analytics consent modal if MSW shows it.
|
||||
*/
|
||||
async function dismissConsentModal(page: Page) {
|
||||
await page
|
||||
.getByRole("button", { name: "Confirm preferences" })
|
||||
.click({ timeout: 3_000 })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire up the base routes every skills test needs.
|
||||
*/
|
||||
async function setupMocks(page: Page) {
|
||||
await seedLocalStorage(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inject skills data directly into the React Query cache via the
|
||||
* window.__OH_QUERY_CLIENT__ handle exposed in dev/mock mode.
|
||||
*/
|
||||
async function seedSkills(page: Page, skills = MOCK_SKILLS) {
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
!!(window as unknown as { __OH_QUERY_CLIENT__?: unknown })
|
||||
.__OH_QUERY_CLIENT__,
|
||||
{ timeout: 10_000 },
|
||||
);
|
||||
await page.evaluate((skillsData) => {
|
||||
// Matches the `useSkills` queryKey: `["skills", projectDir ?? null]`.
|
||||
// The global Skills page calls `useSkills()` with no projectDir, so the
|
||||
// key resolves to `["skills", null]`. Seeding with the bare `["skills"]`
|
||||
// tuple (as the test originally did) misses the cache entry and the
|
||||
// skill cards never render.
|
||||
(
|
||||
window as unknown as {
|
||||
__OH_QUERY_CLIENT__: {
|
||||
setQueryData: (key: unknown[], data: unknown) => void;
|
||||
};
|
||||
}
|
||||
).__OH_QUERY_CLIENT__.setQueryData(["skills", null], skillsData);
|
||||
}, skills);
|
||||
await page.waitForTimeout(200);
|
||||
}
|
||||
|
||||
test.describe("Skills Page Visual Snapshots", () => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
test("empty state renders correctly", async ({ page }) => {
|
||||
// MSW intercepts POST /api/skills and returns { skills: [] } so the
|
||||
// "No skills found" empty state is the reliable baseline here.
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/skills");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const skillsScreen = page.getByTestId("skills-settings-screen");
|
||||
await expect(skillsScreen).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await expect(skillsScreen).toHaveScreenshot("skills-empty.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("skills page with loaded cards renders correctly", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/skills");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const skillsScreen = page.getByTestId("skills-settings-screen");
|
||||
await expect(skillsScreen).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await seedSkills(page);
|
||||
await expect(page.getByTestId("skill-card-code-review")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
await expect(skillsScreen).toHaveScreenshot("skills-loaded.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("search narrows cards to matching skill", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/skills");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const skillsScreen = page.getByTestId("skills-settings-screen");
|
||||
await expect(skillsScreen).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await seedSkills(page);
|
||||
await expect(page.getByTestId("skill-card-code-review")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
await page.getByTestId("skills-search-input").fill("docker");
|
||||
await page.waitForTimeout(300);
|
||||
|
||||
await expect(page.getByTestId("skill-card-docker")).toBeVisible();
|
||||
await expect(page.getByTestId("skill-card-code-review")).toHaveCount(0);
|
||||
|
||||
await expect(skillsScreen).toHaveScreenshot("skills-search-filtered.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("search with no results shows empty message", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/skills");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const skillsScreen = page.getByTestId("skills-settings-screen");
|
||||
await expect(skillsScreen).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await seedSkills(page);
|
||||
await expect(page.getByTestId("skill-card-code-review")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
await page.getByTestId("skills-search-input").fill("xyznonexistent");
|
||||
await page.waitForTimeout(300);
|
||||
|
||||
await expect(page.getByText(/No skills match/i)).toBeVisible();
|
||||
|
||||
await expect(skillsScreen).toHaveScreenshot("skills-no-match.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
|
||||
test("type filter shows only matching skill type", async ({ page }) => {
|
||||
await setupMocks(page);
|
||||
|
||||
await page.goto("/skills");
|
||||
await dismissConsentModal(page);
|
||||
await page.waitForLoadState("networkidle");
|
||||
|
||||
const skillsScreen = page.getByTestId("skills-settings-screen");
|
||||
await expect(skillsScreen).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await seedSkills(page);
|
||||
await expect(page.getByTestId("skill-card-code-review")).toBeVisible({
|
||||
timeout: 5_000,
|
||||
});
|
||||
|
||||
const filter = page.getByTestId("skills-type-filter");
|
||||
await filter.getByTestId("dropdown-trigger").click();
|
||||
await page.getByTestId("skills-type-filter-agentskills").click();
|
||||
await page.waitForTimeout(300);
|
||||
|
||||
await expect(page.getByTestId("skill-card-code-review")).toBeVisible();
|
||||
await expect(page.getByTestId("skill-card-docker")).toBeVisible();
|
||||
await expect(page.getByTestId("skill-card-prd")).toHaveCount(0);
|
||||
|
||||
await expect(skillsScreen).toHaveScreenshot("skills-type-filter.png", {
|
||||
animations: "disabled",
|
||||
maxDiffPixelRatio: 0.01,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,82 +0,0 @@
|
||||
import type { Page } from "@playwright/test";
|
||||
|
||||
export interface SeedLocalStorageOptions {
|
||||
/**
|
||||
* Remove openhands-onboarded instead of setting it.
|
||||
* Use in onboarding modal tests that need the modal to appear.
|
||||
*/
|
||||
removeOnboarded?: boolean;
|
||||
/**
|
||||
* Skip suppressing the analytics consent modal.
|
||||
* Use in tests that specifically snapshot the consent modal UI.
|
||||
*/
|
||||
showConsentModal?: boolean;
|
||||
/**
|
||||
* Additional [key, value] pairs to seed alongside the standard keys.
|
||||
*/
|
||||
extra?: [string, string][];
|
||||
}
|
||||
|
||||
/**
|
||||
* Seeds the standard localStorage keys required by snapshot tests via
|
||||
* `page.addInitScript`, so the values are present before any app code runs.
|
||||
*
|
||||
* Defaults (overridable via options):
|
||||
* - openhands-onboarded = "1" (suppresses onboarding modal)
|
||||
* - openhands-telemetry-consent = "denied" (suppresses analytics consent modal)
|
||||
* - openhands-backends / openhands-active-backend = seeded local backend
|
||||
*/
|
||||
export async function seedLocalStorage(
|
||||
page: Page,
|
||||
{
|
||||
removeOnboarded = false,
|
||||
showConsentModal = false,
|
||||
extra = [],
|
||||
}: SeedLocalStorageOptions = {},
|
||||
) {
|
||||
await page.addInitScript(
|
||||
({
|
||||
removeOnboarded,
|
||||
showConsentModal,
|
||||
extra,
|
||||
}: {
|
||||
removeOnboarded: boolean;
|
||||
showConsentModal: boolean;
|
||||
extra: [string, string][];
|
||||
}) => {
|
||||
if (removeOnboarded) {
|
||||
window.localStorage.removeItem("openhands-onboarded");
|
||||
} else {
|
||||
window.localStorage.setItem("openhands-onboarded", "1");
|
||||
}
|
||||
if (!showConsentModal) {
|
||||
window.localStorage.setItem("analytics-consent", "false");
|
||||
window.localStorage.setItem("openhands-telemetry-consent", "denied");
|
||||
}
|
||||
if (!window.localStorage.getItem("openhands-backends")) {
|
||||
window.localStorage.setItem(
|
||||
"openhands-backends",
|
||||
JSON.stringify([
|
||||
{
|
||||
id: "default-local",
|
||||
name: "Local",
|
||||
host: window.location.origin,
|
||||
apiKey: "test-session-key",
|
||||
kind: "local",
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
if (!window.localStorage.getItem("openhands-active-backend")) {
|
||||
window.localStorage.setItem(
|
||||
"openhands-active-backend",
|
||||
JSON.stringify({ backendId: "default-local", orgId: null }),
|
||||
);
|
||||
}
|
||||
for (const [key, value] of extra) {
|
||||
window.localStorage.setItem(key, value);
|
||||
}
|
||||
},
|
||||
{ removeOnboarded, showConsentModal, extra },
|
||||
);
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
import type { Page } from "@playwright/test";
|
||||
|
||||
/**
|
||||
* Stubs the browser's native `WebSocket` so conversation-page snapshot
|
||||
* tests don't trigger real connection attempts (which fail in CI with
|
||||
* "Unable to connect to server" toasts because no agent-server is
|
||||
* running at the Vite proxy target).
|
||||
*
|
||||
* Must be called **before** `page.goto()` — it uses `page.addInitScript`
|
||||
* so the stub is in place before any app code runs.
|
||||
*/
|
||||
export async function stubWebSocket(page: Page): Promise<void> {
|
||||
await page.addInitScript(() => {
|
||||
const noop = () => {};
|
||||
class StubWebSocket extends EventTarget {
|
||||
static CONNECTING = 0;
|
||||
static OPEN = 1;
|
||||
static CLOSING = 2;
|
||||
static CLOSED = 3;
|
||||
readyState = StubWebSocket.OPEN;
|
||||
url: string;
|
||||
protocol = "";
|
||||
extensions = "";
|
||||
bufferedAmount = 0;
|
||||
binaryType: BinaryType = "blob";
|
||||
onopen: ((ev: Event) => void) | null = null;
|
||||
onclose: ((ev: CloseEvent) => void) | null = null;
|
||||
onmessage: ((ev: MessageEvent) => void) | null = null;
|
||||
onerror: ((ev: Event) => void) | null = null;
|
||||
|
||||
constructor(url: string | URL) {
|
||||
super();
|
||||
this.url = typeof url === "string" ? url : url.toString();
|
||||
setTimeout(() => {
|
||||
const evt = new Event("open");
|
||||
this.onopen?.(evt);
|
||||
this.dispatchEvent(evt);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
send = noop;
|
||||
close = noop;
|
||||
CONNECTING = StubWebSocket.CONNECTING;
|
||||
OPEN = StubWebSocket.OPEN;
|
||||
CLOSING = StubWebSocket.CLOSING;
|
||||
CLOSED = StubWebSocket.CLOSED;
|
||||
}
|
||||
(window as unknown as { WebSocket: unknown }).WebSocket =
|
||||
StubWebSocket as unknown as typeof WebSocket;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user