From e6e61b0fa68d64c5d0f288fc2b115be751c3f847 Mon Sep 17 00:00:00 2001 From: Rohit Malhotra Date: Mon, 8 Jun 2026 11:50:50 -0400 Subject: [PATCH] refactor(e2e): drive mock-LLM test interactions through the UI (#1222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace direct API calls for seeding/configuring state in mock-LLM E2E tests with UI-driven interactions wherever possible, ensuring downstream API calls are covered by the test. Changes: - mock-llm-profile-management.spec.ts: All three scenarios (active profile deletion, same-model identity, litellm_proxy base_url preservation) now create and activate profiles through the Settings → LLM Profiles UI instead of raw POST/activate API calls. Cleanup in afterAll uses the UI delete flow via exported deleteProfileIfExists. - mock-llm-model-switch.spec.ts: The switch-target profile B is now created through the Settings UI (createProfileViaUI) instead of a raw POST to /api/profiles. Cleanup uses UI-driven deletion. - mock-llm-skills.spec.ts: Replaced the inlined configureMockLLM() helper (which did a raw PATCH to /api/settings) with the UI-driven ensureMockLLMProfile(page) that creates and activates the profile through the Settings screen. - mock-llm-acp-agent.spec.ts: afterAll cleanup now resets agent type back to OpenHands via the Settings → Agent UI (resetToOpenHandsAgentViaUI) instead of a raw PATCH to /api/settings. The local selectDropdownOption is removed in favor of the shared export from mock-llm-helpers. - mock-llm-conversation.spec.ts: Removed the redundant API pre-check in step 3 that verified the profile was active via GET /api/profiles. Steps 1+2 already verified this through the UI (Active badge check). - mock-llm-helpers.ts: - Extracted createProfileViaUI() from ensureMockLLMProfile() as a standalone exported helper for tests that need to create profiles without activating them. - Exported deleteProfileIfExists() and activateProfileViaUI() so tests can compose profile lifecycle operations through the UI. - Added selectDropdownOption() (consolidated from ACP spec's local copy). - Added resetToOpenHandsAgentViaUI() for UI-driven agent type reset. - Marked the API-based resetToOpenHandsAgent() as @deprecated. Co-authored-by: openhands --- tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts | 45 +--- .../mock-llm/mock-llm-conversation.spec.ts | 19 +- .../mock-llm/mock-llm-model-switch.spec.ts | 99 +++------ .../mock-llm-profile-management.spec.ts | 195 ++++++++---------- tests/e2e/mock-llm/mock-llm-skills.spec.ts | 52 +---- tests/e2e/mock-llm/utils/mock-llm-helpers.ts | 84 +++++++- 6 files changed, 202 insertions(+), 292 deletions(-) diff --git a/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts b/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts index 704f3e3772..8d38fb2ed4 100644 --- a/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts +++ b/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts @@ -16,7 +16,7 @@ * round-trip without any real LLM. */ -import { test, expect, type Page } from "@playwright/test"; +import { test, expect } from "@playwright/test"; import { ACP_REPLY_TOKEN, MOCK_ACP_COMMAND_PYTHON, @@ -29,9 +29,10 @@ import { getConversationIdFromURL, waitForNonUserMessageText, deleteConversation, - resetToOpenHandsAgent, + resetToOpenHandsAgentViaUI, resetMockLLM, ensureMockLLMProfile, + selectDropdownOption, setChatInput, BACKEND_URL, SESSION_API_KEY, @@ -48,34 +49,6 @@ const USER_MESSAGE = "Hello ACP agent, please reply."; */ const ACP_COMMAND_TEXT = `${MOCK_ACP_COMMAND_PYTHON} ${MOCK_ACP_COMMAND_SCRIPT}`; -// ── UI interaction helpers for HeroUI Autocomplete dropdowns ────────── - -/** - * Select an option from a HeroUI Autocomplete dropdown (SettingsDropdownInput). - * - * HeroUI Autocomplete does NOT forward `data-testid` to the underlying - * ``, so we locate the combobox by its `aria-label` (which the - * component sets to the label prop or the name prop). We then click to - * open the listbox and click the matching option. - */ -async function selectDropdownOption( - page: Page, - /** aria-label of the combobox (= the SettingsDropdownInput label text) */ - comboboxLabel: string | RegExp, - /** Visible text of the option to click */ - optionText: string | RegExp, -) { - const combobox = page.getByRole("combobox", { name: comboboxLabel }); - await expect(combobox).toBeVisible({ timeout: 10_000 }); - // Clear any existing text (so all options show) and open - await combobox.click(); - await combobox.fill(""); - // Wait for the option to appear and click it - const option = page.getByRole("option", { name: optionText }); - await expect(option).toBeVisible({ timeout: 5_000 }); - await option.click(); -} - test.describe.configure({ mode: "serial" }); test.describe("mock-LLM ACP agent conversation", () => { @@ -95,17 +68,13 @@ test.describe("mock-LLM ACP agent conversation", () => { } } - // Reset agent-server back to OpenHands + restore mock LLM profile - // so subsequent test suites (which expect agent_kind=openhands) are - // not affected by our ACP configuration. - try { - await resetToOpenHandsAgent(request); - } catch { - // best-effort - } + // Reset agent-server back to OpenHands via the Settings → Agent UI + // + restore mock LLM profile so subsequent test suites (which expect + // agent_kind=openhands) are not affected by our ACP configuration. const page = await browser.newPage(); try { await seedLocalStorage(page); + await resetToOpenHandsAgentViaUI(page); await ensureMockLLMProfile(page); } catch { // best-effort diff --git a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts index 0c1bea8c56..ed6fa04e23 100644 --- a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts +++ b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts @@ -248,23 +248,8 @@ test.describe("mock-LLM agent-server conversation", () => { // that wasn't fully consumed due to earlier failures. await resetMockLLM(request); - // Verify the mock LLM profile is active before creating a conversation. - // Steps 1+2 configure it via the UI; this API check ensures persistence. - await test.step("verify mock-llm profile is active via API", async () => { - const profilesResp = await request.get(`${BACKEND_URL}/api/profiles`, { - headers: { "X-Session-API-Key": SESSION_API_KEY }, - }); - expect(profilesResp.ok(), `GET /api/profiles returned ${profilesResp.status()}`).toBe(true); - const profiles = await profilesResp.json(); - const activeProfile = profiles?.active_profile; - const profileNames = profiles?.profiles ? Object.keys(profiles.profiles) : []; - expect( - activeProfile, - `Expected active_profile="${PROFILE_NAME}" but got "${activeProfile}". ` + - `Available profiles: [${profileNames.join(", ")}]. ` + - `Full response: ${JSON.stringify(profiles).slice(0, 500)}`, - ).toBe(PROFILE_NAME); - }); + // Steps 1+2 already configured and verified the profile via the UI + // (including the "Active" badge check). No API pre-check needed. // Passively observe POST /api/conversations to capture the request body. // Using page.on('request') instead of page.route() avoids conflicts with diff --git a/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts b/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts index f27f8f047f..94fa833275 100644 --- a/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts +++ b/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts @@ -19,11 +19,8 @@ * working under the new profile. */ -import { test, expect, type APIRequestContext } from "@playwright/test"; +import { test, expect } from "@playwright/test"; import { - BACKEND_URL, - SESSION_API_KEY, - MOCK_LLM_AGENT_URL, seedLocalStorage, routeSessionApiKey, dismissAnalyticsModal, @@ -36,62 +33,18 @@ import { activateTrajectory, resetMockLLM, ensureMockLLMProfile, + createProfileViaUI, + deleteProfileIfExists, setChatInput, } from "./utils/mock-llm-helpers"; -/** Profile B is the switch target — created via the profiles API. */ +/** Profile B is the switch target — created via the Settings UI. */ const PROFILE_B_NAME = "model-switch-profile-b"; const MODEL_B = "openai/mock-model-beta"; const INITIAL_REPLY_TOKEN = "MODEL_SWITCH_INITIAL_REPLY_OK"; const POST_SWITCH_REPLY_TOKEN = "MODEL_SWITCH_POST_SWITCH_REPLY_OK"; -/** - * Create (or overwrite) a named LLM profile via the agent-server profiles API. - * Deletes first so setup is idempotent across re-runs — if a previous test - * crashed before afterAll cleanup, the stale profile won't cause a 409. - */ -async function saveProfile( - request: APIRequestContext, - name: string, - model: string, -) { - // Best-effort delete so a leftover profile doesn't block creation. - await request.delete( - `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`, - { headers: { "X-Session-API-Key": SESSION_API_KEY } }, - ); - const resp = await request.post( - `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`, - { - headers: { - "X-Session-API-Key": SESSION_API_KEY, - "Content-Type": "application/json", - }, - data: { - llm: { - model, - api_key: "mock-api-key-for-testing", - base_url: MOCK_LLM_AGENT_URL, - }, - include_secrets: true, - }, - }, - ); - expect( - resp.ok(), - `POST /api/profiles/${name} returned ${resp.status()}`, - ).toBe(true); -} - -/** Delete a profile (best-effort cleanup). */ -async function deleteProfile(request: APIRequestContext, name: string) { - await request.delete( - `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`, - { headers: { "X-Session-API-Key": SESSION_API_KEY } }, - ); -} - test.describe.configure({ mode: "serial" }); test.describe("mock-LLM /model slash command", () => { @@ -112,12 +65,20 @@ test.describe("mock-LLM /model slash command", () => { } }); - test.afterAll(async ({ request }) => { - // Clean up the profile we created and reset mock LLM + test.afterAll(async ({ request, browser }) => { + // Best-effort cleanup via UI + const page = await browser.newPage(); try { - await deleteProfile(request, PROFILE_B_NAME); + await seedLocalStorage(page); + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + await deleteProfileIfExists(page, PROFILE_B_NAME); } catch { // best-effort + } finally { + await page.close(); } try { await resetMockLLM(request); @@ -136,20 +97,24 @@ test.describe("mock-LLM /model slash command", () => { // flow used by mock-llm-conversation.spec.ts. await ensureMockLLMProfile(page); - // Create profile B as the switch target — it has a different model name - // but the same mock LLM base_url so post-switch inference still works. - await saveProfile(request, PROFILE_B_NAME, MODEL_B); + // Create profile B as the switch target through the Settings UI — it has + // a different model name but the same mock LLM base_url so post-switch + // inference still works. + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); - // Verify profile B was created - const profilesResp = await request.get(`${BACKEND_URL}/api/profiles`, { - headers: { "X-Session-API-Key": SESSION_API_KEY }, - }); - expect(profilesResp.ok()).toBe(true); - const profiles = await profilesResp.json(); - const profileNames: string[] = profiles.profiles.map( - (p: { name: string }) => p.name, - ); - expect(profileNames).toContain(PROFILE_B_NAME); + await deleteProfileIfExists(page, PROFILE_B_NAME); + await createProfileViaUI(page, { profileName: PROFILE_B_NAME, model: MODEL_B }); + + // Verify profile B appears in the list + const profileRows = page.getByTestId("profile-row"); + const profileTexts = await profileRows.allTextContents(); + expect( + profileTexts.some((text) => text.includes(PROFILE_B_NAME)), + `Profile "${PROFILE_B_NAME}" should appear in the list`, + ).toBe(true); // Register a trajectory with THREE entries: // Turn 0: padding — the agent-server makes an internal LLM call diff --git a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts index 9f1556e9a4..706387237b 100644 --- a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts +++ b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts @@ -24,7 +24,7 @@ * stranded and LiteLLM reroutes requests to the wrong endpoint. */ -import { test, expect, type APIRequestContext } from "@playwright/test"; +import { test, expect } from "@playwright/test"; import { BACKEND_URL, SESSION_API_KEY, @@ -41,46 +41,19 @@ import { resetMockLLM, setChatInput, waitForPath, + createProfileViaUI, + deleteProfileIfExists, + activateProfileViaUI, } from "./utils/mock-llm-helpers"; -// ═══════════════════════════════════════════════════════════════════════ -// Profile API helpers -// ═══════════════════════════════════════════════════════════════════════ - const MOCK_MODEL = "openai/mock-test-model"; -async function saveProfile( - request: APIRequestContext, - name: string, - model: string, - baseUrl: string = MOCK_LLM_AGENT_URL, -) { - await request.delete( - `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`, - { headers: { "X-Session-API-Key": SESSION_API_KEY } }, - ); - const resp = await request.post( - `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`, - { - headers: { - "X-Session-API-Key": SESSION_API_KEY, - "Content-Type": "application/json", - }, - data: { - llm: { - model, - api_key: "mock-api-key-for-testing", - base_url: baseUrl, - }, - include_secrets: true, - }, - }, - ); - expect(resp.ok(), `POST /api/profiles/${name}: ${resp.status()}`).toBe(true); -} - +/** + * Read-only helper: fetch a profile's persisted config via the API. + * Used to verify that UI-driven saves persisted the expected values. + */ async function getProfileConfig( - request: APIRequestContext, + request: import("@playwright/test").APIRequestContext, name: string, ): Promise> { const resp = await request.get( @@ -97,24 +70,6 @@ async function getProfileConfig( return (data.config ?? {}) as Record; } -async function activateProfile(request: APIRequestContext, name: string) { - const resp = await request.post( - `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}/activate`, - { headers: { "X-Session-API-Key": SESSION_API_KEY } }, - ); - expect( - resp.ok(), - `POST /api/profiles/${name}/activate: ${resp.status()}`, - ).toBe(true); -} - -async function deleteProfile(request: APIRequestContext, name: string) { - await request.delete( - `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`, - { headers: { "X-Session-API-Key": SESSION_API_KEY } }, - ); -} - test.describe.configure({ mode: "serial" }); // ═══════════════════════════════════════════════════════════════════════ @@ -129,30 +84,44 @@ test.describe("active profile deletion + reconciliation", () => { await seedLocalStorage(page); }); - test.afterAll(async ({ request }) => { - for (const name of [ACTIVE_PROFILE, INACTIVE_PROFILE]) { - try { - await deleteProfile(request, name); - } catch { - // best-effort - } + test.afterAll(async ({ browser }) => { + // Best-effort cleanup via UI + const page = await browser.newPage(); + try { + await seedLocalStorage(page); + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + await deleteProfileIfExists(page, ACTIVE_PROFILE); + await deleteProfileIfExists(page, INACTIVE_PROFILE); + } catch { + // best-effort + } finally { + await page.close(); } }); test("active profile is deletable and reconciliation activates another profile", async ({ page, - request, }) => { - // ── Setup: create two profiles, activate one ── - await saveProfile(request, ACTIVE_PROFILE, MOCK_MODEL); - await saveProfile(request, INACTIVE_PROFILE, MOCK_MODEL); - await activateProfile(request, ACTIVE_PROFILE); - + // ── Setup: create two profiles via the UI, activate one ── await routeSessionApiKey(page); await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); await dismissAnalyticsModal(page); await waitForTestId(page, "add-llm-profile"); + // Clean up any leftover profiles from prior runs + await deleteProfileIfExists(page, ACTIVE_PROFILE); + await deleteProfileIfExists(page, INACTIVE_PROFILE); + + // Create both profiles through the Settings UI + await createProfileViaUI(page, { profileName: ACTIVE_PROFILE, model: MOCK_MODEL }); + await createProfileViaUI(page, { profileName: INACTIVE_PROFILE, model: MOCK_MODEL }); + + // Activate the first profile through the UI + await activateProfileViaUI(page, ACTIVE_PROFILE); + const rowFor = async (name: string) => { const rows = page.getByTestId("profile-row"); const count = await rows.count(); @@ -281,13 +250,21 @@ test.describe("same-model profile identity", () => { } }); - test.afterAll(async ({ request }) => { - for (const name of [PROFILE_ALPHA, PROFILE_BETA]) { - try { - await deleteProfile(request, name); - } catch { - // best-effort - } + test.afterAll(async ({ request, browser }) => { + // Best-effort cleanup via UI + const page = await browser.newPage(); + try { + await seedLocalStorage(page); + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + await deleteProfileIfExists(page, PROFILE_ALPHA); + await deleteProfileIfExists(page, PROFILE_BETA); + } catch { + // best-effort + } finally { + await page.close(); } try { await resetMockLLM(request); @@ -302,10 +279,19 @@ test.describe("same-model profile identity", () => { }) => { test.setTimeout(120_000); - // ── Setup: create both profiles with the same model, activate BETA ── - await saveProfile(request, PROFILE_ALPHA, SHARED_MODEL); - await saveProfile(request, PROFILE_BETA, SHARED_MODEL); - await activateProfile(request, PROFILE_BETA); + // ── Setup: create both profiles with the same model via the UI, + // then activate BETA through the profile menu ── + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + + await deleteProfileIfExists(page, PROFILE_ALPHA); + await deleteProfileIfExists(page, PROFILE_BETA); + + await createProfileViaUI(page, { profileName: PROFILE_ALPHA, model: SHARED_MODEL }); + await createProfileViaUI(page, { profileName: PROFILE_BETA, model: SHARED_MODEL }); + await activateProfileViaUI(page, PROFILE_BETA); // Register a trajectory for the conversation. // Turn 0 is padding: the agent-server makes an internal LLM call @@ -316,19 +302,6 @@ test.describe("same-model profile identity", () => { ]); await activateTrajectory(request, "profile-identity"); - // ── Verify: active_profile is BETA via the API ── - await test.step("verify active profile is BETA via API", async () => { - const resp = await request.get(`${BACKEND_URL}/api/profiles`, { - headers: { "X-Session-API-Key": SESSION_API_KEY }, - }); - expect(resp.ok()).toBe(true); - const data = await resp.json(); - expect( - data.active_profile, - `Expected active_profile="${PROFILE_BETA}" but got "${data.active_profile}"`, - ).toBe(PROFILE_BETA); - }); - // ── Start a conversation ── await routeSessionApiKey(page); await page.goto("/", { waitUntil: "domcontentloaded" }); @@ -392,11 +365,19 @@ test.describe("litellm_proxy proxy base_url preservation", () => { await seedLocalStorage(page); }); - test.afterAll(async ({ request }) => { + test.afterAll(async ({ browser }) => { + const page = await browser.newPage(); try { - await deleteProfile(request, PROXY_PROFILE); + await seedLocalStorage(page); + await routeSessionApiKey(page); + await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "add-llm-profile"); + await deleteProfileIfExists(page, PROXY_PROFILE); } catch { // best-effort + } finally { + await page.close(); } }); @@ -405,29 +386,21 @@ test.describe("litellm_proxy proxy base_url preservation", () => { request, }) => { // ── Setup: create a profile with the SDK-rewritten litellm_proxy - // model + proxy base_url, exactly as the agent-server persists it - // after an openhands/* model selection during onboarding. ── - await saveProfile( - request, - PROXY_PROFILE, - LITELLM_PROXY_MODEL, - OPENHANDS_PROXY_BASE_URL, - ); - - // ── Pre-check: verify the profile's base_url via API before the - // UI round-trip so we know the starting state is correct. ── - await test.step("verify initial profile has proxy base_url", async () => { - const config = await getProfileConfig(request, PROXY_PROFILE); - expect(config.model).toBe(LITELLM_PROXY_MODEL); - expect(config.base_url).toBe(OPENHANDS_PROXY_BASE_URL); - }); - - // ── Navigate to the LLM settings page ── + // model + proxy base_url through the Settings UI, exactly as + // the agent-server persists it after an openhands/* model + // selection during onboarding. ── await routeSessionApiKey(page); await page.goto("/settings/llm", { waitUntil: "domcontentloaded" }); await dismissAnalyticsModal(page); await waitForTestId(page, "add-llm-profile"); + await deleteProfileIfExists(page, PROXY_PROFILE); + await createProfileViaUI(page, { + profileName: PROXY_PROFILE, + model: LITELLM_PROXY_MODEL, + baseUrl: OPENHANDS_PROXY_BASE_URL, + }); + // ── Find the profile row and open edit mode ── await test.step("open profile in edit mode", async () => { const profileRows = page.getByTestId("profile-row"); diff --git a/tests/e2e/mock-llm/mock-llm-skills.spec.ts b/tests/e2e/mock-llm/mock-llm-skills.spec.ts index 23420c84ad..f7708b7aab 100644 --- a/tests/e2e/mock-llm/mock-llm-skills.spec.ts +++ b/tests/e2e/mock-llm/mock-llm-skills.spec.ts @@ -26,7 +26,6 @@ import { test, expect, type APIRequestContext } from "@playwright/test"; import { BACKEND_URL, SESSION_API_KEY, - MOCK_LLM_AGENT_URL, seedLocalStorage, routeSessionApiKey, dismissAnalyticsModal, @@ -35,6 +34,7 @@ import { registerTrajectory, activateTrajectory, resetMockLLM, + ensureMockLLMProfile, setChatInput, waitForPath, getConversationIdFromURL, @@ -48,50 +48,6 @@ import { userSkillDirExists, } from "./utils/skill-test-helpers"; -/** - * Configure the mock LLM profile. Inlined from `ensureMockLLMProfile` - * to work around a CI-specific TS6/Node24 type inference bug (TS2345) - * where importing that function alongside `skill-test-helpers` causes - * TypeScript to incorrectly resolve its signature. - */ -async function configureMockLLM( - request: APIRequestContext, - model = "openai/mock-test-model", -) { - const settingsResp = await request.get(`${BACKEND_URL}/api/settings`, { - headers: { - "X-Session-API-Key": SESSION_API_KEY, - "X-Expose-Secrets": "encrypted", - }, - }); - if (settingsResp.ok()) { - const settings = (await settingsResp.json()) as Record; - const llm = ( - settings?.agent_settings as Record | undefined - )?.llm as Record | undefined; - if (llm?.model === model && llm?.base_url === MOCK_LLM_AGENT_URL) return; - } - const patchResp = await request.patch(`${BACKEND_URL}/api/settings`, { - headers: { - "X-Session-API-Key": SESSION_API_KEY, - "Content-Type": "application/json", - }, - data: { - agent_settings_diff: { - llm: { - model, - api_key: "mock-api-key-for-testing", - base_url: MOCK_LLM_AGENT_URL, - }, - }, - }, - }); - expect( - patchResp.ok(), - `PATCH /api/settings failed: ${patchResp.status()}`, - ).toBe(true); -} - /** * Register a workspace on the agent-server so it appears in the UI dropdown. */ @@ -191,7 +147,7 @@ test.describe("skill loading: project, user, and deletion", () => { page, request, }) => { - await configureMockLLM(request); + await ensureMockLLMProfile(page); // Create a git repo with the skill committed const { agentDir } = await test.step( @@ -291,7 +247,7 @@ test.describe("skill loading: project, user, and deletion", () => { page, request, }) => { - await configureMockLLM(request); + await ensureMockLLMProfile(page); await test.step("create user skill file", () => { writeUserSkill(USER_SKILL_NAME, USER_SKILL_TRIGGER); @@ -362,7 +318,7 @@ test.describe("skill loading: project, user, and deletion", () => { page, request, }) => { - await configureMockLLM(request); + await ensureMockLLMProfile(page); await test.step("delete user skill file", () => { removeUserSkill(USER_SKILL_NAME); diff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts index a3051088ce..5894dae598 100644 --- a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts +++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts @@ -322,7 +322,7 @@ export async function deleteConversation( } // ═══════════════════════════════════════════════════════════════════════ -// LLM profile setup via API (for tests that can't depend on UI setup) +// LLM profile setup via the Settings UI // ═══════════════════════════════════════════════════════════════════════ /** @@ -330,8 +330,7 @@ export async function deleteConversation( * UI — the same flow a real user follows. * * Exercises the full frontend save path (including `include_secrets`) so the - * api_key is persisted correctly. Callers that previously used the API-only - * `ensureMockLLMProfile(request)` should switch to this. + * api_key is persisted correctly. */ export async function ensureMockLLMProfile( page: Page, @@ -357,6 +356,33 @@ export async function ensureMockLLMProfile( await deleteProfileIfExists(page, profileName); // ── Create the profile ────────────────────────────────────────────── + await createProfileViaUI(page, { profileName, model, apiKey, baseUrl }); + + // ── Activate the profile ──────────────────────────────────────────── + await activateProfileViaUI(page, profileName); +} + +/** + * Create a new LLM profile through the Settings UI. + * + * Assumes the page is already on /settings/llm with profiles loaded + * (the "add-llm-profile" button is visible). Does NOT activate the + * profile — call `activateProfileViaUI` separately if needed. + */ +export async function createProfileViaUI( + page: Page, + { + profileName, + model, + apiKey = "mock-api-key-for-testing", + baseUrl = MOCK_LLM_AGENT_URL, + }: { + profileName: string; + model: string; + apiKey?: string; + baseUrl?: string; + }, +) { await page.getByTestId("add-llm-profile").click(); await waitForTestId(page, "profile-editor-title"); @@ -382,16 +408,13 @@ export async function ensureMockLLMProfile( await page.getByTestId("save-profile-btn").click(); await waitForTestId(page, "add-llm-profile"); - - // ── Activate the profile ──────────────────────────────────────────── - await activateProfileViaUI(page, profileName); } /** * Delete a profile by name through the Settings UI if it exists. * Assumes the page is already on /settings/llm with profiles loaded. */ -async function deleteProfileIfExists(page: Page, profileName: string) { +export async function deleteProfileIfExists(page: Page, profileName: string) { // Use the profile name span's `title` attribute for exact matching // to avoid substring collisions (e.g. "mock-llm" vs "mock-llm-e2e"). const row = page @@ -420,7 +443,7 @@ async function deleteProfileIfExists(page: Page, profileName: string) { * Assumes the page is already on /settings/llm with profiles loaded. * Polls until the "Active" badge is visible on the target profile row. */ -async function activateProfileViaUI(page: Page, profileName: string) { +export async function activateProfileViaUI(page: Page, profileName: string) { const exactRow = (p: Page) => p .getByTestId("profile-row") @@ -459,6 +482,46 @@ async function activateProfileViaUI(page: Page, profileName: string) { .toBe(true); } +/** + * Select an option from a HeroUI Autocomplete dropdown (SettingsDropdownInput). + * + * HeroUI Autocomplete does NOT forward `data-testid` to the underlying + * ``, so we locate the combobox by its `aria-label` (which the + * component sets to the label prop or the name prop). We then click to + * open the listbox and click the matching option. + */ +export async function selectDropdownOption( + page: Page, + comboboxLabel: string | RegExp, + optionText: string | RegExp, +) { + const combobox = page.getByRole("combobox", { name: comboboxLabel }); + await expect(combobox).toBeVisible({ timeout: 10_000 }); + await combobox.click(); + await combobox.fill(""); + const option = page.getByRole("option", { name: optionText }); + await expect(option).toBeVisible({ timeout: 5_000 }); + await option.click(); +} + +/** + * Reset agent type back to OpenHands through the Settings → Agent UI. + * Used in afterAll cleanup to restore the default agent for subsequent tests. + */ +export async function resetToOpenHandsAgentViaUI(page: Page) { + await routeSessionApiKey(page); + await page.goto("/settings/agent", { waitUntil: "domcontentloaded" }); + await dismissAnalyticsModal(page); + await waitForTestId(page, "agent-settings-screen"); + + await selectDropdownOption(page, /Agent/, /OpenHands/); + + const saveBtn = page.getByTestId("agent-save-button"); + await expect(saveBtn).toBeEnabled({ timeout: 5_000 }); + await saveBtn.click(); + await expect(saveBtn).toBeDisabled({ timeout: 10_000 }); +} + /** * Register a named trajectory on the mock LLM server. * Each turn is: { tool_call: { name, arguments } } or { text: "..." } @@ -615,8 +678,8 @@ export const MOCK_ACP_COMMAND_SCRIPT = process.env.MOCK_ACP_CONTAINER_SCRIPT || MOCK_ACP_SERVER_PATH; /** - * Reset the agent-server back to the default OpenHands agent. - * Used in afterAll cleanup to avoid polluting other test suites. + * @deprecated Use `resetToOpenHandsAgentViaUI(page)` to exercise the UI path. + * Kept only for callers that cannot open a page (should not exist in new tests). */ export async function resetToOpenHandsAgent( request: APIRequestContext, @@ -632,7 +695,6 @@ export async function resetToOpenHandsAgent( }, }, }); - // Best-effort — don't fail the test if cleanup fails if (!resp.ok()) { console.warn(`[cleanup] Reset to OpenHands failed: ${resp.status()}`); }