From 5d5a0648db4532a6490258a63582a18694f18fd7 Mon Sep 17 00:00:00 2001 From: Hiep Le <69354317+hieptl@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:16:05 +0700 Subject: [PATCH] feat: mock the setup contract from the published fixtures (#16221) --- __tests__/api/automation-handlers.test.ts | 122 ++++ __tests__/manifests/automation-setup.test.ts | 551 ++++++++---------- .../manifests/manifest-capabilities.test.ts | 46 ++ src/mocks/automation-handlers.ts | 104 ++++ tests/e2e/mock-llm/test-mapping.json | 8 +- 5 files changed, 527 insertions(+), 304 deletions(-) diff --git a/__tests__/api/automation-handlers.test.ts b/__tests__/api/automation-handlers.test.ts index e16c6cd77d..d82477344c 100644 --- a/__tests__/api/automation-handlers.test.ts +++ b/__tests__/api/automation-handlers.test.ts @@ -1,12 +1,45 @@ import { http, HttpResponse } from "msw"; import { setupServer } from "msw/node"; import { describe, it, expect, beforeAll, afterAll, afterEach } from "vitest"; +import capabilitiesFixture from "@openhands/extensions/testing/automations/capabilities.json"; +import prReviewerFixture from "@openhands/extensions/testing/automations/github-pr-reviewer.json"; +import repoMonitorFixture from "@openhands/extensions/testing/automations/github-repo-monitor.json"; import { AUTOMATION_HANDLERS, resetAutomationMockData, } from "#/mocks/automation-handlers"; import { MOCK_AUTOMATIONS_RESPONSE } from "#/mocks/automations.mock"; +interface PreflightExchange { + request: { method: string; path: string; body: unknown }; + response: { status: number; body: unknown }; +} + +interface FixtureBundle { + automationId: string; + scenarios: { id: string; preflight?: PreflightExchange }[]; +} + +/** + * Every preflight exchange the published contract fixtures record. The mock + * backend must reproduce each one, so the setup flow exercised against it is + * exercised against the reference contract. + */ +const PREFLIGHT_EXCHANGES = ( + [prReviewerFixture, repoMonitorFixture] as FixtureBundle[] +).flatMap((bundle) => + bundle.scenarios.flatMap((scenario) => + scenario.preflight + ? [ + { + name: `${bundle.automationId}/${scenario.id}`, + exchange: scenario.preflight, + }, + ] + : [], + ), +); + const server = setupServer(...AUTOMATION_HANDLERS); describe("Automation MSW Handlers", () => { @@ -39,6 +72,95 @@ describe("Automation MSW Handlers", () => { }); }); + describe("GET /api/automation/v1/capabilities", () => { + it("answers with the contract fixtures' supported deployment", async () => { + // Act + const res = await fetch("/api/automation/v1/capabilities"); + const data = await res.json(); + + // Assert + expect({ status: res.status, body: data }).toEqual({ + status: 200, + body: capabilitiesFixture.responses.supported.body, + }); + }); + }); + + describe("POST /api/automation/v1/validate", () => { + it.each(PREFLIGHT_EXCHANGES)( + "reproduces the $name exchange from the contract fixtures", + async ({ exchange }) => { + // Act + const res = await fetch(`/api/automation${exchange.request.path}`, { + method: exchange.request.method, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(exchange.request.body), + }); + const data = await res.json(); + + // Assert + expect({ status: res.status, body: data }).toEqual({ + status: exchange.response.status, + body: exchange.response.body, + }); + }, + ); + + it("rejects an every-minute schedule below the deployment minimum", async () => { + // Arrange + const draft = { + trigger: { type: "cron", schedule: "* * * * *", timezone: "UTC" }, + }; + + // Act + const res = await fetch("/api/automation/v1/validate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + automationId: "github-pr-reviewer", + endpoint: "/v1/preset/prompt", + draft, + }), + }); + const data = await res.json(); + + // Assert + expect(data).toEqual({ + valid: false, + errors: [ + { + field: "trigger.schedule", + code: "interval_too_short", + message: "Minimum interval for this deployment is 5 minutes.", + }, + ], + }); + }); + + it("passes a schedule shape its cron reader does not model", async () => { + // Arrange — "0 0 31 2 *" can never fire, but only the real service can + // say so; the fixtures record that rejection at the create stage. + const draft = { + trigger: { type: "cron", schedule: "0 0 31 2 *", timezone: "UTC" }, + }; + + // Act + const res = await fetch("/api/automation/v1/validate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + automationId: "github-pr-reviewer", + endpoint: "/v1/preset/prompt", + draft, + }), + }); + const data = await res.json(); + + // Assert + expect(data).toEqual({ valid: true, errors: [] }); + }); + }); + describe("GET /api/automation/v1/:id", () => { it("returns a single automation by id", async () => { const id = MOCK_AUTOMATIONS_RESPONSE.automations[0].id; diff --git a/__tests__/manifests/automation-setup.test.ts b/__tests__/manifests/automation-setup.test.ts index 74501f9780..aba6986520 100644 --- a/__tests__/manifests/automation-setup.test.ts +++ b/__tests__/manifests/automation-setup.test.ts @@ -1,16 +1,27 @@ import { describe, expect, it, vi } from "vitest"; +import incidentFixture from "@openhands/extensions/testing/automations/incident-retrospective-drafter.json"; +import prReviewerFixture from "@openhands/extensions/testing/automations/github-pr-reviewer.json"; +import repoMonitorFixture from "@openhands/extensions/testing/automations/github-repo-monitor.json"; import { + AUTOMATION_CREATE_ENDPOINT, buildAssistedMessage, buildCreatePayload, buildPreflightBody, deriveErrorMap, } from "#/manifests/automation-setup"; -import { validateSetupEntry } from "#/manifests/manifest-validation"; -import type { SetupEntry } from "#/manifests/types"; +import { + mapServiceErrors, + normalizeServiceErrors, +} from "#/manifests/manifest-error-map"; +import { validateFormValues } from "#/manifests/manifest-local-validation"; +import { SETUP_REGISTRY } from "#/manifests/manifest-sources"; +import type { SetupEntry, SetupFormValues } from "#/manifests/types"; -// The command a skill publishes in its own frontmatter, which the host looks up -// rather than storing. Pinned here so the assertion does not move when the -// packaged catalog does. +// The command a skill publishes in its own frontmatter, which the host looks +// up rather than storing. Pinned so the assertion does not move when the +// packaged skills catalog does; the automation catalog itself is imported for +// real, because pinning its derivation to the contract fixtures published +// beside it is this file's point. vi.mock("@openhands/extensions/skills", () => ({ SKILLS_CATALOG: [ { @@ -23,276 +34,176 @@ vi.mock("@openhands/extensions/skills", () => ({ })); /** - * The setup blocks published in `OpenHands/extensions`, and the request bodies - * its contract fixtures say they must produce. Those bodies were verified - * against the live service, and the create model forbids extra keys, so any - * divergence here is a 422 in production rather than a cosmetic difference. + * The reference fixtures `OpenHands/extensions` publishes with its catalog. + * Their request bodies were verified against the live service, and the create + * model forbids extra keys, so any divergence between the host's derivation + * and a fixture is a 422 in production rather than a cosmetic difference. */ -const PR_REVIEWER: SetupEntry = { - id: "github-pr-reviewer", - name: "GitHub Code Review Agent", - description: "Watch for a configurable label on GitHub pull requests.", - requires: { - integrations: { - github: { message: "Used to read pull requests and post comments." }, - }, - features: ["repoClone", "presetPrompt"], - }, - setup: { - version: "1.0", - mode: "direct", - form: { - triggers: { - cron: { - schedule: { - type: "cron", - label: "Check frequency", - help: "How often to look for newly labelled pull requests.", - default: "*/15 * * * *", - required: true, - }, - timezone: { - type: "timezone", - label: "Timezone", - help: "Timezone the schedule is interpreted in.", - default: "UTC", - required: true, - }, - }, - }, - args: { - repository: { - type: "repo-picker", - label: "Repository", - help: "The repository whose pull requests will be reviewed.", - provider: "github", - required: true, - }, - triggerLabel: { - type: "text", - label: "Trigger label", - help: "Only pull requests carrying this label are reviewed.", - default: "openhands-review", - required: true, - constraints: { minLength: 1, maxLength: 50 }, - }, - reviewTone: { - type: "select", - label: "Review tone", - help: "How detailed the review comments should be.", - default: "concise", - required: true, - options: [ - { value: "concise", label: "Concise" }, - { value: "thorough", label: "Thorough" }, - ], - }, - }, - }, - prompt: - "Review pull requests labeled '{{form.triggerLabel}}' in {{form.repository}}. Review tone: {{form.reviewTone}}.", - }, -}; +interface FixtureExchange { + request: { method: string; path: string; body: Record }; + response: { status: number; body: unknown }; +} -const REPO_MONITOR: SetupEntry = { - id: "github-repo-monitor", - name: "GitHub repository monitor", - description: "Watch a repository for @OpenHands mentions.", - requires: { - integrations: { - github: { message: "Used to read comments and post replies." }, - }, - features: ["repoClone", "presetPrompt", "webhookDelivery"], - }, - setup: { - version: "1.0", - mode: "direct", - form: { - triggers: { - event: { - on: { - type: "select", - label: "Respond to", - help: "Which GitHub comment event starts a conversation.", - default: "issue_comment.created", - required: true, - options: [ - { - value: "issue_comment.created", - label: "Comments on issues and pull requests", - }, - { - value: "pull_request_review_comment.created", - label: "Pull request review comments only", - }, - ], +interface FixtureScenario { + id: string; + formValues?: SetupFormValues; + localValidation?: { valid: boolean }; + preflight?: FixtureExchange; + create?: FixtureExchange; + conversation?: { request: { action: string; message: string } }; + expectedFieldErrors?: Record; + /** False when the recorded request is deliberately not what setup sends. */ + matchesSetupPayload?: boolean; +} + +interface FixtureBundle { + automationId: string; + scenarios: FixtureScenario[]; +} + +const BUNDLES = [ + prReviewerFixture, + repoMonitorFixture, + incidentFixture, +] as FixtureBundle[]; + +function requireEntry(automationId: string): SetupEntry { + const entry = SETUP_REGISTRY.findById(automationId); + if (!entry) throw new Error(`The registry did not admit ${automationId}`); + return entry; +} + +function requireScenario( + bundle: FixtureBundle, + scenarioId: string, +): FixtureScenario { + const scenario = bundle.scenarios.find(({ id }) => id === scenarioId); + if (!scenario) { + throw new Error(`${bundle.automationId} has no scenario ${scenarioId}`); + } + return scenario; +} + +const CREATE_CASES = BUNDLES.flatMap((bundle) => + bundle.scenarios.flatMap((scenario) => + scenario.create && scenario.matchesSetupPayload !== false + ? [ + { + name: `${bundle.automationId}/${scenario.id}`, + automationId: bundle.automationId, + formValues: scenario.formValues ?? {}, + body: scenario.create.request.body, }, - triggerPhrase: { - type: "text", - label: "Trigger phrase", - help: "Only comments containing this phrase start a conversation.", - default: "@openhands", - required: true, - constraints: { - minLength: 2, - maxLength: 50, - format: "safeExpressionLiteral", - }, + ] + : [], + ), +); + +const PREFLIGHT_CASES = BUNDLES.flatMap((bundle) => + bundle.scenarios.flatMap((scenario) => + scenario.preflight + ? [ + { + name: `${bundle.automationId}/${scenario.id}`, + automationId: bundle.automationId, + formValues: scenario.formValues ?? {}, + body: scenario.preflight.request.body, }, - }, - }, - args: { - repository: { - type: "repo-picker", - label: "Repository", - help: "The repository or repositories to watch for mentions.", - provider: "github", - required: true, - }, - ref: { - type: "text", - label: "Base branch", - help: "Branch checked out when the agent responds.", - default: "main", - required: true, - constraints: { minLength: 1, maxLength: 255 }, - }, - }, - }, - prompt: - "A comment in {{form.repository}} mentions '{{form.triggerPhrase}}'. Read the surrounding issue or pull request context from the event payload, then post a helpful reply as a comment on the same thread.", - filter: - "icontains(comment.body, '{{form.triggerPhrase}}') && glob(repository.full_name, '{{form.repository}}')", - }, -}; + ] + : [], + ), +); -const RETRO_DRAFTER: SetupEntry = { - id: "incident-retrospective-drafter", - name: "Incident retrospective drafter", - description: "Collect incident chatter and draft a timeline.", - requires: { - integrations: { - slack: { message: "Reads incident discussion and timestamps." }, - linear: { message: "Reads follow-up tickets, owners, and status." }, - notion: { - message: "Publishes the drafted retrospective.", - required: false, - }, - }, - features: ["mcpTools", "conversationDispatch"], - }, - skill: "incident-retrospective", - setup: { - version: "1.0", - mode: "assisted", - form: { - note: "These answers start the conversation.", - args: { - incidentChannel: { - type: "text", - label: "Incident channel", - help: "The Slack channel where incidents are discussed.", - required: false, - constraints: { maxLength: 100 }, - }, - linearTeam: { - type: "text", - label: "Linear team", - help: "The team whose follow-up tickets belong in the retrospective.", - required: false, - constraints: { maxLength: 100 }, - }, - notionDestination: { - type: "text", - label: "Notion destination", - help: "Page or database where drafts should be published.", - required: false, - constraints: { maxLength: 200 }, - }, - triggerPreference: { - type: "select", - label: "How should it run?", - help: "A starting preference only.", - default: "undecided", - required: false, - options: [ - { value: "undecided", label: "Let the agent recommend one" }, - { value: "scheduled", label: "On a schedule" }, - ], - }, - notes: { - type: "textarea", - label: "Anything else the agent should know?", - help: "Incident naming conventions, who should be notified.", - required: false, - constraints: { maxLength: 2000 }, - }, - }, - }, - message: - "The user supplied these starting points. Confirm each one, ask about anything left blank, then create the automation.\n\n- Incident channel: {{form.incidentChannel}}\n- Linear team: {{form.linearTeam}}\n- Notion destination: {{form.notionDestination}}\n- Preferred trigger: {{form.triggerPreference}}\n- Notes: {{form.notes}}\n\nStill undecided and required before creation: the incident identifier format and the approved Notion retrospective template.", - }, -}; +const CONVERSATION_CASES = BUNDLES.flatMap((bundle) => + bundle.scenarios.flatMap((scenario) => + scenario.conversation + ? [ + { + name: `${bundle.automationId}/${scenario.id}`, + automationId: bundle.automationId, + formValues: scenario.formValues ?? {}, + message: scenario.conversation.request.message, + }, + ] + : [], + ), +); -const PR_REVIEWER_VALUES = { - repository: "OpenHands/agent-server-gui", - triggerLabel: "openhands-review", - reviewTone: "thorough", - schedule: "*/15 * * * *", - timezone: "UTC", -}; +const SERVICE_ERROR_CASES = BUNDLES.flatMap((bundle) => + bundle.scenarios.flatMap((scenario) => { + const exchange = scenario.preflight ?? scenario.create; + if (!scenario.expectedFieldErrors || !exchange) return []; + return [ + { + name: `${bundle.automationId}/${scenario.id}`, + automationId: bundle.automationId, + formValues: scenario.formValues ?? {}, + responseBody: exchange.response.body, + expectedFieldErrors: scenario.expectedFieldErrors, + }, + ]; + }), +); -const PR_REVIEWER_DRAFT = { - name: "GitHub Code Review Agent - OpenHands/agent-server-gui", - prompt: - "Review pull requests labeled 'openhands-review' in OpenHands/agent-server-gui. Review tone: thorough.", - repos: [{ url: "OpenHands/agent-server-gui", provider: "github" }], - trigger: { type: "cron", schedule: "*/15 * * * *", timezone: "UTC" }, -}; +describe("the published catalog", () => { + it.each(BUNDLES.map((bundle) => [bundle.automationId]))( + "admits %s", + (automationId) => { + // Act + const entry = SETUP_REGISTRY.findById(automationId); + + // Assert + expect(entry).not.toBeNull(); + }, + ); +}); + +describe("the contract fixtures", () => { + it("address the endpoints the host calls", () => { + // Act + const createPaths = new Set( + BUNDLES.flatMap((bundle) => + bundle.scenarios.flatMap((scenario) => + scenario.create ? [scenario.create.request.path] : [], + ), + ), + ); + const preflightPaths = new Set( + BUNDLES.flatMap((bundle) => + bundle.scenarios.flatMap((scenario) => + scenario.preflight ? [scenario.preflight.request.path] : [], + ), + ), + ); + + // Assert + expect({ create: [...createPaths], preflight: [...preflightPaths] }).toEqual( + { create: [AUTOMATION_CREATE_ENDPOINT], preflight: ["/v1/validate"] }, + ); + }); +}); describe("buildCreatePayload", () => { - it("derives the scheduled request body the contract fixture pins", () => { - // Act - const payload = buildCreatePayload(PR_REVIEWER, PR_REVIEWER_VALUES); + it.each(CREATE_CASES)( + "derives the $name create body its fixture pins", + ({ automationId, formValues, body }) => { + // Arrange + const entry = requireEntry(automationId); - // Assert - expect(payload).toEqual(PR_REVIEWER_DRAFT); - }); + // Act + const payload = buildCreatePayload(entry, formValues); - it("derives an event trigger's source and filter, which no manifest states", () => { - // Act - const payload = buildCreatePayload(REPO_MONITOR, { - repository: "OpenHands/agent-server-gui", - triggerPhrase: "@openhands", - on: "issue_comment.created", - ref: "main", - }); - - // Assert - expect(payload).toEqual({ - name: "GitHub repository monitor - OpenHands/agent-server-gui", - prompt: - "A comment in OpenHands/agent-server-gui mentions '@openhands'. Read the surrounding issue or pull request context from the event payload, then post a helpful reply as a comment on the same thread.", - repos: [ - { - url: "OpenHands/agent-server-gui", - ref: "main", - provider: "github", - }, - ], - trigger: { - type: "event", - source: "github", - on: "issue_comment.created", - filter: - "icontains(comment.body, '@openhands') && glob(repository.full_name, 'OpenHands/agent-server-gui')", - }, - }); - }); + // Assert + expect(payload).toEqual(body); + }, + ); it("sends no request body for an entry that hands setup to a conversation", () => { + // Arrange + const entry = requireEntry("incident-retrospective-drafter"); + // Act - const payload = buildCreatePayload(RETRO_DRAFTER, {}); + const payload = buildCreatePayload(entry, {}); // Assert expect(payload).toBeNull(); @@ -300,23 +211,93 @@ describe("buildCreatePayload", () => { }); describe("buildPreflightBody", () => { - it("wraps the draft in the envelope the validate endpoint expects", () => { + it.each(PREFLIGHT_CASES)( + "derives the $name preflight envelope its fixture pins", + ({ automationId, formValues, body }) => { + // Arrange + const entry = requireEntry(automationId); + + // Act + const envelope = buildPreflightBody(entry, formValues); + + // Assert + expect(envelope).toEqual(body); + }, + ); +}); + +describe("buildAssistedMessage", () => { + it.each(CONVERSATION_CASES)( + "opens $name with the skill command and the fixture's seed message", + ({ automationId, formValues, message }) => { + // Arrange + const entry = requireEntry(automationId); + + // Act + const seed = buildAssistedMessage(entry, formValues); + + // Assert + expect(seed).toBe(`/incident-retro:setup\n\n${message}`); + }, + ); +}); + +describe("service rejections mapped back to fields", () => { + it.each(SERVICE_ERROR_CASES)( + "maps the $name rejection to the fields the fixture names", + ({ automationId, formValues, responseBody, expectedFieldErrors }) => { + // Arrange + const entry = requireEntry(automationId); + const payload = buildCreatePayload(entry, formValues); + + // Act + const mapped = mapServiceErrors( + normalizeServiceErrors(responseBody, payload), + deriveErrorMap(entry), + ); + + // Assert + expect(mapped).toEqual({ fieldErrors: expectedFieldErrors, formErrors: [] }); + }, + ); +}); + +describe("local validation of fixture form values", () => { + it("blocks the unsafe trigger phrase before any request is made", () => { + // Arrange — the fixture names the failing field; the code is the host's + // own vocabulary, rendered through its translations. + const scenario = requireScenario( + BUNDLES[1], + "quote-in-trigger-phrase-blocked-locally", + ); + const entry = requireEntry("github-repo-monitor"); + // Act - const body = buildPreflightBody(PR_REVIEWER, PR_REVIEWER_VALUES); + const errors = validateFormValues(entry.setup, scenario.formValues ?? {}); // Assert - expect(body).toEqual({ - automationId: "github-pr-reviewer", - endpoint: "/v1/preset/prompt", - draft: PR_REVIEWER_DRAFT, + expect(errors).toEqual({ + triggerPhrase: { code: "unsafeExpressionLiteral" }, }); }); + + it("passes an entirely blank assisted form, as its fixture records", () => { + // Arrange + const scenario = requireScenario(BUNDLES[2], "nothing-filled-in"); + const entry = requireEntry("incident-retrospective-drafter"); + + // Act + const errors = validateFormValues(entry.setup, scenario.formValues ?? {}); + + // Assert + expect(errors).toEqual({}); + }); }); describe("deriveErrorMap", () => { it("recovers which fields built each payload path", () => { // Act - const errorMap = deriveErrorMap(PR_REVIEWER); + const errorMap = deriveErrorMap(requireEntry("github-pr-reviewer")); // Assert expect(errorMap).toEqual({ @@ -328,37 +309,3 @@ describe("deriveErrorMap", () => { }); }); }); - -describe("buildAssistedMessage", () => { - it("opens with the owning skill's command and the interpolated answers", () => { - // Act - const message = buildAssistedMessage(RETRO_DRAFTER, { - incidentChannel: "#incidents", - linearTeam: "Reliability", - notionDestination: "", - triggerPreference: "undecided", - notes: "", - }); - - // Assert - expect(message).toBe( - "/incident-retro:setup\n\nThe user supplied these starting points. Confirm each one, ask about anything left blank, then create the automation.\n\n- Incident channel: #incidents\n- Linear team: Reliability\n- Notion destination: \n- Preferred trigger: undecided\n- Notes: \n\nStill undecided and required before creation: the incident identifier format and the approved Notion retrospective template.", - ); - }); -}); - -describe("the published manifests", () => { - // The host enforces invariants its own derivation depends on, which the - // published schema does not. Every shipped entry has to satisfy them too. - it.each([ - ["github-pr-reviewer", PR_REVIEWER], - ["github-repo-monitor", REPO_MONITOR], - ["incident-retrospective-drafter", RETRO_DRAFTER], - ])("admits %s", (_id, entry) => { - // Act - const result = validateSetupEntry(entry); - - // Assert - expect(result).toEqual({ valid: true, errors: [] }); - }); -}); diff --git a/__tests__/manifests/manifest-capabilities.test.ts b/__tests__/manifests/manifest-capabilities.test.ts index 261f6aba37..3a0406b525 100644 --- a/__tests__/manifests/manifest-capabilities.test.ts +++ b/__tests__/manifests/manifest-capabilities.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; +import capabilitiesFixture from "@openhands/extensions/testing/automations/capabilities.json"; +import incidentFixture from "@openhands/extensions/testing/automations/incident-retrospective-drafter.json"; +import prReviewerFixture from "@openhands/extensions/testing/automations/github-pr-reviewer.json"; +import repoMonitorFixture from "@openhands/extensions/testing/automations/github-repo-monitor.json"; import { assessCapabilityRequirements } from "#/manifests/manifest-capabilities"; +import { SETUP_REGISTRY } from "#/manifests/manifest-sources"; import type { DeploymentCapabilities } from "#/manifests/types"; import { createSetup, createSetupEntry } from "./manifest-test-data"; @@ -77,3 +82,44 @@ describe("assessCapabilityRequirements", () => { expect(assessment).toEqual({ supported: false, unmet: [] }); }); }); + +describe("the published contract fixtures", () => { + // Each fixture bundle lists in `blockedBy` exactly the published deployment + // shapes its entry cannot run under. The host's assessment must agree with + // that record for every entry × shape pair. + const shapes = Object.entries(capabilitiesFixture.responses).map( + ([shapeName, response]) => ({ + shapeName, + capabilities: response.body as DeploymentCapabilities, + }), + ); + + const bundles = [prReviewerFixture, repoMonitorFixture, incidentFixture] as { + automationId: string; + blockedBy: string[]; + }[]; + + it.each( + bundles.flatMap(({ automationId, blockedBy }) => + shapes.map(({ shapeName, capabilities }) => ({ + automationId, + shapeName, + capabilities, + blocked: blockedBy.includes(shapeName), + })), + ), + )( + "$automationId under the $shapeName deployment: blocked=$blocked", + ({ automationId, capabilities, blocked }) => { + // Arrange + const entry = SETUP_REGISTRY.findById(automationId); + if (!entry) throw new Error(`The registry did not admit ${automationId}`); + + // Act + const assessment = assessCapabilityRequirements(entry, capabilities); + + // Assert + expect(assessment.supported).toBe(!blocked); + }, + ); +}); diff --git a/src/mocks/automation-handlers.ts b/src/mocks/automation-handlers.ts index e2d4aac70f..c71894b13b 100644 --- a/src/mocks/automation-handlers.ts +++ b/src/mocks/automation-handlers.ts @@ -1,4 +1,10 @@ import { http, HttpResponse, delay } from "msw"; +import capabilitiesFixture from "@openhands/extensions/testing/automations/capabilities.json"; +import type { + DeploymentCapabilities, + DraftValidationError, + ValidateDraftResponse, +} from "#/manifests/types"; import type { Automation, AutomationsResponse, @@ -9,6 +15,79 @@ import { AutomationRunStatus } from "#/types/automation"; import { MOCK_AUTOMATIONS_RESPONSE } from "./automations.mock"; import { MOCK_AUTOMATION_RUNS } from "./automation-runs.mock"; +// The "supported" deployment from the published contract fixtures. Discovery +// and preflight answer with it, so the setup flow runs against the same +// reference data the extensions contract is verified against. +const CAPABILITIES: DeploymentCapabilities = + capabilitiesFixture.responses.supported.body; + +interface DraftTrigger { + type?: string; + schedule?: string; + on?: string; + source?: string; +} + +// The schedules the mock can read: "* * * * *" and "*/N * * * *". Anything +// else is assumed to satisfy the deployment minimum — this stands in for the +// service's cron parser rather than reimplementing it. +const STEP_SCHEDULE_PATTERN = /^(?:\*|\*\/(\d+)) \* \* \* \*$/; + +function cronIntervalSeconds(schedule: string): number | null { + const match = STEP_SCHEDULE_PATTERN.exec(schedule.trim()); + if (!match) return null; + return (match[1] ? Number(match[1]) : 1) * 60; +} + +// Event deliveries registered for the mock organization. Distinct from +// CAPABILITIES.eventTypes on purpose: a deployment can support an event type +// that no webhook is registered to deliver, and preflight is what catches +// that gap — the fixtures' event-type-not-delivered scenario depends on one +// supported type staying unregistered here. +const REGISTERED_EVENT_TYPES = CAPABILITIES.eventTypes.filter( + (type) => type !== "pull_request_review_comment.created", +); + +// The deployment checks the contract fixtures record: a cron schedule below +// triggers.cron.minIntervalSeconds, and an event type no webhook delivers. +// Errors address the draft by dotted path, exactly as the fixtures do. +function validateDraftTrigger( + trigger: DraftTrigger | undefined, +): DraftValidationError[] { + if (!trigger) return []; + + const cron = CAPABILITIES.triggers.cron; + if (trigger.type === "cron" && trigger.schedule && cron) { + const interval = cronIntervalSeconds(trigger.schedule); + if (interval !== null && interval < cron.minIntervalSeconds) { + return [ + { + field: "trigger.schedule", + code: "interval_too_short", + message: `Minimum interval for this deployment is ${cron.minIntervalSeconds / 60} minutes.`, + }, + ]; + } + } + + if ( + trigger.type === "event" && + trigger.on && + !REGISTERED_EVENT_TYPES.includes(trigger.on) + ) { + const source = trigger.source === "github" ? "GitHub" : trigger.source; + return [ + { + field: "trigger.on", + code: "event_type_not_delivered", + message: `No ${source} webhook delivering ${trigger.on} is registered for this organization.`, + }, + ]; + } + + return []; +} + // Mutable copy for CRUD operations within the mock session const automations = new Map( MOCK_AUTOMATIONS_RESPONSE.automations.map((a) => [a.id, { ...a }]), @@ -47,6 +126,31 @@ export const AUTOMATION_HANDLERS = [ return HttpResponse.json(response); }), + // GET /api/automation/v1/capabilities — What this deployment supports + http.get("*/api/automation/v1/capabilities", async () => { + await delay(200); + return HttpResponse.json(CAPABILITIES); + }), + + // POST /api/automation/v1/validate — Preflight a draft without creating it + http.post("*/api/automation/v1/validate", async ({ request }) => { + await delay(200); + + const body = (await request.clone().json()) as { + automationId?: string; + endpoint?: string; + draft?: { trigger?: DraftTrigger }; + }; + + const errors = validateDraftTrigger(body.draft?.trigger); + const response: ValidateDraftResponse = { + valid: errors.length === 0, + errors, + }; + + return HttpResponse.json(response); + }), + // POST /api/automation/v1/preset/:kind — Create a prompt/plugin automation http.post("*/api/automation/v1/preset/:kind", async ({ params, request }) => { await delay(200); diff --git a/tests/e2e/mock-llm/test-mapping.json b/tests/e2e/mock-llm/test-mapping.json index 14bbc81108..57e758cdbb 100644 --- a/tests/e2e/mock-llm/test-mapping.json +++ b/tests/e2e/mock-llm/test-mapping.json @@ -47,10 +47,14 @@ "tests": ["files"] }, { - "comment": "Automations lifecycle and preset cards", + "comment": "Automations lifecycle, preset cards, and the manifest setup host", "sources": [ "src/components/features/automations/**", - "src/routes/automations*.tsx" + "src/components/features/manifest/**", + "src/manifests/**", + "src/mocks/automation-handlers.ts", + "src/routes/automations*.tsx", + "src/routes/automation-setup-route.tsx" ], "tests": ["automations"] },