From b6727865b711aeb6304bdd92df46016ca89d991b Mon Sep 17 00:00:00 2001 From: Hiep Le <69354317+hieptl@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:26:39 +0700 Subject: [PATCH] feat: instrument PostHog analytics for the onboarding funnel (#1547) * feat: instrument PostHog analytics for the onboarding funnel * test: de-flake onboarding layout probe and profile activation --- .../onboarding/onboarding-modal.test.tsx | 156 ++++++++++++++++++ .../features/onboarding/onboarding-modal.tsx | 99 ++++++++++- src/hooks/use-tracking.ts | 50 ++++++ tests/e2e/mock-llm/utils/mock-llm-helpers.ts | 6 +- tests/e2e/support/onboarding-helpers.ts | 16 ++ 5 files changed, 323 insertions(+), 4 deletions(-) diff --git a/__tests__/components/onboarding/onboarding-modal.test.tsx b/__tests__/components/onboarding/onboarding-modal.test.tsx index 8550abf507..ed79313bdb 100644 --- a/__tests__/components/onboarding/onboarding-modal.test.tsx +++ b/__tests__/components/onboarding/onboarding-modal.test.tsx @@ -20,6 +20,13 @@ import { DEFAULT_SETTINGS } from "#/services/settings"; const llmSettingsScreenMock = vi.hoisted(() => vi.fn()); const getServerInfoMock = vi.hoisted(() => vi.fn()); +const captureMock = vi.hoisted(() => vi.fn()); + +// useTracking depends on PostHog's `usePostHog`. Mock that underlying service +// (not useTracking itself) so the onboarding analytics events can be asserted. +vi.mock("posthog-js/react", () => ({ + usePostHog: () => ({ capture: captureMock }), +})); // Both the backend status badge in the embedded edit form and the // step-1 health probe ride on `useBackendsHealth`, which resolves @@ -194,6 +201,7 @@ function renderModal(onClose = vi.fn()) { beforeEach(() => { window.localStorage.clear(); + window.sessionStorage.clear(); vi.stubEnv("VITE_BACKEND_BASE_URL", "http://localhost:9000"); vi.stubEnv("VITE_SESSION_API_KEY", "session-key"); __resetActiveStoreForTests(); @@ -228,6 +236,7 @@ beforeEach(() => { }); afterEach(() => { window.localStorage.clear(); + window.sessionStorage.clear(); vi.unstubAllEnvs(); __resetActiveStoreForTests(); }); @@ -974,4 +983,151 @@ describe("OnboardingModal", () => { ); expect(onClose).toHaveBeenCalledTimes(1); }); + + describe("analytics events", () => { + // Surface only the captures for a given event; a session emits several. + const eventCalls = (event: string) => + captureMock.mock.calls.filter(([name]) => name === event); + + beforeEach(() => { + // Grant analytics consent so events pass useTracking's consent gate; + // the outer beforeEach seeds settings with consent withheld. + vi.spyOn(SettingsService, "getSettings").mockResolvedValue({ + ...DEFAULT_SETTINGS, + agent_settings: { ...DEFAULT_SETTINGS.agent_settings, llm: {} }, + user_consents_to_analytics: true, + }); + }); + + it("captures onboarding_started once per session even across a remount", async () => { + // Arrange + act: open onboarding and let the first mount settle (a Step + // Viewed capture confirms consent resolved and backend health settled). + const firstMount = renderModal(); + await waitFor(() => + expect( + eventCalls("onboarding_step_viewed").length, + ).toBeGreaterThanOrEqual(1), + ); + + // Act: tear the modal down and mount it again in the same session. The + // remounted modal re-emits Step Viewed from a fresh ref, so waiting for + // the second one proves the Started effect had its chance to re-fire. + firstMount.unmount(); + renderModal(); + await waitFor(() => + expect( + eventCalls("onboarding_step_viewed").length, + ).toBeGreaterThanOrEqual(2), + ); + + // Assert: the sessionStorage guard kept Started at a single capture. + expect(eventCalls("onboarding_started")).toHaveLength(1); + }); + + it("captures no onboarding events while analytics consent is not granted", async () => { + // Arrange: withhold consent. + vi.spyOn(SettingsService, "getSettings").mockResolvedValue({ + ...DEFAULT_SETTINGS, + agent_settings: { ...DEFAULT_SETTINGS.agent_settings, llm: {} }, + user_consents_to_analytics: false, + }); + + // Act: open onboarding and let it settle on the first step. + renderModal(); + await waitForConfiguredBackendToBeSkipped(); + + // Assert: nothing reached PostHog. + expect(captureMock).not.toHaveBeenCalled(); + }); + + it("captures onboarding_step_viewed for each step as the user advances", async () => { + // Arrange: the healthy backend is skipped, so the user lands on the + // agent step (index 0 of the 3-step flow). + renderModal(); + const user = userEvent.setup(); + await waitForConfiguredBackendToBeSkipped(); + + // Assert: the entry step carries the full funnel identity. + await waitFor(() => + expect(captureMock).toHaveBeenCalledWith( + "onboarding_step_viewed", + expect.objectContaining({ + step: "agent", + step_index: 0, + total_steps: 3, + agent: "openhands", + }), + ), + ); + + // Act: advance to the next step. + await completeAgentStep(user); + + // Assert: the new step is captured too — tracking follows transitions, + // not just the initial mount. + await waitFor(() => + expect(captureMock).toHaveBeenCalledWith( + "onboarding_step_viewed", + expect.objectContaining({ + step: "setup", + step_index: 1, + total_steps: 3, + }), + ), + ); + }); + + it("captures onboarding_completed when a conversation is launched from the final step", async () => { + // Arrange: walk through to the final Say Hello step. + renderModal(); + const user = userEvent.setup(); + await waitForConfiguredBackendToBeSkipped(); + await completeAgentStep(user); + await user.click(screen.getByTestId("onboarding-llm-next")); + await waitFor(() => + expect(screen.getByTestId("onboarding-slide-2")).toHaveAttribute( + "data-active", + "true", + ), + ); + + // Act: launch from the final step. The stubbed recommended-automation + // launcher invokes the same onLaunched completion callback as Say Hello. + const recommendations = screen.getByTestId( + "onboarding-recommended-automations", + ); + await user.click( + within(recommendations).getByRole("button", { + name: "launch recommended automation", + }), + ); + + // Assert. + expect(captureMock).toHaveBeenCalledWith( + "onboarding_completed", + expect.objectContaining({ agent: "openhands" }), + ); + }); + + it("captures onboarding_skipped with the current step when the user skips", async () => { + // Arrange: healthy backend skipped → user is on the agent step (0 of 3). + renderModal(); + const user = userEvent.setup(); + await waitForConfiguredBackendToBeSkipped(); + + // Act: skip out of onboarding. + await user.click(screen.getByTestId("onboarding-skip")); + + // Assert. + expect(captureMock).toHaveBeenCalledWith( + "onboarding_skipped", + expect.objectContaining({ + step: "agent", + step_index: 0, + total_steps: 3, + agent: "openhands", + }), + ); + }); + }); }); diff --git a/src/components/features/onboarding/onboarding-modal.tsx b/src/components/features/onboarding/onboarding-modal.tsx index 92db30982f..76e303b525 100644 --- a/src/components/features/onboarding/onboarding-modal.tsx +++ b/src/components/features/onboarding/onboarding-modal.tsx @@ -11,6 +11,8 @@ import { I18nKey } from "#/i18n/declaration"; import { cn } from "#/utils/utils"; import { useActiveBackendContext } from "#/contexts/active-backend-context"; import { useBackendsHealth } from "#/hooks/query/use-backends-health"; +import { useSettings } from "#/hooks/query/use-settings"; +import { useTracking } from "#/hooks/use-tracking"; import { OnboardingProgressBar } from "./onboarding-progress-bar"; import { ChooseAgentStep, @@ -45,6 +47,14 @@ const PHASE_ORDER_WITHOUT_BACKEND: readonly OnboardingPhase[] = [ "hello", ]; +/** + * sessionStorage flag marking that the one-time "onboarding started" analytics + * event has already been captured for this browser session. It survives + * rerenders, remounts and the lazy/Suspense flip so the event fires at most + * once per onboarding session. + */ +const ONBOARDING_STARTED_TRACKED_KEY = "openhands-onboarding-started"; + interface SlideProps { /** Index of this slide in the step sequence. */ index: number; @@ -124,6 +134,14 @@ export function OnboardingModal({ isPreview = false, }: OnboardingModalProps) { const { t } = useTranslation("openhands"); + const { data: settings } = useSettings(); + const analyticsEnabled = settings?.user_consents_to_analytics === true; + const { + trackOnboardingStarted, + trackOnboardingStepViewed, + trackOnboardingCompleted, + trackOnboardingSkipped, + } = useTracking(); const { active } = useActiveBackendContext(); const { backend } = active; const noBackendSelected = isNoBackend(backend); @@ -173,6 +191,15 @@ export function OnboardingModal({ const currentPhase = slideOrder.includes(phase) ? phase : slideOrder[0]; const currentStep = slideOrder.indexOf(currentPhase); + // Backend connectivity is "settled" once we know whether the active backend + // is reachable (or no backend is selected). Until then `skipBackendStep` may + // still flip true and renumber the slides, briefly showing the backend slide + // for an already-healthy backend. Gate Step Viewed on this so we never emit a + // phantom "backend" view that immediately auto-skips to "agent". + const backendHealthSettled = + noBackendSelected || + healthByBackendId[backend.id]?.isConnected !== undefined; + const isOpenHands = selectedAgentId === "openhands"; const hideSkip = currentStep === 0 && getLockedCloudHost() !== null; const goNext = React.useCallback(() => { @@ -194,6 +221,72 @@ export function OnboardingModal({ }); }, [slideOrder]); + // --- Onboarding analytics ------------------------------------------------- + // Every event is routed through `useTracking`, which drops captures unless + // the user has consented; the design-preview harness (`isPreview`) emits + // nothing. + + // Onboarding Started — fire at most once per browser session. The + // sessionStorage flag survives rerenders, remounts and the lazy/Suspense + // flip; the consent gate means we only arm the guard once a capture can + // actually happen, so a user who consents partway through onboarding still + // gets the event instead of having it latched away during the no-consent + // window. + const startedTrackedRef = React.useRef(false); + React.useEffect(() => { + if (isPreview || !analyticsEnabled || startedTrackedRef.current) return; + if (window.sessionStorage.getItem(ONBOARDING_STARTED_TRACKED_KEY)) return; + startedTrackedRef.current = true; + window.sessionStorage.setItem(ONBOARDING_STARTED_TRACKED_KEY, "1"); + trackOnboardingStarted(); + }, [isPreview, analyticsEnabled]); + + // Onboarding Step Viewed — fire once per phase entry. The phase string is the + // stable identity (slide indices renumber), so dedupe on it: this absorbs + // StrictMode's double-invoke and the index renumber, while re-entering a step + // via Back counts as a genuine new view. + const lastViewedPhaseRef = React.useRef(null); + React.useEffect(() => { + if (isPreview || !analyticsEnabled || !backendHealthSettled) return; + if (lastViewedPhaseRef.current === currentPhase) return; + lastViewedPhaseRef.current = currentPhase; + trackOnboardingStepViewed({ + step: currentPhase, + stepIndex: currentStep, + totalSteps, + agent: selectedAgentId, + }); + }, [ + isPreview, + analyticsEnabled, + backendHealthSettled, + currentPhase, + currentStep, + totalSteps, + selectedAgentId, + ]); + + // Onboarding Completed — `onLaunched` runs only after a conversation is + // successfully created (the hello message or a recommended automation), so + // wrapping it captures completion exactly once before the modal closes. + const handleCompleted = () => { + trackOnboardingCompleted({ agent: selectedAgentId }); + onClose(); + }; + + // Onboarding Skipped/Dismissed — the user exited before completing, via the + // skip button (non-final steps) or the final-step Close button. `currentPhase` + // records where they left, which also distinguishes the two affordances. + const handleSkipOrDismiss = () => { + trackOnboardingSkipped({ + step: currentPhase, + stepIndex: currentStep, + totalSteps, + agent: selectedAgentId, + }); + onClose(); + }; + return ( // No `onClose`: the flow must only be dismissed via explicit actions // (the skip button or launching), never by an errant backdrop click or @@ -267,8 +360,8 @@ export function OnboardingModal({ > @@ -279,7 +372,7 @@ export function OnboardingModal({