Compare commits

...

1 Commits

Author SHA1 Message Date
Naiyuan Qing
cfd360597e feat(analytics): add onboarding_runtime_detected event on desktop Step 3
Answers "did the user have an AI CLI installed locally when they hit
Step 3" — currently unanswerable from the existing funnel because the
bundled daemon fails to register at all when zero CLIs are on PATH, so
`runtime_registered` is silent on that cohort. Splits the 40% of
`completion_path=runtime_skipped` into "had CLIs, skipped anyway" vs "no
CLIs available, had no choice" — the two cases need opposite product
fixes.

Fires once per Step 3 mount in `step-runtime-connect.tsx` (desktop
only), when the scanning phase resolves — either immediately on first
runtime registration or after the 5 s empty timeout. Reports
`runtime_count`, `online_count`, sorted `providers`, convenience
booleans (`has_claude` / `has_codex` / `has_cursor`), and `detect_ms`.
Also writes `has_any_cli` + `detected_cli_count` via `$set` as cohort
signals.

Not emitted from the web Step 3 (`step-platform-fork.tsx`) — web users
don't run the bundled daemon, so their runtime list can reflect
daemons on other machines and would corrupt the
"CLI installed locally" signal.

Refs MUL-1250.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-23 15:53:43 +08:00
3 changed files with 300 additions and 0 deletions

View File

@@ -323,6 +323,45 @@ request payload.
`path: "download_desktop"` signals Step 3 path choice specifically,
not actual download start.
- `onboarding_runtime_detected` — fired from
`packages/views/onboarding/steps/step-runtime-connect.tsx` (desktop
Step 3) once per mount, when the scanning phase resolves — either
immediately on first runtime registration, or after the 5 s empty
timeout. Answers the question "did the user have any AI CLI
installed on this machine when they hit Step 3" — currently
unanswerable from the existing funnel because the bundled daemon
fails to register at all when zero CLIs are on PATH, so
`runtime_registered` is silent on that cohort. Splits
`completion_path=runtime_skipped` into "had CLIs, skipped anyway"
vs "no CLIs available, had no choice". Properties:
- `source`: `step3_desktop` (literal; reserved for a future web
emission under a different value).
- `outcome`: `found` (at least one runtime registered before the
5 s grace window expired) or `empty` (none registered by then).
- `runtime_count`: number of runtimes visible to this user at
resolution time.
- `online_count`: subset of `runtime_count` whose `status` is
`online`.
- `providers`: sorted array of distinct provider names (e.g.
`["claude", "codex"]`).
- `has_claude` / `has_codex` / `has_cursor`: convenience booleans
derived from `providers` for funnel breakdowns without array
filtering in HogQL.
- `detect_ms`: wall-clock ms from component mount to resolution.
Surfaces daemon boot latency — `found` events with a high
`detect_ms` approach the timeout threshold and inform whether
to lengthen the grace period.
Person properties set with `$set`:
- `has_any_cli`: boolean — cohort signal for "user has at least
one local AI CLI detected on this machine".
- `detected_cli_count`: number — granular cohort signal.
Not emitted from the web Step 3 (`step-platform-fork.tsx`) — web
users don't run the bundled daemon, so their runtime list reflects
daemons from other machines and would corrupt the
"CLI installed locally" signal.
- `download_intent_expressed` — fired whenever a user clicks a CTA
that points at the `/download` page. Surfaces five sources across
the funnel, letting the top-of-funnel entry be split cleanly.

View File

@@ -0,0 +1,216 @@
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
import { act, render, screen } from "@testing-library/react";
import type { AgentRuntime } from "@multica/core/types";
// Hoisted mocks — replace analytics and the runtime picker before the SUT
// imports them. Tests drive picker state via `mocks.pickerState`; every
// captureEvent / setPersonProperties call lands on `mocks.captureEvent` /
// `mocks.setPersonProperties` so we can assert the payload shape.
const mocks = vi.hoisted(() => ({
captureEvent: vi.fn<(name: string, props?: Record<string, unknown>) => void>(),
setPersonProperties: vi.fn<(props: Record<string, unknown>) => void>(),
pickerState: {
runtimes: [] as AgentRuntime[],
selected: null as AgentRuntime | null,
selectedId: null as string | null,
setSelectedId: vi.fn<(id: string) => void>(),
hasRuntimes: false,
},
}));
vi.mock("@multica/core/analytics", () => ({
captureEvent: mocks.captureEvent,
setPersonProperties: mocks.setPersonProperties,
}));
vi.mock("../components/use-runtime-picker", () => ({
useRuntimePicker: () => mocks.pickerState,
}));
import { StepRuntimeConnect } from "./step-runtime-connect";
function makeRuntime(overrides: Partial<AgentRuntime> = {}): AgentRuntime {
return {
id: "rt_test",
workspace_id: "ws_test",
name: "Claude Code",
provider: "claude",
status: "online",
runtime_mode: "local",
runtime_config: {},
device_info: "",
metadata: {},
daemon_id: null,
last_seen_at: new Date().toISOString(),
created_at: new Date().toISOString(),
updated_at: new Date().toISOString(),
...overrides,
} as unknown as AgentRuntime;
}
function setPicker(patch: Partial<typeof mocks.pickerState> = {}) {
mocks.pickerState.runtimes = patch.runtimes ?? [];
mocks.pickerState.selected = patch.selected ?? null;
mocks.pickerState.selectedId = patch.selectedId ?? null;
mocks.pickerState.hasRuntimes = patch.hasRuntimes ?? false;
mocks.pickerState.setSelectedId = vi.fn();
}
function renderStep() {
const onNext = vi.fn();
const onBack = vi.fn();
render(
<StepRuntimeConnect wsId="ws_test" onNext={onNext} onBack={onBack} />,
);
return { onNext, onBack };
}
describe("StepRuntimeConnect — onboarding_runtime_detected", () => {
beforeEach(() => {
mocks.captureEvent.mockReset();
mocks.setPersonProperties.mockReset();
setPicker();
vi.useFakeTimers({ shouldAdvanceTime: true });
});
afterEach(() => {
vi.useRealTimers();
});
it("fires `outcome: found` when runtimes arrive synchronously on mount", () => {
const rt = makeRuntime({
id: "rt_claude",
provider: "claude",
status: "online",
});
setPicker({ runtimes: [rt], selected: rt, selectedId: rt.id, hasRuntimes: true });
renderStep();
expect(mocks.captureEvent).toHaveBeenCalledTimes(1);
const [name, props] = mocks.captureEvent.mock.calls[0]!;
expect(name).toBe("onboarding_runtime_detected");
expect(props).toMatchObject({
source: "step3_desktop",
outcome: "found",
runtime_count: 1,
online_count: 1,
providers: ["claude"],
has_claude: true,
has_codex: false,
has_cursor: false,
});
expect(typeof (props as Record<string, unknown>).detect_ms).toBe("number");
expect(mocks.setPersonProperties).toHaveBeenCalledWith({
has_any_cli: true,
detected_cli_count: 1,
});
});
it("derives has_claude / has_codex / has_cursor from distinct providers", () => {
setPicker({
runtimes: [
makeRuntime({ id: "rt1", provider: "claude" }),
makeRuntime({ id: "rt2", provider: "codex", status: "offline" }),
makeRuntime({ id: "rt3", provider: "cursor" }),
],
hasRuntimes: true,
});
renderStep();
expect(mocks.captureEvent).toHaveBeenCalledTimes(1);
const props = mocks.captureEvent.mock.calls[0]![1] as Record<string, unknown>;
expect(props.runtime_count).toBe(3);
expect(props.online_count).toBe(2);
expect(props.providers).toEqual(["claude", "codex", "cursor"]);
expect(props.has_claude).toBe(true);
expect(props.has_codex).toBe(true);
expect(props.has_cursor).toBe(true);
});
it("fires `outcome: empty` after the 5s scanning timeout when no runtimes arrive", () => {
setPicker({ runtimes: [] });
renderStep();
// Scanning phase: no event yet.
expect(mocks.captureEvent).not.toHaveBeenCalled();
// Advance past the 5s empty-timeout inside act so the state flip
// flushes React updates before we assert.
act(() => {
vi.advanceTimersByTime(5_001);
});
expect(mocks.captureEvent).toHaveBeenCalledTimes(1);
const props = mocks.captureEvent.mock.calls[0]![1] as Record<string, unknown>;
expect(props).toMatchObject({
source: "step3_desktop",
outcome: "empty",
runtime_count: 0,
online_count: 0,
providers: [],
has_claude: false,
has_codex: false,
has_cursor: false,
});
expect(mocks.setPersonProperties).toHaveBeenCalledWith({
has_any_cli: false,
detected_cli_count: 0,
});
});
it("does not re-emit if the component re-renders after resolution", () => {
const rt = makeRuntime({ id: "rt_claude", provider: "claude" });
setPicker({ runtimes: [rt], selected: rt, selectedId: rt.id, hasRuntimes: true });
const { onNext } = renderStep();
expect(mocks.captureEvent).toHaveBeenCalledTimes(1);
// Simulate a runtime coming online / a second runtime registering:
// the event has already resolved once; it must not re-emit.
setPicker({
runtimes: [rt, makeRuntime({ id: "rt_codex", provider: "codex" })],
selected: rt,
selectedId: rt.id,
hasRuntimes: true,
});
// Force a re-render by firing a timer tick — React will re-read the
// mocked picker state but the ref latch keeps the event unique.
act(() => {
vi.advanceTimersByTime(1_000);
});
expect(mocks.captureEvent).toHaveBeenCalledTimes(1);
expect(onNext).not.toHaveBeenCalled();
});
it("only counts distinct providers (multiple runtimes of the same provider)", () => {
setPicker({
runtimes: [
makeRuntime({ id: "rt1", provider: "claude" }),
makeRuntime({ id: "rt2", provider: "claude", status: "offline" }),
],
hasRuntimes: true,
});
renderStep();
const props = mocks.captureEvent.mock.calls[0]![1] as Record<string, unknown>;
expect(props.runtime_count).toBe(2);
expect(props.online_count).toBe(1);
expect(props.providers).toEqual(["claude"]);
});
it("mounts without touching framework-level globals", () => {
// Sanity: the StepHeader renders and the DragStrip doesn't explode
// under jsdom. Keeps the test file honest if someone refactors the
// shell around the effect.
setPicker({ runtimes: [] });
renderStep();
expect(screen.getByText(/Looking for your tools/i)).toBeInTheDocument();
});
});

View File

@@ -2,6 +2,7 @@
import { useEffect, useRef, useState } from "react";
import { ArrowLeft, ArrowRight, Loader2 } from "lucide-react";
import { captureEvent, setPersonProperties } from "@multica/core/analytics";
import { Button } from "@multica/ui/components/ui/button";
import {
Dialog,
@@ -112,6 +113,50 @@ function FancyView({
const onlineCount = runtimes.filter((r) => r.status === "online").length;
// One-shot analytics event when the scan window resolves. Answers the
// question "did the user actually have any AI CLI installed on this
// machine when they hit Step 3" — currently unanswerable from the
// existing funnel because a zero-CLI daemon fails to register at all,
// so `runtime_registered` is silent on that cohort. Emitting from here
// (rather than the daemon) keeps the signal in sync with what the UI
// actually showed the user: "scanning → found" vs "scanning → empty"
// after the 5s grace period.
const detectStartRef = useRef<number | null>(null);
if (detectStartRef.current === null) {
detectStartRef.current =
typeof performance !== "undefined" ? performance.now() : Date.now();
}
const detectedEmittedRef = useRef(false);
useEffect(() => {
if (detectedEmittedRef.current) return;
if (phase === "scanning") return;
detectedEmittedRef.current = true;
const providers = Array.from(
new Set(runtimes.map((r) => r.provider).filter(Boolean)),
).sort();
const now =
typeof performance !== "undefined" ? performance.now() : Date.now();
const detectMs = Math.round(now - (detectStartRef.current ?? now));
captureEvent("onboarding_runtime_detected", {
source: "step3_desktop",
outcome: phase,
runtime_count: runtimes.length,
online_count: onlineCount,
providers,
has_claude: providers.includes("claude"),
has_codex: providers.includes("codex"),
has_cursor: providers.includes("cursor"),
detect_ms: detectMs,
});
setPersonProperties({
has_any_cli: runtimes.length > 0,
detected_cli_count: runtimes.length,
});
}, [phase, runtimes, onlineCount]);
const [submitting, setSubmitting] = useState(false);
// Cloud waitlist submission state lives here (rather than in EmptyView)
// so it survives phase flips — e.g. a runtime coming online after the