From b12af9d7888c5fc0fb5d06bd11135d76e80d13c9 Mon Sep 17 00:00:00 2001 From: owenkephart Date: Fri, 4 Sep 2026 21:33:41 +0000 Subject: [PATCH 1/2] feat(cli): add telemetry identity and init stages Signed-off-by: owenkephart --- .changeset/track-init-telemetry-stages.md | 5 ++ docs/reference/telemetry.md | 10 ++-- .../eve/src/cli/commands/extension-init.ts | 8 +++ packages/eve/src/cli/commands/init.ts | 24 +++++++- .../eve/src/cli/dev/run-interactive-ui.ts | 4 ++ packages/eve/src/cli/dev/tui/runner.test.ts | 12 ++++ packages/eve/src/cli/dev/tui/runner.ts | 48 ++++++++++++---- packages/eve/src/cli/dev/tui/tui.ts | 5 ++ packages/eve/src/cli/run.test.ts | 38 +++++++++---- packages/eve/src/cli/run.ts | 29 +++++++--- packages/eve/src/cli/telemetry/index.test.ts | 55 ++++++++++++++++++- packages/eve/src/cli/telemetry/index.ts | 26 ++++++++- 12 files changed, 225 insertions(+), 39 deletions(-) create mode 100644 .changeset/track-init-telemetry-stages.md diff --git a/.changeset/track-init-telemetry-stages.md b/.changeset/track-init-telemetry-stages.md new file mode 100644 index 0000000000..13992254d5 --- /dev/null +++ b/.changeset/track-init-telemetry-stages.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Record the furthest stage reached by `eve init`, `eve extension init`, and the interactive `eve dev --onboard` handoff in CLI telemetry, so failed setup runs can be grouped by their stage. Telemetry now also identifies whether its installation and project identifiers are ephemeral or persisted locally. diff --git a/docs/reference/telemetry.md b/docs/reference/telemetry.md index 7d34d4b0d9..0ff233a37b 100644 --- a/docs/reference/telemetry.md +++ b/docs/reference/telemetry.md @@ -12,11 +12,11 @@ eve collects usage data from its CLI to help improve its commands and developmen eve sends the following information to Vercel: - The eve version, operating system, CPU architecture, and whether stdin is a terminal. -- The command you ran and whether it succeeded, had a usage error, or failed. -- For `eve dev`, whether you connected to a local or remote agent and whether the UI was interactive or headless. -- Random identifiers for the CLI session, your eve installation, and the project. +- The command you ran and whether it succeeded, had a usage error, or failed. For `eve init` and `eve extension init`, eve also records the furthest stage reached: target resolution, scaffolding, dependency installation, Git initialization, or post-initialization handoff. +- For `eve dev`, whether you connected to a local or remote agent and whether the UI was interactive or headless. For the internal `eve dev --onboard` handoff, eve also records the furthest onboarding stage: model or registry setup, including whether that phase completed, was cancelled, or errored. +- Random identifiers for the CLI session, your eve installation, and the project, plus whether those identifiers are ephemeral or persistent. -The project identifier lets eve group usage from the same project without sending its name or location. eve derives it from the Git remote when available, otherwise `REPOSITORY_URL` or the working directory, and transforms that value before sending it. +The project identifier lets eve group usage from the same project without sending its name or location. eve derives it from the Git remote when available, otherwise `REPOSITORY_URL` or the working directory, and transforms that value before sending it. `identity_kind` is `persistent` when eve reads the identifiers from its local configuration and `ephemeral` when it creates fresh in-memory identifiers for that invocation. ## What eve does not collect @@ -51,6 +51,6 @@ To disable telemetry for one command without changing the saved setting, set `EV EVE_TELEMETRY_DISABLED=1 eve dev ``` -On an interactive terminal, eve displays this information once before it collects telemetry. eve saves your preference in your platform user configuration directory. In CI and Docker environments, eve uses fresh in-memory identifiers for each invocation instead of saving them. +On an interactive terminal, eve displays this information once before it collects telemetry. eve saves your preference in your platform user configuration directory. In CI and Docker environments, eve uses fresh in-memory identifiers for each invocation instead of saving them and reports `identity_kind` as `ephemeral`. This indicates eve's identifier behavior, not whether the invocation ran in a particular CI provider. Vercel handles CLI telemetry under the [Vercel Privacy Notice](https://vercel.com/legal/privacy-notice). diff --git a/packages/eve/src/cli/commands/extension-init.ts b/packages/eve/src/cli/commands/extension-init.ts index d6dbffce0e..92efe01c70 100644 --- a/packages/eve/src/cli/commands/extension-init.ts +++ b/packages/eve/src/cli/commands/extension-init.ts @@ -39,6 +39,8 @@ export interface ExtensionInitCliLogger { log(message: string): void; } +export type ExtensionInitStage = "target" | "scaffold" | "install" | "git" | "post_init"; + export interface ExtensionInitCommandDependencies { detectInvokingPackageManager: typeof detectInvokingPackageManager; detectPackageManager: typeof detectPackageManager; @@ -221,7 +223,9 @@ export async function runExtensionInitCommand( parentDirectory: string, target: string | undefined, dependencies: ExtensionInitCommandDependencies = defaultDependencies, + trackStage?: (stage: ExtensionInitStage) => void, ): Promise { + trackStage?.("target"); // Coding agent with no target: print a setup guide, same gate as agent init. if (target === undefined && (await dependencies.isCodingAgentLaunch())) { logger.log(initExtensionInstructions()); @@ -256,6 +260,7 @@ export async function runExtensionInitCommand( ); } + trackStage?.("scaffold"); progress.update("Creating extension"); initLog.debug("creating extension"); const agentStartedAt = dependencies.now(); @@ -279,6 +284,7 @@ export async function runExtensionInitCommand( agentElapsedMs = dependencies.now() - agentStartedAt; initLog.debug("creating extension done", { ms: agentElapsedMs }); + trackStage?.("install"); progress.update("Installing dependencies", `${packageManager} install`); initLog.debug(`installing dependencies with ${packageManager}`); const installStartedAt = dependencies.now(); @@ -316,6 +322,7 @@ export async function runExtensionInitCommand( } initLog.debug("dependencies installed", { ms: installElapsedMs }); + trackStage?.("git"); progress.update("Initializing Git repository"); initLog.debug("initializing git repository"); gitResult = await dependencies.tryInitializeGit(projectPath); @@ -323,6 +330,7 @@ export async function runExtensionInitCommand( progress.stop(); } + trackStage?.("post_init"); logger.log( `${pc.green("✓")} Created an ${EVE_WORDMARK} extension in ${pc.bold(projectPath!)} ${pc.dim(`in ${formatElapsed(agentElapsedMs!)}`)}`, ); diff --git a/packages/eve/src/cli/commands/init.ts b/packages/eve/src/cli/commands/init.ts index ec2da0eeb6..edde77c7f4 100644 --- a/packages/eve/src/cli/commands/init.ts +++ b/packages/eve/src/cli/commands/init.ts @@ -59,6 +59,8 @@ import { resolveInitTarget } from "./init-target.js"; export type { InitCliLogger, InitCommandOptions } from "./init-agent-workspace.js"; +export type InitStage = "target" | "scaffold" | "install" | "git" | "post_init"; + export interface InitCommandDependencies { addAgentToProject: typeof addAgentToProject; confirmInitInNonEmptyDirectory: typeof confirmInitInNonEmptyDirectory; @@ -360,8 +362,9 @@ async function runInitSteps(input: { options: InitCommandOptions; parentDirectory: string; target: string | undefined; + trackStage?: (stage: InitStage) => void; }): Promise { - const { dependencies, logger, options, parentDirectory, target } = input; + const { dependencies, logger, options, parentDirectory, target, trackStage } = input; const debug = isLogLevelEnabled("debug"); const agentLaunched = await dependencies.isCodingAgentLaunch(); const initTarget = await resolveInitTarget({ parentDirectory, target }); @@ -371,6 +374,7 @@ async function runInitSteps(input: { progress.update("Preparing project"); try { const scaffoldPhase = initTarget.kind === "fresh" ? "creating agent" : "adding agent"; + trackStage?.("scaffold"); progress.update(initTarget.kind === "fresh" ? "Creating agent" : "Adding agent"); initLog.debug(scaffoldPhase); const agentStartedAt = dependencies.now(); @@ -462,6 +466,7 @@ async function runInitSteps(input: { progress = startCliLiveRow(logger); } + trackStage?.("install"); progress.update("Installing dependencies", `${project.packageManager} install`); initLog.debug(`installing dependencies with ${project.packageManager}`); const installStartedAt = dependencies.now(); @@ -532,6 +537,7 @@ async function runInitSteps(input: { initLog.debug("dependencies installed", { ms: installElapsedMs }); if (project.kind === "created") { + trackStage?.("git"); progress.update("Initializing Git repository"); initLog.debug("initializing git repository"); return { @@ -567,7 +573,9 @@ export async function runInitCommand( target: string | undefined, options: InitCommandOptions, dependencies: InitCommandDependencies = defaultDependencies, + trackStage?: (stage: InitStage) => void, ): Promise { + trackStage?.("target"); if ( await addAgentsToWorkspace( logger, @@ -576,17 +584,27 @@ export async function runInitCommand( options, dependencies.validateModelSlug, ) - ) + ) { + trackStage?.("post_init"); return; + } let result: InitResult; try { - result = await runInitSteps({ dependencies, logger, options, parentDirectory, target }); + result = await runInitSteps({ + dependencies, + logger, + options, + parentDirectory, + target, + trackStage, + }); } catch (error) { if (error instanceof WizardCancelledError) return; throw error; } + trackStage?.("post_init"); if (result.kind === "created") { logger.log( `${pc.green("✓")} Created an ${EVE_WORDMARK} agent in ${pc.bold(result.projectPath)} ${pc.dim(`in ${formatElapsed(result.agentElapsedMs)}`)}`, diff --git a/packages/eve/src/cli/dev/run-interactive-ui.ts b/packages/eve/src/cli/dev/run-interactive-ui.ts index b1c92d15e6..6bba881c20 100644 --- a/packages/eve/src/cli/dev/run-interactive-ui.ts +++ b/packages/eve/src/cli/dev/run-interactive-ui.ts @@ -7,6 +7,8 @@ import { suspendDevelopmentRuntimeArtifacts, } from "#services/dev-client/runtime-artifacts.js"; +import type { EveCliOnboardingStage } from "#cli/telemetry/index.js"; + import type { DevelopmentCliOptions } from "./command-options.js"; import { resolveTuiDisplayOptions } from "./ui-options.js"; import type { DevelopmentTuiStartup, RunDevelopmentTuiInput } from "./tui/tui.js"; @@ -22,6 +24,7 @@ export async function runInteractiveDevelopmentUi(input: { readonly remoteTarget?: DevelopmentUrlTarget; readonly report?: DevBootProgressReporter; readonly runDevelopmentTui?: (input: RunDevelopmentTuiInput) => Promise; + readonly onOnboardingStage?: (stage: EveCliOnboardingStage) => void; readonly server: { readonly appRoot?: string; readonly serverUrl: string }; readonly startup?: DevelopmentTuiStartup; }): Promise { @@ -48,6 +51,7 @@ export async function runInteractiveDevelopmentUi(input: { initialInput: input.options.input, onboard: input.options.onboard, onBootProgress: input.report, + onOnboardingStage: input.onOnboardingStage, lifecycle: input.lifecycle, ...display, }; diff --git a/packages/eve/src/cli/dev/tui/runner.test.ts b/packages/eve/src/cli/dev/tui/runner.test.ts index 87ad07e61f..efadab0aa6 100644 --- a/packages/eve/src/cli/dev/tui/runner.test.ts +++ b/packages/eve/src/cli/dev/tui/runner.test.ts @@ -3562,6 +3562,7 @@ describe("EveTUIRunner boot setup detection", () => { it("runs initial onboarding as one-way model and registry phases", async () => { const order: string[] = []; const results: string[] = []; + const stages: string[] = []; const handle = vi.fn(async (command: { name: string }) => { order.push(command.name); return command.name === "model" @@ -3584,12 +3585,14 @@ describe("EveTUIRunner boot setup detection", () => { appRoot: "/tmp/weather-agent", onboard: true, bootDetections: [], + onOnboardingStage: (stage) => stages.push(stage), promptCommandHandler: { handle }, }); await runner.run(); expect(order).toEqual(["model", "prompt"]); + expect(stages).toEqual(["model", "model_error"]); expect(results).toContain("/model failed: provider unavailable"); expect(handle).toHaveBeenCalledWith( { type: "extension", name: "model", argument: "" }, @@ -3619,6 +3622,7 @@ describe("EveTUIRunner boot setup detection", () => { it("moves from Model to Channels and preserves diagnostics after a failed registry phase", async () => { const order: string[] = []; + const stages: string[] = []; const end = vi.fn(() => order.push("end")); const handle = vi.fn(async (command: { name: string }) => { order.push(command.name); @@ -3640,12 +3644,14 @@ describe("EveTUIRunner boot setup detection", () => { appRoot: "/tmp/weather-agent", onboard: true, bootDetections: [], + onOnboardingStage: (stage) => stages.push(stage), promptCommandHandler: { handle }, }); await runner.run(); expect(order).toEqual(["model", "add", "end", "prompt"]); + expect(stages).toEqual(["model", "add", "add_error"]); expect(end).toHaveBeenCalledWith({ preserveDiagnostics: true }); expect(handle).toHaveBeenNthCalledWith( 2, @@ -3669,6 +3675,7 @@ describe("EveTUIRunner boot setup detection", () => { it("keeps the completed /add result after onboarding", async () => { const renderCommandInvocation = vi.fn(); const renderCommandResult = vi.fn(); + const stages: string[] = []; const runner = new EveTUIRunner({ session: sessionYielding([]), renderer: fakeRenderer({ @@ -3681,6 +3688,7 @@ describe("EveTUIRunner boot setup detection", () => { appRoot: "/tmp/weather-agent", onboard: true, bootDetections: [], + onOnboardingStage: (stage) => stages.push(stage), promptCommandHandler: { handle: async (command) => command.name === "model" @@ -3691,12 +3699,14 @@ describe("EveTUIRunner boot setup detection", () => { await runner.run(); + expect(stages).toEqual(["model", "add", "completed"]); expect(renderCommandInvocation).toHaveBeenCalledWith("/add", undefined); expect(renderCommandResult).toHaveBeenCalledWith("Added Web Chat", "success"); }); it("does not render a detached /add dismissed result when onboarding is cancelled", async () => { const renderCommandResult = vi.fn(); + const stages: string[] = []; const renderer = fakeRenderer({ readPrompt: vi.fn(async () => undefined), renderCommandResult, @@ -3709,6 +3719,7 @@ describe("EveTUIRunner boot setup detection", () => { appRoot: "/tmp/weather-agent", onboard: true, bootDetections: [], + onOnboardingStage: (stage) => stages.push(stage), getVercelAuthStatus: vi.fn(async () => "authenticated" as const), promptCommandHandler: { handle: async (command) => @@ -3720,6 +3731,7 @@ describe("EveTUIRunner boot setup detection", () => { await runner.run(); + expect(stages).toEqual(["model", "add", "add_cancelled"]); expect(renderCommandResult).not.toHaveBeenCalledWith("/add dismissed.", expect.anything()); }); diff --git a/packages/eve/src/cli/dev/tui/runner.ts b/packages/eve/src/cli/dev/tui/runner.ts index 945d40fb67..85cde167d8 100644 --- a/packages/eve/src/cli/dev/tui/runner.ts +++ b/packages/eve/src/cli/dev/tui/runner.ts @@ -20,6 +20,7 @@ import { ClientSession, } from "#client/index.js"; import { renderApplicationInfo } from "#cli/commands/info.js"; +import type { EveCliOnboardingStage } from "#cli/telemetry/index.js"; import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js"; import { subscribeDevelopmentSandboxPrewarmLogs } from "#execution/sandbox/development-prewarm.js"; import { createEventDeduper } from "#protocol/event-dedupe.js"; @@ -451,6 +452,8 @@ export type EveTUIRunnerOptions = TuiDisplayOptions & { initialInput?: string; /** Explicit fresh-agent onboarding handoff from `eve init`. */ onboard?: boolean; + /** Reports the furthest stage reached by fresh-agent onboarding. */ + onOnboardingStage?: (stage: EveCliOnboardingStage) => void; /** Handles non-core slash commands without adding feature branches to the runner. */ promptCommandHandler?: PromptCommandHandler; /** Commands shown in discovery for this local or remote session. */ @@ -508,6 +511,7 @@ export class EveTUIRunner { readonly #startup?: TuiStartup; /** Explicit fresh-agent onboarding handoff from `eve init`. */ readonly #onboard: boolean; + readonly #onOnboardingStage?: (stage: EveCliOnboardingStage) => void; #initialOnboardingActive = false; readonly #promptCommandHandler?: PromptCommandHandler; readonly #availablePromptCommands: readonly PromptCommandSpec[]; @@ -623,6 +627,7 @@ export class EveTUIRunner { if (options.initialInput !== undefined) this.#initialInput = options.initialInput; if (options.startup !== undefined) this.#startup = options.startup; this.#onboard = options.onboard === true; + this.#onOnboardingStage = options.onOnboardingStage; if (options.appRoot !== undefined) { this.#appRoot = options.appRoot; const trackerOptions: VercelStatusTrackerOptions = { @@ -1791,23 +1796,34 @@ export class EveTUIRunner { trigger: "startup" as const, }; - const modelOutcome = await this.#executeExtensionCommand( - { type: "extension", name: "model", argument: "" }, - title, - { - ...journey, - suppressSuccessfulTranscript: true, - initialModelStep: "provider", - setupFlowNavigation: this.#onboardingNavigation(0), - }, - ); + this.#onOnboardingStage?.("model"); + let modelOutcome: PromptCommandOutcome | undefined; + try { + modelOutcome = await this.#executeExtensionCommand( + { type: "extension", name: "model", argument: "" }, + title, + { + ...journey, + suppressSuccessfulTranscript: true, + initialModelStep: "provider", + setupFlowNavigation: this.#onboardingNavigation(0), + }, + ); + } catch (error) { + this.#onOnboardingStage?.("model_error"); + throw error; + } if (modelOutcome?.tone === "error" || modelOutcome?.cancelled === true) { + this.#onOnboardingStage?.( + modelOutcome.cancelled === true ? "model_cancelled" : "model_error", + ); this.#renderer.setupFlow?.end({ preserveDiagnostics: modelOutcome.tone === "error" }); return; } let addOutcome: PromptCommandOutcome | undefined; try { + this.#onOnboardingStage?.("add"); addOutcome = await this.#executeExtensionCommand( { type: "extension", name: "add", argument: "" }, title, @@ -1821,7 +1837,19 @@ export class EveTUIRunner { }, }, ); + } catch (error) { + this.#onOnboardingStage?.("add_error"); + throw error; } finally { + if (addOutcome !== undefined) { + this.#onOnboardingStage?.( + addOutcome.cancelled === true + ? "add_cancelled" + : addOutcome.tone === "error" + ? "add_error" + : "completed", + ); + } // `/add` is the final onboarding phase. Release the shared panel so the // ordinary chat prompt can own input, retaining deploy/setup evidence on failure. this.#renderer.setupFlow?.end({ preserveDiagnostics: addOutcome?.tone === "error" }); diff --git a/packages/eve/src/cli/dev/tui/tui.ts b/packages/eve/src/cli/dev/tui/tui.ts index 44a4134089..ae56cad6e5 100644 --- a/packages/eve/src/cli/dev/tui/tui.ts +++ b/packages/eve/src/cli/dev/tui/tui.ts @@ -1,3 +1,4 @@ +import type { EveCliOnboardingStage } from "#cli/telemetry/index.js"; import { Client } from "#client/index.js"; import type { DevBootProgressReporter } from "#internal/dev-boot-progress.js"; import { resolveInstalledPackageInfo } from "#internal/application/package.js"; @@ -38,6 +39,8 @@ export interface RunDevelopmentTuiInput extends TuiDisplayOptions { readonly initialInput?: string; /** Explicit fresh-agent onboarding handoff from `eve init`. */ readonly onboard?: boolean; + /** Reports the furthest stage reached by fresh-agent onboarding. */ + readonly onOnboardingStage?: (stage: EveCliOnboardingStage) => void; /** Reports local CLI boot phases. Omitted for remote and programmatic TUI runs. */ readonly onBootProgress?: DevBootProgressReporter; /** Gives setup subprocesses exclusive terminal and development-host ownership. */ @@ -141,6 +144,7 @@ export async function runDevelopmentTui(input: RunDevelopmentTuiInput): Promise< headers, initialInput, onboard, + onOnboardingStage, onBootProgress, lifecycle, startup, @@ -189,6 +193,7 @@ export async function runDevelopmentTui(input: RunDevelopmentTuiInput): Promise< options.startup = startup; } if (onboard !== undefined) options.onboard = onboard; + if (onOnboardingStage !== undefined) options.onOnboardingStage = onOnboardingStage; if (onBootProgress !== undefined) options.onBootProgress = onBootProgress; if (lifecycle !== undefined) options.lifecycle = lifecycle; if (withExclusiveTerminal !== undefined) options.withExclusiveTerminal = withExclusiveTerminal; diff --git a/packages/eve/src/cli/run.test.ts b/packages/eve/src/cli/run.test.ts index ceed53fb7b..966951dc9b 100644 --- a/packages/eve/src/cli/run.test.ts +++ b/packages/eve/src/cli/run.test.ts @@ -76,7 +76,7 @@ describe("CLI command registration", () => { resolveAgent: async () => undefined as never, root: process.cwd(), }, - { trackDevContext: () => {} }, + { trackDevContext: () => {}, trackInitStage: () => {}, trackOnboardingStage: () => {} }, ); const paths: string[] = []; const visit = (command: (typeof program.commands)[number], parentPath = ""): void => { @@ -311,11 +311,19 @@ describe("bare eve command", () => { await runCli([], logger, { findApplicationRoot }); expect(findApplicationRoot).toHaveBeenCalledWith(resolve(process.cwd())); - expect(runInitCommand).toHaveBeenCalledWith(logger, resolve(process.cwd()), undefined, { - channelWebNextjs: undefined, - model: undefined, - reasoning: undefined, - }); + expect(runInitCommand).toHaveBeenCalledWith( + logger, + resolve(process.cwd()), + undefined, + { + agents: undefined, + channelWebNextjs: undefined, + model: undefined, + reasoning: undefined, + }, + undefined, + expect.any(Function), + ); }); it("runs dev from the enclosing eve application", async () => { @@ -384,11 +392,19 @@ describe("eve init compatibility flags", () => { logger, ); - expect(runInitCommand).toHaveBeenCalledWith(logger, resolve(process.cwd()), "my-agent", { - channelWebNextjs: undefined, - model: "openai/gpt-5.6-sol", - reasoning: "high", - }); + expect(runInitCommand).toHaveBeenCalledWith( + logger, + resolve(process.cwd()), + "my-agent", + { + agents: undefined, + channelWebNextjs: undefined, + model: "openai/gpt-5.6-sol", + reasoning: "high", + }, + undefined, + expect.any(Function), + ); }); it("rejects unsupported reasoning before running the init command", async () => { diff --git a/packages/eve/src/cli/run.ts b/packages/eve/src/cli/run.ts index 47243f9f82..b1791cdc7f 100644 --- a/packages/eve/src/cli/run.ts +++ b/packages/eve/src/cli/run.ts @@ -137,7 +137,7 @@ export function createCliProgram( logger: CliLogger, runtime: CliRuntimeOverrides, applicationContext: CliApplicationContext, - telemetry: Pick, + telemetry: Pick, ): Command { const packageVersion = resolveInstalledPackageInfo().version; const program = new Command(); @@ -196,7 +196,9 @@ export function createCliProgram( } const { runExtensionInitCommand } = await import("#cli/commands/extension-init.js"); - await runExtensionInitCommand(logger, applicationContext.root, target); + await runExtensionInitCommand(logger, applicationContext.root, target, undefined, (stage) => { + telemetry.trackInitStage(stage); + }); }); extension @@ -246,12 +248,21 @@ export function createCliProgram( } const { runInitCommand } = await import("#cli/commands/init.js"); - await runInitCommand(logger, applicationContext.root, target, { - agents: options.agents, - channelWebNextjs: options.channelWebNextjs, - model: options.model, - reasoning: options.reasoning, - }); + await runInitCommand( + logger, + applicationContext.root, + target, + { + agents: options.agents, + channelWebNextjs: options.channelWebNextjs, + model: options.model, + reasoning: options.reasoning, + }, + undefined, + (stage) => { + telemetry.trackInitStage(stage); + }, + ); }, ); @@ -417,6 +428,7 @@ export function createCliProgram( applicationRoot: applicationContext.root, existingLocalServer: existingLocalDevelopmentServer, lifecycle, + onOnboardingStage: telemetry.trackOnboardingStage, options, remoteTarget, runDevelopmentTui: runtime.runDevelopmentTui, @@ -518,6 +530,7 @@ export function createCliProgram( applicationRoot: applicationContext.root, existingLocalServer: false, lifecycle, + onOnboardingStage: telemetry.trackOnboardingStage, options, report: onBootProgress, runDevelopmentTui: runtime.runDevelopmentTui, diff --git a/packages/eve/src/cli/telemetry/index.test.ts b/packages/eve/src/cli/telemetry/index.test.ts index 25bd66c060..0728db1d05 100644 --- a/packages/eve/src/cli/telemetry/index.test.ts +++ b/packages/eve/src/cli/telemetry/index.test.ts @@ -134,6 +134,9 @@ describe("createEveCliTelemetry", () => { }>; expect(events).toContainEqual(expect.objectContaining({ key: "target", value: "remote" })); expect(events).toContainEqual(expect.objectContaining({ key: "ui", value: "headless" })); + expect(events).toContainEqual( + expect.objectContaining({ key: "identity_kind", value: "persistent" }), + ); expect(events).toContainEqual( expect.objectContaining({ key: "installation_id", value: "installation_123" }), ); @@ -145,6 +148,48 @@ describe("createEveCliTelemetry", () => { }); }); + it("records the furthest init stage without error details", async () => { + vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("EVE_TELEMETRY_DEBUG", "1"); + const write = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + const telemetry = createEveCliTelemetry("1.0.0"); + telemetry.trackCommand("init"); + telemetry.trackInitStage("target"); + telemetry.trackInitStage("install"); + + await telemetry.flush(); + + const events = JSON.parse( + String(write.mock.calls[0]?.[0]).replace("[eve telemetry] ", ""), + ) as Array<{ key: string; value: string }>; + expect(events).toContainEqual(expect.objectContaining({ key: "init_stage", value: "install" })); + expect(events).not.toContainEqual( + expect.objectContaining({ key: "init_stage", value: "target" }), + ); + }); + + it("records the final onboarding stage without user selections", async () => { + vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("EVE_TELEMETRY_DEBUG", "1"); + const write = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + const telemetry = createEveCliTelemetry("1.0.0"); + telemetry.trackCommand("dev"); + telemetry.trackOnboardingStage("model"); + telemetry.trackOnboardingStage("add_error"); + + await telemetry.flush(); + + const events = JSON.parse( + String(write.mock.calls[0]?.[0]).replace("[eve telemetry] ", ""), + ) as Array<{ key: string; value: string }>; + expect(events).toContainEqual( + expect.objectContaining({ key: "onboarding_stage", value: "add_error" }), + ); + expect(events).not.toContainEqual( + expect.objectContaining({ key: "onboarding_stage", value: "model" }), + ); + }); + it("skips telemetry when identity initialization fails", async () => { vi.stubEnv("NODE_ENV", "production"); vi.mocked(readOrCreateEveTelemetryIdentity).mockRejectedValue(new Error("read-only config")); @@ -168,7 +213,15 @@ describe("createEveCliTelemetry", () => { expect(readOrCreateEveTelemetryIdentity).not.toHaveBeenCalled(); expect(createEveTelemetryIdentity).toHaveBeenCalledOnce(); - expect(String(write.mock.calls[0]?.[0])).toContain("ephemeral_installation_123"); + const events = JSON.parse( + String(write.mock.calls[0]?.[0]).replace("[eve telemetry] ", ""), + ) as Array<{ key: string; value: string }>; + expect(events).toContainEqual( + expect.objectContaining({ key: "identity_kind", value: "ephemeral" }), + ); + expect(events).toContainEqual( + expect.objectContaining({ key: "installation_id", value: "ephemeral_installation_123" }), + ); }); it("flushes an allowlisted outcome through a telemetry-disabled child process", async () => { diff --git a/packages/eve/src/cli/telemetry/index.ts b/packages/eve/src/cli/telemetry/index.ts index 4f85d2a050..70edfbde5b 100644 --- a/packages/eve/src/cli/telemetry/index.ts +++ b/packages/eve/src/cli/telemetry/index.ts @@ -20,9 +20,21 @@ export type EveCliTelemetryEvent = { readonly value: string; }; +export type EveCliInitStage = "target" | "scaffold" | "install" | "git" | "post_init"; +export type EveCliOnboardingStage = + | "model" + | "model_cancelled" + | "model_error" + | "add" + | "add_cancelled" + | "add_error" + | "completed"; + export type EveCliTelemetry = { trackCommand(command: string): void; trackDevContext(context: { target: "local" | "remote"; ui: "tui" | "headless" }): void; + trackInitStage(stage: EveCliInitStage): void; + trackOnboardingStage(stage: EveCliOnboardingStage): void; trackOutcome(outcome: "success" | "usage_error" | "error"): void; notify(logger: { error(message: string): void }): Promise; flush(): Promise; @@ -109,6 +121,8 @@ export function createEveCliTelemetry(version: string): EveCliTelemetry { event("stdin_is_tty", process.stdin.isTTY ? "true" : "false"), ]; const sessionId = randomUUID(); + let initStage: EveCliInitStage | undefined; + let onboardingStage: EveCliOnboardingStage | undefined; return { trackCommand(command) { @@ -117,6 +131,12 @@ export function createEveCliTelemetry(version: string): EveCliTelemetry { trackDevContext(context) { events.push(event("target", context.target), event("ui", context.ui)); }, + trackInitStage(stage) { + initStage = stage; + }, + trackOnboardingStage(stage) { + onboardingStage = stage; + }, trackOutcome(outcome) { events.push(event("outcome", outcome)); }, @@ -145,13 +165,17 @@ export function createEveCliTelemetry(version: string): EveCliTelemetry { async flush() { if (!(await isEnabled()) || events.length === 0) return; try { - const identity = isEphemeralEveTelemetryEnvironment() + const ephemeralIdentity = isEphemeralEveTelemetryEnvironment(); + const identity = ephemeralIdentity ? createEveTelemetryIdentity() : await readOrCreateEveTelemetryIdentity(); events.push( + event("identity_kind", ephemeralIdentity ? "ephemeral" : "persistent"), event("installation_id", identity.installationId), event("project_id", await resolveEveTelemetryProjectId({ identity })), ); + if (initStage !== undefined) events.push(event("init_stage", initStage)); + if (onboardingStage !== undefined) events.push(event("onboarding_stage", onboardingStage)); } catch { return; } From 5f69e2d8a59671284813829525901dc589b8e94b Mon Sep 17 00:00:00 2001 From: owenkephart Date: Fri, 4 Sep 2026 21:55:21 +0000 Subject: [PATCH 2/2] docs: simplify telemetry policy Signed-off-by: owenkephart --- .changeset/track-init-telemetry-stages.md | 2 +- docs/reference/telemetry.md | 10 +-- .../eve/src/cli/commands/extension-init.ts | 19 +++-- packages/eve/src/cli/commands/init.ts | 52 ++++--------- .../eve/src/cli/dev/run-interactive-ui.ts | 8 +- .../src/cli/dev/tui/prompt-command-handler.ts | 3 + packages/eve/src/cli/dev/tui/runner.test.ts | 20 +++-- packages/eve/src/cli/dev/tui/runner.ts | 77 ++++++++++++++----- .../eve/src/cli/dev/tui/setup-commands.ts | 19 ++++- packages/eve/src/cli/dev/tui/tui.ts | 13 ++-- packages/eve/src/cli/run.test.ts | 4 +- packages/eve/src/cli/run.ts | 29 ++++--- packages/eve/src/cli/telemetry/index.test.ts | 22 +++--- packages/eve/src/cli/telemetry/index.ts | 70 ++++++++++++----- packages/eve/src/setup/flows/model.ts | 3 + packages/eve/src/setup/flows/registry.test.ts | 8 ++ packages/eve/src/setup/flows/registry.ts | 14 ++++ 17 files changed, 241 insertions(+), 132 deletions(-) diff --git a/.changeset/track-init-telemetry-stages.md b/.changeset/track-init-telemetry-stages.md index 13992254d5..ac9a24826b 100644 --- a/.changeset/track-init-telemetry-stages.md +++ b/.changeset/track-init-telemetry-stages.md @@ -2,4 +2,4 @@ "eve": patch --- -Record the furthest stage reached by `eve init`, `eve extension init`, and the interactive `eve dev --onboard` handoff in CLI telemetry, so failed setup runs can be grouped by their stage. Telemetry now also identifies whether its installation and project identifiers are ephemeral or persisted locally. +Add CLI telemetry for setup and onboarding flows. diff --git a/docs/reference/telemetry.md b/docs/reference/telemetry.md index 0ff233a37b..a9b5097420 100644 --- a/docs/reference/telemetry.md +++ b/docs/reference/telemetry.md @@ -12,11 +12,11 @@ eve collects usage data from its CLI to help improve its commands and developmen eve sends the following information to Vercel: - The eve version, operating system, CPU architecture, and whether stdin is a terminal. -- The command you ran and whether it succeeded, had a usage error, or failed. For `eve init` and `eve extension init`, eve also records the furthest stage reached: target resolution, scaffolding, dependency installation, Git initialization, or post-initialization handoff. -- For `eve dev`, whether you connected to a local or remote agent and whether the UI was interactive or headless. For the internal `eve dev --onboard` handoff, eve also records the furthest onboarding stage: model or registry setup, including whether that phase completed, was cancelled, or errored. -- Random identifiers for the CLI session, your eve installation, and the project, plus whether those identifiers are ephemeral or persistent. +- The command you ran, its outcome, and setup or onboarding steps when applicable. +- For `eve dev`, whether you connected to a local or remote agent and whether the UI was interactive or headless. +- Random identifiers for the CLI session, installation, and project, plus whether the installation and project identifiers are ephemeral or persistent. -The project identifier lets eve group usage from the same project without sending its name or location. eve derives it from the Git remote when available, otherwise `REPOSITORY_URL` or the working directory, and transforms that value before sending it. `identity_kind` is `persistent` when eve reads the identifiers from its local configuration and `ephemeral` when it creates fresh in-memory identifiers for that invocation. +The project identifier lets eve group usage from the same project without sending its name or location. eve derives it from the Git remote when available, otherwise `REPOSITORY_URL` or the working directory, and transforms that value before sending it. ## What eve does not collect @@ -51,6 +51,6 @@ To disable telemetry for one command without changing the saved setting, set `EV EVE_TELEMETRY_DISABLED=1 eve dev ``` -On an interactive terminal, eve displays this information once before it collects telemetry. eve saves your preference in your platform user configuration directory. In CI and Docker environments, eve uses fresh in-memory identifiers for each invocation instead of saving them and reports `identity_kind` as `ephemeral`. This indicates eve's identifier behavior, not whether the invocation ran in a particular CI provider. +On an interactive terminal, eve displays this information once before it collects telemetry. eve saves your preference in your platform user configuration directory. In CI and Docker environments, eve uses fresh in-memory identifiers for each invocation instead of saving them. Vercel handles CLI telemetry under the [Vercel Privacy Notice](https://vercel.com/legal/privacy-notice). diff --git a/packages/eve/src/cli/commands/extension-init.ts b/packages/eve/src/cli/commands/extension-init.ts index 92efe01c70..ffe077fd2e 100644 --- a/packages/eve/src/cli/commands/extension-init.ts +++ b/packages/eve/src/cli/commands/extension-init.ts @@ -39,7 +39,12 @@ export interface ExtensionInitCliLogger { log(message: string): void; } -export type ExtensionInitStage = "target" | "scaffold" | "install" | "git" | "post_init"; +export type ExtensionInitSetupStep = + | "resolve_target" + | "scaffold" + | "install_dependencies" + | "initialize_git" + | "handoff"; export interface ExtensionInitCommandDependencies { detectInvokingPackageManager: typeof detectInvokingPackageManager; @@ -223,9 +228,9 @@ export async function runExtensionInitCommand( parentDirectory: string, target: string | undefined, dependencies: ExtensionInitCommandDependencies = defaultDependencies, - trackStage?: (stage: ExtensionInitStage) => void, + trackStep?: (step: ExtensionInitSetupStep) => void, ): Promise { - trackStage?.("target"); + trackStep?.("resolve_target"); // Coding agent with no target: print a setup guide, same gate as agent init. if (target === undefined && (await dependencies.isCodingAgentLaunch())) { logger.log(initExtensionInstructions()); @@ -260,7 +265,7 @@ export async function runExtensionInitCommand( ); } - trackStage?.("scaffold"); + trackStep?.("scaffold"); progress.update("Creating extension"); initLog.debug("creating extension"); const agentStartedAt = dependencies.now(); @@ -284,7 +289,7 @@ export async function runExtensionInitCommand( agentElapsedMs = dependencies.now() - agentStartedAt; initLog.debug("creating extension done", { ms: agentElapsedMs }); - trackStage?.("install"); + trackStep?.("install_dependencies"); progress.update("Installing dependencies", `${packageManager} install`); initLog.debug(`installing dependencies with ${packageManager}`); const installStartedAt = dependencies.now(); @@ -322,7 +327,7 @@ export async function runExtensionInitCommand( } initLog.debug("dependencies installed", { ms: installElapsedMs }); - trackStage?.("git"); + trackStep?.("initialize_git"); progress.update("Initializing Git repository"); initLog.debug("initializing git repository"); gitResult = await dependencies.tryInitializeGit(projectPath); @@ -330,7 +335,7 @@ export async function runExtensionInitCommand( progress.stop(); } - trackStage?.("post_init"); + trackStep?.("handoff"); logger.log( `${pc.green("✓")} Created an ${EVE_WORDMARK} extension in ${pc.bold(projectPath!)} ${pc.dim(`in ${formatElapsed(agentElapsedMs!)}`)}`, ); diff --git a/packages/eve/src/cli/commands/init.ts b/packages/eve/src/cli/commands/init.ts index edde77c7f4..bef13c2b02 100644 --- a/packages/eve/src/cli/commands/init.ts +++ b/packages/eve/src/cli/commands/init.ts @@ -5,6 +5,7 @@ import { performance } from "node:perf_hooks"; import pc from "#compiled/picocolors/index.js"; import { isCodingAgentLaunch } from "#cli/agent-detection.js"; +import type { EveCliSetupStep, EveCliSetupTerminalResult } from "#cli/telemetry/index.js"; import { EVE_WORDMARK } from "#cli/banner.js"; import { formatElapsed } from "#cli/format-elapsed.js"; import { startCliLiveRow } from "#cli/ui/live-row.js"; @@ -59,8 +60,6 @@ import { resolveInitTarget } from "./init-target.js"; export type { InitCliLogger, InitCommandOptions } from "./init-agent-workspace.js"; -export type InitStage = "target" | "scaffold" | "install" | "git" | "post_init"; - export interface InitCommandDependencies { addAgentToProject: typeof addAgentToProject; confirmInitInNonEmptyDirectory: typeof confirmInitInNonEmptyDirectory; @@ -130,10 +129,6 @@ function formatWorkspaceRootMutationWarning(mutation: WorkspaceRootMutation): st return `Updated workspace root ${target} at ${mutation.path}${suffix}`; } -/** - * Adds the agent to an existing project and returns the - * detected manager, which drives the install and dev handoff. - */ async function addToExistingProject( targetPath: string, options: InitCommandOptions, @@ -175,10 +170,6 @@ async function addToExistingProject( }; } -/** - * The manager a fresh scaffold will be owned by: an existing ancestor project - * manager first, then the package runner that launched the CLI, then pnpm. - */ async function resolveScaffoldPackageManager( projectPath: string, dependencies: InitCommandDependencies, @@ -362,9 +353,9 @@ async function runInitSteps(input: { options: InitCommandOptions; parentDirectory: string; target: string | undefined; - trackStage?: (stage: InitStage) => void; + trackStep?: (step: EveCliSetupStep) => void; }): Promise { - const { dependencies, logger, options, parentDirectory, target, trackStage } = input; + const { dependencies, logger, options, parentDirectory, target, trackStep } = input; const debug = isLogLevelEnabled("debug"); const agentLaunched = await dependencies.isCodingAgentLaunch(); const initTarget = await resolveInitTarget({ parentDirectory, target }); @@ -374,7 +365,7 @@ async function runInitSteps(input: { progress.update("Preparing project"); try { const scaffoldPhase = initTarget.kind === "fresh" ? "creating agent" : "adding agent"; - trackStage?.("scaffold"); + trackStep?.("scaffold"); progress.update(initTarget.kind === "fresh" ? "Creating agent" : "Adding agent"); initLog.debug(scaffoldPhase); const agentStartedAt = dependencies.now(); @@ -466,7 +457,7 @@ async function runInitSteps(input: { progress = startCliLiveRow(logger); } - trackStage?.("install"); + trackStep?.("install_dependencies"); progress.update("Installing dependencies", `${project.packageManager} install`); initLog.debug(`installing dependencies with ${project.packageManager}`); const installStartedAt = dependencies.now(); @@ -537,7 +528,7 @@ async function runInitSteps(input: { initLog.debug("dependencies installed", { ms: installElapsedMs }); if (project.kind === "created") { - trackStage?.("git"); + trackStep?.("initialize_git"); progress.update("Initializing Git repository"); initLog.debug("initializing git repository"); return { @@ -555,27 +546,16 @@ async function runInitSteps(input: { } } -/** - * Creates a new eve agent (`target` is a project name), or adds one to an - * existing project (`target` is a directory), without external provisioning. - * A fresh in-place scaffold asks whether to use the current directory or a new - * subdirectory when the current directory is not empty. Coding-agent launches - * must pass an explicit subdirectory instead. - * - * Runs launched by a coding agent get the dev command printed instead of - * spawned after scaffolding, since the dev TUI would wedge the launching agent. - * - * For extension packages, use `eve extension init` instead. - */ export async function runInitCommand( logger: InitCliLogger, parentDirectory: string, target: string | undefined, options: InitCommandOptions, dependencies: InitCommandDependencies = defaultDependencies, - trackStage?: (stage: InitStage) => void, + trackStep?: (step: EveCliSetupStep) => void, + trackTerminal?: (step: EveCliSetupStep, result: EveCliSetupTerminalResult) => void, ): Promise { - trackStage?.("target"); + trackStep?.("resolve_target"); if ( await addAgentsToWorkspace( logger, @@ -585,7 +565,8 @@ export async function runInitCommand( dependencies.validateModelSlug, ) ) { - trackStage?.("post_init"); + trackStep?.("handoff"); + trackTerminal?.("handoff", "completed"); return; } @@ -597,14 +578,17 @@ export async function runInitCommand( options, parentDirectory, target, - trackStage, + trackStep, }); } catch (error) { - if (error instanceof WizardCancelledError) return; + if (error instanceof WizardCancelledError) { + trackTerminal?.("resolve_target", "cancelled"); + return; + } throw error; } - trackStage?.("post_init"); + trackStep?.("handoff"); if (result.kind === "created") { logger.log( `${pc.green("✓")} Created an ${EVE_WORDMARK} agent in ${pc.bold(result.projectPath)} ${pc.dim(`in ${formatElapsed(result.agentElapsedMs)}`)}`, @@ -636,8 +620,6 @@ export async function runInitCommand( } } - // A workspace has no implicit primary agent. Let `eve dev` ask the person - // which member to run, rather than silently selecting the first --agents value. const baseDevArguments = eveDevArguments(result.packageManager); const agentDevCommand = [result.packageManager, ...baseDevArguments].join(" "); const agentHandoff = initAgentDevHandoff({ diff --git a/packages/eve/src/cli/dev/run-interactive-ui.ts b/packages/eve/src/cli/dev/run-interactive-ui.ts index 6bba881c20..bc35d269d9 100644 --- a/packages/eve/src/cli/dev/run-interactive-ui.ts +++ b/packages/eve/src/cli/dev/run-interactive-ui.ts @@ -7,7 +7,7 @@ import { suspendDevelopmentRuntimeArtifacts, } from "#services/dev-client/runtime-artifacts.js"; -import type { EveCliOnboardingStage } from "#cli/telemetry/index.js"; +import type { EveCliSetupStepEvent, EveCliSetupTerminalEvent } from "#cli/telemetry/index.js"; import type { DevelopmentCliOptions } from "./command-options.js"; import { resolveTuiDisplayOptions } from "./ui-options.js"; @@ -24,7 +24,8 @@ export async function runInteractiveDevelopmentUi(input: { readonly remoteTarget?: DevelopmentUrlTarget; readonly report?: DevBootProgressReporter; readonly runDevelopmentTui?: (input: RunDevelopmentTuiInput) => Promise; - readonly onOnboardingStage?: (stage: EveCliOnboardingStage) => void; + readonly onOnboardingStep?: (input: EveCliSetupStepEvent) => void; + readonly onOnboardingTerminal?: (input: EveCliSetupTerminalEvent) => void; readonly server: { readonly appRoot?: string; readonly serverUrl: string }; readonly startup?: DevelopmentTuiStartup; }): Promise { @@ -51,7 +52,8 @@ export async function runInteractiveDevelopmentUi(input: { initialInput: input.options.input, onboard: input.options.onboard, onBootProgress: input.report, - onOnboardingStage: input.onOnboardingStage, + onOnboardingStep: input.onOnboardingStep, + onOnboardingTerminal: input.onOnboardingTerminal, lifecycle: input.lifecycle, ...display, }; diff --git a/packages/eve/src/cli/dev/tui/prompt-command-handler.ts b/packages/eve/src/cli/dev/tui/prompt-command-handler.ts index 65298ddf31..18ca9fe913 100644 --- a/packages/eve/src/cli/dev/tui/prompt-command-handler.ts +++ b/packages/eve/src/cli/dev/tui/prompt-command-handler.ts @@ -127,6 +127,9 @@ export function createPromptCommandHandler( if (context.registryPlannerContext !== undefined) { commandInput.registryPlannerContext = context.registryPlannerContext; } + if (context.onOnboardingScreen !== undefined) { + commandInput.onOnboardingScreen = context.onOnboardingScreen; + } // `/add ` confirms and installs that address; bare `/add` opens the planner. if (command.name === "add" && command.argument.length > 0) { commandInput.initialRegistryAddress = command.argument; diff --git a/packages/eve/src/cli/dev/tui/runner.test.ts b/packages/eve/src/cli/dev/tui/runner.test.ts index efadab0aa6..a5bd48288a 100644 --- a/packages/eve/src/cli/dev/tui/runner.test.ts +++ b/packages/eve/src/cli/dev/tui/runner.test.ts @@ -3585,14 +3585,15 @@ describe("EveTUIRunner boot setup detection", () => { appRoot: "/tmp/weather-agent", onboard: true, bootDetections: [], - onOnboardingStage: (stage) => stages.push(stage), + onOnboardingStep: ({ step }) => stages.push(step), + onOnboardingTerminal: ({ step, result }) => stages.push(`${step}_${result}`), promptCommandHandler: { handle }, }); await runner.run(); expect(order).toEqual(["model", "prompt"]); - expect(stages).toEqual(["model", "model_error"]); + expect(stages).toEqual(["model_provider", "model_provider_error"]); expect(results).toContain("/model failed: provider unavailable"); expect(handle).toHaveBeenCalledWith( { type: "extension", name: "model", argument: "" }, @@ -3644,14 +3645,15 @@ describe("EveTUIRunner boot setup detection", () => { appRoot: "/tmp/weather-agent", onboard: true, bootDetections: [], - onOnboardingStage: (stage) => stages.push(stage), + onOnboardingStep: ({ step }) => stages.push(step), + onOnboardingTerminal: ({ step, result }) => stages.push(`${step}_${result}`), promptCommandHandler: { handle }, }); await runner.run(); expect(order).toEqual(["model", "add", "end", "prompt"]); - expect(stages).toEqual(["model", "add", "add_error"]); + expect(stages).toEqual(["model_provider", "registry_channels", "registry_channels_error"]); expect(end).toHaveBeenCalledWith({ preserveDiagnostics: true }); expect(handle).toHaveBeenNthCalledWith( 2, @@ -3688,7 +3690,8 @@ describe("EveTUIRunner boot setup detection", () => { appRoot: "/tmp/weather-agent", onboard: true, bootDetections: [], - onOnboardingStage: (stage) => stages.push(stage), + onOnboardingStep: ({ step }) => stages.push(step), + onOnboardingTerminal: ({ step, result }) => stages.push(`${step}_${result}`), promptCommandHandler: { handle: async (command) => command.name === "model" @@ -3699,7 +3702,7 @@ describe("EveTUIRunner boot setup detection", () => { await runner.run(); - expect(stages).toEqual(["model", "add", "completed"]); + expect(stages).toEqual(["model_provider", "registry_channels", "registry_channels_completed"]); expect(renderCommandInvocation).toHaveBeenCalledWith("/add", undefined); expect(renderCommandResult).toHaveBeenCalledWith("Added Web Chat", "success"); }); @@ -3719,7 +3722,8 @@ describe("EveTUIRunner boot setup detection", () => { appRoot: "/tmp/weather-agent", onboard: true, bootDetections: [], - onOnboardingStage: (stage) => stages.push(stage), + onOnboardingStep: ({ step }) => stages.push(step), + onOnboardingTerminal: ({ step, result }) => stages.push(`${step}_${result}`), getVercelAuthStatus: vi.fn(async () => "authenticated" as const), promptCommandHandler: { handle: async (command) => @@ -3731,7 +3735,7 @@ describe("EveTUIRunner boot setup detection", () => { await runner.run(); - expect(stages).toEqual(["model", "add", "add_cancelled"]); + expect(stages).toEqual(["model_provider", "registry_channels", "registry_channels_cancelled"]); expect(renderCommandResult).not.toHaveBeenCalledWith("/add dismissed.", expect.anything()); }); diff --git a/packages/eve/src/cli/dev/tui/runner.ts b/packages/eve/src/cli/dev/tui/runner.ts index 85cde167d8..8d3a79256e 100644 --- a/packages/eve/src/cli/dev/tui/runner.ts +++ b/packages/eve/src/cli/dev/tui/runner.ts @@ -20,7 +20,12 @@ import { ClientSession, } from "#client/index.js"; import { renderApplicationInfo } from "#cli/commands/info.js"; -import type { EveCliOnboardingStage } from "#cli/telemetry/index.js"; +import type { + EveCliSetupStep, + EveCliSetupStepEvent, + EveCliSetupTerminalEvent, +} from "#cli/telemetry/index.js"; +import type { OnboardingScreenEvent } from "./setup-commands.js"; import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js"; import { subscribeDevelopmentSandboxPrewarmLogs } from "#execution/sandbox/development-prewarm.js"; import { createEventDeduper } from "#protocol/event-dedupe.js"; @@ -377,6 +382,7 @@ export interface PromptCommandHandlerContext { /** Provider entry authorized by confirmed boot-time model-access evidence. */ readonly initialModelStep?: "provider"; readonly registryPlannerContext?: RegistryPlannerContext; + readonly onOnboardingScreen?: (input: OnboardingScreenEvent) => void; /** Overrides the standalone command title inside a composed setup journey. */ readonly setupFlowTitle?: string; /** Progress owned by an enclosing journey while this command runs. */ @@ -452,8 +458,9 @@ export type EveTUIRunnerOptions = TuiDisplayOptions & { initialInput?: string; /** Explicit fresh-agent onboarding handoff from `eve init`. */ onboard?: boolean; - /** Reports the furthest stage reached by fresh-agent onboarding. */ - onOnboardingStage?: (stage: EveCliOnboardingStage) => void; + /** Reports timestamped steps and terminal result for fresh-agent onboarding. */ + onOnboardingStep?: (input: EveCliSetupStepEvent) => void; + onOnboardingTerminal?: (input: EveCliSetupTerminalEvent) => void; /** Handles non-core slash commands without adding feature branches to the runner. */ promptCommandHandler?: PromptCommandHandler; /** Commands shown in discovery for this local or remote session. */ @@ -511,7 +518,8 @@ export class EveTUIRunner { readonly #startup?: TuiStartup; /** Explicit fresh-agent onboarding handoff from `eve init`. */ readonly #onboard: boolean; - readonly #onOnboardingStage?: (stage: EveCliOnboardingStage) => void; + readonly #onOnboardingStep?: EveTUIRunnerOptions["onOnboardingStep"]; + readonly #onOnboardingTerminal?: EveTUIRunnerOptions["onOnboardingTerminal"]; #initialOnboardingActive = false; readonly #promptCommandHandler?: PromptCommandHandler; readonly #availablePromptCommands: readonly PromptCommandSpec[]; @@ -627,7 +635,8 @@ export class EveTUIRunner { if (options.initialInput !== undefined) this.#initialInput = options.initialInput; if (options.startup !== undefined) this.#startup = options.startup; this.#onboard = options.onboard === true; - this.#onOnboardingStage = options.onOnboardingStage; + this.#onOnboardingStep = options.onOnboardingStep; + this.#onOnboardingTerminal = options.onOnboardingTerminal; if (options.appRoot !== undefined) { this.#appRoot = options.appRoot; const trackerOptions: VercelStatusTrackerOptions = { @@ -1754,6 +1763,7 @@ export class EveTUIRunner { readonly trigger: "startup" | "command"; readonly initialModelStep?: "provider"; readonly registryPlannerContext?: RegistryPlannerContext; + readonly onOnboardingScreen?: PromptCommandHandlerContext["onOnboardingScreen"]; readonly setupFlowTitle?: string; readonly setupFlowNavigation?: PlannerNavigation; readonly keepSetupFlowOpen?: true; @@ -1796,7 +1806,18 @@ export class EveTUIRunner { trigger: "startup" as const, }; - this.#onOnboardingStage?.("model"); + let activeStep: EveCliSetupStep = "model_provider"; + const onOnboardingScreen: NonNullable = ( + input, + ) => { + activeStep = input.screen; + this.#onOnboardingStep?.({ + flow: "onboarding", + step: input.screen, + registrySelectedCount: input.registrySelectedCount, + }); + }; + onOnboardingScreen({ screen: "model_provider" }); let modelOutcome: PromptCommandOutcome | undefined; try { modelOutcome = await this.#executeExtensionCommand( @@ -1807,28 +1828,43 @@ export class EveTUIRunner { suppressSuccessfulTranscript: true, initialModelStep: "provider", setupFlowNavigation: this.#onboardingNavigation(0), + onOnboardingScreen, }, ); } catch (error) { - this.#onOnboardingStage?.("model_error"); + this.#onOnboardingTerminal?.({ flow: "onboarding", step: activeStep, result: "error" }); throw error; } if (modelOutcome?.tone === "error" || modelOutcome?.cancelled === true) { - this.#onOnboardingStage?.( - modelOutcome.cancelled === true ? "model_cancelled" : "model_error", - ); + this.#onOnboardingTerminal?.({ + flow: "onboarding", + step: activeStep, + result: modelOutcome.cancelled === true ? "cancelled" : "error", + }); this.#renderer.setupFlow?.end({ preserveDiagnostics: modelOutcome.tone === "error" }); return; } let addOutcome: PromptCommandOutcome | undefined; + activeStep = "registry_channels"; + const onRegistryScreen: NonNullable = ( + input, + ) => { + activeStep = input.screen; + this.#onOnboardingStep?.({ + flow: "onboarding", + step: input.screen, + registrySelectedCount: input.registrySelectedCount, + }); + }; try { - this.#onOnboardingStage?.("add"); + onRegistryScreen({ screen: "registry_channels" }); addOutcome = await this.#executeExtensionCommand( { type: "extension", name: "add", argument: "" }, title, { ...journey, + onOnboardingScreen: onRegistryScreen, registryPlannerContext: { prefixSteps: [{ label: "Model", complete: true }], reviewMessage: "Review your agent", @@ -1838,17 +1874,20 @@ export class EveTUIRunner { }, ); } catch (error) { - this.#onOnboardingStage?.("add_error"); + this.#onOnboardingTerminal?.({ flow: "onboarding", step: activeStep, result: "error" }); throw error; } finally { if (addOutcome !== undefined) { - this.#onOnboardingStage?.( - addOutcome.cancelled === true - ? "add_cancelled" - : addOutcome.tone === "error" - ? "add_error" - : "completed", - ); + this.#onOnboardingTerminal?.({ + flow: "onboarding", + step: activeStep, + result: + addOutcome.cancelled === true + ? "cancelled" + : addOutcome.tone === "error" + ? "error" + : "completed", + }); } // `/add` is the final onboarding phase. Release the shared panel so the // ordinary chat prompt can own input, retaining deploy/setup evidence on failure. diff --git a/packages/eve/src/cli/dev/tui/setup-commands.ts b/packages/eve/src/cli/dev/tui/setup-commands.ts index f46d48d1b3..385e1a4ea1 100644 --- a/packages/eve/src/cli/dev/tui/setup-commands.ts +++ b/packages/eve/src/cli/dev/tui/setup-commands.ts @@ -41,7 +41,6 @@ export const SETUP_FLOW_CONFIG = { deploy: { title: "Deploy to Vercel", indicator: "spinner" }, } satisfies Record; -/** The prompter surface plus the working-state interrupt trap a command races against. */ export type TuiSetupCommandRenderer = TuiPrompterRenderer & Pick< SetupFlowRenderer, @@ -51,6 +50,17 @@ export type TuiSetupCommandRenderer = TuiPrompterRenderer & type MuteableSetupRenderer = TuiPrompterRenderer & Pick; +export type OnboardingScreenEvent = { + screen: + | "model_provider" + | "model_settings" + | "registry_channels" + | "registry_integrations" + | "registry_review" + | "registry_install"; + registrySelectedCount?: number; +}; + export interface TuiSetupCommandInput { command: TuiSetupCommand; /** Project root for setup that changes shared dependencies, links, or environment files. */ @@ -65,17 +75,16 @@ export interface TuiSetupCommandInput { initialRegistryAddress?: string; /** Presentation and navigation supplied by an enclosing setup journey. */ registryPlannerContext?: RegistryPlannerContext; + onOnboardingScreen?: (input: OnboardingScreenEvent) => void; /** Live ChatGPT identity shown only inside model configuration UI. */ chatGptAccountLabel?: string; /** Suspends development runtime artifacts while registry installation and setup mutate them. */ withExclusiveTerminal?(task: () => Promise): Promise; - /** Test seam; defaults to the real TUI-native prompter over `renderer`. */ createPrompter?: (renderer: TuiPrompterRenderer) => Prompter; /** Test seam; defaults to the real setup flows. */ flows?: Partial; } -/** The flow entry points the commands dispatch to, injectable for tests. */ export interface TuiSetupFlows { runInstallVercelCliFlow: typeof runInstallVercelCliFlow; runLoginFlow: typeof runLoginFlow; @@ -281,6 +290,9 @@ async function executeSetupCommand( if (input.initialModelStep !== undefined) { modelInput.initialStep = input.initialModelStep; } + if (input.onOnboardingScreen !== undefined) { + modelInput.onScreen = (screen) => input.onOnboardingScreen?.({ screen }); + } modelInput.withExclusiveTerminal = (task) => renderer.withInheritedStdio(() => input.withExclusiveTerminal?.(task) ?? task()); const result = await flows.runModelFlow(modelInput); @@ -320,6 +332,7 @@ async function executeSetupCommand( signal, initialAddress: input.initialRegistryAddress, plannerContext: input.registryPlannerContext, + onScreen: input.onOnboardingScreen, onItemStart: registryItemProgress(renderer), runItem: runRegistryItem, }); diff --git a/packages/eve/src/cli/dev/tui/tui.ts b/packages/eve/src/cli/dev/tui/tui.ts index ae56cad6e5..1854cfdfc9 100644 --- a/packages/eve/src/cli/dev/tui/tui.ts +++ b/packages/eve/src/cli/dev/tui/tui.ts @@ -1,4 +1,4 @@ -import type { EveCliOnboardingStage } from "#cli/telemetry/index.js"; +import type { EveCliSetupStepEvent, EveCliSetupTerminalEvent } from "#cli/telemetry/index.js"; import { Client } from "#client/index.js"; import type { DevBootProgressReporter } from "#internal/dev-boot-progress.js"; import { resolveInstalledPackageInfo } from "#internal/application/package.js"; @@ -39,8 +39,9 @@ export interface RunDevelopmentTuiInput extends TuiDisplayOptions { readonly initialInput?: string; /** Explicit fresh-agent onboarding handoff from `eve init`. */ readonly onboard?: boolean; - /** Reports the furthest stage reached by fresh-agent onboarding. */ - readonly onOnboardingStage?: (stage: EveCliOnboardingStage) => void; + /** Reports timestamped steps and terminal result for fresh-agent onboarding. */ + readonly onOnboardingStep?: (input: EveCliSetupStepEvent) => void; + readonly onOnboardingTerminal?: (input: EveCliSetupTerminalEvent) => void; /** Reports local CLI boot phases. Omitted for remote and programmatic TUI runs. */ readonly onBootProgress?: DevBootProgressReporter; /** Gives setup subprocesses exclusive terminal and development-host ownership. */ @@ -144,7 +145,8 @@ export async function runDevelopmentTui(input: RunDevelopmentTuiInput): Promise< headers, initialInput, onboard, - onOnboardingStage, + onOnboardingStep, + onOnboardingTerminal, onBootProgress, lifecycle, startup, @@ -193,7 +195,8 @@ export async function runDevelopmentTui(input: RunDevelopmentTuiInput): Promise< options.startup = startup; } if (onboard !== undefined) options.onboard = onboard; - if (onOnboardingStage !== undefined) options.onOnboardingStage = onOnboardingStage; + if (onOnboardingStep !== undefined) options.onOnboardingStep = onOnboardingStep; + if (onOnboardingTerminal !== undefined) options.onOnboardingTerminal = onOnboardingTerminal; if (onBootProgress !== undefined) options.onBootProgress = onBootProgress; if (lifecycle !== undefined) options.lifecycle = lifecycle; if (withExclusiveTerminal !== undefined) options.withExclusiveTerminal = withExclusiveTerminal; diff --git a/packages/eve/src/cli/run.test.ts b/packages/eve/src/cli/run.test.ts index 966951dc9b..c847b31af2 100644 --- a/packages/eve/src/cli/run.test.ts +++ b/packages/eve/src/cli/run.test.ts @@ -76,7 +76,7 @@ describe("CLI command registration", () => { resolveAgent: async () => undefined as never, root: process.cwd(), }, - { trackDevContext: () => {}, trackInitStage: () => {}, trackOnboardingStage: () => {} }, + { trackDevContext: () => {}, trackSetupStep: () => {}, trackSetupTerminal: () => {} }, ); const paths: string[] = []; const visit = (command: (typeof program.commands)[number], parentPath = ""): void => { @@ -323,6 +323,7 @@ describe("bare eve command", () => { }, undefined, expect.any(Function), + expect.any(Function), ); }); @@ -404,6 +405,7 @@ describe("eve init compatibility flags", () => { }, undefined, expect.any(Function), + expect.any(Function), ); }); diff --git a/packages/eve/src/cli/run.ts b/packages/eve/src/cli/run.ts index b1791cdc7f..b61c86f0bb 100644 --- a/packages/eve/src/cli/run.ts +++ b/packages/eve/src/cli/run.ts @@ -137,7 +137,7 @@ export function createCliProgram( logger: CliLogger, runtime: CliRuntimeOverrides, applicationContext: CliApplicationContext, - telemetry: Pick, + telemetry: Pick, ): Command { const packageVersion = resolveInstalledPackageInfo().version; const program = new Command(); @@ -185,8 +185,6 @@ export function createCliProgram( .description("Create and build reusable eve extension packages."); extension - // Optional: a missing target scaffolds the current directory, matching - // `eve extension init .`. .command("init [target]") .description("Create a new eve extension package.") .option("-y, --yes", "Accepted for compatibility; has no effect") @@ -196,8 +194,8 @@ export function createCliProgram( } const { runExtensionInitCommand } = await import("#cli/commands/extension-init.js"); - await runExtensionInitCommand(logger, applicationContext.root, target, undefined, (stage) => { - telemetry.trackInitStage(stage); + await runExtensionInitCommand(logger, applicationContext.root, target, undefined, (step) => { + telemetry.trackSetupStep({ flow: "extension_init", step }); }); }); @@ -215,8 +213,6 @@ export function createCliProgram( registerRegistryCommands({ program, logger, applicationContext }); program - // Optional: a missing target scaffolds or updates the current directory, - // matching `eve init .`. .command("init [target]") .description("Create a new eve agent, or add one to an existing project directory.") .option("--channel-web-nextjs", "Add the Web Chat application (Next.js)") @@ -259,8 +255,11 @@ export function createCliProgram( reasoning: options.reasoning, }, undefined, - (stage) => { - telemetry.trackInitStage(stage); + (step) => { + telemetry.trackSetupStep({ flow: "init", step }); + }, + (step, result) => { + telemetry.trackSetupTerminal({ flow: "init", step, result }); }, ); }, @@ -428,7 +427,8 @@ export function createCliProgram( applicationRoot: applicationContext.root, existingLocalServer: existingLocalDevelopmentServer, lifecycle, - onOnboardingStage: telemetry.trackOnboardingStage, + onOnboardingStep: telemetry.trackSetupStep, + onOnboardingTerminal: telemetry.trackSetupTerminal, options, remoteTarget, runDevelopmentTui: runtime.runDevelopmentTui, @@ -530,7 +530,8 @@ export function createCliProgram( applicationRoot: applicationContext.root, existingLocalServer: false, lifecycle, - onOnboardingStage: telemetry.trackOnboardingStage, + onOnboardingStep: telemetry.trackSetupStep, + onOnboardingTerminal: telemetry.trackSetupTerminal, options, report: onBootProgress, runDevelopmentTui: runtime.runDevelopmentTui, @@ -679,11 +680,7 @@ export async function runCli( telemetry.trackOutcome(error instanceof CommanderError ? "usage_error" : "error"); if (error instanceof CommanderError) { - // A coding agent that fumbles `eve init` can trip commander before the - // init action runs, so the action's own agent detection never fires. - // Commander has already written its usage error to stderr; add the setup - // guide on stdout so the agent gets actionable next steps, but still fall - // through to throw so the malformed invocation keeps its nonzero exit. + // Commander can reject `eve init` before its action detects the coding agent. const detectCodingAgentLaunch = runtime.isCodingAgentLaunch ?? isCodingAgentLaunch; const agentLaunched = await detectCodingAgentLaunch(); if (input[0] === "init" && agentLaunched) { diff --git a/packages/eve/src/cli/telemetry/index.test.ts b/packages/eve/src/cli/telemetry/index.test.ts index 0728db1d05..438fec7540 100644 --- a/packages/eve/src/cli/telemetry/index.test.ts +++ b/packages/eve/src/cli/telemetry/index.test.ts @@ -154,17 +154,19 @@ describe("createEveCliTelemetry", () => { const write = vi.spyOn(process.stderr, "write").mockImplementation(() => true); const telemetry = createEveCliTelemetry("1.0.0"); telemetry.trackCommand("init"); - telemetry.trackInitStage("target"); - telemetry.trackInitStage("install"); + telemetry.trackSetupStep({ flow: "init", step: "resolve_target" }); + telemetry.trackSetupStep({ flow: "init", step: "install_dependencies" }); await telemetry.flush(); const events = JSON.parse( String(write.mock.calls[0]?.[0]).replace("[eve telemetry] ", ""), ) as Array<{ key: string; value: string }>; - expect(events).toContainEqual(expect.objectContaining({ key: "init_stage", value: "install" })); - expect(events).not.toContainEqual( - expect.objectContaining({ key: "init_stage", value: "target" }), + expect(events).toContainEqual( + expect.objectContaining({ key: "setup_step", value: "install_dependencies" }), + ); + expect(events).toContainEqual( + expect.objectContaining({ key: "setup_step", value: "resolve_target" }), ); }); @@ -174,8 +176,8 @@ describe("createEveCliTelemetry", () => { const write = vi.spyOn(process.stderr, "write").mockImplementation(() => true); const telemetry = createEveCliTelemetry("1.0.0"); telemetry.trackCommand("dev"); - telemetry.trackOnboardingStage("model"); - telemetry.trackOnboardingStage("add_error"); + telemetry.trackSetupStep({ flow: "onboarding", step: "model_provider" }); + telemetry.trackSetupTerminal({ flow: "onboarding", step: "registry_install", result: "error" }); await telemetry.flush(); @@ -183,10 +185,10 @@ describe("createEveCliTelemetry", () => { String(write.mock.calls[0]?.[0]).replace("[eve telemetry] ", ""), ) as Array<{ key: string; value: string }>; expect(events).toContainEqual( - expect.objectContaining({ key: "onboarding_stage", value: "add_error" }), + expect.objectContaining({ key: "setup_terminal_step", value: "registry_install" }), ); - expect(events).not.toContainEqual( - expect.objectContaining({ key: "onboarding_stage", value: "model" }), + expect(events).toContainEqual( + expect.objectContaining({ key: "setup_step", value: "model_provider" }), ); }); diff --git a/packages/eve/src/cli/telemetry/index.ts b/packages/eve/src/cli/telemetry/index.ts index 70edfbde5b..19f6a2b172 100644 --- a/packages/eve/src/cli/telemetry/index.ts +++ b/packages/eve/src/cli/telemetry/index.ts @@ -20,21 +20,38 @@ export type EveCliTelemetryEvent = { readonly value: string; }; -export type EveCliInitStage = "target" | "scaffold" | "install" | "git" | "post_init"; -export type EveCliOnboardingStage = - | "model" - | "model_cancelled" - | "model_error" - | "add" - | "add_cancelled" - | "add_error" - | "completed"; +export type EveCliSetupFlow = "init" | "extension_init" | "onboarding"; +export type EveCliSetupStep = + | "resolve_target" + | "scaffold" + | "install_dependencies" + | "initialize_git" + | "handoff" + | "model_provider" + | "model_settings" + | "registry_channels" + | "registry_integrations" + | "registry_review" + | "registry_install"; +export type EveCliSetupTerminalResult = "completed" | "cancelled" | "error"; + +export type EveCliSetupStepEvent = { + flow: EveCliSetupFlow; + step: EveCliSetupStep; + registrySelectedCount?: number; +}; + +export type EveCliSetupTerminalEvent = { + flow: EveCliSetupFlow; + step: EveCliSetupStep; + result: EveCliSetupTerminalResult; +}; export type EveCliTelemetry = { trackCommand(command: string): void; trackDevContext(context: { target: "local" | "remote"; ui: "tui" | "headless" }): void; - trackInitStage(stage: EveCliInitStage): void; - trackOnboardingStage(stage: EveCliOnboardingStage): void; + trackSetupStep(input: EveCliSetupStepEvent): void; + trackSetupTerminal(input: EveCliSetupTerminalEvent): void; trackOutcome(outcome: "success" | "usage_error" | "error"): void; notify(logger: { error(message: string): void }): Promise; flush(): Promise; @@ -121,8 +138,9 @@ export function createEveCliTelemetry(version: string): EveCliTelemetry { event("stdin_is_tty", process.stdin.isTTY ? "true" : "false"), ]; const sessionId = randomUUID(); - let initStage: EveCliInitStage | undefined; - let onboardingStage: EveCliOnboardingStage | undefined; + const setupEvents: EveCliTelemetryEvent[] = []; + let activeSetup: { flow: EveCliSetupFlow; step: EveCliSetupStep } | undefined; + let setupTerminalRecorded = false; return { trackCommand(command) { @@ -131,13 +149,28 @@ export function createEveCliTelemetry(version: string): EveCliTelemetry { trackDevContext(context) { events.push(event("target", context.target), event("ui", context.ui)); }, - trackInitStage(stage) { - initStage = stage; + trackSetupStep(input) { + activeSetup = { flow: input.flow, step: input.step }; + setupEvents.push(event("setup_flow", input.flow), event("setup_step", input.step)); + if (input.registrySelectedCount !== undefined) { + setupEvents.push(event("registry_selected_count", String(input.registrySelectedCount))); + } }, - trackOnboardingStage(stage) { - onboardingStage = stage; + trackSetupTerminal(input) { + setupTerminalRecorded = true; + setupEvents.push( + event("setup_flow", input.flow), + event("setup_terminal_step", input.step), + event("setup_terminal_result", input.result), + ); }, trackOutcome(outcome) { + if (activeSetup !== undefined && !setupTerminalRecorded) { + this.trackSetupTerminal({ + ...activeSetup, + result: outcome === "success" ? "completed" : "error", + }); + } events.push(event("outcome", outcome)); }, async notify(logger) { @@ -174,8 +207,7 @@ export function createEveCliTelemetry(version: string): EveCliTelemetry { event("installation_id", identity.installationId), event("project_id", await resolveEveTelemetryProjectId({ identity })), ); - if (initStage !== undefined) events.push(event("init_stage", initStage)); - if (onboardingStage !== undefined) events.push(event("onboarding_stage", onboardingStage)); + events.push(...setupEvents); } catch { return; } diff --git a/packages/eve/src/setup/flows/model.ts b/packages/eve/src/setup/flows/model.ts index 689690ce23..b8bf58087e 100644 --- a/packages/eve/src/setup/flows/model.ts +++ b/packages/eve/src/setup/flows/model.ts @@ -289,6 +289,7 @@ export async function runModelFlow(input: { prompter: Prompter; /** Opens provider setup before the root menu when runtime evidence requires it. */ initialStep?: "provider"; + onScreen?: (screen: "model_provider" | "model_settings") => void; signal?: AbortSignal; /** Gives Codex uncontested inherited stdio while it performs login. */ withExclusiveTerminal?: (task: () => Promise) => Promise; @@ -414,6 +415,7 @@ export async function runModelFlow(input: { } if (pick === "model") { + input.onScreen?.("model_settings"); const pickModelSettings = deps.pickModelSettings; if (pickModelSettings === undefined) { throw new Error("runModelFlow requires a pickModelSettings dep to open the model screen."); @@ -460,6 +462,7 @@ export async function runModelFlow(input: { continue; } + input.onScreen?.("model_provider"); const result = await deps.runProviderFlow({ appRoot: environmentRoot, prompter, diff --git a/packages/eve/src/setup/flows/registry.test.ts b/packages/eve/src/setup/flows/registry.test.ts index 9021f78535..17ab8e1a48 100644 --- a/packages/eve/src/setup/flows/registry.test.ts +++ b/packages/eve/src/setup/flows/registry.test.ts @@ -206,6 +206,7 @@ describe("runRegistryFlow", () => { it("starts bare /add on channels and keeps empty Review open", async () => { const prompts: unknown[] = []; + const screens: Array<{ screen: string; registrySelectedCount?: number }> = []; const fake = createFakePrompter({ single: (options) => { prompts.push(options); @@ -220,9 +221,16 @@ describe("runRegistryFlow", () => { await runRegistryFlow({ appRoot: APP_ROOT, prompter: fake.prompter, + onScreen: (screen) => screens.push(screen), deps: deps(), }); + expect(screens).toEqual([ + { screen: "registry_channels" }, + { screen: "registry_integrations" }, + { screen: "registry_review", registrySelectedCount: 0 }, + { screen: "registry_install" }, + ]); expect(prompts).toMatchObject([ { message: "Where should people reach your agent?" }, { message: "What should your agent be able to work with?" }, diff --git a/packages/eve/src/setup/flows/registry.ts b/packages/eve/src/setup/flows/registry.ts index 00b9e62bba..f73b0ae1fc 100644 --- a/packages/eve/src/setup/flows/registry.ts +++ b/packages/eve/src/setup/flows/registry.ts @@ -194,11 +194,18 @@ async function editPlan(input: { selected: Set; notices?: readonly SelectNotice[]; plannerContext?: RegistryPlannerContext; + onScreen?: (input: { + screen: "registry_channels" | "registry_integrations" | "registry_review" | "registry_install"; + registrySelectedCount?: number; + }) => void; }): Promise<"install" | "cancelled"> { let screen: PlannerScreen = "channels"; let notices = input.notices; while (true) { if (screen !== "review") { + input.onScreen?.({ + screen: screen === "channels" ? "registry_channels" : "registry_integrations", + }); try { const direction = await editSection({ ...input, @@ -219,6 +226,7 @@ async function editPlan(input: { } try { + input.onScreen?.({ screen: "registry_review", registrySelectedCount: input.selected.size }); const hasSelections = input.selected.size > 0; const selectedItems = [...input.selected].map((address) => { const item = input.itemsByAddress.get(address); @@ -284,6 +292,10 @@ export async function runRegistryFlow(input: { /** Registry item supplied by `/add `, confirmed and installed directly. */ initialAddress?: string; plannerContext?: RegistryPlannerContext; + onScreen?: (input: { + screen: "registry_channels" | "registry_integrations" | "registry_review" | "registry_install"; + registrySelectedCount?: number; + }) => void; onItemStart?: (item: Item, index: number, total: number) => void; /** Gives each installation its own cancellation boundary without ending the batch. */ runItem?(task: (signal?: AbortSignal) => Promise): Promise; @@ -324,6 +336,7 @@ export async function runRegistryFlow(input: { selected, notices, plannerContext: input.plannerContext, + onScreen: input.onScreen, }); if (plan !== "install") return { kind: "cancelled" }; items = [...selected].map((address) => { @@ -342,6 +355,7 @@ export async function runRegistryFlow(input: { (await import("#setup/project-resolution.js")).detectDeployment; const runDeployFlow = input.deps?.runDeployFlow ?? (await import("./deploy.js")).runDeployFlow; session = createRegistrySession({ detectDeployment, runDeployFlow }); + input.onScreen?.({ screen: "registry_install" }); const activeSession = session; for (const [index, item] of items.entries()) { input.signal?.throwIfAborted();