feat(cli): clarify telemetry identity and onboarding outcomes

.changeset/track-init-telemetry-stages.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Add CLI telemetry for setup and onboarding flows.

docs/reference/telemetry.md

@@ -12,9 +12,9 @@ eve collects usage data from its CLI to help improve its commands and developmen
 eve sends the following information to Vercel:
 
 - The eve version, operating system, CPU architecture, and whether stdin is a terminal.
-- The command you ran and whether it succeeded, had a usage error, or failed.
+- The command you ran, its outcome, and setup or onboarding steps when applicable.
 - For `eve dev`, whether you connected to a local or remote agent and whether the UI was interactive or headless.
-- Random identifiers for the CLI session, your eve installation, and the project.
+- Random identifiers for the CLI session, installation, and project, plus whether the installation and project identifiers are ephemeral or persistent.
 
 The project identifier lets eve group usage from the same project without sending its name or location. eve derives it from the Git remote when available, otherwise `REPOSITORY_URL` or the working directory, and transforms that value before sending it.
 

packages/eve/src/cli/commands/extension-init.ts

@@ -39,6 +39,13 @@ export interface ExtensionInitCliLogger {
   log(message: string): void;
 }
 
+export type ExtensionInitSetupStep =
+  | "resolve_target"
+  | "scaffold"
+  | "install_dependencies"
+  | "initialize_git"
+  | "handoff";
+
 export interface ExtensionInitCommandDependencies {
   detectInvokingPackageManager: typeof detectInvokingPackageManager;
   detectPackageManager: typeof detectPackageManager;
@@ -221,7 +228,9 @@ export async function runExtensionInitCommand(
   parentDirectory: string,
   target: string | undefined,
   dependencies: ExtensionInitCommandDependencies = defaultDependencies,
+  trackStep?: (step: ExtensionInitSetupStep) => void,
 ): Promise<void> {
+  trackStep?.("resolve_target");
   // Coding agent with no target: print a setup guide, same gate as agent init.
   if (target === undefined && (await dependencies.isCodingAgentLaunch())) {
     logger.log(initExtensionInstructions());
@@ -256,6 +265,7 @@ export async function runExtensionInitCommand(
       );
     }
 
+    trackStep?.("scaffold");
     progress.update("Creating extension");
     initLog.debug("creating extension");
     const agentStartedAt = dependencies.now();
@@ -279,6 +289,7 @@ export async function runExtensionInitCommand(
     agentElapsedMs = dependencies.now() - agentStartedAt;
     initLog.debug("creating extension done", { ms: agentElapsedMs });
 
+    trackStep?.("install_dependencies");
     progress.update("Installing dependencies", `${packageManager} install`);
     initLog.debug(`installing dependencies with ${packageManager}`);
     const installStartedAt = dependencies.now();
@@ -316,13 +327,15 @@ export async function runExtensionInitCommand(
     }
     initLog.debug("dependencies installed", { ms: installElapsedMs });
 
+    trackStep?.("initialize_git");
     progress.update("Initializing Git repository");
     initLog.debug("initializing git repository");
     gitResult = await dependencies.tryInitializeGit(projectPath);
   } finally {
     progress.stop();
   }
 
+  trackStep?.("handoff");
   logger.log(
     `${pc.green("✓")} Created an ${EVE_WORDMARK} extension in ${pc.bold(projectPath!)} ${pc.dim(`in ${formatElapsed(agentElapsedMs!)}`)}`,
   );

packages/eve/src/cli/commands/init.ts

@@ -5,6 +5,7 @@ import { performance } from "node:perf_hooks";
 import pc from "#compiled/picocolors/index.js";
 
 import { isCodingAgentLaunch } from "#cli/agent-detection.js";
+import type { EveCliSetupStep, EveCliSetupTerminalResult } from "#cli/telemetry/index.js";
 import { EVE_WORDMARK } from "#cli/banner.js";
 import { formatElapsed } from "#cli/format-elapsed.js";
 import { startCliLiveRow } from "#cli/ui/live-row.js";
@@ -128,10 +129,6 @@ function formatWorkspaceRootMutationWarning(mutation: WorkspaceRootMutation): st
   return `Updated workspace root ${target} at ${mutation.path}${suffix}`;
 }
 
-/**
- * Adds the agent to an existing project and returns the
- * detected manager, which drives the install and dev handoff.
- */
 async function addToExistingProject(
   targetPath: string,
   options: InitCommandOptions,
@@ -173,10 +170,6 @@ async function addToExistingProject(
   };
 }
 
-/**
- * The manager a fresh scaffold will be owned by: an existing ancestor project
- * manager first, then the package runner that launched the CLI, then pnpm.
- */
 async function resolveScaffoldPackageManager(
   projectPath: string,
   dependencies: InitCommandDependencies,
@@ -360,8 +353,9 @@ async function runInitSteps(input: {
   options: InitCommandOptions;
   parentDirectory: string;
   target: string | undefined;
+  trackStep?: (step: EveCliSetupStep) => void;
 }): Promise<InitResult> {
-  const { dependencies, logger, options, parentDirectory, target } = input;
+  const { dependencies, logger, options, parentDirectory, target, trackStep } = input;
   const debug = isLogLevelEnabled("debug");
   const agentLaunched = await dependencies.isCodingAgentLaunch();
   const initTarget = await resolveInitTarget({ parentDirectory, target });
@@ -371,6 +365,7 @@ async function runInitSteps(input: {
   progress.update("Preparing project");
   try {
     const scaffoldPhase = initTarget.kind === "fresh" ? "creating agent" : "adding agent";
+    trackStep?.("scaffold");
     progress.update(initTarget.kind === "fresh" ? "Creating agent" : "Adding agent");
     initLog.debug(scaffoldPhase);
     const agentStartedAt = dependencies.now();
@@ -462,6 +457,7 @@ async function runInitSteps(input: {
       progress = startCliLiveRow(logger);
     }
 
+    trackStep?.("install_dependencies");
     progress.update("Installing dependencies", `${project.packageManager} install`);
     initLog.debug(`installing dependencies with ${project.packageManager}`);
     const installStartedAt = dependencies.now();
@@ -532,6 +528,7 @@ async function runInitSteps(input: {
     initLog.debug("dependencies installed", { ms: installElapsedMs });
 
     if (project.kind === "created") {
+      trackStep?.("initialize_git");
       progress.update("Initializing Git repository");
       initLog.debug("initializing git repository");
       return {
@@ -549,25 +546,16 @@ async function runInitSteps(input: {
   }
 }
 
-/**
- * Creates a new eve agent (`target` is a project name), or adds one to an
- * existing project (`target` is a directory), without external provisioning.
- * A fresh in-place scaffold asks whether to use the current directory or a new
- * subdirectory when the current directory is not empty. Coding-agent launches
- * must pass an explicit subdirectory instead.
- *
- * Runs launched by a coding agent get the dev command printed instead of
- * spawned after scaffolding, since the dev TUI would wedge the launching agent.
- *
- * For extension packages, use `eve extension init` instead.
- */
 export async function runInitCommand(
   logger: InitCliLogger,
   parentDirectory: string,
   target: string | undefined,
   options: InitCommandOptions,
   dependencies: InitCommandDependencies = defaultDependencies,
+  trackStep?: (step: EveCliSetupStep) => void,
+  trackTerminal?: (step: EveCliSetupStep, result: EveCliSetupTerminalResult) => void,
 ): Promise<void> {
+  trackStep?.("resolve_target");
   if (
     await addAgentsToWorkspace(
       logger,
@@ -576,17 +564,31 @@ export async function runInitCommand(
       options,
       dependencies.validateModelSlug,
     )
-  )
+  ) {
+    trackStep?.("handoff");
+    trackTerminal?.("handoff", "completed");
     return;
+  }
 
   let result: InitResult;
   try {
-    result = await runInitSteps({ dependencies, logger, options, parentDirectory, target });
+    result = await runInitSteps({
+      dependencies,
+      logger,
+      options,
+      parentDirectory,
+      target,
+      trackStep,
+    });
   } catch (error) {
-    if (error instanceof WizardCancelledError) return;
+    if (error instanceof WizardCancelledError) {
+      trackTerminal?.("resolve_target", "cancelled");
+      return;
+    }
     throw error;
   }
 
+  trackStep?.("handoff");
   if (result.kind === "created") {
     logger.log(
       `${pc.green("✓")} Created an ${EVE_WORDMARK} agent in ${pc.bold(result.projectPath)} ${pc.dim(`in ${formatElapsed(result.agentElapsedMs)}`)}`,
@@ -618,8 +620,6 @@ export async function runInitCommand(
     }
   }
 
-  // A workspace has no implicit primary agent. Let `eve dev` ask the person
-  // which member to run, rather than silently selecting the first --agents value.
   const baseDevArguments = eveDevArguments(result.packageManager);
   const agentDevCommand = [result.packageManager, ...baseDevArguments].join(" ");
   const agentHandoff = initAgentDevHandoff({

packages/eve/src/cli/dev/run-interactive-ui.ts

@@ -7,6 +7,8 @@ import {
   suspendDevelopmentRuntimeArtifacts,
 } from "#services/dev-client/runtime-artifacts.js";
 
+import type { EveCliSetupStepEvent, EveCliSetupTerminalEvent } from "#cli/telemetry/index.js";
+
 import type { DevelopmentCliOptions } from "./command-options.js";
 import { resolveTuiDisplayOptions } from "./ui-options.js";
 import type { DevelopmentTuiStartup, RunDevelopmentTuiInput } from "./tui/tui.js";
@@ -22,6 +24,8 @@ export async function runInteractiveDevelopmentUi(input: {
   readonly remoteTarget?: DevelopmentUrlTarget;
   readonly report?: DevBootProgressReporter;
   readonly runDevelopmentTui?: (input: RunDevelopmentTuiInput) => Promise<void>;
+  readonly onOnboardingStep?: (input: EveCliSetupStepEvent) => void;
+  readonly onOnboardingTerminal?: (input: EveCliSetupTerminalEvent) => void;
   readonly server: { readonly appRoot?: string; readonly serverUrl: string };
   readonly startup?: DevelopmentTuiStartup;
 }): Promise<void> {
@@ -48,6 +52,8 @@ export async function runInteractiveDevelopmentUi(input: {
     initialInput: input.options.input,
     onboard: input.options.onboard,
     onBootProgress: input.report,
+    onOnboardingStep: input.onOnboardingStep,
+    onOnboardingTerminal: input.onOnboardingTerminal,
     lifecycle: input.lifecycle,
     ...display,
   };

packages/eve/src/cli/dev/tui/prompt-command-handler.ts

@@ -127,6 +127,9 @@ export function createPromptCommandHandler(
         if (context.registryPlannerContext !== undefined) {
           commandInput.registryPlannerContext = context.registryPlannerContext;
         }
+        if (context.onOnboardingScreen !== undefined) {
+          commandInput.onOnboardingScreen = context.onOnboardingScreen;
+        }
         // `/add <item>` confirms and installs that address; bare `/add` opens the planner.
         if (command.name === "add" && command.argument.length > 0) {
           commandInput.initialRegistryAddress = command.argument;

packages/eve/src/cli/dev/tui/runner.test.ts

@@ -3562,6 +3562,7 @@ describe("EveTUIRunner boot setup detection", () => {
   it("runs initial onboarding as one-way model and registry phases", async () => {
     const order: string[] = [];
     const results: string[] = [];
+    const stages: string[] = [];
     const handle = vi.fn(async (command: { name: string }) => {
       order.push(command.name);
       return command.name === "model"
@@ -3584,12 +3585,15 @@ describe("EveTUIRunner boot setup detection", () => {
       appRoot: "/tmp/weather-agent",
       onboard: true,
       bootDetections: [],
+      onOnboardingStep: ({ step }) => stages.push(step),
+      onOnboardingTerminal: ({ step, result }) => stages.push(`${step}_${result}`),
       promptCommandHandler: { handle },
     });
 
     await runner.run();
 
     expect(order).toEqual(["model", "prompt"]);
+    expect(stages).toEqual(["model_provider", "model_provider_error"]);
     expect(results).toContain("/model failed: provider unavailable");
     expect(handle).toHaveBeenCalledWith(
       { type: "extension", name: "model", argument: "" },
@@ -3619,6 +3623,7 @@ describe("EveTUIRunner boot setup detection", () => {
 
   it("moves from Model to Channels and preserves diagnostics after a failed registry phase", async () => {
     const order: string[] = [];
+    const stages: string[] = [];
     const end = vi.fn(() => order.push("end"));
     const handle = vi.fn(async (command: { name: string }) => {
       order.push(command.name);
@@ -3640,12 +3645,15 @@ describe("EveTUIRunner boot setup detection", () => {
       appRoot: "/tmp/weather-agent",
       onboard: true,
       bootDetections: [],
+      onOnboardingStep: ({ step }) => stages.push(step),
+      onOnboardingTerminal: ({ step, result }) => stages.push(`${step}_${result}`),
       promptCommandHandler: { handle },
     });
 
     await runner.run();
 
     expect(order).toEqual(["model", "add", "end", "prompt"]);
+    expect(stages).toEqual(["model_provider", "registry_channels", "registry_channels_error"]);
     expect(end).toHaveBeenCalledWith({ preserveDiagnostics: true });
     expect(handle).toHaveBeenNthCalledWith(
       2,
@@ -3669,6 +3677,7 @@ describe("EveTUIRunner boot setup detection", () => {
   it("keeps the completed /add result after onboarding", async () => {
     const renderCommandInvocation = vi.fn();
     const renderCommandResult = vi.fn();
+    const stages: string[] = [];
     const runner = new EveTUIRunner({
       session: sessionYielding([]),
       renderer: fakeRenderer({
@@ -3681,6 +3690,8 @@ describe("EveTUIRunner boot setup detection", () => {
       appRoot: "/tmp/weather-agent",
       onboard: true,
       bootDetections: [],
+      onOnboardingStep: ({ step }) => stages.push(step),
+      onOnboardingTerminal: ({ step, result }) => stages.push(`${step}_${result}`),
       promptCommandHandler: {
         handle: async (command) =>
           command.name === "model"
@@ -3691,12 +3702,14 @@ describe("EveTUIRunner boot setup detection", () => {
 
     await runner.run();
 
+    expect(stages).toEqual(["model_provider", "registry_channels", "registry_channels_completed"]);
     expect(renderCommandInvocation).toHaveBeenCalledWith("/add", undefined);
     expect(renderCommandResult).toHaveBeenCalledWith("Added Web Chat", "success");
   });
 
   it("does not render a detached /add dismissed result when onboarding is cancelled", async () => {
     const renderCommandResult = vi.fn();
+    const stages: string[] = [];
     const renderer = fakeRenderer({
       readPrompt: vi.fn(async () => undefined),
       renderCommandResult,
@@ -3709,6 +3722,8 @@ describe("EveTUIRunner boot setup detection", () => {
       appRoot: "/tmp/weather-agent",
       onboard: true,
       bootDetections: [],
+      onOnboardingStep: ({ step }) => stages.push(step),
+      onOnboardingTerminal: ({ step, result }) => stages.push(`${step}_${result}`),
       getVercelAuthStatus: vi.fn(async () => "authenticated" as const),
       promptCommandHandler: {
         handle: async (command) =>
@@ -3720,6 +3735,7 @@ describe("EveTUIRunner boot setup detection", () => {
 
     await runner.run();
 
+    expect(stages).toEqual(["model_provider", "registry_channels", "registry_channels_cancelled"]);
     expect(renderCommandResult).not.toHaveBeenCalledWith("/add dismissed.", expect.anything());
   });