diff --git a/apps/account-directory/src/directory.ts b/apps/account-directory/src/directory.ts index 563780d49..09093782e 100644 --- a/apps/account-directory/src/directory.ts +++ b/apps/account-directory/src/directory.ts @@ -64,6 +64,8 @@ type AccountRoute = export const DEFAULT_ONLINE_WINDOW_MS = 90_000; const MAX_PUBKEY_CHARS = 128; const remoteJwksByUrl = new Map>(); +const CORRELATION_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; type CallerTokenFailureReason = | "authentication unavailable" @@ -464,11 +466,56 @@ function trustedWebClientOrigin(env: Env): string | null { function withCors(response: Response, origin: string): Response { const headers = new Headers(response.headers); headers.set("access-control-allow-origin", origin); - headers.set("access-control-expose-headers", "Server-Timing"); + headers.set( + "access-control-expose-headers", + "Server-Timing, X-ADE-Correlation-ID", + ); headers.set("vary", "Origin"); return new Response(response.body, { status: response.status, statusText: response.statusText, headers }); } +function requestCorrelationId(request: Request): string { + const provided = request.headers.get("x-ade-correlation-id")?.trim() ?? ""; + return CORRELATION_ID_PATTERN.test(provided) + ? provided.toLowerCase() + : crypto.randomUUID(); +} + +function withCorrelationId(response: Response, correlationId: string): Response { + const headers = new Headers(response.headers); + headers.set("x-ade-correlation-id", correlationId); + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }); +} + +function logDirectoryLifecycle(args: { + correlationId: string; + route: AccountRoute | null; + method: string; + status: number; + durationMs: number; +}): void { + const outcome = args.status < 400 + ? "ok" + : args.status < 500 + ? "client_error" + : "server_error"; + console.log(JSON.stringify({ + ts: new Date().toISOString(), + svc: "ade-account-directory", + kind: "request_completed", + correlationId: args.correlationId, + route: args.route?.kind ?? "other", + method: args.method, + status: args.status, + outcome, + durationMs: Math.max(0, Math.round(args.durationMs)), + })); +} + async function handleRequestCore( request: Request, env: Env, @@ -511,42 +558,65 @@ export async function handleRequest( env: Env, options: DeviceAuthorizationRequestOptions = {}, ): Promise { + const startedAt = performance.now(); + const correlationId = requestCorrelationId(request); const url = new URL(request.url); + const route = routeAccount(url.pathname); const requestOrigin = request.headers.get("origin"); const allowedOrigin = trustedWebClientOrigin(env); const corsOrigin = requestOrigin && allowedOrigin && requestOrigin === allowedOrigin ? allowedOrigin : null; + const finish = (response: Response, applyCors = false): Response => { + const correlatedResponse = withCorrelationId(response, correlationId); + logDirectoryLifecycle({ + correlationId, + route, + method: request.method, + status: correlatedResponse.status, + durationMs: performance.now() - startedAt, + }); + return applyCors && corsOrigin + ? withCors(correlatedResponse, corsOrigin) + : correlatedResponse; + }; if (request.method === "OPTIONS") { - const route = routeAccount(url.pathname); - if (!route || route.kind !== "list") return text("not found", 404); - if (!corsOrigin) return text("origin not allowed", 403); - if (request.headers.get("access-control-request-method")?.toUpperCase() !== "GET") { - return text("method not allowed", 405); + const allowedMethod = route?.kind === "list" + ? "GET" + : route?.kind === "delete" + ? "DELETE" + : null; + if (!allowedMethod) return finish(text("not found", 404)); + if (!corsOrigin) return finish(text("origin not allowed", 403)); + if (request.headers.get("access-control-request-method")?.toUpperCase() !== allowedMethod) { + return finish(text("method not allowed", 405)); } const requestedHeaders = (request.headers.get("access-control-request-headers") ?? "") .split(",") .map((header) => header.trim().toLowerCase()) .filter(Boolean); - if (requestedHeaders.some((header) => header !== "authorization")) { - return text("headers not allowed", 403); + if (requestedHeaders.some((header) => + header !== "authorization" && header !== "x-ade-correlation-id" + )) { + return finish(text("headers not allowed", 403)); } - return new Response(null, { + return finish(new Response(null, { status: 204, headers: { "access-control-allow-origin": corsOrigin, - "access-control-allow-headers": "authorization", - "access-control-allow-methods": "GET, OPTIONS", + "access-control-allow-headers": "authorization, x-ade-correlation-id", + "access-control-expose-headers": "X-ADE-Correlation-ID", + "access-control-allow-methods": `${allowedMethod}, OPTIONS`, "access-control-max-age": "600", vary: "Origin", }, - }); + })); } // Daemon/native callers omit Origin. Browser callers must match the one // configured hosted client exactly; reject hostile origins before auth or D1. if (requestOrigin && routeAccount(url.pathname) && !corsOrigin) { - return text("origin not allowed", 403); + return finish(text("origin not allowed", 403)); } const response = await handleRequestCore(request, env, options); - return corsOrigin ? withCors(response, corsOrigin) : response; + return finish(response, Boolean(corsOrigin)); } diff --git a/apps/account-directory/test/directory.test.ts b/apps/account-directory/test/directory.test.ts index aed2d8cde..6551e852c 100644 --- a/apps/account-directory/test/directory.test.ts +++ b/apps/account-directory/test/directory.test.ts @@ -1001,14 +1001,37 @@ describe("machine directory", () => { }), env); expect(preflight.status).toBe(204); expect(preflight.headers.get("access-control-allow-origin")).toBe("https://app.ade.dev"); - expect(preflight.headers.get("access-control-allow-headers")).toBe("authorization"); + expect(preflight.headers.get("access-control-allow-headers")).toBe( + "authorization, x-ade-correlation-id", + ); + expect(preflight.headers.get("x-ade-correlation-id")).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + + const deletePreflight = await handleRequest(new Request( + "https://directory.test/account/machines/machine-a", + { + method: "OPTIONS", + headers: { + origin: "https://app.ade.dev", + "access-control-request-method": "DELETE", + "access-control-request-headers": "authorization, x-ade-correlation-id", + }, + }, + ), env); + expect(deletePreflight.status).toBe(204); + expect(deletePreflight.headers.get("access-control-allow-methods")).toBe( + "DELETE, OPTIONS", + ); const allowed = await handleRequest(new Request("https://directory.test/account/machines", { headers: { origin: "https://app.ade.dev", authorization: `Bearer ${token}` }, }), env); expect(allowed.status).toBe(200); expect(allowed.headers.get("access-control-allow-origin")).toBe("https://app.ade.dev"); - expect(allowed.headers.get("access-control-expose-headers")).toBe("Server-Timing"); + expect(allowed.headers.get("access-control-expose-headers")).toBe( + "Server-Timing, X-ADE-Correlation-ID", + ); const hostilePreflight = await handleRequest(new Request("https://directory.test/account/machines", { method: "OPTIONS", @@ -1054,6 +1077,61 @@ describe("machine directory", () => { expect(await otherUserList.json()).toEqual({ machines: [] }); }); + it("echoes safe correlation ids in responses and structured logs", async () => { + const env = makeEnv(); + const token = await mintToken(); + const correlationId = "123e4567-e89b-42d3-a456-426614174000"; + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + const response = await handleRequest(new Request( + "https://directory.test/account/machines?ignored=secret", + { + headers: { + authorization: `Bearer ${token}`, + "x-ade-correlation-id": correlationId.toUpperCase(), + }, + }, + ), env); + + expect(response.status).toBe(200); + expect(response.headers.get("x-ade-correlation-id")).toBe(correlationId); + const lifecycle = log.mock.calls + .map(([line]) => String(line)) + .find((line) => line.includes('"kind":"request_completed"')); + expect(lifecycle).toBeDefined(); + expect(JSON.parse(lifecycle ?? "{}")).toMatchObject({ + svc: "ade-account-directory", + kind: "request_completed", + correlationId, + route: "list", + method: "GET", + status: 200, + outcome: "ok", + }); + expect(lifecycle).not.toContain(token); + expect(lifecycle).not.toContain("ignored=secret"); + log.mockRestore(); + }); + + it("replaces invalid correlation ids instead of reflecting them", async () => { + const token = await mintToken(); + const response = await handleRequest(new Request( + "https://directory.test/account/machines", + { + headers: { + authorization: `Bearer ${token}`, + "x-ade-correlation-id": "unsafe-value", + }, + }, + ), makeEnv()); + + expect(response.status).toBe(200); + expect(response.headers.get("x-ade-correlation-id")).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + expect(response.headers.get("x-ade-correlation-id")).not.toBe("unsafe-value"); + }); + it("retains the authenticated machine's verified Relay route during a transient health dip", async () => { const env = makeEnv(); const token = await mintToken({ sub: "user_1" }); diff --git a/apps/ade-cli/README.md b/apps/ade-cli/README.md index 57992c919..df4b882a1 100644 --- a/apps/ade-cli/README.md +++ b/apps/ade-cli/README.md @@ -418,7 +418,8 @@ ade chat scheduled-work create --at "2026-07-23T01:05:00-04:00" --prompt "Check ade chat scheduled-work create --cron "9,29,49 * * * *" --prompt "Check CI and report" --once # five-field cron uses the ADE brain machine's local timezone ade chat scheduled-work cancel session-id job-id # cancel one job; Claude-native jobs request CronDelete in the owning chat ade chat wait session-id --for idle --timeout-ms 600000 -ade chat recover session-id --turn turn-id --action nudge # wait | nudge | retry | resume +ade chat recover session-id --turn turn-id --action nudge # provider-neutral wait | nudge | retry | resume; falls back for older Codex brains +ade chat resolve-unprocessed session-id --steer steer-id --action run-next # durable/idempotent; action is run-next | dismiss ade chat handoff session-id --model openai/gpt-5.6-sol --note "focus on tests" # brief handoff; add --target-lane to hand off into another lane ade chat fork session-id --model openai/gpt-5.6-sol # fork provider history (claude/codex/opencode/droid); stays in source lane ade chat models --provider codex --json # model order + supported reasoning tiers diff --git a/apps/ade-cli/src/chatRecovery.ts b/apps/ade-cli/src/chatRecovery.ts new file mode 100644 index 000000000..585cf7101 --- /dev/null +++ b/apps/ade-cli/src/chatRecovery.ts @@ -0,0 +1,19 @@ +import { + isUnsupportedAgentChatRecoveryActionError, + type AgentChatRecoverCodexTurnArgs, + type AgentChatRecoverTurnArgs, +} from "../../desktop/src/shared/types/chat"; + +export { + isUnsupportedAgentChatRecoveryActionError as isUnsupportedRecoveryActionError, +}; + +export const LEGACY_RECOVERY_ACTION_BY_NEUTRAL: Readonly> = { + wait: "wait", + nudge: "steer", + retry_same_runtime: "interrupt_retry_same_thread", + restart_resume: "restart_resume_thread", +}; diff --git a/apps/ade-cli/src/cli.test.ts b/apps/ade-cli/src/cli.test.ts index 9cf9263f1..75cef138d 100644 --- a/apps/ade-cli/src/cli.test.ts +++ b/apps/ade-cli/src/cli.test.ts @@ -2986,11 +2986,11 @@ describe("ADE CLI", () => { it.each([ ["wait", "wait"], - ["nudge", "steer"], - ["retry", "interrupt_retry_same_thread"], - ["resume", "restart_resume_thread"], + ["nudge", "nudge"], + ["retry", "retry_same_runtime"], + ["resume", "restart_resume"], ] as const)("maps chat recovery action %s to %s", (cliAction, action) => { - const executePlan = expectExecutePlan(buildCliPlan([ + const plan = buildCliPlan([ "chat", "recover", "chat-1", @@ -2998,20 +2998,44 @@ describe("ADE CLI", () => { "turn-1", "--action", cliAction, - ])); + ]); - expect(executePlan.label).toBe("chat recover"); - expect(executePlan.steps[0]?.params).toMatchObject({ + expect(plan).toEqual({ + kind: "chat-recover", + sessionId: "chat-1", + turnId: "turn-1", + action, + }); + }); + + it("builds durable unprocessed-message resolution actions", () => { + const runNext = expectExecutePlan(buildCliPlan([ + "chat", + "resolve-unprocessed", + "chat-1", + "--steer", + "steer-1", + "--action", + "run-next", + ])); + expect(runNext.steps[0]?.params).toMatchObject({ arguments: { domain: "chat", - action: "recoverCodexTurn", + action: "resolveUnprocessedMessage", args: { sessionId: "chat-1", - turnId: "turn-1", - action, + steerId: "steer-1", + action: "run_next", }, }, }); + + expect(() => buildCliPlan([ + "chat", "resolve-unprocessed", "chat-1", "--action", "dismiss", + ])).toThrow(/steerId/); + expect(() => buildCliPlan([ + "chat", "resolve-unprocessed", "chat-1", "--steer", "steer-1", "--action", "retry", + ])).toThrow(/run-next or dismiss/); }); it("rejects incomplete or unknown chat recovery requests", () => { @@ -3023,6 +3047,81 @@ describe("ADE CLI", () => { ])).toThrow(/wait, nudge, retry, or resume/); }); + posixIt("prefers provider-neutral recovery and falls back for an older Codex brain", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "ade-cli-chat-recover-sock-")); + const socketPath = path.join(root, "ade.sock"); + const actions: Array<{ action: string; args: unknown }> = []; + const stop = await startHeadlessRpcSocketServer({ + socketPath, + createHandler: () => (async (request: any) => { + if (request.method === "ade/initialize") return {}; + if (request.method === "ade/actions/call") { + const action = request.params?.arguments?.action; + const args = request.params?.arguments?.args; + actions.push({ action, args }); + if (action === "recoverTurn") { + throw new Error("Unknown chat action: recoverTurn"); + } + if (action === "recoverCodexTurn") { + return { + domain: "chat", + action, + result: { + turnId: "turn-1", + action: "interrupt_retry_same_thread", + status: "retrying", + }, + }; + } + } + throw new Error(`Unexpected method: ${request.method}`); + }) as any, + }); + + try { + const result = await runCli([ + "--socket", + socketPath, + "chat", + "recover", + "chat-1", + "--turn", + "turn-1", + "--action", + "retry", + "--json", + ]); + + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.output)).toMatchObject({ + turnId: "turn-1", + action: "retry_same_runtime", + status: "retrying", + }); + expect(actions).toEqual([ + { + action: "recoverTurn", + args: { + sessionId: "chat-1", + turnId: "turn-1", + action: "retry_same_runtime", + }, + }, + { + action: "recoverCodexTurn", + args: { + sessionId: "chat-1", + turnId: "turn-1", + action: "interrupt_retry_same_thread", + }, + }, + ]); + } finally { + stop?.(); + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("filters the typed chat model inventory by provider", () => { const executePlan = expectExecutePlan(buildCliPlan([ "chat", @@ -3380,6 +3479,48 @@ describe("ADE CLI", () => { }, }); + const recover = expectExecutePlan(buildCliPlan([ + "chat", + "recover", + "personal-1", + "--personal", + "--turn", + "turn-1", + "--action", + "retry", + ])); + expect(recover.steps[0]).toMatchObject({ + params: { + action: "recoverTurn", + args: { + sessionId: "personal-1", + turnId: "turn-1", + action: "retry_same_runtime", + }, + }, + }); + + const resolveUnprocessed = expectExecutePlan(buildCliPlan([ + "chat", + "resolve-unprocessed", + "personal-1", + "--personal", + "--steer", + "steer-1", + "--action", + "dismiss", + ])); + expect(resolveUnprocessed.steps[0]).toMatchObject({ + params: { + action: "resolveUnprocessedMessage", + args: { + sessionId: "personal-1", + steerId: "steer-1", + action: "dismiss", + }, + }, + }); + const models = expectExecutePlan(buildCliPlan([ "chat", "models", @@ -6092,6 +6233,7 @@ describe("ADE CLI", () => { expect(chatHelp.text).toContain("ade chat steer "); expect(chatHelp.text).toContain("ade chat wait "); expect(chatHelp.text).toContain("ade chat recover "); + expect(chatHelp.text).toContain("ade chat resolve-unprocessed "); expect(chatHelp.text).toContain("ade chat models --provider codex"); expect(chatHelp.text).toContain("ade chat read "); expect(chatHelp.text).toContain("ade new chat --mode cli"); @@ -6119,9 +6261,15 @@ describe("ADE CLI", () => { const chatRecoveryHelp = buildCliPlan(["help", "chat", "recover"]); expect(chatRecoveryHelp.kind).toBe("help"); if (chatRecoveryHelp.kind !== "help") return; - expect(chatRecoveryHelp.text).toContain("same actions as the desktop"); + expect(chatRecoveryHelp.text).toContain("desktop and mobile recovery cards"); expect(chatRecoveryHelp.text).toContain("--action resume"); + const chatResolutionHelp = buildCliPlan(["help", "chat", "resolve-unprocessed"]); + expect(chatResolutionHelp.kind).toBe("help"); + if (chatResolutionHelp.kind !== "help") return; + expect(chatResolutionHelp.text).toContain("durable and idempotent"); + expect(chatResolutionHelp.text).toContain("--action dismiss"); + const agentSpawnHelp = buildCliPlan(["agent", "spawn", "--help"]); expect(agentSpawnHelp.kind).toBe("help"); if (agentSpawnHelp.kind !== "help") return; diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index 90c9f394a..a2d86f39d 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -85,6 +85,10 @@ import { } from "./rpcAuth"; import { isAdeRuntimeNamedPipePath } from "../../desktop/src/shared/adeRuntimeIpc"; import { headlessMobileProjectSummary } from "./services/sync/headlessMobileProjectSummary"; +import { + isUnsupportedRecoveryActionError, + LEGACY_RECOVERY_ACTION_BY_NEUTRAL, +} from "./chatRecovery"; import { isLaunchProfile, isTrackedCliPermissionMode, @@ -317,6 +321,12 @@ type CliPlan = timeoutMs: number; pollIntervalMs: number; } + | { + kind: "chat-recover"; + sessionId: string; + turnId: string; + action: ChatRecoveryCliAction; + } | { kind: "github-app-login"; maxWaitSec: number | null } | { kind: "account-login"; maxWaitSec: number | null; explicitHeadless: boolean } | { kind: "account-machine-connect"; machine: string; remoteArgs: string[] }; @@ -1673,7 +1683,9 @@ const HELP_BY_COMMAND: Record = { $ ade chat wait --for idle --timeout-ms 600000 Wait for idle, active, awaiting-input, or terminal $ ade chat recover --turn --action nudge - Recover a stalled Codex turn: wait, nudge, retry, or resume + Recover a stalled provider turn: wait, nudge, retry, or resume + $ ade chat resolve-unprocessed --steer --action run-next + Run an accepted-but-unprocessed follow-up next, or dismiss it $ ade chat models --provider codex --json List models and supported reasoning tiers $ ade chat read --limit 20 --text Read recent chat messages $ ade chat goal --objective "Ship it" Set or inspect a Codex goal @@ -1792,8 +1804,9 @@ const HELP_BY_COMMAND: Record = { "chat recover": `${ADE_BANNER} Chat recovery - Recover a stalled Codex Work-chat turn using the same actions as the desktop - recovery card. The session and turn must still be the active Codex turn. + Recover a stalled Work-chat turn using the same provider-neutral actions as + the desktop and mobile recovery cards. Older Codex brains automatically use + the compatible legacy action. $ ade chat recover --turn --action wait $ ade chat recover --turn --action nudge @@ -1802,9 +1815,22 @@ const HELP_BY_COMMAND: Record = { Actions: wait Keep the current turn alive and restart its stalled-turn watchdog. - nudge Steer a short status request into the current turn. - retry Interrupt, then retry on the same Codex thread. - resume Restart the app server, resume the thread, then retry the turn. + nudge Ask the current provider turn for a short status update. + retry Interrupt, then retry on the same provider runtime. + resume Restart the provider runtime, resume its thread, then retry. +`, + "chat resolve-unprocessed": `${ADE_BANNER} + Resolve an unprocessed chat message + + Resolve a message ADE accepted while a turn was active but the provider did + not process. The operation is durable and idempotent across retries. + + $ ade chat resolve-unprocessed --steer --action run-next + $ ade chat resolve-unprocessed --steer --action dismiss + + Actions: + run-next Send the accepted message as the next turn. + dismiss Mark the message handled without sending it. `, agent: `${ADE_BANNER} Agent sessions @@ -2679,39 +2705,55 @@ type ToolClaimArgs = { type CodexGoalCliStatus = "active" | "paused" | "blocked" | "complete"; -type CodexRecoveryCliAction = +type ChatRecoveryCliAction = | "wait" - | "steer" - | "interrupt_retry_same_thread" - | "restart_resume_thread"; + | "nudge" + | "retry_same_runtime" + | "restart_resume"; + +type UnprocessedMessageCliAction = "run_next" | "dismiss"; function isCodexGoalCliStatus(value: string | null): value is CodexGoalCliStatus { return value === "active" || value === "paused" || value === "blocked" || value === "complete"; } -function normalizeCodexRecoveryCliAction(value: string | null): CodexRecoveryCliAction { +function normalizeChatRecoveryCliAction(value: string | null): ChatRecoveryCliAction { const normalized = value?.trim().toLowerCase().replace(/-/g, "_") ?? ""; if (normalized === "wait") return "wait"; - if (normalized === "nudge" || normalized === "steer") return "steer"; + if (normalized === "nudge" || normalized === "steer") return "nudge"; if ( normalized === "retry" || normalized === "interrupt_retry" || normalized === "interrupt_retry_same_thread" + || normalized === "retry_same_runtime" ) { - return "interrupt_retry_same_thread"; + return "retry_same_runtime"; } if ( normalized === "resume" || normalized === "restart_resume" || normalized === "restart_resume_thread" ) { - return "restart_resume_thread"; + return "restart_resume"; } throw new CliUsageError( "chat recover --action must be wait, nudge, retry, or resume.", ); } +function normalizeUnprocessedMessageCliAction( + value: string | null, +): UnprocessedMessageCliAction { + const normalized = value?.trim().toLowerCase().replace(/-/g, "_") ?? ""; + if (normalized === "run_next" || normalized === "run" || normalized === "next") { + return "run_next"; + } + if (normalized === "dismiss") return "dismiss"; + throw new CliUsageError( + "chat resolve-unprocessed --action must be run-next or dismiss.", + ); +} + function readToolClaimArgs(args: string[]): ToolClaimArgs { const laneId = asString( readValue(args, ["--lane", "--lane-id"]) ?? process.env.ADE_LANE_ID, @@ -7253,21 +7295,41 @@ function buildChatPlan(args: string[]): CliPlan { }; if (sub === "recover" || sub === "recovery") { const turnId = requireValue(readValue(args, ["--turn", "--turn-id"]), "turnId"); - const action = normalizeCodexRecoveryCliAction( + const action = normalizeChatRecoveryCliAction( readValue(args, ["--action", "--recovery-action"]) ?? firstStandalonePositional(args), ); + return { + kind: "chat-recover", + sessionId: requireValue(sessionId, "sessionId"), + turnId, + action, + }; + } + if ( + sub === "resolve-unprocessed" + || sub === "resolve-message" + || sub === "unprocessed" + ) { + const steerId = requireValue( + readValue(args, ["--steer", "--steer-id", "--message", "--message-id"]), + "steerId", + ); + const action = normalizeUnprocessedMessageCliAction( + readValue(args, ["--action", "--resolution-action"]) + ?? firstStandalonePositional(args), + ); return { kind: "execute", - label: "chat recover", + label: "chat resolve unprocessed message", steps: [ actionStep( "result", "chat", - "recoverCodexTurn", + "resolveUnprocessedMessage", withSession({ sessionId: requireValue(sessionId, "sessionId"), - turnId, + steerId, action, }), ), @@ -7735,6 +7797,11 @@ function buildPersonalChatPlan(sub: string, args: string[]): CliPlan { "configure", "interrupt", "stop", + "recover", + "recovery", + "resolve-unprocessed", + "resolve-message", + "unprocessed", "archive", "unarchive", "delete", @@ -7743,7 +7810,7 @@ function buildPersonalChatPlan(sub: string, args: string[]): CliPlan { "status", ]); if (!sessionSubcommands.has(sub)) { - throw new CliUsageError(`Personal chats support actions, action, list, create, show, read, send, steer, update, models, model-catalog, interrupt, archive, unarchive, or delete; got '${sub}'.`); + throw new CliUsageError(`Personal chats support actions, action, list, create, show, read, send, steer, update, models, model-catalog, interrupt, recover, resolve-unprocessed, archive, unarchive, or delete; got '${sub}'.`); } const sessionId = requireValue( @@ -7822,6 +7889,45 @@ function buildPersonalChatPlan(sub: string, args: string[]): CliPlan { steps: [personalChatStep("interrupt", collectGenericObjectArgs(args, { sessionId }))], }; } + if (sub === "recover" || sub === "recovery") { + const turnId = requireValue(readValue(args, ["--turn", "--turn-id"]), "turnId"); + const action = normalizeChatRecoveryCliAction( + readValue(args, ["--action", "--recovery-action"]) + ?? firstStandalonePositional(args), + ); + return { + ...base, + label: "personal chat recover", + steps: [personalChatStep("recoverTurn", collectGenericObjectArgs(args, { + sessionId, + turnId, + action, + }))], + }; + } + if ( + sub === "resolve-unprocessed" + || sub === "resolve-message" + || sub === "unprocessed" + ) { + const steerId = requireValue( + readValue(args, ["--steer", "--steer-id", "--message", "--message-id"]), + "steerId", + ); + const action = normalizeUnprocessedMessageCliAction( + readValue(args, ["--action", "--resolution-action"]) + ?? firstStandalonePositional(args), + ); + return { + ...base, + label: "personal chat resolve unprocessed message", + steps: [personalChatStep("resolveUnprocessedMessage", collectGenericObjectArgs(args, { + sessionId, + steerId, + action, + }))], + }; + } if (sub === "archive" || sub === "unarchive" || sub === "delete" || sub === "rm") { const action = sub === "rm" ? "delete" : sub; return { @@ -19331,6 +19437,64 @@ async function runChatWaitCommand( } } +async function runChatRecoverCommand( + plan: CliPlan & { kind: "chat-recover" }, + options: GlobalOptions, +): Promise<{ output: string; exitCode: number }> { + let connection: CliConnection; + try { + connection = await createConnection(options, { autoRegisterProject: true }); + } catch (error) { + throw new CliExecutionError( + "Failed to initialize ADE CLI connection for chat recovery.", + { + cause: error instanceof Error ? error.message : String(error), + nextAction: + "Verify --project-root points at an ADE project and run ade doctor --json.", + }, + ); + } + + const runRecoveryAction = async ( + action: "recoverTurn" | "recoverCodexTurn", + args: JsonObject, + ): Promise => { + const raw = await connection.request("ade/actions/call", { + name: "run_ade_action", + arguments: { + domain: "chat", + action, + args, + }, + }); + return unwrapActionEnvelope(unwrapToolResult(raw)); + }; + + try { + let result: unknown; + try { + result = await runRecoveryAction("recoverTurn", { + sessionId: plan.sessionId, + turnId: plan.turnId, + action: plan.action, + }); + } catch (error) { + if (!isUnsupportedRecoveryActionError(error)) throw error; + const legacyResult = await runRecoveryAction("recoverCodexTurn", { + sessionId: plan.sessionId, + turnId: plan.turnId, + action: LEGACY_RECOVERY_ACTION_BY_NEUTRAL[plan.action], + }); + result = isRecord(legacyResult) + ? { ...legacyResult, action: plan.action } + : legacyResult; + } + return { output: formatOutput(result, options), exitCode: 0 }; + } finally { + await connection.close(); + } +} + function formatOutput( value: unknown, options: GlobalOptions, @@ -19564,6 +19728,9 @@ async function runCli( if (plan.kind === "chat-wait") { return await runChatWaitCommand(plan, parsed.options); } + if (plan.kind === "chat-recover") { + return await runChatRecoverCommand(plan, parsed.options); + } if (plan.kind === "init") { const result = await runInit(plan.targetPath); return { diff --git a/apps/ade-cli/src/services/account/accountMachineDirectoryService.test.ts b/apps/ade-cli/src/services/account/accountMachineDirectoryService.test.ts index 3a46f4e50..0205fa520 100644 --- a/apps/ade-cli/src/services/account/accountMachineDirectoryService.test.ts +++ b/apps/ade-cli/src/services/account/accountMachineDirectoryService.test.ts @@ -33,7 +33,28 @@ function directoryFetch(machines: AdeAccountMachine[], status = 200): typeof fet } describe("AccountMachineDirectoryService", () => { - it("removes account-owned client trust together and preserves direct pairings", () => { + it("preserves device-bound trust on sign-out", () => { + const pairedPrune = vi.fn(); + const targetPrune = vi.fn(); + const remove = vi.fn(); + + expect(reconcileAccountOwnedMachineTrust(null, { + pairedStore: { pruneAccountOwned: pairedPrune }, + targetRegistry: { + pruneAccountOwned: targetPrune, + list: vi.fn(() => []), + remove, + }, + })).toEqual({ + removedTargetIds: [], + removedCredentialHostIds: [], + }); + expect(pairedPrune).not.toHaveBeenCalled(); + expect(targetPrune).not.toHaveBeenCalled(); + expect(remove).not.toHaveBeenCalled(); + }); + + it("removes a different account's client trust together and preserves direct pairings", () => { const removedCredential = { hostIdentity: { deviceId: "account-host" }, machineKey: "account-key", @@ -47,7 +68,7 @@ describe("AccountMachineDirectoryService", () => { pairedMachine: { hostIdentity: "account-host", machineKey: "account-key" }, } as RemoteRuntimeTarget; const remove = vi.fn((id: string) => id === orphanedHistoricalTarget.id); - const result = reconcileAccountOwnedMachineTrust(null, { + const result = reconcileAccountOwnedMachineTrust("account-b", { pairedStore: { pruneAccountOwned: vi.fn(() => [removedCredential]), }, @@ -206,6 +227,8 @@ describe("AccountMachineDirectoryService", () => { redirect: "error", }); expect(new Headers(init?.headers).get("authorization")).toBe("Bearer account-token"); + expect(new Headers(init?.headers).get("x-ade-correlation-id")) + .toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); }); it("uses the hosted directory when the machine override is blank", async () => { diff --git a/apps/ade-cli/src/services/account/accountMachineDirectoryService.ts b/apps/ade-cli/src/services/account/accountMachineDirectoryService.ts index 45fcbd310..38952207d 100644 --- a/apps/ade-cli/src/services/account/accountMachineDirectoryService.ts +++ b/apps/ade-cli/src/services/account/accountMachineDirectoryService.ts @@ -14,6 +14,7 @@ import type { } from "../../../../desktop/src/shared/types/account"; import { accountMachineSecureSyncEndpoints, + createAccountDirectoryCorrelationId, fetchAccountMachines, resolveTrustedAccountDirectoryBaseUrl, selectAccountMachine, @@ -54,8 +55,10 @@ export type AccountMachineTrustReconciliationResult = { }; /** - * Remove client-side machine trust that belongs to a different (or signed-out) - * ADE account. Ownerless PIN/address/SSH credentials are deliberately kept. + * Remove client-side machine trust that belongs to a different signed-in ADE + * account. Sign-out keeps host-issued paired secrets for LAN/Tailscale while + * Relay and directory access remain disabled. Ownerless PIN/address/SSH + * credentials are deliberately kept. * Any target left pointing at a removed credential is unusable and is removed * in the same pass so a partial historical write cannot leak its machine name. */ @@ -69,7 +72,9 @@ export function reconcileAccountOwnedMachineTrust( const currentOwnerUserId = currentOwnerUserIdValue?.trim() || null; const pairedStore = options.pairedStore ?? new DesktopPairedMachineStore(); const targetRegistry = options.targetRegistry ?? new RemoteTargetRegistry(); - const removedCredentials = pairedStore.pruneAccountOwned(currentOwnerUserId); + const removedCredentials = currentOwnerUserId + ? pairedStore.pruneAccountOwned(currentOwnerUserId) + : []; const removedCredentialIds = new Set(); for (const credentials of removedCredentials) { removedCredentialIds.add(credentials.hostIdentity.deviceId); @@ -77,7 +82,9 @@ export function reconcileAccountOwnedMachineTrust( } const removedTargetIds = new Set( - targetRegistry.pruneAccountOwned(currentOwnerUserId).map((target) => target.id), + currentOwnerUserId + ? targetRegistry.pruneAccountOwned(currentOwnerUserId).map((target) => target.id) + : [], ); for (const target of targetRegistry.list()) { const reference = target.pairedMachine; @@ -202,6 +209,7 @@ export class AccountMachineDirectoryService { controller.abort(); }, timeoutMs); timer.unref?.(); + const correlationId = createAccountDirectoryCorrelationId(); try { const sendDelete = (accessToken: string): Promise => (this.options.fetchImpl ?? fetch)( @@ -211,6 +219,7 @@ export class AccountMachineDirectoryService { headers: { accept: "application/json", authorization: `Bearer ${accessToken}`, + "x-ade-correlation-id": correlationId, }, credentials: "omit", referrerPolicy: "no-referrer", diff --git a/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts b/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts index bdefcf734..bab18c63d 100644 --- a/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts +++ b/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts @@ -1178,6 +1178,8 @@ describe("account machine registration publisher", () => { const [url, init] = fetchImpl.mock.calls[0]!; expect(url).toBe("https://directory.example/account/machines/register"); expect(new Headers(init?.headers).get("authorization")).toBe("Bearer account-secret-token"); + expect(new Headers(init?.headers).get("x-ade-correlation-id")) + .toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); expect(init).toMatchObject({ method: "POST", credentials: "omit", diff --git a/apps/ade-cli/src/services/account/accountMachinePublisherService.ts b/apps/ade-cli/src/services/account/accountMachinePublisherService.ts index 979f65c29..7d34811e6 100644 --- a/apps/ade-cli/src/services/account/accountMachinePublisherService.ts +++ b/apps/ade-cli/src/services/account/accountMachinePublisherService.ts @@ -9,6 +9,7 @@ import { } from "../../../../desktop/src/shared/types"; import type { ProductAnalyticsCapture } from "../../../../desktop/src/shared/types/productAnalytics"; import { + createAccountDirectoryCorrelationId, readAccountDirectoryHttpReason, resolveTrustedAccountDirectoryBaseUrl, shouldIgnoreDevelopmentAccountDirectoryUrl, @@ -528,6 +529,7 @@ export function createAccountMachinePublisherService(options: { addLegDuration("token", startedAt); } }; + const correlationId = createAccountDirectoryCorrelationId(); const sendRegistration = async ( token: string, registration: AccountMachineRegistration, @@ -562,6 +564,7 @@ export function createAccountMachinePublisherService(options: { accept: "application/json", authorization: `Bearer ${token}`, "content-type": "application/json", + "x-ade-correlation-id": correlationId, }, body: JSON.stringify(registration), credentials: "omit", diff --git a/apps/ade-cli/src/services/personalChats/personalChatScope.test.ts b/apps/ade-cli/src/services/personalChats/personalChatScope.test.ts index 704f04e38..88f314ff2 100644 --- a/apps/ade-cli/src/services/personalChats/personalChatScope.test.ts +++ b/apps/ade-cli/src/services/personalChats/personalChatScope.test.ts @@ -51,6 +51,16 @@ describe("PersonalChatScope", () => { readTranscript: vi.fn(async () => []), steer: vi.fn(async () => undefined), interrupt: vi.fn(async () => undefined), + recoverTurn: vi.fn(async ({ turnId, action }) => ({ + turnId, + action, + status: action === "nudge" ? "nudged" : "waiting", + })), + resolveUnprocessedMessage: vi.fn(async ({ steerId, action }) => ({ + steerId, + action, + status: "completed", + })), respondToInput: vi.fn(async () => undefined), approveToolUse: vi.fn(async () => undefined), createScheduledWork: vi.fn(async ({ sessionId, cron, runAt, prompt }: { @@ -207,6 +217,39 @@ describe("PersonalChatScope", () => { expect(service.sendMessage).toHaveBeenCalledWith({ sessionId: "chat-1", text: "continue" }); }); + it("routes recovery and durable message resolution only for personal sessions", async () => { + const { createRuntime, service } = fixture(); + const scope = new PersonalChatScope({ createRuntime }); + + await expect(scope.call("recoverTurn", { + sessionId: "chat-1", + turnId: "turn-1", + action: "nudge", + })).resolves.toMatchObject({ + action: "recoverTurn", + result: { turnId: "turn-1", action: "nudge", status: "nudged" }, + }); + await expect(scope.call("resolveUnprocessedMessage", { + sessionId: "chat-1", + steerId: "steer-1", + action: "run_next", + })).resolves.toMatchObject({ + action: "resolveUnprocessedMessage", + result: { steerId: "steer-1", action: "run_next", status: "completed" }, + }); + + expect(service.recoverTurn).toHaveBeenCalledWith({ + sessionId: "chat-1", + turnId: "turn-1", + action: "nudge", + }); + expect(service.resolveUnprocessedMessage).toHaveBeenCalledWith({ + sessionId: "chat-1", + steerId: "steer-1", + action: "run_next", + }); + }); + it("rejects session-scoped calls when the session row is missing", async () => { const { createRuntime, service } = fixture(); service.getSessionSummary.mockResolvedValueOnce(null as never); @@ -333,6 +376,8 @@ describe("PersonalChatScope", () => { "list", "create", "send", + "recoverTurn", + "resolveUnprocessedMessage", "createScheduledWork", "cancelScheduledWork", "setScheduledWorkPaused", diff --git a/apps/ade-cli/src/services/personalChats/personalChatScope.ts b/apps/ade-cli/src/services/personalChats/personalChatScope.ts index 185d326a4..bb964b8dd 100644 --- a/apps/ade-cli/src/services/personalChats/personalChatScope.ts +++ b/apps/ade-cli/src/services/personalChats/personalChatScope.ts @@ -229,6 +229,14 @@ export class PersonalChatScope { await this.requirePersonalSession(service, readSessionId(args)); result = await service.interrupt(args as never); break; + case "recoverTurn": + await this.requirePersonalSession(service, readSessionId(args)); + result = await service.recoverTurn(args as never); + break; + case "resolveUnprocessedMessage": + await this.requirePersonalSession(service, readSessionId(args)); + result = await service.resolveUnprocessedMessage(args as never); + break; case "respondToInput": await this.requirePersonalSession(service, readSessionId(args)); result = await service.respondToInput(args as never); diff --git a/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts b/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts index a3ac654a8..7891e8b04 100644 --- a/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts +++ b/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts @@ -889,6 +889,55 @@ describe("createSyncRemoteCommandService", () => { ]); }); + it("routes provider-neutral recovery and durable unprocessed-message actions", async () => { + const recoverTurn = vi.fn(async (args) => ({ + action: args.action, + turnId: args.turnId, + status: args.action === "nudge" ? "nudged" : "waiting", + })); + const resolveUnprocessedMessage = vi.fn(async (args) => ({ + steerId: args.steerId, + action: args.action, + status: "completed", + })); + const { service } = createService({ + agentChatService: { recoverTurn, resolveUnprocessedMessage }, + }); + + expect(service.getDescriptor("chat.recoverTurn")).toEqual({ + action: "chat.recoverTurn", + scope: "project", + policy: { viewerAllowed: true, queueable: false }, + }); + expect(service.getDescriptor("chat.resolveUnprocessedMessage")).toEqual({ + action: "chat.resolveUnprocessedMessage", + scope: "project", + policy: { viewerAllowed: true, queueable: false }, + }); + + await service.execute(makePayload("chat.recoverTurn", { + sessionId: "chat-1", + turnId: "turn-1", + action: "nudge", + })); + await service.execute(makePayload("chat.resolveUnprocessedMessage", { + sessionId: "chat-1", + steerId: "steer-1", + action: "run_next", + })); + + expect(recoverTurn).toHaveBeenCalledWith({ + sessionId: "chat-1", + turnId: "turn-1", + action: "nudge", + }); + expect(resolveUnprocessedMessage).toHaveBeenCalledWith({ + sessionId: "chat-1", + steerId: "steer-1", + action: "run_next", + }); + }); + it("routes scheduled-work cancellation through the non-queueable mobile command", async () => { const cancelScheduledWork = vi.fn(async ({ sessionId, scheduleId }: { sessionId: string; @@ -1038,6 +1087,27 @@ describe("createSyncRemoteCommandService", () => { expect(recoverCodexTurn).not.toHaveBeenCalled(); }); + it("rejects unsupported provider-neutral recovery and message-resolution actions", async () => { + const recoverTurn = vi.fn(); + const resolveUnprocessedMessage = vi.fn(); + const { service } = createService({ + agentChatService: { recoverTurn, resolveUnprocessedMessage }, + }); + + await expect(service.execute(makePayload("chat.recoverTurn", { + sessionId: "chat-1", + turnId: "turn-1", + action: "replace", + }))).rejects.toThrow("unsupported action 'replace'"); + await expect(service.execute(makePayload("chat.resolveUnprocessedMessage", { + sessionId: "chat-1", + steerId: "steer-1", + action: "retry", + }))).rejects.toThrow("unsupported action 'retry'"); + expect(recoverTurn).not.toHaveBeenCalled(); + expect(resolveUnprocessedMessage).not.toHaveBeenCalled(); + }); + it("preserves Claude priority steering and guarded queue cancellation", async () => { const steer = vi.fn().mockResolvedValue({ steerId: "steer-1", queued: false }); const cancelSteer = vi.fn().mockResolvedValue(undefined); diff --git a/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts b/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts index 17dff956e..a2566c22b 100644 --- a/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts +++ b/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { createHash, randomUUID } from "node:crypto"; import { fileURLToPath } from "node:url"; +import { isAgentChatTurnRecoveryAction } from "../../../../desktop/src/shared/types/chat"; import { runWithAbortSignal } from "./abortSignal"; import type { AgentChatCreateArgs, @@ -57,6 +58,8 @@ import type { AgentChatCancelDispatchedSteerArgs, AgentChatInterruptArgs, AgentChatRecoverCodexTurnArgs, + AgentChatRecoverTurnArgs, + AgentChatResolveUnprocessedMessageArgs, AgentChatUpdateSessionArgs, AddPrCommentArgs, AiReviewSummaryArgs, @@ -2310,6 +2313,41 @@ function parseAgentChatRecoverCodexTurnArgs(value: Record): Age }; } +function parseAgentChatRecoverTurnArgs(value: Record): AgentChatRecoverTurnArgs { + const action = requireString(value.action, "chat.recoverTurn requires action."); + if (!isAgentChatTurnRecoveryAction(action)) { + throw new Error(`chat.recoverTurn received unsupported action '${action}'.`); + } + return { + sessionId: requireString(value.sessionId, "chat.recoverTurn requires sessionId."), + turnId: requireString(value.turnId, "chat.recoverTurn requires turnId."), + action, + }; +} + +function parseAgentChatResolveUnprocessedMessageArgs( + value: Record, +): AgentChatResolveUnprocessedMessageArgs { + const action = requireString( + value.action, + "chat.resolveUnprocessedMessage requires action.", + ); + if (action !== "run_next" && action !== "dismiss") { + throw new Error(`chat.resolveUnprocessedMessage received unsupported action '${action}'.`); + } + return { + sessionId: requireString( + value.sessionId, + "chat.resolveUnprocessedMessage requires sessionId.", + ), + steerId: requireString( + value.steerId, + "chat.resolveUnprocessedMessage requires steerId.", + ), + action, + }; +} + function parseAgentChatApproveArgs(value: Record): AgentChatApproveArgs { return { sessionId: requireString(value.sessionId, "chat.approve requires sessionId."), @@ -4190,6 +4228,12 @@ function registerChatRemoteCommands({ args, register }: RemoteCommandRegistratio register("chat.recoverCodexTurn", { viewerAllowed: true, queueable: false }, async (payload) => requireService(args.agentChatService, "Agent chat service not available.") .recoverCodexTurn(parseAgentChatRecoverCodexTurnArgs(payload))); + register("chat.recoverTurn", { viewerAllowed: true, queueable: false }, async (payload) => + requireService(args.agentChatService, "Agent chat service not available.") + .recoverTurn(parseAgentChatRecoverTurnArgs(payload))); + register("chat.resolveUnprocessedMessage", { viewerAllowed: true, queueable: false }, async (payload) => + requireService(args.agentChatService, "Agent chat service not available.") + .resolveUnprocessedMessage(parseAgentChatResolveUnprocessedMessageArgs(payload))); register("chat.steer", { viewerAllowed: true, queueable: false }, async (payload) => { const result = await requireService(args.agentChatService, "Agent chat service not available.").steer(parseAgentChatSteerArgs(payload)); return isRecord(result) ? { ...result, ok: true } : { ok: true }; diff --git a/apps/ade-cli/src/tuiClient/__tests__/ChatView.test.tsx b/apps/ade-cli/src/tuiClient/__tests__/ChatView.test.tsx index ef5fb1aa4..32bfe29ce 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/ChatView.test.tsx +++ b/apps/ade-cli/src/tuiClient/__tests__/ChatView.test.tsx @@ -491,6 +491,64 @@ describe("ChatView", () => { expect(frame).not.toContain("queued version"); expect(frame).not.toContain("staged message"); expect(frame).toContain("delivered version"); + expect(frame).toContain("accepted · waiting to be processed"); + }); + + it("renders one user bubble for steer lifecycle updates with the latest state", () => { + const frame = renderEvents([ + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:00.000Z", + sequence: 1, + event: { type: "user_message", text: "run release checks", steerId: "steer-1", deliveryState: "accepted", turnId: "turn-active" }, + }, + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:01.000Z", + sequence: 2, + event: { type: "user_message", text: "run release checks", steerId: "steer-1", deliveryState: "processed", processed: true, turnId: "turn-active" }, + }, + ], { width: 80 }); + + expect(frame.match(/run release checks/g)).toHaveLength(1); + expect(frame).toContain("processed"); + expect(frame).not.toContain("accepted · waiting"); + }); + + it("keeps raw moderation quiet and renders cumulative turn diagnostics", () => { + const frame = renderEvents([ + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:00.000Z", + sequence: 1, + event: { + type: "codex_moderation_metadata", + metadata: { turnId: "turn-1", metadata: { is_blocked: false } }, + turnId: "turn-1", + }, + }, + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:01.000Z", + sequence: 2, + event: { + type: "turn_diagnostics", + turnId: "turn-1", + moderationChecks: 3, + optionalIntegrationFailures: [{ integration: "unityMCP" }], + }, + }, + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:02.000Z", + sequence: 3, + event: { type: "user_message", text: "continue", turnId: "turn-1" }, + }, + ], { width: 80 }); + + expect(frame).not.toContain("moderation checked"); + expect(frame).toContain("turn details · 3 safety checks"); + expect(frame).toContain("unityMCP"); }); it("keeps steer lifecycle notices out of visible chat blocks", () => { diff --git a/apps/ade-cli/src/tuiClient/__tests__/adeApi.test.ts b/apps/ade-cli/src/tuiClient/__tests__/adeApi.test.ts index 0515ecef1..2ddc1c0d3 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/adeApi.test.ts +++ b/apps/ade-cli/src/tuiClient/__tests__/adeApi.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { AgentChatEventEnvelope } from "../../../../desktop/src/shared/types/chat"; -import { archiveChatSession, buildPtyContinuationLaunchFields, cancelSteerMessage, createChatSession, DEFAULT_CODEX_REASONING_EFFORT, deleteChatSession, deriveClaudeGoalFromEvents, dispatchSteerMessage, discoverProjectSlashCommands, editSteerMessage, enrichChatSessionsWithLifecycle, enrichTerminalSessionsWithLifecycle, getAvailableModels, getChatHistoryPage, getMainTranscript, latestGoal, latestTokenStats, listChatSessions, listLaneDiffStats, listPrsByLane, listSessionSummaries, listTerminalSessions, messageChatSession, recoverCodexTurn, requestSessionAttention, resumeTerminalSession, runDefaultLaneSetup, sendChatMessage, setSessionStatusNote, settleSession, signalTerminal, startCliTerminalSession, steerChatMessage, trackedCliTerminalProvider, unarchiveChatSession, unsettleSession } from "../adeApi"; +import { archiveChatSession, buildPtyContinuationLaunchFields, cancelSteerMessage, createChatSession, DEFAULT_CODEX_REASONING_EFFORT, deleteChatSession, deriveClaudeGoalFromEvents, dispatchSteerMessage, discoverProjectSlashCommands, editSteerMessage, enrichChatSessionsWithLifecycle, enrichTerminalSessionsWithLifecycle, getAvailableModels, getChatHistoryPage, getMainTranscript, latestGoal, latestTokenStats, listChatSessions, listLaneDiffStats, listPrsByLane, listSessionSummaries, listTerminalSessions, messageChatSession, recoverCodexTurn, recoverTurn, requestSessionAttention, resolveUnprocessedMessage, resumeTerminalSession, runDefaultLaneSetup, sendChatMessage, setSessionStatusNote, settleSession, signalTerminal, startCliTerminalSession, steerChatMessage, trackedCliTerminalProvider, unarchiveChatSession, unsettleSession } from "../adeApi"; import type { ChatTerminalSession, TerminalSessionSummary } from "../../../../desktop/src/shared/types/sessions"; import type { AdeCodeConnection } from "../types"; @@ -1465,4 +1465,127 @@ describe("steer helpers", () => { args: { sessionId: "chat-1", turnId: "turn-1", action: "steer" }, }]); }); + + it("prefers provider-neutral turn recovery", async () => { + const calls: Array<{ domain: string; action: string; args: Record }> = []; + const connection = { + action: async (domain: string, action: string, args: Record) => { + calls.push({ domain, action, args }); + return { action: "nudge", turnId: "turn-1", status: "nudged" }; + }, + } as unknown as AdeCodeConnection; + + await expect(recoverTurn(connection, { + sessionId: "chat-1", + turnId: "turn-1", + action: "nudge", + }, { allowLegacyCodexFallback: true })).resolves.toEqual({ action: "nudge", turnId: "turn-1", status: "nudged" }); + expect(calls).toEqual([{ + domain: "chat", + action: "recoverTurn", + args: { sessionId: "chat-1", turnId: "turn-1", action: "nudge" }, + }]); + }); + + it.each([ + "Unsupported chat method: recoverTurn", + "Action not supported", + "Unknown action", + "chat.recoverTurn is not available", + ])("falls back to legacy Codex recovery for compatibility errors: %s", async (message) => { + const calls: Array<{ domain: string; action: string; args: Record }> = []; + const connection = { + action: async (domain: string, action: string, args: Record) => { + calls.push({ domain, action, args }); + if (action === "recoverTurn") throw new Error(message); + return { action: "restart_resume_thread", turnId: "turn-1", status: "resumed" }; + }, + } as unknown as AdeCodeConnection; + + await expect(recoverTurn(connection, { + sessionId: "chat-1", + turnId: "turn-1", + action: "restart_resume", + }, { allowLegacyCodexFallback: true })).resolves.toEqual({ action: "restart_resume", turnId: "turn-1", status: "resumed" }); + expect(calls.map((call) => call.action)).toEqual(["recoverTurn", "recoverCodexTurn"]); + expect(calls[1]?.args).toEqual({ + sessionId: "chat-1", + turnId: "turn-1", + action: "restart_resume_thread", + }); + }); + + it("does not hide operational recovery failures behind the legacy fallback", async () => { + const connection = { + action: async () => { + throw new Error("Relay disconnected"); + }, + } as unknown as AdeCodeConnection; + + await expect(recoverTurn(connection, { + sessionId: "chat-1", + turnId: "turn-1", + action: "wait", + }, { allowLegacyCodexFallback: true })).rejects.toThrow("Relay disconnected"); + }); + + it("never applies the Codex fallback to another provider", async () => { + const calls: string[] = []; + const connection = { + action: async (_domain: string, action: string) => { + calls.push(action); + throw new Error("Unknown action"); + }, + } as unknown as AdeCodeConnection; + + await expect(recoverTurn(connection, { + sessionId: "chat-1", + turnId: "turn-1", + action: "retry_same_runtime", + })).rejects.toThrow("Unknown action"); + expect(calls).toEqual(["recoverTurn"]); + }); + + it("does not treat an unknown host identity as an unsupported action", async () => { + const calls: string[] = []; + const connection = { + action: async (_domain: string, action: string) => { + calls.push(action); + throw new Error("Unknown host identity"); + }, + } as unknown as AdeCodeConnection; + + await expect(recoverTurn(connection, { + sessionId: "chat-1", + turnId: "turn-1", + action: "wait", + }, { allowLegacyCodexFallback: true })).rejects.toThrow("Unknown host identity"); + expect(calls).toEqual(["recoverTurn"]); + }); + + it("routes durable unprocessed-message resolution through chat actions", async () => { + const calls: Array<{ domain: string; action: string; args: Record }> = []; + const connection = { + action: async (domain: string, action: string, args: Record) => { + calls.push({ domain, action, args }); + return { steerId: "steer-1", action: "run_next", status: "completed", replacementMessageId: "message-2" }; + }, + } as unknown as AdeCodeConnection; + + await expect(resolveUnprocessedMessage(connection, { + sessionId: "chat-1", + steerId: "steer-1", + action: "run_next", + })).resolves.toEqual({ + steerId: "steer-1", + action: "run_next", + status: "completed", + replacementMessageId: "message-2", + }); + expect(calls).toEqual([{ + domain: "chat", + action: "resolveUnprocessedMessage", + args: { sessionId: "chat-1", steerId: "steer-1", action: "run_next" }, + }]); + }); }); diff --git a/apps/ade-cli/src/tuiClient/__tests__/aggregate.test.ts b/apps/ade-cli/src/tuiClient/__tests__/aggregate.test.ts index 0be6a2084..d9c04db10 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/aggregate.test.ts +++ b/apps/ade-cli/src/tuiClient/__tests__/aggregate.test.ts @@ -806,4 +806,31 @@ describe("aggregateChatBlocks claude history accuracy", () => { expect(texts).toHaveLength(1); expect(texts[0]!.line.body).toBe("Both explorations are complete — what docs exist."); }); + + it("folds a durable resolution into its original unprocessed user block", () => { + const events: AgentChatEventEnvelope[] = [ + env("2026-01-01T12:00:00.000Z", { + type: "user_message", + text: "Continue", + steerId: "steer-1", + deliveryState: "unprocessed", + }), + env("2026-01-01T12:00:01.000Z", { + type: "user_message_resolution", + steerId: "steer-1", + action: "dismiss", + state: "completed", + resolvedAt: "2026-01-01T12:00:01.000Z", + }), + ]; + + const blocks = aggregate(events); + expect(blocks).toEqual([expect.objectContaining({ + kind: "user-bubble", + line: expect.objectContaining({ + header: "not processed · dismissed", + body: "Continue", + }), + })]); + }); }); diff --git a/apps/ade-cli/src/tuiClient/__tests__/appInput.test.ts b/apps/ade-cli/src/tuiClient/__tests__/appInput.test.ts index 90aae3057..2a2581141 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/appInput.test.ts +++ b/apps/ade-cli/src/tuiClient/__tests__/appInput.test.ts @@ -60,8 +60,10 @@ import { resolveContextDefault, resolveDrawerPaneWidth, resolvePromptChatSubmitTarget, - resolveTuiCodexRecoveryRequest, - resolveTuiCodexRecoveryTargetProvider, + resolveTuiRecoveryRequest, + resolveTuiRecoveryTargetProvider, + resolveTuiUnprocessedMessageRequest, + resolveTuiUnprocessedMessageDraft, shouldHandlePendingQuestionKey, resolveModelPickerEscape, nextModelPickerProviderTabKey, @@ -1271,47 +1273,134 @@ describe("interface draft setup", () => { }, ]; - expect(resolveTuiCodexRecoveryRequest({ input: "nudge", sessionId: "chat-1", events })).toEqual({ - action: "steer", + expect(resolveTuiRecoveryRequest({ input: "nudge", sessionId: "chat-1", events })).toEqual({ + action: "nudge", turnId: "turn-1", sessionId: "chat-1", + provider: "codex", }); - expect(resolveTuiCodexRecoveryRequest({ input: "resume explicit-turn", sessionId: "chat-1", events })).toEqual({ - action: "restart_resume_thread", + expect(resolveTuiRecoveryRequest({ input: "resume explicit-turn", sessionId: "chat-1", events })).toEqual({ + action: "restart_resume", turnId: "explicit-turn", sessionId: "chat-1", + provider: null, }); - expect(resolveTuiCodexRecoveryRequest({ + expect(resolveTuiRecoveryRequest({ input: "retry", sessionId: "chat-1", events: events.slice(0, 1), })).toEqual({ - action: "interrupt_retry_same_thread", + action: "retry_same_runtime", turnId: "child-turn", sessionId: "child-chat", + provider: "codex", }); - expect(resolveTuiCodexRecoveryRequest({ input: "unknown", sessionId: "chat-1", events })).toBeNull(); + expect(resolveTuiRecoveryRequest({ input: "unknown", sessionId: "chat-1", events })).toBeNull(); + }); + + it("prefers provider-neutral turn health when resolving recovery", () => { + const events: AgentChatEventEnvelope[] = [{ + sessionId: "chat-1", + timestamp: "2026-01-01T00:00:00.000Z", + sequence: 1, + event: { + type: "turn_health", + provider: "codex", + turnId: "turn-1", + state: "stalled", + reason: "no_output", + message: "No output", + turnStartedAt: "2026-01-01T00:00:00.000Z", + lastProgressAt: "2026-01-01T00:00:00.000Z", + detectedAt: "2026-01-01T00:02:00.000Z", + recoveryCount: 0, + supportedActions: ["wait", "nudge", "retry_same_runtime", "restart_resume"], + automaticRecoveryAttempted: false, + sourceSessionId: "chat-child", + }, + }]; + + expect(resolveTuiRecoveryRequest({ input: "retry", sessionId: "chat-1", events })).toEqual({ + action: "retry_same_runtime", + turnId: "turn-1", + sessionId: "chat-child", + provider: "codex", + }); + }); + + it("resolves unprocessed-message commands against the active or explicit session", () => { + expect(resolveTuiUnprocessedMessageRequest({ + input: "steer-1", + sessionId: "chat-1", + })).toEqual({ steerId: "steer-1", sessionId: "chat-1" }); + expect(resolveTuiUnprocessedMessageRequest({ + input: "steer-1 chat-2", + sessionId: "chat-1", + })).toEqual({ steerId: "steer-1", sessionId: "chat-2" }); + expect(resolveTuiUnprocessedMessageRequest({ + input: "steer-1 chat-2 extra", + sessionId: "chat-1", + })).toBeNull(); + expect(resolveTuiUnprocessedMessageRequest({ input: "", sessionId: "chat-1" })).toBeNull(); + expect(resolveTuiUnprocessedMessageRequest({ input: "steer-1", sessionId: null })).toBeNull(); + }); + + it("restores only unresolved unprocessed messages to the composer", () => { + const message = { + sessionId: "chat-1", + timestamp: "2026-01-01T00:00:00.000Z", + sequence: 1, + event: { + type: "user_message" as const, + text: "original text", + displayText: "editable text", + steerId: "steer-1", + deliveryState: "unprocessed" as const, + }, + }; + expect(resolveTuiUnprocessedMessageDraft({ + steerId: "steer-1", + events: [message], + })).toBe("editable text"); + expect(resolveTuiUnprocessedMessageDraft({ + steerId: "steer-1", + events: [ + message, + { + sessionId: "chat-1", + timestamp: "2026-01-01T00:00:01.000Z", + sequence: 2, + event: { + type: "user_message_resolution" as const, + steerId: "steer-1", + action: "dismiss" as const, + state: "completed" as const, + resolvedAt: "2026-01-01T00:00:01.000Z", + }, + }, + ], + })).toBeNull(); }); - it("gates recovery on the resolved Codex child instead of its visible Claude parent", () => { + it("resolves the recovery provider from a child or visible chat", () => { const sessions = [ { sessionId: "parent-chat", provider: "claude" as const }, { sessionId: "child-chat", provider: "codex" as const }, ]; - expect(resolveTuiCodexRecoveryTargetProvider({ + expect(resolveTuiRecoveryTargetProvider({ targetSessionId: "child-chat", visibleSessionId: "parent-chat", visibleProvider: "claude", sessions, })).toBe("codex"); - expect(resolveTuiCodexRecoveryTargetProvider({ + expect(resolveTuiRecoveryTargetProvider({ targetSessionId: "parent-chat", visibleSessionId: "parent-chat", visibleProvider: "claude", sessions, })).toBe("claude"); - expect(resolveTuiCodexRecoveryTargetProvider({ + expect(resolveTuiRecoveryTargetProvider({ targetSessionId: "child-not-yet-listed", visibleSessionId: "parent-chat", visibleProvider: "claude", diff --git a/apps/ade-cli/src/tuiClient/__tests__/commands.test.ts b/apps/ade-cli/src/tuiClient/__tests__/commands.test.ts index a4748b188..b9ebffe37 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/commands.test.ts +++ b/apps/ade-cli/src/tuiClient/__tests__/commands.test.ts @@ -110,20 +110,38 @@ describe("commands", () => { })); }); - it("routes Codex stalled-turn recovery from Codex or orchestration-parent chats", () => { + it("routes provider-neutral stalled-turn recovery", () => { const parsed = parseCommand("/recover nudge turn-1"); expect(parsed?.name).toBe("/recover"); expect(parsed?.args).toBe("nudge turn-1"); expect(parsed ? commandPlacement(parsed) : null).toBe("right"); expect(paletteCommands("/rec", [], { provider: "codex" })).toContainEqual(expect.objectContaining({ name: "/recover", - description: "Recover the latest stalled Codex turn", + description: "Recover the latest stalled turn", })); expect(paletteCommands("/rec", [], { provider: "claude" })).toContainEqual(expect.objectContaining({ name: "/recover", })); }); + it("routes durable unprocessed-message actions through the ADE right pane", () => { + expect(parseCommand("/run-next steer-1 chat-2")).toMatchObject({ + name: "/run-next", + args: "steer-1 chat-2", + spec: { placement: "right" }, + }); + expect(parseCommand("/edit-message steer-1")).toMatchObject({ + name: "/edit-message", + args: "steer-1", + spec: { placement: "right" }, + }); + expect(parseCommand("/dismiss-message steer-1")).toMatchObject({ + name: "/dismiss-message", + args: "steer-1", + spec: { placement: "right" }, + }); + }); + it("routes /feedback to the ADE Code right pane", () => { const parsed = parseCommand("/feedback"); expect(parsed?.spec?.name).toBe("/feedback"); diff --git a/apps/ade-cli/src/tuiClient/__tests__/format.test.ts b/apps/ade-cli/src/tuiClient/__tests__/format.test.ts index 08c3a3056..10ff7b936 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/format.test.ts +++ b/apps/ade-cli/src/tuiClient/__tests__/format.test.ts @@ -522,12 +522,162 @@ describe("renderChatLines", () => { expect(body).toContain("Codex app server docs — example.com"); expect(body).toContain("Deep dive — docs.example.org +2 more"); expect(body).toContain("image generated"); - expect(body).toContain("/recover wait · nudge · retry · resume"); + expect(body).toContain("/recover resume (restart runtime + resume)"); + expect(body).toContain("/recover retry (keep current runtime)"); // Goal/token-usage events are suppressed in the chat transcript. expect(body).not.toContain("goal active"); expect(body).not.toContain("tokens · last"); }); + it("folds durable resolution into the original unprocessed user message", () => { + const baseEvents = [{ + sessionId: "s1", + timestamp: "2026-01-01T12:00:00.000Z", + sequence: 1, + event: { + type: "user_message" as const, + text: "Please continue", + steerId: "steer-1", + deliveryState: "unprocessed" as const, + }, + }]; + + const unresolved = renderChatLines({ + activeSession: null, + notices: [], + events: baseEvents, + }); + expect(unresolved).toEqual([expect.objectContaining({ + header: "not processed · /run-next steer-1 · /edit-message steer-1 · /dismiss-message steer-1", + body: "Please continue", + })]); + + const resolved = renderChatLines({ + activeSession: null, + notices: [], + events: [ + ...baseEvents, + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:01.000Z", + sequence: 2, + event: { + type: "user_message_resolution" as const, + steerId: "steer-1", + action: "run_next" as const, + state: "completed" as const, + resolvedAt: "2026-01-01T12:00:01.000Z", + }, + }, + ], + }); + expect(resolved).toEqual([expect.objectContaining({ + header: "not processed · started as the next turn", + body: "Please continue", + })]); + }); + + it("prefers provider-neutral health and recovery events over legacy duplicates", () => { + const healthLines = renderChatLines({ + activeSession: null, + notices: [], + events: [ + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:00.000Z", + sequence: 1, + event: { + type: "turn_health", + provider: "codex", + turnId: "turn-1", + state: "stalled", + reason: "no_output", + message: "No output yet", + turnStartedAt: "2026-01-01T11:58:00.000Z", + lastProgressAt: "2026-01-01T11:58:00.000Z", + detectedAt: "2026-01-01T12:00:00.000Z", + recoveryCount: 0, + supportedActions: ["wait", "nudge", "retry_same_runtime", "restart_resume"], + automaticRecoveryAttempted: false, + }, + }, + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:00.001Z", + sequence: 2, + event: { + type: "turn_health", + provider: "codex", + turnId: "turn-1", + state: "stalled", + reason: "no_progress", + message: "Still no output", + turnStartedAt: "2026-01-01T11:58:00.000Z", + lastProgressAt: "2026-01-01T11:58:00.000Z", + detectedAt: "2026-01-01T12:00:00.001Z", + recoveryCount: 0, + supportedActions: ["wait", "nudge", "retry_same_runtime", "restart_resume"], + automaticRecoveryAttempted: false, + }, + }, + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:00.002Z", + sequence: 3, + event: { + type: "codex_turn_stalled", + turnId: "turn-1", + reason: "no_output", + message: "No output yet", + }, + }, + ], + }); + expect(healthLines.filter((line) => line.body.startsWith("recovery ·"))).toHaveLength(1); + expect(healthLines[0]?.body).toContain("Still no output"); + expect(healthLines[0]?.body).toContain("keep current runtime"); + + const recoveryLines = renderChatLines({ + activeSession: null, + notices: [], + events: [ + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:01.000Z", + sequence: 3, + event: { + type: "turn_recovery", + provider: "codex", + turnId: "turn-1", + action: "restart_resume", + state: "recovered", + message: "Thread resumed", + automatic: true, + at: "2026-01-01T12:00:01.000Z", + recoveryCount: 1, + }, + }, + { + sessionId: "s1", + timestamp: "2026-01-01T12:00:01.001Z", + sequence: 4, + event: { + type: "codex_turn_recovery", + turnId: "turn-1", + action: "restart_resume_thread", + state: "recovered", + message: "Thread resumed", + automatic: true, + at: "2026-01-01T12:00:01.000Z", + }, + }, + ], + }); + expect(recoveryLines).toEqual([expect.objectContaining({ + body: "recovered automatically · Thread resumed", + })]); + }); + it("renders the new event variants (status, error, done, todo, subagent, completion_report, turn_diff_summary, codex_context_compaction)", () => { const lines = renderChatLines({ activeSession: null, diff --git a/apps/ade-cli/src/tuiClient/__tests__/remoteLauncher.test.ts b/apps/ade-cli/src/tuiClient/__tests__/remoteLauncher.test.ts index 874020163..015832c44 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/remoteLauncher.test.ts +++ b/apps/ade-cli/src/tuiClient/__tests__/remoteLauncher.test.ts @@ -2,7 +2,9 @@ import { EventEmitter } from "node:events"; import { PassThrough } from "node:stream"; import { describe, expect, it, vi } from "vitest"; import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import { PairedRuntimeRelayAuthRequiredError } from "../../../../desktop/src/main/services/remoteRuntime/pairedRuntimeErrors"; import type { RemoteRuntimeTarget } from "../../../../desktop/src/shared/types/remoteRuntime"; +import { relayAuthErrorWithDiagnostic } from "../pairedRemoteConnector"; import { assertRelayAccountUnchanged, buildSshArgs, @@ -237,6 +239,38 @@ describe("ade code remote launcher", () => { )).rejects.toThrow(/account changed.*same account/i); }); + it("preserves Relay auth semantics and bounded route diagnostics", () => { + const cause = new Error("account proof expired"); + const authError = new PairedRuntimeRelayAuthRequiredError( + "Sign in to ADE to connect through Relay.", + cause, + ); + const attempts = [{ + kind: "relay" as const, + host: "relay.example", + startedAt: 100, + durationMs: 25, + outcome: "failed" as const, + failure: "authentication" as const, + }]; + + const diagnosed = relayAuthErrorWithDiagnostic(authError, { + correlationId: "connection-reference", + attempts, + omittedAttemptCount: 3, + }); + + expect(diagnosed).toBeInstanceOf(PairedRuntimeRelayAuthRequiredError); + expect(diagnosed.code).toBe("PAIRED_RUNTIME_RELAY_AUTH_REQUIRED"); + expect(diagnosed.message).toBe(authError.message); + expect(diagnosed.cause).toBe(cause); + expect(diagnosed.diagnostic).toEqual({ + correlationId: "connection-reference", + attempts, + omittedAttemptCount: 3, + }); + }); + it("never invents SSH fallback from a paired LAN or Tailscale route", () => { const paired = { ...legacyAccountTarget(), diff --git a/apps/ade-cli/src/tuiClient/adeApi.ts b/apps/ade-cli/src/tuiClient/adeApi.ts index d61522cba..a9bf8cc54 100644 --- a/apps/ade-cli/src/tuiClient/adeApi.ts +++ b/apps/ade-cli/src/tuiClient/adeApi.ts @@ -14,6 +14,10 @@ import type { AgentChatCodexConfigSource, AgentChatRecoverCodexTurnArgs, AgentChatRecoverCodexTurnResult, + AgentChatRecoverTurnArgs, + AgentChatRecoverTurnResult, + AgentChatResolveUnprocessedMessageArgs, + AgentChatResolveUnprocessedMessageResult, AgentChatCodexSandbox, AgentChatContextUsage, AgentChatCursorConfigValue, @@ -69,6 +73,10 @@ import type { } from "../../../desktop/src/shared/types"; import { discoverAllProjectSlashCommands } from "../../../desktop/src/main/services/chat/projectSlashCommandDiscovery"; import type { AdeCodeConnection, AdeCodeInterfaceMode, AdeCodeProvider, ChatHistorySnapshot, CreatedChat, NavigateRequest, NavigateResult } from "./types"; +import { + isUnsupportedRecoveryActionError, + LEGACY_RECOVERY_ACTION_BY_NEUTRAL, +} from "../chatRecovery"; export const DEFAULT_CODEX_REASONING_EFFORT = "low"; export { buildPtyContinuationLaunchFields }; @@ -790,6 +798,49 @@ export async function recoverCodexTurn( ); } +/** + * Prefer the provider-neutral recovery action, while retaining compatibility + * with brains that predate chat.recoverTurn. + */ +export async function recoverTurn( + connection: AdeCodeConnection, + args: AgentChatRecoverTurnArgs, + options: { allowLegacyCodexFallback?: boolean } = {}, +): Promise { + try { + return await connection.action("chat", "recoverTurn", args); + } catch (error) { + if ( + options.allowLegacyCodexFallback !== true + || !isUnsupportedRecoveryActionError(error) + ) { + throw error; + } + const legacyAction = LEGACY_RECOVERY_ACTION_BY_NEUTRAL[args.action]; + const result = await recoverCodexTurn(connection, { + sessionId: args.sessionId, + turnId: args.turnId, + action: legacyAction, + }); + return { + action: args.action, + turnId: result.turnId, + status: result.status, + }; + } +} + +export async function resolveUnprocessedMessage( + connection: AdeCodeConnection, + args: AgentChatResolveUnprocessedMessageArgs, +): Promise { + return await connection.action( + "chat", + "resolveUnprocessedMessage", + args, + ); +} + /** * Pull a subagent's real child transcript from the daemon. Only meaningful for * runtimes with `canViewFullTranscript` (Codex app-server threads, OpenCode diff --git a/apps/ade-cli/src/tuiClient/aggregate.ts b/apps/ade-cli/src/tuiClient/aggregate.ts index 1b7d039ed..357c9835c 100644 --- a/apps/ade-cli/src/tuiClient/aggregate.ts +++ b/apps/ade-cli/src/tuiClient/aggregate.ts @@ -442,6 +442,7 @@ const SILENCED_EVENT_TYPES = new Set([ "codex_token_usage", "codex_goal_updated", "codex_goal_cleared", + "codex_moderation_metadata", "pending_input_resolved", // Droid AGI mission lifecycle drives the Missions section in the chat-info // pane (see chatMission), not the transcript — keep it out of the timeline. @@ -832,6 +833,14 @@ export function aggregateChatBlocks(args: { if (a.kind !== b.kind) return a.kind === "event" ? -1 : 1; return a.index - b.index; }); + const latestUserMessageIndexBySteer = new Map(); + for (const entry of timeline) { + if (entry.kind !== "event") continue; + const event = entry.envelope.event; + if (event.type === "user_message" && event.steerId) { + latestUserMessageIndexBySteer.set(event.steerId, entry.index); + } + } const passthrough = (id: string, kind: "user-bubble" | "assistant-text" | "approval" | "error" | "notice"): void => { const line = linesById.get(id); @@ -873,6 +882,9 @@ export function aggregateChatBlocks(args: { } if (event.type === "user_message") { + if (event.steerId && latestUserMessageIndexBySteer.get(event.steerId) !== index) { + continue; + } if (event.steerId && event.deliveryState === "queued") { if (pendingSteerIds.has(event.steerId)) { blocks.push({ @@ -888,6 +900,9 @@ export function aggregateChatBlocks(args: { passthrough(id, "user-bubble"); continue; } + if (event.type === "user_message_resolution") { + continue; + } if (event.type === "text") { if (event.text.length === 0) continue; const line = linesById.get(id); @@ -1112,6 +1127,22 @@ export function aggregateChatBlocks(args: { }); continue; } + if ( + event.type === "turn_diagnostics" + || event.type === "turn_recovery" + || event.type === "turn_health" + || event.type === "codex_turn_recovery" + || event.type === "codex_turn_stalled" + ) { + const line = linesById.get(id); + if (!line) continue; + blocks.push({ + kind: line.tone === "error" ? "error" : "notice", + id, + line, + }); + continue; + } if (isSteerLifecycleNotice(event)) { continue; } diff --git a/apps/ade-cli/src/tuiClient/app.tsx b/apps/ade-cli/src/tuiClient/app.tsx index 479f59d9f..eb2a030a0 100644 --- a/apps/ade-cli/src/tuiClient/app.tsx +++ b/apps/ade-cli/src/tuiClient/app.tsx @@ -23,7 +23,7 @@ import { import { findSmartLinks } from "../../../desktop/src/shared/smartLinks"; import type { AgentChatClaudePlugin, - AgentChatCodexRecoveryAction, + AgentChatTurnRecoveryAction, AgentChatReloadClaudePluginsResult, AgentChatEventEnvelope, AgentChatFileRef, @@ -94,7 +94,8 @@ import { normalizeChatTerminalSession, previewTerminal, renameChat, - recoverCodexTurn, + recoverTurn, + resolveUnprocessedMessage, requestSessionAttention, resumeTerminalSession, resizeTerminal, @@ -1718,47 +1719,118 @@ function splitFirstArg(input: string): { first: string; rest: string } { }; } -const CODEX_RECOVERY_ACTION_ALIASES: Readonly> = { +const TURN_RECOVERY_ACTION_ALIASES: Readonly> = { wait: "wait", - nudge: "steer", - steer: "steer", - retry: "interrupt_retry_same_thread", - interrupt_retry_same_thread: "interrupt_retry_same_thread", - resume: "restart_resume_thread", - restart_resume_thread: "restart_resume_thread", + nudge: "nudge", + steer: "nudge", + retry: "retry_same_runtime", + retry_same_runtime: "retry_same_runtime", + interrupt_retry_same_thread: "retry_same_runtime", + resume: "restart_resume", + restart_resume: "restart_resume", + restart_resume_thread: "restart_resume", }; -export function resolveTuiCodexRecoveryRequest(args: { +export function resolveTuiRecoveryRequest(args: { input: string; sessionId: string; events: readonly AgentChatEventEnvelope[]; -}): { action: AgentChatCodexRecoveryAction; turnId: string; sessionId: string } | null { +}): { + action: AgentChatTurnRecoveryAction; + turnId: string; + sessionId: string; + provider: string | null; +} | null { const parsed = splitFirstArg(args.input); - const action = CODEX_RECOVERY_ACTION_ALIASES[parsed.first.trim().toLowerCase().replace(/-/g, "_")]; + const action = TURN_RECOVERY_ACTION_ALIASES[parsed.first.trim().toLowerCase().replace(/-/g, "_")]; if (!action) return null; const explicitTurnId = splitFirstArg(parsed.rest).first; if (explicitTurnId) { const matchingEnvelope = [...args.events].reverse().find((envelope) => - envelope.event.type === "codex_turn_stalled" && envelope.event.turnId === explicitTurnId + (envelope.event.type === "turn_health" || envelope.event.type === "codex_turn_stalled") + && envelope.event.turnId === explicitTurnId ); - const targetSessionId = matchingEnvelope?.event.type === "codex_turn_stalled" - ? matchingEnvelope.event.sourceSessionId?.trim() || matchingEnvelope.sessionId - : args.sessionId; - return { action, turnId: explicitTurnId, sessionId: targetSessionId }; + const sourceSessionId = matchingEnvelope + && ( + matchingEnvelope.event.type === "turn_health" + || matchingEnvelope.event.type === "codex_turn_stalled" + ) + ? matchingEnvelope.event.sourceSessionId?.trim() + : ""; + const targetSessionId = sourceSessionId + || matchingEnvelope?.sessionId + || args.sessionId; + const provider = matchingEnvelope?.event.type === "turn_health" + ? matchingEnvelope.event.provider + : matchingEnvelope?.event.type === "codex_turn_stalled" + ? "codex" + : null; + return { + action, + turnId: explicitTurnId, + sessionId: targetSessionId, + provider, + }; } for (let index = args.events.length - 1; index >= 0; index -= 1) { const envelope = args.events[index]; - if (envelope?.event.type !== "codex_turn_stalled") continue; + if ( + envelope?.event.type !== "turn_health" + && envelope?.event.type !== "codex_turn_stalled" + ) continue; return { action, turnId: envelope.event.turnId, - sessionId: envelope.event.sourceSessionId?.trim() || envelope.sessionId, + sessionId: envelope.event.type === "codex_turn_stalled" + ? envelope.event.sourceSessionId?.trim() || envelope.sessionId + : envelope.event.sourceSessionId?.trim() || envelope.sessionId, + provider: envelope.event.type === "turn_health" + ? envelope.event.provider + : "codex", }; } return null; } -export function resolveTuiCodexRecoveryTargetProvider(args: { +export function resolveTuiUnprocessedMessageRequest(args: { + input: string; + sessionId: string | null; +}): { steerId: string; sessionId: string } | null { + const steer = splitFirstArg(args.input); + if (!steer.first) return null; + const session = splitFirstArg(steer.rest); + if (session.rest) return null; + const targetSessionId = session.first || args.sessionId?.trim() || ""; + if (!targetSessionId) return null; + return { steerId: steer.first, sessionId: targetSessionId }; +} + +export function resolveTuiUnprocessedMessageDraft(args: { + steerId: string; + events: readonly AgentChatEventEnvelope[]; +}): string | null { + for (let index = args.events.length - 1; index >= 0; index -= 1) { + const event = args.events[index]?.event; + if ( + event?.type === "user_message_resolution" + && event.steerId === args.steerId + ) { + return null; + } + if ( + event?.type !== "user_message" + || event.steerId !== args.steerId + || event.deliveryState !== "unprocessed" + ) { + continue; + } + const text = event.displayText?.trim() || event.text.trim(); + return text || null; + } + return null; +} + +export function resolveTuiRecoveryTargetProvider(args: { targetSessionId: string; visibleSessionId: string; visibleProvider: AgentChatSessionSummary["provider"] | null | undefined; @@ -1767,8 +1839,7 @@ export function resolveTuiCodexRecoveryTargetProvider(args: { const targetSession = args.sessions.find((session) => session.sessionId === args.targetSessionId); if (targetSession) return targetSession.provider; // An orchestration child may not have reached the TUI's session inventory yet. - // In that case the forwarded stalled event remains authoritative and the - // service performs the final Codex-provider validation. + // The forwarded health event remains authoritative for that case. return args.targetSessionId === args.visibleSessionId ? args.visibleProvider ?? null : null; } @@ -9994,16 +10065,83 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, } return; } + if ( + name === "/run-next" + || name === "/edit-message" + || name === "/dismiss-message" + ) { + if (activeTerminalSessionRef.current) { + setRightPane({ + kind: "details", + title: "Unprocessed message", + body: "Message recovery is available for Work chats, not CLI terminals.", + }); + return; + } + const request = resolveTuiUnprocessedMessageRequest({ + input: args, + sessionId, + }); + if (!request) { + setRightPane({ + kind: "details", + title: "Unprocessed message", + body: `Usage: ${name} [session-id]`, + }); + return; + } + if (name === "/edit-message") { + const targetEvents = eventsBySessionIdRef.current[request.sessionId] + ?? (request.sessionId === sessionId ? eventsRef.current : []); + const restoredText = resolveTuiUnprocessedMessageDraft({ + steerId: request.steerId, + events: targetEvents, + }); + if (!restoredText) { + const message = "That unresolved message is not available in the loaded chat transcript."; + setRightPane({ kind: "details", title: "Unprocessed message", body: message }); + addNotice(message, "error"); + return; + } + chatDraftRef.current = restoredText; + focusChat(); + addNotice("Restored unprocessed message to the composer.", "success"); + return; + } + const action = name === "/run-next" ? "run_next" : "dismiss"; + try { + const result = await resolveUnprocessedMessage(conn, { ...request, action }); + const label = action === "run_next" + ? result.status === "already_completed" + ? "Message was already started as the next turn" + : "Started message as the next turn" + : result.status === "already_completed" + ? "Message was already dismissed" + : "Dismissed unprocessed message"; + setRightPane({ + kind: "details", + title: "Unprocessed message", + body: `${label}\nMessage ${result.steerId}`, + }); + addNotice(label, "success"); + await refreshState(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + setRightPane({ kind: "details", title: "Unprocessed message", body: message }); + addNotice(message, "error"); + } + return; + } if (name === "/recover") { if (!sessionId || activeTerminalSessionRef.current) { setRightPane({ kind: "details", - title: "Codex recovery", - body: "Recovery is available for an active Codex Work chat, not a CLI terminal.", + title: "Turn recovery", + body: "Recovery is available for an active Work chat, not a CLI terminal.", }); return; } - const request = resolveTuiCodexRecoveryRequest({ + const request = resolveTuiRecoveryRequest({ input: args, sessionId, events: eventsRef.current, @@ -10011,48 +10149,48 @@ export function AdeCodeApp({ project, forceEmbedded, requireSocket, socketPath, if (!request) { setRightPane({ kind: "details", - title: "Codex recovery", + title: "Turn recovery", body: [ "Usage: /recover [turn-id]", "", + "resume restarts the provider runtime and resumes the turn.", + "retry keeps the current provider runtime and retries the same turn.", "The turn id is optional when this chat has a recent stalled-turn notice.", ].join("\n"), }); return; } - const targetProvider = resolveTuiCodexRecoveryTargetProvider({ + const targetProvider = request.provider ?? resolveTuiRecoveryTargetProvider({ targetSessionId: request.sessionId, visibleSessionId: sessionId, visibleProvider: activeSession?.provider, sessions, }); - if (targetProvider && targetProvider !== "codex") { - setRightPane({ - kind: "details", - title: "Codex recovery", - body: "/recover target is not a Codex chat.", - }); - return; - } try { - const result = await recoverCodexTurn(conn, request); + const result = await recoverTurn(conn, { + sessionId: request.sessionId, + turnId: request.turnId, + action: request.action, + }, { + allowLegacyCodexFallback: targetProvider === "codex", + }); const label = request.action === "wait" - ? "Waiting" - : request.action === "steer" + ? "Keeping the current turn open" + : request.action === "nudge" ? "Status nudge sent" - : request.action === "interrupt_retry_same_thread" - ? "Retrying the same thread" - : "App server restarted; thread resumed"; + : request.action === "retry_same_runtime" + ? "Retrying on the same provider runtime" + : "Provider runtime restarted; turn resumed"; setRightPane({ kind: "details", - title: "Codex recovery", + title: "Turn recovery", body: `${label} · ${result.status}\nTurn ${result.turnId}`, }); addNotice(label, "success"); await refreshState(); } catch (error) { const message = error instanceof Error ? error.message : String(error); - setRightPane({ kind: "details", title: "Codex recovery", body: message }); + setRightPane({ kind: "details", title: "Turn recovery", body: message }); addNotice(message, "error"); } return; diff --git a/apps/ade-cli/src/tuiClient/commands.ts b/apps/ade-cli/src/tuiClient/commands.ts index 62c3f7075..4d0332f00 100644 --- a/apps/ade-cli/src/tuiClient/commands.ts +++ b/apps/ade-cli/src/tuiClient/commands.ts @@ -77,9 +77,12 @@ export const BUILTIN_COMMANDS: BuiltinCommand[] = [ { name: "/insights", description: "Generate Claude session insights through the active SDK session", placement: "chat", providers: ["claude"], category: "Model" }, { name: "/fast", description: "Toggle Claude fast mode through the active SDK session", placement: "chat", argumentHint: "[on|off]", providers: ["claude"], category: "Model" }, { name: "/goal", description: "Set, clear, or inspect the active chat goal", placement: "chat", argumentHint: "[|clear|status active|paused|complete]", providers: ["claude", "codex"], category: "Model" }, - // A non-Codex orchestration parent can surface a stalled Codex child's event, - // so recovery availability is determined from that event's target session. - { name: "/recover", description: "Recover the latest stalled Codex turn", placement: "right", argumentHint: " [turn-id]", category: "Chats" }, + // An orchestration parent can surface a stalled child turn, so recovery + // availability is determined from that event's target session. + { name: "/recover", description: "Recover the latest stalled turn", placement: "right", argumentHint: " [turn-id]", category: "Chats" }, + { name: "/run-next", description: "Run an accepted but unprocessed message next", placement: "right", argumentHint: " [session-id]", category: "Chats" }, + { name: "/edit-message", description: "Restore an unprocessed message to the composer", placement: "right", argumentHint: " [session-id]", category: "Chats" }, + { name: "/dismiss-message", description: "Dismiss an accepted but unprocessed message", placement: "right", argumentHint: " [session-id]", category: "Chats" }, { name: "/diff", description: "Show active lane diff", placement: "right", category: "Lanes" }, { name: "/log", description: "Show recent commits", placement: "right", category: "Lanes" }, { name: "/reparent", description: "Move the active lane under another lane", placement: "right", argumentHint: " [stack-base-ref]", category: "Lanes" }, @@ -287,7 +290,7 @@ export function paletteCommands( } return a.name.localeCompare(b.name); }); - return filtered.slice(0, 80); + return filtered.slice(0, 100); } export function commandPlacement(command: ParsedCommand): CommandPlacement { diff --git a/apps/ade-cli/src/tuiClient/format.ts b/apps/ade-cli/src/tuiClient/format.ts index e682b69b7..d53839cb9 100644 --- a/apps/ade-cli/src/tuiClient/format.ts +++ b/apps/ade-cli/src/tuiClient/format.ts @@ -1,6 +1,6 @@ import path from "node:path"; import { Lexer, type Token, type Tokens } from "marked"; -import type { AgentChatEventEnvelope, AgentChatSessionSummary } from "../../../desktop/src/shared/types/chat"; +import type { AgentChatEvent, AgentChatEventEnvelope, AgentChatSessionSummary } from "../../../desktop/src/shared/types/chat"; import type { LaneSummary } from "../../../desktop/src/shared/types/lanes"; import { highlightCode, type HighlightedToken } from "./highlightCache"; import { glyphFor } from "./theme"; @@ -556,6 +556,37 @@ export function renderChatLines(args: { if (a.kind !== b.kind) return a.kind === "event" ? -1 : 1; return a.index - b.index; }); + const latestUserMessageIndexBySteer = new Map(); + const latestUserMessageResolutionBySteer = new Map< + string, + Extract + >(); + const latestDiagnosticsIndexByTurn = new Map(); + const latestLegacyRecoveryIndexByTurn = new Map(); + const latestNeutralRecoveryIndexByTurn = new Map(); + const latestRecoveryStateByTurn = new Map(); + const neutralHealthTurnIds = new Set(); + const latestHealthIndexByTurn = new Map(); + for (const entry of timeline) { + if (entry.kind !== "event") continue; + const event = entry.envelope.event; + if (event.type === "user_message" && event.steerId) { + latestUserMessageIndexBySteer.set(event.steerId, entry.index); + } else if (event.type === "user_message_resolution") { + latestUserMessageResolutionBySteer.set(event.steerId, event); + } else if (event.type === "turn_diagnostics") { + latestDiagnosticsIndexByTurn.set(event.turnId ?? "__session_startup__", entry.index); + } else if (event.type === "codex_turn_recovery") { + latestLegacyRecoveryIndexByTurn.set(event.turnId, entry.index); + latestRecoveryStateByTurn.set(event.turnId, event.state); + } else if (event.type === "turn_recovery") { + latestNeutralRecoveryIndexByTurn.set(event.turnId, entry.index); + latestRecoveryStateByTurn.set(event.turnId, event.state); + } else if (event.type === "turn_health") { + neutralHealthTurnIds.add(event.turnId); + latestHealthIndexByTurn.set(event.turnId, entry.index); + } + } const pushLine = (line: RenderedChatLine): void => { const last = lines[lines.length - 1]; @@ -586,13 +617,40 @@ export function renderChatLines(args: { const id = chatEventLineId(envelope, index); const expanded = args.expandedLineIds?.has(id) ?? false; if (event.type === "user_message") { + if (event.steerId && latestUserMessageIndexBySteer.get(event.steerId) !== index) { + continue; + } + const resolution = event.steerId + ? latestUserMessageResolutionBySteer.get(event.steerId) + : undefined; + const deliveryHeader = resolution?.action === "run_next" + ? "not processed · started as the next turn" + : resolution?.action === "dismiss" + ? "not processed · dismissed" + : event.deliveryState === "processed" || event.processed + ? "processed" + : event.deliveryState === "unprocessed" + ? event.steerId + ? `not processed · /run-next ${event.steerId} · /edit-message ${event.steerId} · /dismiss-message ${event.steerId}` + : "not processed · send again when ready" + : event.deliveryState === "accepted" || event.deliveryState === "delivered" + ? "accepted · waiting to be processed" + : event.deliveryState === "inline" + ? "accepted during active turn" + : event.deliveryState === "failed" + ? "send failed" + : undefined; lines.push({ id, tone: "user", + header: deliveryHeader, body: event.displayText ?? event.text, }); continue; } + if (event.type === "user_message_resolution") { + continue; + } if (event.type === "transcript_retraction") { const retractedIds = new Set(event.messageIds.map((messageId) => messageId.trim()).filter(Boolean)); if (!retractedIds.size) continue; @@ -712,7 +770,58 @@ export function renderChatLines(args: { continue; } if (event.type === "codex_moderation_metadata") { - lines.push({ id, tone: "notice", body: "moderation checked" }); + continue; + } + if (event.type === "turn_diagnostics") { + if (latestDiagnosticsIndexByTurn.get(event.turnId ?? "__session_startup__") !== index) { + continue; + } + const checks = Math.max(0, event.moderationChecks ?? 0); + const integrations = event.optionalIntegrationFailures ?? []; + const parts = [ + checks ? `${checks} safety ${checks === 1 ? "check" : "checks"}` : null, + integrations.length + ? `optional integrations unavailable: ${integrations.map((entry) => entry.integration).join(", ")}` + : null, + ].filter((part): part is string => Boolean(part)); + if (parts.length) { + lines.push({ id, tone: "notice", body: `turn details · ${parts.join(" · ")}` }); + } + continue; + } + if (event.type === "turn_recovery") { + if (latestNeutralRecoveryIndexByTurn.get(event.turnId) !== index) { + continue; + } + const label = event.state === "recovered" + ? "recovered" + : event.state === "failed" + ? "recovery failed" + : "recovering"; + lines.push({ + id, + tone: event.state === "failed" ? "error" : "notice", + body: `${label}${event.automatic ? " automatically" : ""} · ${singleLine(event.message, 140)}`, + }); + continue; + } + if (event.type === "codex_turn_recovery") { + if ( + latestNeutralRecoveryIndexByTurn.has(event.turnId) + || latestLegacyRecoveryIndexByTurn.get(event.turnId) !== index + ) { + continue; + } + const label = event.state === "recovered" + ? "recovered" + : event.state === "failed" + ? "recovery failed" + : "recovering"; + lines.push({ + id, + tone: event.state === "failed" ? "error" : "notice", + body: `${label}${event.automatic ? " automatically" : ""} · ${singleLine(event.message, 140)}`, + }); continue; } if (event.type === "codex_sleep") { @@ -732,11 +841,39 @@ export function renderChatLines(args: { lines.push({ id, tone: "error", body: "thread deleted upstream · next message starts fresh" }); continue; } + if (event.type === "turn_health") { + if (latestHealthIndexByTurn.get(event.turnId) !== index) { + continue; + } + if (latestRecoveryStateByTurn.get(event.turnId) === "recovered") { + continue; + } + lines.push({ + id, + tone: "error", + body: [ + `recovery · ${singleLine(event.message, 140)}`, + " primary: /recover resume (restart runtime + resume) · /recover wait", + " more: /recover nudge · /recover retry (keep current runtime)", + ].join("\n"), + }); + continue; + } if (event.type === "codex_turn_stalled") { + if (neutralHealthTurnIds.has(event.turnId)) { + continue; + } + if (latestRecoveryStateByTurn.get(event.turnId) === "recovered") { + continue; + } lines.push({ id, tone: "error", - body: `recovery · ${singleLine(event.message, 140)}\n /recover wait · nudge · retry · resume`, + body: [ + `recovery · ${singleLine(event.message, 140)}`, + " primary: /recover resume (restart runtime + resume) · /recover wait", + " more: /recover nudge · /recover retry (keep current runtime)", + ].join("\n"), }); continue; } diff --git a/apps/ade-cli/src/tuiClient/pairedRemoteConnector.ts b/apps/ade-cli/src/tuiClient/pairedRemoteConnector.ts index 23337443c..d1a6fe492 100644 --- a/apps/ade-cli/src/tuiClient/pairedRemoteConnector.ts +++ b/apps/ade-cli/src/tuiClient/pairedRemoteConnector.ts @@ -1,9 +1,16 @@ +import { randomUUID } from "node:crypto"; import { PairedRuntimeCompatibilityError, PairedRuntimeRelayAuthRequiredError, + type PairedRuntimeRouteDiagnostic, } from "../../../desktop/src/main/services/remoteRuntime/pairedRuntimeErrors"; import { buildPairedEndpointCandidates, + classifyPairedRuntimeFailure, + createRouteAttemptRecorder, + MAX_ROUTE_ATTEMPTS, + orderPairedCandidates, + pairedRuntimeRouteHost, type PairedRuntimeEndpointCandidate, } from "../../../desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes"; import { DesktopPairedMachineStore } from "../../../desktop/src/main/services/remoteRuntime/syncPairedMachineStore"; @@ -11,7 +18,10 @@ import { openSyncRuntimeTransport, type SyncRuntimeTransport, } from "../../../desktop/src/main/services/remoteRuntime/syncRuntimeTransport"; -import type { RemoteRuntimeTarget } from "../../../desktop/src/shared/types/remoteRuntime"; +import type { + RemoteRuntimeConnectionAttempt, + RemoteRuntimeTarget, +} from "../../../desktop/src/shared/types/remoteRuntime"; import type { DesktopPairedMachineCredentials } from "../../../desktop/src/shared/types/pairedRuntime"; import { withBoundedAttempt, @@ -109,16 +119,35 @@ export class PairedRemoteConnectionUnavailableError extends Error { readonly reason: PairedConnectionFailureReason, readonly failures: readonly string[], message: string, + readonly diagnostic?: { + correlationId: string; + attempts: RemoteRuntimeConnectionAttempt[]; + omittedAttemptCount?: number; + }, ) { super(message); this.name = "PairedRemoteConnectionUnavailableError"; } } +export function relayAuthErrorWithDiagnostic( + error: PairedRuntimeRelayAuthRequiredError, + diagnostic: PairedRuntimeRouteDiagnostic, +): PairedRuntimeRelayAuthRequiredError { + return new PairedRuntimeRelayAuthRequiredError( + error.message, + error.cause, + diagnostic, + ); +} + export type OpenedPairedCandidate = { value: T; candidate: PairedRuntimeEndpointCandidate; connectedAt: number; + correlationId: string; + attempts: RemoteRuntimeConnectionAttempt[]; + omittedAttemptCount: number; }; export async function openPairedCandidate(args: { @@ -156,9 +185,15 @@ export async function openPairedCandidate(args: { ); } + const correlationId = randomUUID(); const failures: string[] = []; + const attemptRecorder = createRouteAttemptRecorder(); + const { attempts, record: recordAttempt } = attemptRecorder; let relayAuthError: PairedRuntimeRelayAuthRequiredError | null = null; - for (const candidate of candidates) { + const orderedCandidates = orderPairedCandidates(candidates); + for (const candidate of orderedCandidates) { + const attemptStartedAt = Date.now(); + const safeHost = pairedRuntimeRouteHost(candidate.endpoint); let transport: SyncRuntimeTransport | null = null; try { const relayProof = await pairedRouteAccountProof({ @@ -175,6 +210,7 @@ export async function openPairedCandidate(args: { authTimeoutMs: attempt.timeoutMs, signal: attempt.signal, relayAccountToken: relayProof?.token ?? null, + ...(candidate.kind === "relay" ? { correlationId } : {}), }) ); await assertRelayAccountUnchanged( @@ -194,23 +230,71 @@ export async function openPairedCandidate(args: { `Warning: could not save paired endpoint metadata: ${errorMessage(error)}\n`, ); } - return { value, candidate, connectedAt }; + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, connectedAt - attemptStartedAt), + outcome: "connected", + }); + return { + value, + candidate, + connectedAt, + correlationId, + attempts, + omittedAttemptCount: attemptRecorder.omittedAttemptCount, + }; } catch (error) { try { transport?.close(); } catch {} if (error instanceof PairedRuntimeRelayAuthRequiredError) { relayAuthError = error; + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "failed", + failure: "authentication", + }); continue; } if (error instanceof PairedRuntimeCompatibilityError) throw error; - failures.push(`${pairedConnectionLabel(candidate)}: ${errorMessage(error)}`); + const failure = classifyPairedRuntimeFailure(error); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "failed", + failure, + }); + if (failures.length < MAX_ROUTE_ATTEMPTS) { + failures.push(`${pairedConnectionLabel(candidate)}: ${failure}`); + } } } - if (relayAuthError) throw relayAuthError; + if (relayAuthError) { + throw relayAuthErrorWithDiagnostic(relayAuthError, { + correlationId, + attempts, + ...(attemptRecorder.omittedAttemptCount > 0 + ? { omittedAttemptCount: attemptRecorder.omittedAttemptCount } + : {}), + }); + } throw new PairedRemoteConnectionUnavailableError( "all_paths_failed", failures, `Could not open a paired ADE runtime connection to ${args.target.name}. ${failures .slice(0, 4) .join(" | ")}`, + { + correlationId, + attempts, + ...(attemptRecorder.omittedAttemptCount > 0 + ? { omittedAttemptCount: attemptRecorder.omittedAttemptCount } + : {}), + }, ); } diff --git a/apps/ade-cli/src/tuiClient/remoteLauncher.ts b/apps/ade-cli/src/tuiClient/remoteLauncher.ts index a0157a831..f6cb1a2c3 100644 --- a/apps/ade-cli/src/tuiClient/remoteLauncher.ts +++ b/apps/ade-cli/src/tuiClient/remoteLauncher.ts @@ -744,7 +744,14 @@ async function openPairedRemoteSession( }); } catch (error) { if (error instanceof PairedRemoteConnectionUnavailableError) { - throw new PairedRuntimeTransportUnavailableError(error.message, error); + const reference = error.diagnostic?.correlationId + ? ` Connection reference: ${error.diagnostic.correlationId}.` + : ""; + throw new PairedRuntimeTransportUnavailableError( + `${error.message}${reference}`, + error, + error.diagnostic, + ); } throw error; } diff --git a/apps/desktop/src/main/services/adeActions/registry.test.ts b/apps/desktop/src/main/services/adeActions/registry.test.ts index 18400918c..97175aae1 100644 --- a/apps/desktop/src/main/services/adeActions/registry.test.ts +++ b/apps/desktop/src/main/services/adeActions/registry.test.ts @@ -627,7 +627,7 @@ describe("ADE_ACTION_ALLOWLIST shape", () => { expect(warmQuickOpenIndex).toHaveBeenCalledWith({ workspaceId: "lane-warm" }); }); - it("falls back to headless chat transcript reads when readTranscript is unavailable", async () => { + it("keeps headless chat transcript reads shape-compatible with socket-backed reads", async () => { const getChatTranscript = vi.fn(async () => ({ sessionId: "chat-1", entries: [ @@ -651,12 +651,9 @@ describe("ADE_ACTION_ALLOWLIST shape", () => { sessionId: " chat-1 ", limit: "25", since: "2026-06-29T00:00:00.000Z", - })).resolves.toMatchObject({ - sessionId: "chat-1", - entries: [ - { role: "assistant", text: "new", timestamp: "2026-06-29T00:00:00.000Z" }, - ], - }); + })).resolves.toEqual([ + { role: "assistant", text: "new", timestamp: "2026-06-29T00:00:00.000Z" }, + ]); expect(getChatTranscript).toHaveBeenCalledWith({ sessionId: "chat-1", limit: 25 }); }); diff --git a/apps/desktop/src/main/services/adeActions/registry.ts b/apps/desktop/src/main/services/adeActions/registry.ts index 210c1c1bc..a69383c5f 100644 --- a/apps/desktop/src/main/services/adeActions/registry.ts +++ b/apps/desktop/src/main/services/adeActions/registry.ts @@ -523,7 +523,9 @@ export const ADE_ACTION_ALLOWLIST: Partial { - if (!isRecord(entry) || typeof entry.timestamp !== "string") return true; - const timestampMs = Date.parse(entry.timestamp); - return !Number.isFinite(timestampMs) || timestampMs >= sinceMs; - }), - }; + if (!Number.isFinite(sinceMs)) return entries; + return entries.filter((entry) => { + if (!isRecord(entry) || typeof entry.timestamp !== "string") return true; + const timestampMs = Date.parse(entry.timestamp); + return !Number.isFinite(timestampMs) || timestampMs >= sinceMs; + }); } throw new Error("Chat transcript reads are not available in this runtime."); }, diff --git a/apps/desktop/src/main/services/chat/agentChatService.test.ts b/apps/desktop/src/main/services/chat/agentChatService.test.ts index 0b2146381..cde8335a9 100644 --- a/apps/desktop/src/main/services/chat/agentChatService.test.ts +++ b/apps/desktop/src/main/services/chat/agentChatService.test.ts @@ -16295,7 +16295,7 @@ describe("createAgentChatService", () => { await service.sendMessage({ sessionId: session.id, text: "Inspect the repo", - }); + }, { awaitDispatch: true }); await service.dispose({ sessionId: session.id }); @@ -17420,16 +17420,11 @@ describe("createAgentChatService", () => { ); expect(events.filter((event) => event.event.type === "user_message")).toHaveLength(1); - mockState.emitCodexPayload({ - jsonrpc: "2.0", - method: "turn/started", - params: { - turn: { - id: "turn-1", - status: "in_progress", - }, - }, + await vi.waitFor(() => { + expect(mockState.codexRequestPayloads.some((payload) => payload.method === "turn/start")).toBe(true); }); + mockState.flushCodexResponses(); + await sendPromise; mockState.emitCodexPayload({ jsonrpc: "2.0", method: "item/agentMessage/delta", @@ -17439,9 +17434,6 @@ describe("createAgentChatService", () => { }, }); - mockState.flushCodexResponses(); - await sendPromise; - await waitForEvent( events, (event): event is AgentChatEventEnvelope => @@ -18153,7 +18145,8 @@ describe("createAgentChatService", () => { event: expect.objectContaining({ type: "user_message", text: "Focus on the shared chat UI.", - deliveryState: "delivered", + deliveryState: "accepted", + processed: false, steerId: result.steerId, turnId: "turn-1", }), @@ -18220,7 +18213,8 @@ describe("createAgentChatService", () => { event: expect.objectContaining({ type: "user_message", text: "Keep going with the real turn.", - deliveryState: "delivered", + deliveryState: "accepted", + processed: false, steerId: result.steerId, turnId: "turn-real", }), @@ -18424,7 +18418,8 @@ describe("createAgentChatService", () => { type: "user_message", text: "Use this screenshot while you keep going.", attachments: [{ path: imagePath, type: "image" }], - deliveryState: "delivered", + deliveryState: "accepted", + processed: false, steerId: result.steerId, turnId: "turn-1", }), @@ -24071,7 +24066,7 @@ describe("createAgentChatService", () => { && event.event.message === "Continuing to wait for Codex output.")).toBe(true); expect(mockState.codexRequestPayloads.some((payload) => payload.method === "turn/interrupt")).toBe(false); - await vi.advanceTimersByTimeAsync(120_000); + await vi.advanceTimersByTimeAsync(10 * 60_000); await vi.waitFor(() => { expect(events.some((event) => event.event.type === "codex_turn_stalled" && event.event.turnId === "turn-1")).toBe(true); @@ -24156,7 +24151,7 @@ describe("createAgentChatService", () => { expect(mockState.codexRequestPayloads.filter((payload) => payload.method === "turn/start")).toHaveLength(2); }); - it("surfaces Codex MCP startup failures without treating them as turn progress", async () => { + it("aggregates optional Codex MCP startup failures and auto-recovers a silent first attempt once", async () => { vi.useFakeTimers(); try { const events: AgentChatEventEnvelope[] = []; @@ -24192,163 +24187,124 @@ describe("createAgentChatService", () => { }); await Promise.resolve(); - const mcpNotices = events.filter((event) => + expect(events.some((event) => event.event.type === "system_notice" && event.event.message.includes("Codex MCP server 'local-tools' is unavailable") - ); - expect(mcpNotices).toHaveLength(1); + )).toBe(false); + expect(events.filter((event) => + event.event.type === "turn_diagnostics" + && event.event.optionalIntegrationFailures?.some((failure) => + failure.integration === "local-tools" + ) + )).toHaveLength(1); await vi.advanceTimersByTimeAsync(120_000); await vi.waitFor(() => { expect(events.some((event) => - event.event.type === "codex_turn_stalled" - && event.event.reason === "no_output" + event.event.type === "codex_turn_recovery" + && event.event.state === "recovered" + && event.event.automatic )).toBe(true); }); - expect(events.some((event) => - event.event.type === "system_notice" - && event.event.message.includes("has not streamed model or tool output yet") - )).toBe(true); + expect(mockState.codexRequestPayloads.some((payload) => payload.method === "thread/resume")).toBe(true); + expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); } finally { vi.useRealTimers(); } }); - it("clears the Codex no-output watchdog when an approval request is surfaced", async () => { + it("persists the Codex automatic-recovery guard across restart while keeping a new turn eligible", async () => { vi.useFakeTimers(); try { - const events: AgentChatEventEnvelope[] = []; - const { service } = createService({ - onEvent: (event: AgentChatEventEnvelope) => events.push(event), - }); - const session = await service.createSession({ + const first = createService(); + const session = await first.service.createSession({ laneId: "lane-1", provider: "codex", model: "gpt-5.5", }); - - await service.sendMessage({ + await first.service.sendMessage({ sessionId: session.id, - text: "Keep working.", + text: "Keep this turn running.", }, { awaitDispatch: true }); + await first.service.recoverCodexTurn({ + sessionId: session.id, + turnId: "turn-1", + action: "wait", + }); + + expect(readPersistedChatState(session.id).codexAutomaticRecoveryAttempted) + .toBe(true); + first.service.forceDisposeAll(); + const restartedEvents: AgentChatEventEnvelope[] = []; + const restarted = createService({ + onEvent: (event: AgentChatEventEnvelope) => restartedEvents.push(event), + }); + await restarted.service.resumeSession({ sessionId: session.id }); mockState.emitCodexPayload({ - id: "approval-1", - method: "item/commandExecution/requestApproval", + method: "turn/started", params: { - itemId: "cmd-1", - turnId: "turn-1", - command: "npm test", - cwd: ".", - reason: "Run tests", + threadId: "thread-1", + turn: { id: "turn-1", status: "inProgress" }, }, }); - - await vi.waitFor(() => { - expect(events.some((event) => - event.event.type === "approval_request" - && event.event.itemId === "cmd-1" - )).toBe(true); - }); + await Promise.resolve(); + const resumeRequestsBeforeWatchdog = mockState.codexRequestPayloads + .filter((payload) => payload.method === "thread/resume").length; await vi.advanceTimersByTimeAsync(120_000); + await waitForFakeTimerCondition( + () => restartedEvents.some((event) => + event.event.type === "codex_turn_stalled" + && event.event.turnId === "turn-1" + && event.event.automaticRecoveryAttempted === true), + "the resumed turn to remain stalled without another automatic recovery", + ); - expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); - expect(events.some((event) => - event.event.type === "system_notice" - && event.event.message.includes("has not streamed model or tool output yet") - )).toBe(false); - } finally { - vi.useRealTimers(); - } - }); + expect(restartedEvents.some((event) => + event.event.type === "codex_turn_recovery" + && event.event.turnId === "turn-1" + && event.event.automatic)).toBe(false); + expect(mockState.codexRequestPayloads + .filter((payload) => payload.method === "thread/resume")).toHaveLength( + resumeRequestsBeforeWatchdog, + ); - it("reconciles a completed silent Codex turn from app-server state before reporting a stall", async () => { - vi.useFakeTimers(); - try { - const events: AgentChatEventEnvelope[] = []; - mockState.codexResponseOverrides.set("thread/turns/list", () => ({ - data: [ - { - id: "turn-1", - status: "completed", - usage: { inputTokens: 7, outputTokens: 3 }, - items: [ - { - id: "msg-1", - type: "agentMessage", - text: "Recovered assistant output.", - }, - ], - }, - ], - nextCursor: null, - })); - const { service } = createService({ - onEvent: (event: AgentChatEventEnvelope) => events.push(event), - }); - const session = await service.createSession({ - laneId: "lane-1", - provider: "codex", - model: "gpt-5.5", + mockState.emitCodexPayload({ + method: "turn/completed", + params: { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }, }); - - await service.sendMessage({ + await Promise.resolve(); + await restarted.service.sendMessage({ sessionId: session.id, - text: "Keep working.", + text: "Start a genuinely new turn.", }, { awaitDispatch: true }); + expect(readPersistedChatState(session.id).codexAutomaticRecoveryAttempted) + .not.toBe(true); await vi.advanceTimersByTimeAsync(120_000); - await vi.waitFor(() => { - expect(events.some((event) => - event.event.type === "done" - && event.event.turnId === "turn-1" - && event.event.status === "completed" - )).toBe(true); - }); - - expect(mockState.codexRequestPayloads.some((payload) => payload.method === "thread/read")).toBe(true); - expect(mockState.codexRequestPayloads.some((payload) => payload.method === "thread/turns/list")).toBe(true); - expect(events.some((event) => - event.event.type === "text" - && event.event.text.includes("Recovered assistant output.") - )).toBe(true); - expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); + await waitForFakeTimerCondition( + () => restartedEvents.some((event) => + event.event.type === "codex_turn_recovery" + && event.event.turnId === "turn-2" + && event.event.state === "recovered" + && event.event.automatic), + "the new turn to complete its first automatic recovery", + ); + expect(readPersistedChatState(session.id).codexAutomaticRecoveryAttempted) + .toBe(true); } finally { vi.useRealTimers(); } }); - it("does not complete a reconciled MCP tool call while app-server still reports it running", async () => { + it("warns without killing a Codex turn after ten minutes of mid-turn inactivity", async () => { vi.useFakeTimers(); try { const events: AgentChatEventEnvelope[] = []; - mockState.codexResponseOverrides.set("thread/turns/list", () => ({ - data: [ - { - id: "turn-1", - status: "inProgress", - items: [ - { - id: "mcp-1", - type: "mcpToolCall", - server: "local-tools", - tool: "probe", - pluginId: "local-plugin", - appContext: { - connectorId: "local", - appName: "Local tools", - actionName: "Probe file", - resourceUri: "ui://local/probe", - }, - status: "running", - arguments: { path: "README.md" }, - }, - ], - }, - ], - nextCursor: null, - })); const { service } = createService({ onEvent: (event: AgentChatEventEnvelope) => events.push(event), }); @@ -24357,33 +24313,37 @@ describe("createAgentChatService", () => { provider: "codex", model: "gpt-5.5", }); + await service.sendMessage({ sessionId: session.id, text: "Run the task." }, { awaitDispatch: true }); - await service.sendMessage({ - sessionId: session.id, - text: "Keep working.", - }, { awaitDispatch: true }); + mockState.emitCodexPayload({ + method: "item/started", + params: { + turnId: "turn-1", + item: { + id: "collab-1", + type: "collabAgentToolCall", + tool: "spawn_agent", + prompt: "Inspect one bounded area.", + status: "inProgress", + }, + }, + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(10 * 60_000); - await vi.advanceTimersByTimeAsync(120_000); await vi.waitFor(() => { expect(events.some((event) => - event.event.type === "tool_call" - && event.event.itemId === "mcp-1" - && event.event.mcp?.pluginId === "local-plugin" - && event.event.mcp?.appContext?.appName === "Local tools" + event.event.type === "codex_turn_stalled" + && event.event.reason === "no_progress" )).toBe(true); }); - - expect(events.some((event) => - event.event.type === "tool_result" - && event.event.itemId === "mcp-1" - )).toBe(false); - expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); + expect(mockState.codexRequestPayloads.some((payload) => payload.method === "turn/interrupt")).toBe(false); } finally { vi.useRealTimers(); } }); - it("preserves the Codex imageGeneration lifecycle and local output path", async () => { + it("tracks accepted Codex follow-ups until the app-server proves they were processed", async () => { const events: AgentChatEventEnvelope[] = []; const { service } = createService({ onEvent: (event: AgentChatEventEnvelope) => events.push(event), @@ -24393,53 +24353,52 @@ describe("createAgentChatService", () => { provider: "codex", model: "gpt-5.5", }); - await service.sendMessage({ sessionId: session.id, text: "Generate a tiny moon icon." }, { awaitDispatch: true }); + await service.sendMessage({ sessionId: session.id, text: "Start." }, { awaitDispatch: true }); - const item = { - id: "image-1", - type: "imageGeneration", - status: "inProgress", - prompt: "A tiny moon icon", - }; - mockState.emitCodexPayload({ - jsonrpc: "2.0", - method: "item/started", - params: { turnId: "turn-1", item }, - }); - const started = await waitForEvent( - events, - (event): event is AgentChatEventEnvelope & { event: Extract } => - event.event.type === "codex_image_generation" && event.event.itemId === "image-1", - ); - expect(started.event).toMatchObject({ - prompt: "A tiny moon icon", - status: "running", - }); + const first = await service.steer({ sessionId: session.id, text: "First follow-up." }); + expect(events.some((event) => + event.event.type === "user_message" + && event.event.steerId === first.steerId + && event.event.deliveryState === "accepted" + && event.event.processed === false + )).toBe(true); mockState.emitCodexPayload({ - jsonrpc: "2.0", - method: "item/completed", + method: "item/started", params: { turnId: "turn-1", item: { - ...item, - status: "completed", - revisedPrompt: "A crisp crescent moon icon", - result: "/tmp/generated-moon.png", + id: "user-followup-1", + type: "userMessage", + content: [{ type: "text", text: "First follow-up." }], }, }, }); await vi.waitFor(() => { expect(events.some((event) => - event.event.type === "codex_image_generation" - && event.event.itemId === "image-1" - && event.event.status === "completed" - && event.event.savedPath === "/tmp/generated-moon.png" + event.event.type === "user_message" + && event.event.steerId === first.steerId + && event.event.deliveryState === "processed" + && event.event.processed === true + )).toBe(true); + }); + + const second = await service.steer({ sessionId: session.id, text: "Second follow-up." }); + mockState.emitCodexPayload({ + method: "turn/aborted", + params: { turnId: "turn-1" }, + }); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "user_message" + && event.event.steerId === second.steerId + && event.event.deliveryState === "unprocessed" + && event.event.processed === false )).toBe(true); }); }); - it("preserves live Codex MCP app metadata for Sources aggregation", async () => { + it("correlates combined Codex user-message content without consuming another accepted steer", async () => { const events: AgentChatEventEnvelope[] = []; const { service } = createService({ onEvent: (event: AgentChatEventEnvelope) => events.push(event), @@ -24447,33 +24406,1022 @@ describe("createAgentChatService", () => { const session = await service.createSession({ laneId: "lane-1", provider: "codex", - model: "gpt-5.6-sol", - modelId: "openai/gpt-5.6-sol", + model: "gpt-5.5", }); - await service.sendMessage({ sessionId: session.id, text: "Use the docs connector." }, { awaitDispatch: true }); + await service.sendMessage({ sessionId: session.id, text: "Start." }, { awaitDispatch: true }); + + const first = await service.steer({ sessionId: session.id, text: "First follow-up." }); + const second = await service.steer({ sessionId: session.id, text: "Second follow-up." }); - const item = { - id: "mcp-live-1", - type: "mcpToolCall", - server: "openaiDeveloperDocs", - tool: "search", - status: "inProgress", - arguments: { query: "GPT-5.6" }, - pluginId: "openai-docs", - appContext: { - connectorId: "openai-docs", - linkId: "docs-link", - resourceUri: "ui://openai-docs/search", - appName: "OpenAI Docs", - templateId: "search-results", - actionName: "Search documentation", - }, - }; mockState.emitCodexPayload({ - jsonrpc: "2.0", method: "item/started", - params: { turnId: "turn-1", item }, - }); + params: { + turnId: "turn-1", + item: { + id: "unmatched-user-followup", + type: "userMessage", + content: [{ type: "text", text: "A provider message unrelated to either steer." }], + }, + }, + }); + mockState.emitCodexPayload({ + method: "item/started", + params: { + turnId: "turn-1", + item: { + id: "combined-user-followup", + type: "userMessage", + content: [ + { type: "text", text: "System context supplied by ADE." }, + { type: "text", text: "Second follow-up." }, + ], + }, + }, + }); + + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "user_message" + && event.event.steerId === second.steerId + && event.event.deliveryState === "processed" + )).toBe(true); + }); + expect(events.some((event) => + event.event.type === "user_message" + && event.event.steerId === first.steerId + && event.event.deliveryState === "processed" + )).toBe(false); + + mockState.emitCodexPayload({ + method: "turn/aborted", + params: { turnId: "turn-1" }, + }); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "user_message" + && event.event.steerId === first.steerId + && event.event.deliveryState === "unprocessed" + )).toBe(true); + }); + expect(events.some((event) => + event.event.type === "user_message" + && event.event.steerId === second.steerId + && event.event.deliveryState === "unprocessed" + )).toBe(false); + }); + + it("restores accepted Codex follow-ups from durable history after a runtime restart", async () => { + installRealTranscriptParser(); + const first = createService(); + const session = await first.service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + const transcriptPath = first.sessionService.get(session.id)?.transcriptPath; + expect(transcriptPath).toBeTruthy(); + first.service.forceDisposeAll(); + fs.mkdirSync(path.dirname(String(transcriptPath)), { recursive: true }); + fs.writeFileSync(String(transcriptPath), [ + JSON.stringify({ + sessionId: session.id, + timestamp: "2026-07-25T05:20:00.000Z", + sequence: 1, + event: { + type: "user_message", + text: "Persist this follow-up.", + displayText: "Persist this follow-up.", + steerId: "steer-restart-1", + turnId: "turn-old", + deliveryState: "accepted", + processed: false, + }, + }), + JSON.stringify({ + sessionId: session.id, + timestamp: "2026-07-25T05:21:00.000Z", + sequence: 2, + event: { + type: "done", + status: "interrupted", + turnId: "turn-old", + }, + }), + ].join("\n") + "\n", "utf8"); + + const events: AgentChatEventEnvelope[] = []; + const second = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + await second.service.resumeSession({ sessionId: session.id }); + + await vi.waitFor(() => { + expect(events.some((entry) => + entry.event.type === "user_message" + && entry.event.steerId === "steer-restart-1" + && entry.event.deliveryState === "unprocessed" + )).toBe(true); + }); + }); + + it("runs an unprocessed Codex follow-up once and records an idempotent durable resolution", async () => { + installRealTranscriptParser(); + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + await service.sendMessage({ sessionId: session.id, text: "Start." }, { awaitDispatch: true }); + const followUp = await service.steer({ sessionId: session.id, text: "Run this exactly once." }); + mockState.emitCodexPayload({ + method: "turn/aborted", + params: { turnId: "turn-1" }, + }); + await vi.waitFor(() => { + expect(events.some((entry) => + entry.event.type === "user_message" + && entry.event.steerId === followUp.steerId + && entry.event.deliveryState === "unprocessed" + )).toBe(true); + }); + const turnStartsBefore = mockState.codexRequestPayloads.filter((payload) => payload.method === "turn/start").length; + + const first = await service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "run_next", + }); + const second = await service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "run_next", + }); + + expect(first).toMatchObject({ + steerId: followUp.steerId, + action: "run_next", + status: "completed", + replacementMessageId: expect.any(String), + }); + expect(second).toEqual({ + ...first, + status: "already_completed", + }); + expect(mockState.codexRequestPayloads.filter((payload) => payload.method === "turn/start")).toHaveLength(turnStartsBefore + 1); + expect(events.filter((entry) => + entry.event.type === "user_message" + && entry.event.metadata?.replayedFromUnprocessedSteer?.sourceSteerId === followUp.steerId + )).toHaveLength(1); + expect(events.filter((entry) => + entry.event.type === "user_message_resolution" + && entry.event.steerId === followUp.steerId + && entry.event.action === "run_next" + )).toHaveLength(1); + }); + + it("does not treat optimistic replay rows as a durable backend dispatch after restart", async () => { + installRealTranscriptParser(); + const first = createService(); + const session = await first.service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + first.service.forceDisposeAll(); + + const sourceSteerId = "steer-restart-1"; + const optimisticReplacementMessageId = "replacement-before-backend-ack"; + writeTestTranscriptEnvelopes(session.id, [ + { + sessionId: session.id, + timestamp: "2026-07-25T05:20:00.000Z", + sequence: 1, + event: { + type: "user_message", + text: "Run this after the current turn.", + steerId: sourceSteerId, + deliveryState: "unprocessed", + processed: false, + turnId: "turn-old", + }, + }, + { + sessionId: session.id, + timestamp: "2026-07-25T05:20:01.000Z", + sequence: 2, + event: { + type: "user_message", + text: "Run this after the current turn.", + turnId: "optimistic-turn", + metadata: { + replayedFromUnprocessedSteer: { + sourceSteerId, + action: "run_next", + replacementMessageId: optimisticReplacementMessageId, + }, + }, + }, + }, + { + sessionId: session.id, + timestamp: "2026-07-25T05:20:01.001Z", + sequence: 3, + event: { + type: "status", + turnStatus: "started", + turnId: "optimistic-turn", + }, + }, + ]); + + const emitted: AgentChatEventEnvelope[] = []; + const second = createService({ + onEvent: (event: AgentChatEventEnvelope) => emitted.push(event), + }); + const turnStartsBefore = mockState.codexRequestPayloads + .filter((payload) => payload.method === "turn/start").length; + const retried = await second.service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: sourceSteerId, + action: "run_next", + }); + + expect(retried).toMatchObject({ + steerId: sourceSteerId, + action: "run_next", + status: "completed", + replacementMessageId: expect.any(String), + }); + expect(retried.replacementMessageId).not.toBe(optimisticReplacementMessageId); + expect(emitted.some((entry) => + entry.event.type === "user_message_resolution" + && entry.event.steerId === sourceSteerId + && entry.event.action === "run_next" + )).toBe(true); + expect(emitted.some((entry) => + entry.event.type === "user_message_resolution" + && entry.event.replacementMessageId === retried.replacementMessageId + )).toBe(true); + expect(mockState.codexRequestPayloads + .filter((payload) => payload.method === "turn/start")).toHaveLength(turnStartsBefore + 1); + }); + + it("reconstructs a missing replay resolution from the durable backend dispatch receipt", async () => { + installRealTranscriptParser(); + const firstEvents: AgentChatEventEnvelope[] = []; + const first = createService({ + onEvent: (event: AgentChatEventEnvelope) => firstEvents.push(event), + }); + const session = await first.service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + await first.service.sendMessage( + { sessionId: session.id, text: "Start." }, + { awaitDispatch: true }, + ); + const followUp = await first.service.steer({ + sessionId: session.id, + text: "Run this once after restart.", + }); + mockState.emitCodexPayload({ + method: "turn/aborted", + params: { turnId: "turn-1" }, + }); + await vi.waitFor(() => { + expect(firstEvents.some((entry) => + entry.event.type === "user_message" + && entry.event.steerId === followUp.steerId + && entry.event.deliveryState === "unprocessed" + )).toBe(true); + }); + + const dispatched = await first.service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "run_next", + }); + const persistedReceipt = readPersistedChatState(session.id) + .unprocessedMessageResolutionReceipts + ?.find((receipt: Record) => receipt.steerId === followUp.steerId); + expect(persistedReceipt).toMatchObject({ + steerId: followUp.steerId, + action: "run_next", + state: "completed", + replacementMessageId: dispatched.replacementMessageId, + }); + + first.service.forceDisposeAll(); + await new Promise((resolve) => setTimeout(resolve, 250)); + writeTestTranscriptEnvelopes(session.id, [ + { + sessionId: session.id, + timestamp: "2026-07-25T05:20:00.000Z", + sequence: 1, + event: { + type: "user_message", + text: "Run this once after restart.", + steerId: followUp.steerId, + deliveryState: "unprocessed", + processed: false, + turnId: "turn-old", + }, + }, + { + sessionId: session.id, + timestamp: "2026-07-25T05:20:01.000Z", + sequence: 2, + event: { + type: "user_message", + text: "Run this once after restart.", + metadata: { + replayedFromUnprocessedSteer: { + sourceSteerId: followUp.steerId, + action: "run_next", + replacementMessageId: dispatched.replacementMessageId!, + }, + }, + }, + }, + ]); + + const restartedEvents: AgentChatEventEnvelope[] = []; + const restarted = createService({ + onEvent: (event: AgentChatEventEnvelope) => restartedEvents.push(event), + }); + const turnStartsBeforeRetry = mockState.codexRequestPayloads + .filter((payload) => payload.method === "turn/start").length; + const retried = await restarted.service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "run_next", + }); + + expect(retried).toEqual({ + steerId: followUp.steerId, + action: "run_next", + status: "already_completed", + replacementMessageId: dispatched.replacementMessageId, + }); + expect(mockState.codexRequestPayloads + .filter((payload) => payload.method === "turn/start")).toHaveLength(turnStartsBeforeRetry); + expect(restartedEvents).toEqual(expect.arrayContaining([ + expect.objectContaining({ + event: expect.objectContaining({ + type: "user_message_resolution", + steerId: followUp.steerId, + action: "run_next", + replacementMessageId: dispatched.replacementMessageId, + }), + }), + ])); + }); + + it("allows Run next to retry when the optimistic replacement never reached the provider", async () => { + installRealTranscriptParser(); + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + await service.sendMessage( + { sessionId: session.id, text: "Start." }, + { awaitDispatch: true }, + ); + const followUp = await service.steer({ + sessionId: session.id, + text: "Retry me if the provider rejects the dispatch.", + }); + mockState.emitCodexPayload({ + method: "turn/aborted", + params: { turnId: "turn-1" }, + }); + await vi.waitFor(() => { + expect(events.some((entry) => + entry.event.type === "user_message" + && entry.event.steerId === followUp.steerId + && entry.event.deliveryState === "unprocessed" + )).toBe(true); + }); + + mockState.codexResponseOverrides.set("turn/start", { + error: { code: -32_000, message: "replay start exploded" }, + }); + await expect(service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "run_next", + })).rejects.toThrow(/replay start exploded/i); + await vi.waitFor(() => { + expect(events.some((entry) => + entry.event.type === "done" + && entry.event.status === "failed" + )).toBe(true); + }); + + mockState.codexResponseOverrides.delete("turn/start"); + const turnStartsBeforeRetry = mockState.codexRequestPayloads + .filter((payload) => payload.method === "turn/start").length; + await expect(service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "run_next", + })).resolves.toMatchObject({ + steerId: followUp.steerId, + action: "run_next", + status: "completed", + replacementMessageId: expect.any(String), + }); + expect(mockState.codexRequestPayloads + .filter((payload) => payload.method === "turn/start")).toHaveLength(turnStartsBeforeRetry + 1); + }); + + it("keeps Run next retryable when storage pressure prevents backend dispatch", async () => { + installRealTranscriptParser(); + let allowTurns = true; + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + diskPressureMonitor: { + canPerform: vi.fn(() => allowTurns + ? { allowed: true, state: "normal" } + : { + allowed: false, + state: "exhausted", + code: "disk_full", + message: "Your Mac is almost out of storage.", + }), + }, + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + await service.sendMessage( + { sessionId: session.id, text: "Start." }, + { awaitDispatch: true }, + ); + const followUp = await service.steer({ + sessionId: session.id, + text: "Run this when storage is ready.", + }); + mockState.emitCodexPayload({ + method: "turn/aborted", + params: { turnId: "turn-1" }, + }); + await vi.waitFor(() => { + expect(events.some((entry) => + entry.event.type === "user_message" + && entry.event.steerId === followUp.steerId + && entry.event.deliveryState === "unprocessed" + )).toBe(true); + }); + + allowTurns = false; + await expect(service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "run_next", + })).rejects.toThrow(/provider did not accept/i); + expect(events.some((entry) => + entry.event.type === "user_message_resolution" + && entry.event.steerId === followUp.steerId + )).toBe(false); + + allowTurns = true; + await expect(service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "run_next", + })).resolves.toMatchObject({ + steerId: followUp.steerId, + action: "run_next", + status: "completed", + }); + }); + + it("dismisses an unprocessed Codex follow-up idempotently without starting a turn", async () => { + installRealTranscriptParser(); + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + await service.sendMessage({ sessionId: session.id, text: "Start." }, { awaitDispatch: true }); + const followUp = await service.steer({ sessionId: session.id, text: "Dismiss me." }); + mockState.emitCodexPayload({ + method: "turn/aborted", + params: { turnId: "turn-1" }, + }); + await vi.waitFor(() => { + expect(events.some((entry) => + entry.event.type === "user_message" + && entry.event.steerId === followUp.steerId + && entry.event.deliveryState === "unprocessed" + )).toBe(true); + }); + const turnStartsBefore = mockState.codexRequestPayloads.filter((payload) => payload.method === "turn/start").length; + + await expect(service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "dismiss", + })).resolves.toMatchObject({ status: "completed", action: "dismiss" }); + await expect(service.resolveUnprocessedMessage({ + sessionId: session.id, + steerId: followUp.steerId, + action: "dismiss", + })).resolves.toMatchObject({ status: "already_completed", action: "dismiss" }); + + expect(mockState.codexRequestPayloads.filter((payload) => payload.method === "turn/start")).toHaveLength(turnStartsBefore); + expect(events.filter((entry) => + entry.event.type === "user_message_resolution" + && entry.event.steerId === followUp.steerId + && entry.event.action === "dismiss" + )).toHaveLength(1); + }); + + it("maps the provider-neutral recovery contract onto Codex recovery", async () => { + const { service } = createService(); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.6-sol", + }); + await service.sendMessage({ sessionId: session.id, text: "Keep working." }, { awaitDispatch: true }); + + await expect(service.recoverTurn({ + sessionId: session.id, + turnId: "turn-1", + action: "nudge", + })).resolves.toEqual({ + action: "nudge", + turnId: "turn-1", + status: "nudged", + }); + expect(mockState.codexRequestPayloads.some((payload) => payload.method === "turn/steer")).toBe(true); + }); + + it("clears the Codex no-output watchdog when an approval request is surfaced", async () => { + vi.useFakeTimers(); + try { + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + + await service.sendMessage({ + sessionId: session.id, + text: "Keep working.", + }, { awaitDispatch: true }); + + mockState.emitCodexPayload({ + id: "approval-1", + method: "item/commandExecution/requestApproval", + params: { + itemId: "cmd-1", + turnId: "turn-1", + command: "npm test", + cwd: ".", + reason: "Run tests", + }, + }); + + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "approval_request" + && event.event.itemId === "cmd-1" + )).toBe(true); + }); + + await vi.advanceTimersByTimeAsync(120_000); + + expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); + expect(events.some((event) => + event.event.type === "system_notice" + && event.event.message.includes("has not streamed model or tool output yet") + )).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it("re-arms the Codex watchdog after the user answers a suspended approval", async () => { + vi.useFakeTimers(); + try { + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + await service.sendMessage( + { sessionId: session.id, text: "Keep working." }, + { awaitDispatch: true }, + ); + mockState.emitCodexPayload({ + id: "approval-rearm-1", + method: "item/commandExecution/requestApproval", + params: { + itemId: "cmd-rearm-1", + turnId: "turn-1", + command: "npm test", + cwd: ".", + reason: "Run tests", + }, + }); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "approval_request" + && event.event.itemId === "cmd-rearm-1" + )).toBe(true); + }); + + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); + + await service.respondToInput({ + sessionId: session.id, + itemId: "cmd-rearm-1", + decision: "accept", + }); + await vi.advanceTimersByTimeAsync(10 * 60_000); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "codex_turn_stalled" + && event.event.turnId === "turn-1" + && event.event.reason === "no_progress" + )).toBe(true); + }); + } finally { + vi.useRealTimers(); + } + }); + + it("re-arms the Codex watchdog after full-auto resolves a suspended approval", async () => { + vi.useFakeTimers(); + try { + vi.mocked(mapPermissionToCodex).mockImplementation((mode) => { + if (mode === "full-auto") { + return { approvalPolicy: "never", sandbox: "danger-full-access" }; + } + return { approvalPolicy: "on-request", sandbox: "workspace-write" }; + }); + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + permissionMode: "edit", + }); + await service.sendMessage( + { sessionId: session.id, text: "Keep working." }, + { awaitDispatch: true }, + ); + mockState.emitCodexPayload({ + id: "auto-resolved-approval-1", + method: "item/commandExecution/requestApproval", + params: { + itemId: "cmd-auto-resolved-1", + turnId: "turn-1", + command: "npm test", + cwd: tmpRoot, + reason: "Run tests", + }, + }); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "approval_request" + && event.event.itemId === "cmd-auto-resolved-1" + )).toBe(true); + }); + + await service.updateSession({ + sessionId: session.id, + permissionMode: "full-auto", + }); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "pending_input_resolved" + && event.event.itemId === "cmd-auto-resolved-1" + && event.event.resolution === "accepted" + )).toBe(true); + }); + + await vi.advanceTimersByTimeAsync(10 * 60_000); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "codex_turn_stalled" + && event.event.turnId === "turn-1" + && event.event.reason === "no_progress" + )).toBe(true); + }); + } finally { + vi.useRealTimers(); + } + }); + + it("re-arms the Codex watchdog when the app server resolves a suspended approval", async () => { + vi.useFakeTimers(); + try { + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + await service.sendMessage( + { sessionId: session.id, text: "Keep working." }, + { awaitDispatch: true }, + ); + mockState.emitCodexPayload({ + id: "server-resolved-approval-1", + method: "item/commandExecution/requestApproval", + params: { + itemId: "cmd-server-resolved-1", + turnId: "turn-1", + command: "npm test", + cwd: ".", + reason: "Run tests", + }, + }); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "approval_request" + && event.event.itemId === "cmd-server-resolved-1" + )).toBe(true); + }); + + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); + + mockState.emitCodexPayload({ + jsonrpc: "2.0", + method: "serverRequest/resolved", + params: { + threadId: "thread-1", + turnId: "turn-1", + requestId: "server-resolved-approval-1", + }, + }); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "pending_input_resolved" + && event.event.itemId === "cmd-server-resolved-1" + )).toBe(true); + }); + await vi.advanceTimersByTimeAsync(10 * 60_000); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "codex_turn_stalled" + && event.event.turnId === "turn-1" + && event.event.reason === "no_progress" + )).toBe(true); + }); + } finally { + vi.useRealTimers(); + } + }); + + it("reconciles a completed silent Codex turn from app-server state before reporting a stall", async () => { + vi.useFakeTimers(); + try { + const events: AgentChatEventEnvelope[] = []; + mockState.codexResponseOverrides.set("thread/turns/list", () => ({ + data: [ + { + id: "turn-1", + status: "completed", + usage: { inputTokens: 7, outputTokens: 3 }, + items: [ + { + id: "msg-1", + type: "agentMessage", + text: "Recovered assistant output.", + }, + ], + }, + ], + nextCursor: null, + })); + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + + await service.sendMessage({ + sessionId: session.id, + text: "Keep working.", + }, { awaitDispatch: true }); + + await vi.advanceTimersByTimeAsync(120_000); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "done" + && event.event.turnId === "turn-1" + && event.event.status === "completed" + )).toBe(true); + }); + + expect(mockState.codexRequestPayloads.some((payload) => payload.method === "thread/read")).toBe(true); + expect(mockState.codexRequestPayloads.some((payload) => payload.method === "thread/turns/list")).toBe(true); + expect(events.some((event) => + event.event.type === "text" + && event.event.text.includes("Recovered assistant output.") + )).toBe(true); + expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it("does not complete a reconciled MCP tool call while app-server still reports it running", async () => { + vi.useFakeTimers(); + try { + const events: AgentChatEventEnvelope[] = []; + mockState.codexResponseOverrides.set("thread/turns/list", () => ({ + data: [ + { + id: "turn-1", + status: "inProgress", + items: [ + { + id: "mcp-1", + type: "mcpToolCall", + server: "local-tools", + tool: "probe", + pluginId: "local-plugin", + appContext: { + connectorId: "local", + appName: "Local tools", + actionName: "Probe file", + resourceUri: "ui://local/probe", + }, + status: "running", + arguments: { path: "README.md" }, + }, + ], + }, + ], + nextCursor: null, + })); + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + + await service.sendMessage({ + sessionId: session.id, + text: "Keep working.", + }, { awaitDispatch: true }); + + await vi.advanceTimersByTimeAsync(120_000); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "tool_call" + && event.event.itemId === "mcp-1" + && event.event.mcp?.pluginId === "local-plugin" + && event.event.mcp?.appContext?.appName === "Local tools" + )).toBe(true); + }); + + expect(events.some((event) => + event.event.type === "tool_result" + && event.event.itemId === "mcp-1" + )).toBe(false); + expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it("preserves the Codex imageGeneration lifecycle and local output path", async () => { + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.5", + }); + await service.sendMessage({ sessionId: session.id, text: "Generate a tiny moon icon." }, { awaitDispatch: true }); + + const item = { + id: "image-1", + type: "imageGeneration", + status: "inProgress", + prompt: "A tiny moon icon", + }; + mockState.emitCodexPayload({ + jsonrpc: "2.0", + method: "item/started", + params: { turnId: "turn-1", item }, + }); + const started = await waitForEvent( + events, + (event): event is AgentChatEventEnvelope & { event: Extract } => + event.event.type === "codex_image_generation" && event.event.itemId === "image-1", + ); + expect(started.event).toMatchObject({ + prompt: "A tiny moon icon", + status: "running", + }); + + mockState.emitCodexPayload({ + jsonrpc: "2.0", + method: "item/completed", + params: { + turnId: "turn-1", + item: { + ...item, + status: "completed", + revisedPrompt: "A crisp crescent moon icon", + result: "/tmp/generated-moon.png", + }, + }, + }); + await vi.waitFor(() => { + expect(events.some((event) => + event.event.type === "codex_image_generation" + && event.event.itemId === "image-1" + && event.event.status === "completed" + && event.event.savedPath === "/tmp/generated-moon.png" + )).toBe(true); + }); + }); + + it("preserves live Codex MCP app metadata for Sources aggregation", async () => { + const events: AgentChatEventEnvelope[] = []; + const { service } = createService({ + onEvent: (event: AgentChatEventEnvelope) => events.push(event), + }); + const session = await service.createSession({ + laneId: "lane-1", + provider: "codex", + model: "gpt-5.6-sol", + modelId: "openai/gpt-5.6-sol", + }); + await service.sendMessage({ sessionId: session.id, text: "Use the docs connector." }, { awaitDispatch: true }); + + const item = { + id: "mcp-live-1", + type: "mcpToolCall", + server: "openaiDeveloperDocs", + tool: "search", + status: "inProgress", + arguments: { query: "GPT-5.6" }, + pluginId: "openai-docs", + appContext: { + connectorId: "openai-docs", + linkId: "docs-link", + resourceUri: "ui://openai-docs/search", + appName: "OpenAI Docs", + templateId: "search-results", + actionName: "Search documentation", + }, + }; + mockState.emitCodexPayload({ + jsonrpc: "2.0", + method: "item/started", + params: { turnId: "turn-1", item }, + }); await waitForEvent( events, (event): event is AgentChatEventEnvelope & { event: Extract } => @@ -24562,11 +25510,12 @@ describe("createAgentChatService", () => { }); expect(events.some((event) => event.event.type === "codex_turn_stalled")).toBe(false); - await vi.advanceTimersByTimeAsync(120_000); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(10 * 60_000); await vi.waitFor(() => { expect(events.some((event) => event.event.type === "codex_turn_stalled" - && event.event.reason === "no_output" + && event.event.reason === "no_progress" )).toBe(true); }); expect(events.filter((event) => @@ -24748,6 +25697,11 @@ describe("createAgentChatService", () => { sessionId: child.id, text: "Keep working.", }, { awaitDispatch: true }); + await service.recoverCodexTurn({ + sessionId: child.id, + turnId: "turn-1", + action: "wait", + }); await vi.advanceTimersByTimeAsync(120_000); await vi.waitFor(() => { @@ -24765,8 +25719,8 @@ describe("createAgentChatService", () => { )).toBe(true); expect(events.some((event) => event.sessionId === parent.id - && event.event.type === "system_notice" - && event.event.message.includes("Child Codex session") + && event.event.type === "turn_health" + && event.event.sourceSessionId === child.id )).toBe(true); expect(mockState.codexRequestPayloads.some((payload) => payload.method === "thread/start")).toBe(true); expect(mockState.codexRequestPayloads.filter((payload) => payload.method === "turn/interrupt")).toHaveLength(0); diff --git a/apps/desktop/src/main/services/chat/agentChatService.ts b/apps/desktop/src/main/services/chat/agentChatService.ts index 6a8bda1aa..fc7ec0440 100644 --- a/apps/desktop/src/main/services/chat/agentChatService.ts +++ b/apps/desktop/src/main/services/chat/agentChatService.ts @@ -191,6 +191,8 @@ import type { AgentChatClaudePluginsArgs, AgentChatReloadClaudePluginsArgs, AgentChatReloadClaudePluginsResult, + AgentChatRecoverTurnArgs, + AgentChatRecoverTurnResult, AgentChatRecoverCodexTurnArgs, AgentChatRecoverCodexTurnResult, AgentChatClaudePermissionMode, @@ -255,6 +257,8 @@ import type { AgentChatPrepareCrossMachineHandoffResult, AgentChatValidateCrossMachineSourceArgs, AgentChatRespondToInputArgs, + AgentChatResolveUnprocessedMessageArgs, + AgentChatResolveUnprocessedMessageResult, AgentChatRewindFilesArgs, AgentChatRewindFilesResult, AgentChatSession, @@ -833,6 +837,14 @@ type PersistedRecentConversationEntry = { turnId?: string; }; +type PersistedUnprocessedMessageResolutionReceipt = { + steerId: string; + action: "run_next" | "dismiss"; + state: "completed"; + resolvedAt: string; + replacementMessageId?: string; +}; + type PersistedChatState = { version: 1 | 2; sessionId: string; @@ -904,6 +916,17 @@ type PersistedChatState = { runtimeMode?: AgentChatRuntimeMode; /** Recent terminal Codex turn ids, used to suppress late replayed lifecycle events. */ codexTerminalTurnIds?: string[]; + /** + * True once ADE consumes the automatic recovery attempt for the current + * user-request chain (or the user explicitly chooses to keep waiting). + */ + codexAutomaticRecoveryAttempted?: boolean; + /** + * Fsynced terminal receipts for accepted-but-unprocessed message recovery. + * These close the crash window between provider dispatch and the async + * transcript append that normally carries user_message_resolution. + */ + unprocessedMessageResolutionReceipts?: PersistedUnprocessedMessageResolutionReceipt[]; /** Persisted "Allow for Session" tool approval overrides (Claude runtime). */ approvalOverrides?: string[]; /** Queued mid-turn steers for the Claude runtime, restored on app restart. */ @@ -961,6 +984,35 @@ function isPersistedChatStateShape(value: unknown): value is PersistedChatState && record.sessionId.trim().length > 0; } +function normalizeUnprocessedMessageResolutionReceipts( + value: unknown, +): PersistedUnprocessedMessageResolutionReceipt[] { + if (!Array.isArray(value)) return []; + const bySteerId = new Map(); + for (const candidate of value) { + if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) continue; + const record = candidate as Record; + const steerId = typeof record.steerId === "string" ? record.steerId.trim() : ""; + const action = record.action === "run_next" || record.action === "dismiss" + ? record.action + : null; + const resolvedAt = typeof record.resolvedAt === "string" ? record.resolvedAt.trim() : ""; + const replacementMessageId = typeof record.replacementMessageId === "string" + ? record.replacementMessageId.trim() + : ""; + if (!steerId || !action || !resolvedAt || record.state !== "completed") continue; + bySteerId.delete(steerId); + bySteerId.set(steerId, { + steerId, + action, + state: "completed", + resolvedAt, + ...(replacementMessageId ? { replacementMessageId } : {}), + }); + } + return [...bySteerId.values()].slice(-64); +} + function normalizedPersistedPointer(value: unknown): string | null { return typeof value === "string" && value.trim().length ? value.trim() : null; } @@ -1127,10 +1179,19 @@ type CodexRuntime = { noFirstEventWatchdog: { turnId: string; timer: NodeJS.Timeout; + startedAt: number; + lastProgressAt: number; + firstUsefulProgressSeen: boolean; } | null; stalledTurnIds: Set; stallReconcileInFlight: Set; mcpStartupNoticeKeys: Set; + acceptedSteersByTurnId: Map>; + acceptedSteersHydrationReady: Promise; + turnDiagnosticStateByTurnId: Map; + }>; /** * Plan-approval follow-ups deferred until the planning turn idles. Calling * sendMessage while a planning turn is still active would race the busy @@ -1174,6 +1235,17 @@ type QueuedSteer = { interactionMode?: AgentChatInteractionMode | null; }; +type AcceptedCodexSteer = { + steerId: string; + text: string; + displayText: string; + attachments: AgentChatFileRef[]; + contextAttachments: AgentChatContextAttachment[]; + metadata?: AgentChatEventMetadata | null | undefined; + turnId: string; + acceptedAt: string; +}; + type ClaudeActiveSubagent = { taskId: string; description: string; @@ -2407,6 +2479,8 @@ type ManagedChatSession = { /** Set after we've emitted the once-per-session Claude plan-limit notice. */ claudeRateLimitWarningEmitted: boolean; codexTerminalTurnIds: Set; + codexAutomaticRecoveryAttempted: boolean; + unprocessedMessageResolutionReceipts: Map; todoItems: Extract["items"]; localPendingInputs: Map(); const codexRecoveryInFlight = new Set(); const continuityRecoveryInFlight = new Set(); + const unprocessedMessageResolutionInFlight = new Map< + string, + Promise + >(); const recordLinearIssueContextForLane = ( managed: ManagedChatSession, @@ -11188,10 +11267,13 @@ export function createAgentChatService(args: { } }; - const persistChatState = (managed: ManagedChatSession): void => { + const persistChatState = ( + managed: ManagedChatSession, + options: { fsync?: boolean } = {}, + ): boolean => { // Tombstoned sessions (deleted while async work was in flight) must not be // re-persisted — otherwise the file recreates after deleteSession removed it. - if (managed.deleted) return; + if (managed.deleted) return false; // When runtime has been torn down (null) but NOT intentionally invalidated, // fall back to the last persisted state so that provider session ids and // lastLaneDirectiveKey survive a transient teardown (e.g. app backgrounding). @@ -11354,6 +11436,16 @@ export function createAgentChatService(args: { ...(managed.codexTerminalTurnIds.size ? { codexTerminalTurnIds: [...managed.codexTerminalTurnIds].slice(-64) } : prevPersisted?.codexTerminalTurnIds?.length ? { codexTerminalTurnIds: prevPersisted.codexTerminalTurnIds.slice(-64) } : {}), + ...(managed.codexAutomaticRecoveryAttempted + ? { codexAutomaticRecoveryAttempted: true } + : {}), + ...(managed.unprocessedMessageResolutionReceipts.size + ? { + unprocessedMessageResolutionReceipts: [ + ...managed.unprocessedMessageResolutionReceipts.values(), + ].slice(-64), + } + : {}), ...collectOrchestrationFields(managed.session, prevPersisted), updatedAt: nowIso() }; @@ -11378,12 +11470,14 @@ export function createAgentChatService(args: { } const pointerChanged = currentPointerFingerprint !== previousPointerFingerprint; let lkgUpdated = false; + let persisted = false; try { fs.mkdirSync(path.dirname(managed.metadataPath), { recursive: true }); lkgUpdated = writeJsonWithPrevious(managed.metadataPath, payload, { validate: isPersistedChatStateShape, - fsync: pointerChanged, + fsync: pointerChanged || options.fsync === true, }); + persisted = true; if (pointerChanged) { lastPersistedPointerFingerprints.set(managed.session.id, currentPointerFingerprint); try { @@ -11420,6 +11514,21 @@ export function createAgentChatService(args: { } mirrorClaudeSessionPointer(managed, payload.sdkSessionId); + return persisted; + }; + + const persistUnprocessedMessageResolutionReceipt = ( + managed: ManagedChatSession, + receipt: PersistedUnprocessedMessageResolutionReceipt, + ): void => { + managed.unprocessedMessageResolutionReceipts.delete(receipt.steerId); + managed.unprocessedMessageResolutionReceipts.set(receipt.steerId, receipt); + evictOldestEntries(managed.unprocessedMessageResolutionReceipts, 64); + if (!persistChatState(managed, { fsync: true })) { + throw new Error( + "The provider accepted this message, but ADE could not save its delivery receipt. Retry without closing this chat.", + ); + } }; const readPersistedState = (sessionId: string): PersistedChatState | null => { @@ -11561,6 +11670,10 @@ export function createAgentChatService(args: { 64, ) : undefined; + const unprocessedMessageResolutionReceipts = + normalizeUnprocessedMessageResolutionReceipts( + record.unprocessedMessageResolutionReceipts, + ); const approvalOverrides = Array.isArray(record.approvalOverrides) ? record.approvalOverrides.filter((v): v is string => typeof v === "string" && v.trim().length > 0) : undefined; @@ -11664,6 +11777,12 @@ export function createAgentChatService(args: { ? { idleSinceAt: null } : {}), ...(codexTerminalTurnIds?.length ? { codexTerminalTurnIds } : {}), + ...(record.codexAutomaticRecoveryAttempted === true + ? { codexAutomaticRecoveryAttempted: true } + : {}), + ...(unprocessedMessageResolutionReceipts.length + ? { unprocessedMessageResolutionReceipts } + : {}), ...(record.runtimeMode === "print" || record.runtimeMode === "interactive" ? { runtimeMode: record.runtimeMode } : {}), @@ -14152,57 +14271,113 @@ export function createAgentChatService(args: { runtime.noFirstEventWatchdog = null; }; - const scheduleCodexNoFirstEventWatchdog = ( + const armCodexTurnProgressWatchdog = ( managed: ManagedChatSession, runtime: CodexRuntime, - turnId: string | null | undefined, + state: Omit, "timer">, + delayMs: number, ): void => { - const normalizedTurnId = turnId?.trim() || null; - if (!normalizedTurnId) return; - if (runtime.noFirstEventWatchdog?.turnId === normalizedTurnId) return; - clearCodexNoFirstEventWatchdog(runtime); const timer = setTimeout(() => { if ( managed.deleted || managed.closed || managed.runtime !== runtime - || (runtime.activeTurnId ?? runtime.startedTurnId) !== normalizedTurnId + || (runtime.activeTurnId ?? runtime.startedTurnId) !== state.turnId ) { - if (runtime.noFirstEventWatchdog?.turnId === normalizedTurnId) { + if (runtime.noFirstEventWatchdog?.turnId === state.turnId) { runtime.noFirstEventWatchdog = null; } return; } runtime.noFirstEventWatchdog = null; - void reconcileCodexSilentTurn(managed, runtime, normalizedTurnId).catch((error) => { + void reconcileCodexSilentTurn( + managed, + runtime, + state.turnId, + state.firstUsefulProgressSeen ? "no_progress" : "no_output", + state.startedAt, + state.lastProgressAt, + ).catch((error) => { logger.warn("agent_chat.codex_stall_reconcile_failed", { sessionId: managed.session.id, - turnId: normalizedTurnId, + turnId: state.turnId, error: error instanceof Error ? error.message : String(error), }); emitCodexTurnStalled(managed, runtime, { - turnId: normalizedTurnId, + turnId: state.turnId, reason: "app_server_state_unknown", - message: "Codex accepted this turn but ADE could not confirm its app-server state. You can keep waiting, send a status nudge, or interrupt and retry this thread if it stays stalled.", + message: "ADE could not confirm the Codex app-server state. Keep waiting or restart and resume the thread.", + turnStartedAt: state.startedAt, + lastProgressAt: state.lastProgressAt, }); }); - }, CODEX_NO_FIRST_EVENT_WATCHDOG_MS); + }, Math.max(1, delayMs)); timer.unref?.(); - runtime.noFirstEventWatchdog = { + runtime.noFirstEventWatchdog = { ...state, timer }; + }; + + const scheduleCodexNoFirstEventWatchdog = ( + managed: ManagedChatSession, + runtime: CodexRuntime, + turnId: string | null | undefined, + ): void => { + const normalizedTurnId = turnId?.trim() || null; + if (!normalizedTurnId) return; + if (runtime.noFirstEventWatchdog?.turnId === normalizedTurnId) return; + clearCodexNoFirstEventWatchdog(runtime); + const now = Date.now(); + armCodexTurnProgressWatchdog(managed, runtime, { turnId: normalizedTurnId, - timer, - }; + startedAt: now, + lastProgressAt: now, + firstUsefulProgressSeen: false, + }, CODEX_NO_FIRST_EVENT_WATCHDOG_MS); }; const markCodexTurnProgress = ( + managed: ManagedChatSession, runtime: CodexRuntime, turnId: string | null | undefined, ): void => { const normalizedTurnId = turnId?.trim() || runtime.activeTurnId || runtime.startedTurnId || null; - if (!runtime.noFirstEventWatchdog) return; - if (!normalizedTurnId || runtime.noFirstEventWatchdog.turnId === normalizedTurnId) { - clearCodexNoFirstEventWatchdog(runtime); + const watchdog = runtime.noFirstEventWatchdog; + if (!watchdog || (normalizedTurnId && watchdog.turnId !== normalizedTurnId)) return; + clearTimeout(watchdog.timer); + armCodexTurnProgressWatchdog(managed, runtime, { + turnId: watchdog.turnId, + startedAt: watchdog.startedAt, + lastProgressAt: Date.now(), + firstUsefulProgressSeen: true, + }, CODEX_MID_TURN_INACTIVITY_WATCHDOG_MS); + }; + + const resumeCodexTurnWatchdogAfterInput = ( + managed: ManagedChatSession, + runtime: CodexRuntime, + turnId: string | null | undefined, + ): void => { + const normalizedTurnId = turnId?.trim() + || runtime.activeTurnId + || runtime.startedTurnId + || null; + if ( + !normalizedTurnId + || managed.runtime !== runtime + || (runtime.activeTurnId ?? runtime.startedTurnId) !== normalizedTurnId + ) { + return; } + const prior = runtime.noFirstEventWatchdog?.turnId === normalizedTurnId + ? runtime.noFirstEventWatchdog + : null; + clearCodexNoFirstEventWatchdog(runtime); + const now = Date.now(); + armCodexTurnProgressWatchdog(managed, runtime, { + turnId: normalizedTurnId, + startedAt: prior?.startedAt ?? now, + lastProgressAt: now, + firstUsefulProgressSeen: true, + }, CODEX_MID_TURN_INACTIVITY_WATCHDOG_MS); }; const isCodexUsefulProgressNotification = ( @@ -14273,6 +14448,51 @@ export function createAgentChatService(args: { return { serverName, status, message, failed }; }; + const emitCodexTurnDiagnostics = ( + managed: ManagedChatSession, + runtime: CodexRuntime, + turnId: string | null | undefined, + ): void => { + const key = turnId?.trim() || "__session_startup__"; + const state = runtime.turnDiagnosticStateByTurnId.get(key); + if (!state) return; + emitChatEvent(managed, { + type: "turn_diagnostics", + ...(turnId?.trim() ? { turnId: turnId.trim() } : {}), + ...(state.moderationChecks > 0 ? { moderationChecks: state.moderationChecks } : {}), + ...(state.optionalIntegrationFailures.size > 0 + ? { + optionalIntegrationFailures: Array.from(state.optionalIntegrationFailures, ([integration, message]) => ({ + integration, + message, + })), + } + : {}), + }); + }; + + const codexTurnDiagnosticState = ( + runtime: CodexRuntime, + turnId: string | null | undefined, + ): { + moderationChecks: number; + optionalIntegrationFailures: Map; + } => { + const key = turnId?.trim() || "__session_startup__"; + const existing = runtime.turnDiagnosticStateByTurnId.get(key); + if (existing) return existing; + const created = { + moderationChecks: 0, + optionalIntegrationFailures: new Map(), + }; + runtime.turnDiagnosticStateByTurnId.set(key, created); + evictOldestEntries( + runtime.turnDiagnosticStateByTurnId, + MAX_SESSION_MAP_ENTRIES, + ); + return created; + }; + const handleCodexMcpStartupStatus = ( managed: ManagedChatSession, runtime: CodexRuntime, @@ -14294,20 +14514,16 @@ export function createAgentChatService(args: { const [first] = runtime.mcpStartupNoticeKeys; if (first) runtime.mcpStartupNoticeKeys.delete(first); } - const suffix = status.message ? `: ${status.message}` : status.status ? ` (${status.status})` : ""; logger.warn("agent_chat.codex_mcp_startup_failed", { sessionId: managed.session.id, serverName: status.serverName, status: status.status, message: status.message, }); - emitChatEvent(managed, { - type: "system_notice", - noticeKind: "warning", - severity: "warning", - message: `Codex MCP server '${status.serverName}' is unavailable${suffix}.`, - ...(runtime.activeTurnId ? { turnId: runtime.activeTurnId } : {}), - }); + const turnId = runtime.activeTurnId ?? runtime.startedTurnId; + const diagnosticState = codexTurnDiagnosticState(runtime, turnId); + diagnosticState.optionalIntegrationFailures.set(status.serverName, status.message ?? status.status); + emitCodexTurnDiagnostics(managed, runtime, turnId); persistChatState(managed); }; @@ -14463,6 +14679,7 @@ export function createAgentChatService(args: { decision: "accept", turnId: pendingTurnId, }); + resumeCodexTurnWatchdogAfterInput(managed, runtime, pendingTurnId); }; const autoApprovePendingCodexRuntimeApprovals = ( @@ -14998,6 +15215,7 @@ export function createAgentChatService(args: { runtime.approvals.clear(); runtime.codexAgentIndexByTurn.clear(); if (shouldMarkInterrupted) { + markAcceptedCodexSteersUnprocessed(managed, runtime, interruptedTurnId); emitChatEvent(managed, { type: "system_notice", noticeKind: "info", @@ -15475,6 +15693,12 @@ export function createAgentChatService(args: { runtimeInvalidated: false, claudeRateLimitWarningEmitted: false, codexTerminalTurnIds: new Set(persisted?.codexTerminalTurnIds ?? []), + codexAutomaticRecoveryAttempted: persisted?.codexAutomaticRecoveryAttempted === true, + unprocessedMessageResolutionReceipts: new Map( + normalizeUnprocessedMessageResolutionReceipts( + persisted?.unprocessedMessageResolutionReceipts, + ).map((receipt) => [receipt.steerId, receipt]), + ), todoItems: [], activeAssistantMessageId: null, lastActivitySignature: null, @@ -21397,7 +21621,7 @@ export function createAgentChatService(args: { elicitationSchema: requestedSchema, elicitationPersistenceAllowed: persistenceAllowed, }); - markCodexTurnProgress(runtime, requestTurnId); + markCodexTurnProgress(managed, runtime, requestTurnId); emitPendingInputRequest(managed, request, { kind: "tool_call", description: message, @@ -21460,7 +21684,7 @@ export function createAgentChatService(args: { turnId: requestTurnId, }; runtime.approvals.set(itemId, { requestId: id, kind: "command", request }); - markCodexTurnProgress(runtime, request.turnId); + markCodexTurnProgress(managed, runtime, request.turnId); emitPendingInputRequest(managed, request, { kind: "command", description, @@ -21509,7 +21733,7 @@ export function createAgentChatService(args: { turnId: requestTurnId, }; runtime.approvals.set(itemId, { requestId: id, kind: "file_change", request }); - markCodexTurnProgress(runtime, request.turnId); + markCodexTurnProgress(managed, runtime, request.turnId); emitPendingInputRequest(managed, request, { kind: "file_change", description, @@ -21603,7 +21827,7 @@ export function createAgentChatService(args: { permissions: params.permissions ?? null, request, }); - markCodexTurnProgress(runtime, request.turnId); + markCodexTurnProgress(managed, runtime, request.turnId); emitPendingInputRequest(managed, request, { kind: "tool_call", description, @@ -21688,7 +21912,7 @@ export function createAgentChatService(args: { request, questionResponseKind: "native_request_user_input", }); - markCodexTurnProgress(runtime, request.turnId); + markCodexTurnProgress(managed, runtime, request.turnId); emitPendingInputRequest(managed, request, { kind: "tool_call", description: request.description ?? "Codex requested input", @@ -21841,7 +22065,7 @@ export function createAgentChatService(args: { kind: "plan_approval", request, }); - markCodexTurnProgress(runtime, request.turnId); + markCodexTurnProgress(managed, runtime, request.turnId); emitPendingInputRequest(managed, request, { kind: "tool_call", description: "Plan ready for approval", @@ -21930,7 +22154,7 @@ export function createAgentChatService(args: { kind: "plan_approval", request, }); - markCodexTurnProgress(runtime, request.turnId); + markCodexTurnProgress(managed, runtime, request.turnId); emitPendingInputRequest(managed, request, { kind: "tool_call", description: "Plan ready for approval", @@ -22061,6 +22285,7 @@ export function createAgentChatService(args: { ): void => { const interruptedTurnId = turnId?.trim() || runtime.activeTurnId || runtime.startedTurnId || randomUUID(); clearCodexNoFirstEventWatchdog(runtime); + markAcceptedCodexSteersUnprocessed(managed, runtime, interruptedTurnId); rememberInterruptedCodexTurn(runtime, interruptedTurnId); rememberTerminalCodexTurn(runtime, interruptedTurnId, managed); runtime.awaitingTurnStart = false; @@ -23072,6 +23297,148 @@ export function createAgentChatService(args: { return true; } + const codexUserMessageItemText = (item: Record): string => { + const direct = stringOrNull(item.text); + if (direct) return direct; + if (!Array.isArray(item.content)) return ""; + return item.content + .flatMap((entry) => { + if (typeof entry === "string") return [entry]; + const record = asRecord(entry); + const text = stringOrNull(record?.text ?? record?.content); + return text ? [text] : []; + }) + .join("\n") + .trim(); + }; + + const emitAcceptedCodexSteerState = ( + managed: ManagedChatSession, + steer: AcceptedCodexSteer, + deliveryState: "accepted" | "processed" | "unprocessed", + ): void => { + emitChatEvent(managed, { + type: "user_message", + text: steer.text, + ...(steer.displayText !== steer.text ? { displayText: steer.displayText } : {}), + ...(steer.attachments.length ? { attachments: steer.attachments } : {}), + ...(steer.contextAttachments.length ? { contextAttachments: steer.contextAttachments } : {}), + ...(steer.metadata ? { metadata: steer.metadata } : {}), + steerId: steer.steerId, + deliveryState, + processed: deliveryState === "processed", + turnId: steer.turnId, + }); + }; + + const markAcceptedCodexSteerProcessed = ( + managed: ManagedChatSession, + runtime: CodexRuntime, + turnId: string | null | undefined, + providerText: string, + ): boolean => { + const normalizedTurnId = turnId?.trim(); + if (!normalizedTurnId) return false; + const accepted = runtime.acceptedSteersByTurnId.get(normalizedTurnId); + if (!accepted?.size) return false; + const normalizedProviderText = providerText.trim(); + if (!normalizedProviderText) return false; + const candidates = Array.from(accepted.values()); + const exactMatches = candidates.filter( + (candidate) => candidate.text.trim() === normalizedProviderText, + ); + const matches = exactMatches.length + ? exactMatches + : candidates.filter((candidate) => { + const candidateText = candidate.text.trim(); + return candidateText.length > 0 + && normalizedProviderText.endsWith(`\n${candidateText}`); + }); + if (matches.length !== 1) return false; + const [matched] = matches; + accepted.delete(matched.steerId); + if (accepted.size === 0) runtime.acceptedSteersByTurnId.delete(normalizedTurnId); + emitAcceptedCodexSteerState(managed, matched, "processed"); + return true; + }; + + const markAcceptedCodexSteersUnprocessed = ( + managed: ManagedChatSession, + runtime: CodexRuntime, + turnId: string | null | undefined, + ): void => { + const normalizedTurnId = turnId?.trim(); + if (!normalizedTurnId) return; + const accepted = runtime.acceptedSteersByTurnId.get(normalizedTurnId); + if (!accepted?.size) return; + runtime.acceptedSteersByTurnId.delete(normalizedTurnId); + for (const steer of accepted.values()) { + emitAcceptedCodexSteerState(managed, steer, "unprocessed"); + } + }; + + const hydrateAcceptedCodexSteers = async ( + managed: ManagedChatSession, + runtime: CodexRuntime, + ): Promise => { + const history = await readTranscriptEnvelopesForSessionIdAsync( + managed.session.id, + 8 * 1024 * 1024, + ).catch(() => ({ envelopes: [] as AgentChatEventEnvelope[] })); + const latestBySteerId = new Map(); + const terminalTurnIds = new Set(); + const resolvedSteerIds = new Set(); + + for (const envelope of history.envelopes) { + const event = envelope.event; + if (event.type === "done" && event.turnId) { + terminalTurnIds.add(event.turnId); + continue; + } + if (event.type === "user_message_resolution") { + resolvedSteerIds.add(event.steerId); + continue; + } + if (event.type !== "user_message" || !event.steerId || !event.turnId) continue; + latestBySteerId.set(event.steerId, { + steer: { + steerId: event.steerId, + turnId: event.turnId, + text: event.text, + displayText: event.displayText ?? event.text, + attachments: event.attachments ?? [], + contextAttachments: event.contextAttachments ?? [], + ...(event.metadata ? { metadata: event.metadata } : {}), + acceptedAt: envelope.timestamp, + }, + state: event.deliveryState ?? "", + }); + } + + if ( + managed.deleted + || managed.closed + || (managed.runtime && managed.runtime !== runtime) + ) { + return; + } + for (const { steer, state } of latestBySteerId.values()) { + if (state !== "accepted" || resolvedSteerIds.has(steer.steerId)) continue; + if (terminalTurnIds.has(steer.turnId)) { + emitAcceptedCodexSteerState(managed, steer, "unprocessed"); + continue; + } + const acceptedForTurn = runtime.acceptedSteersByTurnId.get(steer.turnId) + ?? new Map(); + acceptedForTurn.set(steer.steerId, steer); + runtime.acceptedSteersByTurnId.set(steer.turnId, acceptedForTurn); + } + evictOldestEntries(runtime.acceptedSteersByTurnId, 64); + }; + const handleCodexItemEvent = ( managed: ManagedChatSession, runtime: CodexRuntime, @@ -23100,6 +23467,16 @@ export function createAgentChatService(args: { return completedTurnId; })(); + if (itemType === "userMessage") { + markAcceptedCodexSteerProcessed( + managed, + runtime, + turnId, + codexUserMessageItemText(item), + ); + return; + } + if (itemType === "contextCompaction") { const compactionTurnId = turnId ?? ""; if (eventKind === "started") { @@ -23958,6 +24335,26 @@ export function createAgentChatService(args: { return flags.filter((flag): flag is string => typeof flag === "string"); } + function persistCodexAutomaticRecoveryAttempt( + managed: ManagedChatSession, + ): boolean { + if (managed.codexAutomaticRecoveryAttempted) return true; + managed.codexAutomaticRecoveryAttempted = true; + if (persistChatState(managed, { fsync: true })) return true; + managed.codexAutomaticRecoveryAttempted = false; + return false; + } + + function clearCodexAutomaticRecoveryAttempt( + managed: ManagedChatSession, + ): boolean { + if (!managed.codexAutomaticRecoveryAttempted) return true; + managed.codexAutomaticRecoveryAttempted = false; + if (persistChatState(managed, { fsync: true })) return true; + managed.codexAutomaticRecoveryAttempted = true; + return false; + } + function emitCodexTurnStalled( managed: ManagedChatSession, runtime: CodexRuntime, @@ -23965,6 +24362,8 @@ export function createAgentChatService(args: { turnId: string; reason: Extract["reason"]; message: string; + turnStartedAt?: number; + lastProgressAt?: number; }, ): void { if (runtime.stalledTurnIds.has(args.turnId)) return; @@ -23975,12 +24374,37 @@ export function createAgentChatService(args: { "interrupt_retry_same_thread", "restart_resume_thread", ]; + const detectedAt = nowIso(); + const turnStartedAt = new Date(args.turnStartedAt ?? Date.now()).toISOString(); + const lastProgressAt = new Date(args.lastProgressAt ?? args.turnStartedAt ?? Date.now()).toISOString(); + const automaticRecoveryAttempted = managed.codexAutomaticRecoveryAttempted; + const providerNeutralReason = args.reason === "app_server_state_unknown" + ? "runtime_state_unknown" + : args.reason; logger.warn("agent_chat.codex_turn_stalled", { sessionId: managed.session.id, threadId: managed.session.threadId ?? null, turnId: args.turnId, reason: args.reason, - timeoutMs: CODEX_NO_FIRST_EVENT_WATCHDOG_MS, + timeoutMs: args.reason === "no_progress" + ? CODEX_MID_TURN_INACTIVITY_WATCHDOG_MS + : CODEX_NO_FIRST_EVENT_WATCHDOG_MS, + automaticRecoveryAttempted, + }); + emitChatEvent(managed, { + type: "turn_health", + provider: managed.session.provider, + turnId: args.turnId, + state: "stalled", + reason: providerNeutralReason, + message: args.message, + turnStartedAt, + lastProgressAt, + detectedAt, + recoveryCount: automaticRecoveryAttempted ? 1 : 0, + supportedActions: ["wait", "nudge", "retry_same_runtime", "restart_resume"], + automaticRecoveryAttempted, + sourceSessionId: managed.session.id, }); emitChatEvent(managed, { type: "codex_turn_stalled", @@ -23991,20 +24415,31 @@ export function createAgentChatService(args: { recoveryOptions, sourceSessionId: managed.session.id, ...(managed.session.orchestrationParentSessionId ? { parentSessionId: managed.session.orchestrationParentSessionId } : {}), - }); - emitChatEvent(managed, { - type: "system_notice", - noticeKind: "warning", - severity: "warning", - message: args.message, - turnId: args.turnId, + detectedAt, + turnStartedAt, + lastProgressAt, + automaticRecoveryAttempted, }); const parentSessionId = managed.session.orchestrationParentSessionId; if (parentSessionId && parentSessionId !== managed.session.id) { try { const parent = managedSessions.get(parentSessionId) ?? (sessionService.get(parentSessionId) ? ensureManagedSession(parentSessionId) : null); if (parent) { - const childLabel = managed.preview?.trim() || managed.session.id; + emitChatEvent(parent, { + type: "turn_health", + provider: managed.session.provider, + turnId: args.turnId, + state: "stalled", + reason: providerNeutralReason, + message: args.message, + turnStartedAt, + lastProgressAt, + detectedAt, + recoveryCount: automaticRecoveryAttempted ? 1 : 0, + supportedActions: ["wait", "nudge", "retry_same_runtime", "restart_resume"], + automaticRecoveryAttempted, + sourceSessionId: managed.session.id, + }); emitChatEvent(parent, { type: "codex_turn_stalled", turnId: args.turnId, @@ -24014,13 +24449,10 @@ export function createAgentChatService(args: { recoveryOptions, sourceSessionId: managed.session.id, parentSessionId, - }); - emitChatEvent(parent, { - type: "system_notice", - noticeKind: "warning", - severity: "warning", - message: `Child Codex session '${childLabel}' stalled: ${args.message}`, - turnId: args.turnId, + detectedAt, + turnStartedAt, + lastProgressAt, + automaticRecoveryAttempted, }); persistChatState(parent); } @@ -24035,6 +24467,39 @@ export function createAgentChatService(args: { persistChatState(managed); } + function emitCodexTurnRecovery( + managed: ManagedChatSession, + args: { + turnId: string; + action: "restart_resume_thread"; + state: "recovering" | "recovered" | "failed"; + message: string; + automatic: boolean; + }, + ): void { + const at = nowIso(); + emitChatEvent(managed, { + type: "turn_recovery", + provider: managed.session.provider, + turnId: args.turnId, + action: "restart_resume", + state: args.state, + message: args.message, + automatic: args.automatic, + at, + recoveryCount: 1, + }); + emitChatEvent(managed, { + type: "codex_turn_recovery", + turnId: args.turnId, + action: args.action, + state: args.state, + message: args.message, + automatic: args.automatic, + at, + }); + } + function emitCodexErrorOnce( managed: ManagedChatSession, runtime: CodexRuntime, @@ -24074,6 +24539,7 @@ export function createAgentChatService(args: { fallbackTurnId: string, ): Promise { const turnId = stringOrNull(turn.id) ?? fallbackTurnId; + markAcceptedCodexSteersUnprocessed(managed, runtime, turnId); rememberTerminalCodexTurn(runtime, turnId, managed); runtime.awaitingTurnStart = false; runtime.canAttachResumedTurnStart = false; @@ -24159,6 +24625,9 @@ export function createAgentChatService(args: { managed: ManagedChatSession, runtime: CodexRuntime, turnId: string, + stallReason: "no_output" | "no_progress", + turnStartedAt: number, + lastProgressAt: number, ): Promise { if (runtime.stallReconcileInFlight.has(turnId)) return; runtime.stallReconcileInFlight.add(turnId); @@ -24166,6 +24635,14 @@ export function createAgentChatService(args: { if (!isCodexSilentTurnStillCurrent(managed, runtime, turnId)) { return; } + if (runtime.approvals.size > 0 || managed.localPendingInputs.size > 0) { + logger.info("agent_chat.codex_watchdog_suspended", { + sessionId: managed.session.id, + turnId, + reason: "waiting_on_local_input", + }); + return; + } const threadId = managed.session.threadId?.trim(); let activeFlags: string[] = []; let stateProbeFailed = false; @@ -24231,6 +24708,7 @@ export function createAgentChatService(args: { }); persistChatState(managed); scheduleCodexNoFirstEventWatchdog(managed, runtime, turnId); + markCodexTurnProgress(managed, runtime, turnId); return; } } @@ -24240,26 +24718,66 @@ export function createAgentChatService(args: { } if (activeFlags.includes("waitingOnApproval")) { - emitCodexTurnStalled(managed, runtime, { + logger.info("agent_chat.codex_watchdog_suspended", { + sessionId: managed.session.id, turnId, reason: "waiting_on_approval", - message: "Codex is waiting for an approval decision, but ADE did not receive a visible approval item yet. You can keep waiting, send a status nudge, or interrupt and retry this thread.", }); return; } if (activeFlags.includes("waitingOnUserInput")) { - emitCodexTurnStalled(managed, runtime, { + logger.info("agent_chat.codex_watchdog_suspended", { + sessionId: managed.session.id, turnId, reason: "waiting_on_input", - message: "Codex is waiting for user input, but ADE did not receive a visible input request yet. You can answer with a status nudge, keep waiting, or interrupt and retry this thread.", }); return; } + if ( + stallReason === "no_output" + && !managed.codexAutomaticRecoveryAttempted + ) { + if (persistCodexAutomaticRecoveryAttempt(managed)) { + emitCodexTurnRecovery(managed, { + turnId, + action: "restart_resume_thread", + state: "recovering", + message: "No Codex output arrived. ADE is restarting the app-server and resuming this thread once.", + automatic: true, + }); + try { + await recoverCodexTurn({ + sessionId: managed.session.id, + turnId, + action: "restart_resume_thread", + }, { automatic: true }); + return; + } catch (error) { + logger.warn("agent_chat.codex_automatic_recovery_failed", { + sessionId: managed.session.id, + turnId, + error: error instanceof Error ? error.message : String(error), + }); + } + } else { + logger.warn("agent_chat.codex_automatic_recovery_marker_persist_failed", { + sessionId: managed.session.id, + turnId, + }); + } + } + emitCodexTurnStalled(managed, runtime, { turnId, - reason: threadId && !stateProbeFailed ? "no_output" : "app_server_state_unknown", - message: "Codex accepted this turn but has not streamed model or tool output yet. You can keep waiting, send a status nudge, or interrupt and retry this thread if it stays stalled.", + reason: threadId && !stateProbeFailed ? stallReason : "app_server_state_unknown", + message: stallReason === "no_progress" + ? "Codex has not produced model or tool activity for 10 minutes. ADE left the turn running so long work is not destroyed." + : managed.codexAutomaticRecoveryAttempted + ? "Codex still has not produced output after ADE's one automatic restart. Restart and resume again, or keep waiting." + : "Codex has not produced output. ADE could not safely record an automatic recovery attempt, so it left the turn running.", + turnStartedAt, + lastProgressAt, }); } finally { runtime.stallReconcileInFlight.delete(turnId); @@ -24308,11 +24826,13 @@ export function createAgentChatService(args: { for (const [itemId, pending] of runtime.approvals) { if (String(pending.requestId) !== String(requestId)) continue; runtime.approvals.delete(itemId); + const resolvedTurnId = pending.request?.turnId ?? turnIdFromParams; emitPendingInputResolved(managed, { itemId, decision: "cancel", - turnId: pending.request?.turnId ?? turnIdFromParams, + turnId: resolvedTurnId, }); + resumeCodexTurnWatchdogAfterInput(managed, runtime, resolvedTurnId); persistChatState(managed); break; } @@ -24442,7 +24962,7 @@ export function createAgentChatService(args: { } if (isCodexUsefulProgressNotification(method, params)) { - markCodexTurnProgress(runtime, turnIdFromParams); + markCodexTurnProgress(managed, runtime, turnIdFromParams); } if (method === "turn/started") { @@ -24511,6 +25031,7 @@ export function createAgentChatService(args: { return; } const turnId = resolvedTurnId ?? randomUUID(); + markAcceptedCodexSteersUnprocessed(managed, runtime, turnId); rememberTerminalCodexTurn(runtime, turnId, managed); runtime.awaitingTurnStart = false; runtime.canAttachResumedTurnStart = false; @@ -24808,6 +25329,7 @@ export function createAgentChatService(args: { return; } const turnId = resolvedAbortTurnId ?? randomUUID(); + markAcceptedCodexSteersUnprocessed(managed, runtime, turnId); rememberInterruptedCodexTurn(runtime, turnId); rememberTerminalCodexTurn(runtime, turnId, managed); runtime.awaitingTurnStart = false; @@ -24915,11 +25437,18 @@ export function createAgentChatService(args: { if (method === "turn/moderationMetadata") { const metadata = normalizeCodexModerationMetadataPayload(params); + if (!metadata.metadata || Object.keys(metadata.metadata).length === 0) { + return; + } + const turnId = metadata.turnId ?? turnIdFromParams ?? runtime.activeTurnId ?? undefined; emitChatEvent(managed, { type: "codex_moderation_metadata", metadata, - turnId: metadata.turnId ?? turnIdFromParams ?? runtime.activeTurnId ?? undefined, + turnId, }); + const diagnosticState = codexTurnDiagnosticState(runtime, turnId); + diagnosticState.moderationChecks += 1; + emitCodexTurnDiagnostics(managed, runtime, turnId); return; } @@ -25199,6 +25728,9 @@ export function createAgentChatService(args: { stalledTurnIds: new Set(), stallReconcileInFlight: new Set(), mcpStartupNoticeKeys: new Set(), + acceptedSteersByTurnId: new Map>(), + acceptedSteersHydrationReady: Promise.resolve(), + turnDiagnosticStateByTurnId: new Map(), pendingPlanFollowups: [], pendingSteers: [], slashCommands: [], @@ -25431,6 +25963,15 @@ export function createAgentChatService(args: { ]).then(() => undefined); runtime.notify("initialized"); + runtime.acceptedSteersHydrationReady = hydrateAcceptedCodexSteers( + managed, + runtime, + ).catch((error) => { + logger.warn("agent_chat.codex_accepted_steer_hydration_failed", { + sessionId: managed.session.id, + error: error instanceof Error ? error.message : String(error), + }); + }); return runtime; }; @@ -27041,6 +27582,8 @@ export function createAgentChatService(args: { runtimeInvalidated: false, claudeRateLimitWarningEmitted: false, codexTerminalTurnIds: new Set(), + codexAutomaticRecoveryAttempted: false, + unprocessedMessageResolutionReceipts: new Map(), todoItems: [], activeAssistantMessageId: null, previewTextBuffer: null, @@ -27707,6 +28250,8 @@ export function createAgentChatService(args: { runtimeInvalidated: false, claudeRateLimitWarningEmitted: false, codexTerminalTurnIds: new Set(), + codexAutomaticRecoveryAttempted: false, + unprocessedMessageResolutionReceipts: new Map(), todoItems: [], activeAssistantMessageId: null, lastActivitySignature: null, @@ -33436,6 +33981,7 @@ export function createAgentChatService(args: { awaitBackendDispatch?: boolean; onBackendDispatched?: () => void; preparedMessage?: PreparedSendMessage; + automaticRecovery?: boolean; routeActiveToSteer: true; }, ): Promise; @@ -33446,6 +33992,7 @@ export function createAgentChatService(args: { awaitBackendDispatch?: boolean; onBackendDispatched?: () => void; preparedMessage?: PreparedSendMessage; + automaticRecovery?: boolean; routeActiveToSteer?: false; }, ): Promise; @@ -33456,6 +34003,7 @@ export function createAgentChatService(args: { awaitBackendDispatch?: boolean; onBackendDispatched?: () => void; preparedMessage?: PreparedSendMessage; + automaticRecovery?: boolean; routeActiveToSteer?: boolean; }, ): Promise { @@ -33506,6 +34054,13 @@ export function createAgentChatService(args: { if (await maybeHandleClaudeOutputStyleSlashCommand(args)) return; const prepared = options?.preparedMessage ?? prepareSendMessage(args); if (!prepared) return; + if ( + prepared.managed.session.provider === "codex" + && !options?.automaticRecovery + && !clearCodexAutomaticRecoveryAttempt(prepared.managed) + ) { + throw new Error("ADE could not save the new Codex turn state. Retry before closing this chat."); + } clearUserTurnMarkers(); prepared.managed.lastActivityTimestamp = Date.now(); let rejectDispatch: ((error: Error) => void) | null = null; @@ -33893,6 +34448,7 @@ export function createAgentChatService(args: { if (managed.session.provider === "codex") { const runtime = await ensureCodexSessionRuntime(managed); await runtime.collaborationModesReady?.catch(() => {}); + await runtime.acceptedSteersHydrationReady; const preparedSteer = prepareSendMessage({ sessionId, @@ -33969,16 +34525,22 @@ export function createAgentChatService(args: { deliveredTurnId = mismatch.foundTurnId; await steerActiveTurn(deliveredTurnId); } - emitChatEvent(managed, { - type: "user_message", - text: preparedSteer.visibleText, - ...(preparedSteer.attachments.length ? { attachments: preparedSteer.attachments } : {}), - ...(preparedSteer.contextAttachments.length ? { contextAttachments: preparedSteer.contextAttachments } : {}), - ...(preparedSteer.metadata ? { metadata: preparedSteer.metadata } : {}), + const acceptedSteer: AcceptedCodexSteer = { steerId, - deliveryState: "delivered", turnId: deliveredTurnId, - }); + text: preparedSteer.visibleText, + displayText: preparedSteer.visibleText, + attachments: preparedSteer.attachments, + contextAttachments: preparedSteer.contextAttachments, + ...(preparedSteer.metadata ? { metadata: preparedSteer.metadata } : {}), + acceptedAt: nowIso(), + }; + const acceptedForTurn = runtime.acceptedSteersByTurnId.get(deliveredTurnId) + ?? new Map(); + acceptedForTurn.set(steerId, acceptedSteer); + runtime.acceptedSteersByTurnId.set(deliveredTurnId, acceptedForTurn); + evictOldestEntries(runtime.acceptedSteersByTurnId, 64); + emitAcceptedCodexSteerState(managed, acceptedSteer, "accepted"); return { steerId, queued: false }; } @@ -35281,7 +35843,9 @@ export function createAgentChatService(args: { sessionId, turnId, action, - }: AgentChatRecoverCodexTurnArgs): Promise => { + }: AgentChatRecoverCodexTurnArgs, internal?: { + automatic?: boolean; + }): Promise => { const normalizedTurnId = turnId.trim(); if (!normalizedTurnId) throw new Error("Codex recovery requires a turn id."); const managed = ensureManagedSession(sessionId); @@ -35302,6 +35866,11 @@ export function createAgentChatService(args: { try { if (action === "wait") { + // The user explicitly chose to keep this exact turn alive. Do not + // surprise them with ADE's automatic restart on the next watch cycle. + if (!persistCodexAutomaticRecoveryAttempt(managed)) { + throw new Error("ADE could not save the recovery choice. Retry before closing this chat."); + } scheduleCodexNoFirstEventWatchdog(managed, runtime, normalizedTurnId); emitChatEvent(managed, { type: "system_notice", @@ -35309,7 +35878,6 @@ export function createAgentChatService(args: { message: "Continuing to wait for Codex output.", turnId: normalizedTurnId, }); - persistChatState(managed); return { action, turnId: normalizedTurnId, status: "waiting" }; } @@ -35349,17 +35917,227 @@ export function createAgentChatService(args: { text: action === "restart_resume_thread" ? "Resume the interrupted work from the user's last request. Re-check the current workspace state, continue safely, and report progress." : "Retry the interrupted work from the user's last request in this thread. Re-check the current workspace state, continue safely, and report progress.", - }, { awaitDispatch: true }); + }, { awaitDispatch: true, automaticRecovery: internal?.automatic === true }); + if (action === "restart_resume_thread") { + emitCodexTurnRecovery(managed, { + turnId: normalizedTurnId, + action, + state: "recovered", + message: internal?.automatic + ? "ADE restarted the Codex app-server and resumed the thread." + : "Codex restarted and the thread resumed.", + automatic: internal?.automatic === true, + }); + persistChatState(managed); + } return { action, turnId: normalizedTurnId, status: action === "restart_resume_thread" ? "resumed" : "retrying", }; + } catch (error) { + if (action === "restart_resume_thread") { + emitCodexTurnRecovery(managed, { + turnId: normalizedTurnId, + action, + state: "failed", + message: error instanceof Error ? error.message : String(error), + automatic: internal?.automatic === true, + }); + persistChatState(managed); + } + throw error; } finally { codexRecoveryInFlight.delete(recoveryKey); } }; + const recoverTurn = async ( + args: AgentChatRecoverTurnArgs, + ): Promise => { + const legacyAction = { + wait: "wait", + nudge: "steer", + retry_same_runtime: "interrupt_retry_same_thread", + restart_resume: "restart_resume_thread", + }[args.action] as AgentChatRecoverCodexTurnArgs["action"]; + const result = await recoverCodexTurn({ + sessionId: args.sessionId, + turnId: args.turnId, + action: legacyAction, + }); + return { + action: args.action, + turnId: result.turnId, + status: result.status, + }; + }; + + const assertRecoveryTargetOwned = async (args: { + sessionId: string; + turnId?: string; + steerId?: string; + }): Promise => { + const managed = ensureManagedSession(args.sessionId); + if (args.turnId) { + const activeTurnId = managed.runtime?.kind === "codex" + ? managed.runtime.activeTurnId ?? managed.runtime.startedTurnId + : null; + if (activeTurnId !== args.turnId) { + throw new Error("This recovery turn does not belong to the active chat session."); + } + } + if (args.steerId) { + const history = await readTranscriptEnvelopesForSessionIdAsync( + managed.session.id, + 8 * 1024 * 1024, + ); + const belongsToSession = history.envelopes.some((envelope) => + envelope.event.type === "user_message" + && envelope.event.steerId === args.steerId + ); + if (!belongsToSession) { + throw new Error("This unprocessed message does not belong to the chat session."); + } + } + }; + + const resolveUnprocessedMessage = async ( + args: AgentChatResolveUnprocessedMessageArgs, + ): Promise => { + const sessionId = args.sessionId.trim(); + const steerId = args.steerId.trim(); + if (!sessionId || !steerId) { + throw new Error("Resolving an unprocessed message requires a session and steer id."); + } + // Run-next and dismiss are mutually exclusive terminal actions for the + // same durable message. Serialize by steer id so two clients cannot race + // different choices and accidentally dispatch after a dismissal. + const resolutionKey = `${sessionId}:${steerId}`; + const existing = unprocessedMessageResolutionInFlight.get(resolutionKey); + if (existing) return await existing; + + const resolution = (async (): Promise => { + const managed = ensureManagedSession(sessionId); + const history = await readTranscriptEnvelopesForSessionIdAsync( + sessionId, + 8 * 1024 * 1024, + ); + let original: Extract | null = null; + let alreadyCompleted: Extract | null = null; + + for (const envelope of history.envelopes) { + const event = envelope.event; + if (event.type === "user_message" && event.steerId === steerId) { + original = event; + } + if ( + event.type === "user_message_resolution" + && event.steerId === steerId + ) { + alreadyCompleted = event; + } + } + + const persistedReceipt = managed.unprocessedMessageResolutionReceipts.get(steerId) ?? null; + const existingResolution = alreadyCompleted ?? (persistedReceipt + ? { + type: "user_message_resolution" as const, + ...persistedReceipt, + } + : null); + if (existingResolution) { + if (!alreadyCompleted) { + // The provider acknowledgement was fsynced before the asynchronous + // transcript append. Rebuild the missing terminal row after a crash + // without dispatching the replacement message a second time. + emitChatEvent(managed, existingResolution); + } + return { + steerId, + action: existingResolution.action, + status: "already_completed", + ...(existingResolution.replacementMessageId + ? { replacementMessageId: existingResolution.replacementMessageId } + : {}), + }; + } + + if (!original || original.deliveryState !== "unprocessed") { + throw new Error("This follow-up is no longer waiting to be resolved."); + } + + if (args.action === "dismiss") { + const completedResolution = { + type: "user_message_resolution", + steerId, + action: "dismiss", + state: "completed", + resolvedAt: nowIso(), + } as const; + persistUnprocessedMessageResolutionReceipt(managed, completedResolution); + emitChatEvent(managed, completedResolution); + return { steerId, action: args.action, status: "completed" }; + } + + const runtime = managed.runtime; + const activeTurnId = runtime?.kind === "codex" + ? runtime.activeTurnId ?? runtime.startedTurnId + : null; + if (managed.session.status === "active" || activeTurnId) { + throw new Error("A turn is already active. Wait for it to finish, then run this message."); + } + + const dispatchedReplacementMessageId = randomUUID(); + let dispatchReceiptWritten = false; + await sendMessage({ + sessionId, + text: original.text, + ...(original.displayText ? { displayText: original.displayText } : {}), + ...(original.attachments?.length ? { attachments: original.attachments } : {}), + ...(original.contextAttachments?.length ? { contextAttachments: original.contextAttachments } : {}), + metadata: { + ...(original.metadata ?? {}), + replayedFromUnprocessedSteer: { + sourceSteerId: steerId, + action: "run_next", + replacementMessageId: dispatchedReplacementMessageId, + }, + }, + }, { + awaitBackendDispatch: true, + onBackendDispatched: () => { + const replayResolutionReceipt = { + type: "user_message_resolution", + steerId, + action: "run_next", + state: "completed", + resolvedAt: nowIso(), + replacementMessageId: dispatchedReplacementMessageId, + } as const; + persistUnprocessedMessageResolutionReceipt(managed, replayResolutionReceipt); + emitChatEvent(managed, replayResolutionReceipt); + dispatchReceiptWritten = true; + }, + }); + if (!dispatchReceiptWritten) { + throw new Error("The provider did not accept this replay. Run the message again when the chat is ready."); + } + return { + steerId, + action: args.action, + status: "completed", + replacementMessageId: dispatchedReplacementMessageId, + }; + })(); + unprocessedMessageResolutionInFlight.set(resolutionKey, resolution); + try { + return await resolution; + } finally { + unprocessedMessageResolutionInFlight.delete(resolutionKey); + } + }; + const latestLivePendingInputItemId = (managed: ManagedChatSession | null | undefined): string | null => { if (!managed) return null; const localPending = managed.localPendingInputs.keys().next().value; @@ -36445,6 +37223,13 @@ export function createAgentChatService(args: { turnId: localPending.request.turnId ?? null, }); localPending.resolve({ decision: resolvedDecision, answers, responseText }); + if (managed.runtime?.kind === "codex") { + resumeCodexTurnWatchdogAfterInput( + managed, + managed.runtime, + localPending.request.turnId, + ); + } return; } @@ -36487,6 +37272,19 @@ export function createAgentChatService(args: { throw new Error("Codex app-server connection is unavailable. Retry after the session reconnects."); } }; + const completeCodexPendingInput = (): void => { + runtime.approvals.delete(itemId); + emitPendingInputResolved(managed, { + itemId, + decision: resolvedDecision, + turnId: pending.request?.turnId ?? null, + }); + resumeCodexTurnWatchdogAfterInput( + managed, + runtime, + pending.request?.turnId, + ); + }; // Plan approval is created locally (not a JSON-RPC server request). // The planning turn may still be running when the user decides, so we @@ -36524,12 +37322,7 @@ export function createAgentChatService(args: { ? { persist: "always" } : null, }); - runtime.approvals.delete(itemId); - emitPendingInputResolved(managed, { - itemId, - decision: resolvedDecision, - turnId: pending.request?.turnId ?? null, - }); + completeCodexPendingInput(); return; } @@ -36540,12 +37333,7 @@ export function createAgentChatService(args: { permissions: approved ? (pending.permissions ?? {}) : {}, scope: resolvedDecision === "accept_for_session" ? "session" : "turn", }); - runtime.approvals.delete(itemId); - emitPendingInputResolved(managed, { - itemId, - decision: resolvedDecision, - turnId: pending.request?.turnId ?? null, - }); + completeCodexPendingInput(); return; } if (pending.kind === "structured_question") { @@ -36555,12 +37343,7 @@ export function createAgentChatService(args: { // interrupting the surrounding turn. ensureWritable(); runtime.sendResponse(pending.requestId, { answers: {} }); - runtime.approvals.delete(itemId); - emitPendingInputResolved(managed, { - itemId, - decision: resolvedDecision, - turnId: pending.request?.turnId ?? null, - }); + completeCodexPendingInput(); return; } const normalizedAnswers = normalizePendingInputAnswers(pending.request, answers, responseText); @@ -36570,24 +37353,14 @@ export function createAgentChatService(args: { Object.entries(normalizedAnswers).map(([questionId, values]) => [questionId, { answers: values }]), ), }); - runtime.approvals.delete(itemId); - emitPendingInputResolved(managed, { - itemId, - decision: resolvedDecision, - turnId: pending.request?.turnId ?? null, - }); + completeCodexPendingInput(); return; } const mapped = mapApprovalDecisionForCodex(resolvedDecision); ensureWritable(); runtime.sendResponse(pending.requestId, { decision: mapped }); - runtime.approvals.delete(itemId); - emitPendingInputResolved(managed, { - itemId, - decision: resolvedDecision, - turnId: pending.request?.turnId ?? null, - }); + completeCodexPendingInput(); return; } @@ -37525,7 +38298,6 @@ export function createAgentChatService(args: { if (!trimmedSessionId.length) { throw new Error("Chat session id is required."); } - const existing = sessionService.get(trimmedSessionId); if (!existing) { throw new Error(`Chat session '${trimmedSessionId}' was not found.`); @@ -40636,7 +41408,10 @@ export function createAgentChatService(args: { dispatchSteer, cancelDispatchedSteer, interrupt, + recoverTurn, recoverCodexTurn, + assertRecoveryTargetOwned, + resolveUnprocessedMessage, recoverContinuity, resumeSession, listSessions, diff --git a/apps/desktop/src/main/services/ipc/registerIpc.ts b/apps/desktop/src/main/services/ipc/registerIpc.ts index 11152e25a..21930b366 100644 --- a/apps/desktop/src/main/services/ipc/registerIpc.ts +++ b/apps/desktop/src/main/services/ipc/registerIpc.ts @@ -21,6 +21,7 @@ import { encodeCodedErrorMessage, parseCodedErrorMessage } from "../../../shared import { areAutomationsEnabledForPackagedState } from "../../../shared/automationAvailability"; import { findRecentProjectForRepo } from "../projects/repoProjectResolver"; import { getModelById } from "../../../shared/modelRegistry"; +import { isAgentChatTurnRecoveryAction } from "../../../shared/types/chat"; import { appendEvent as perfAppend, isRunActive as isPerfRunActive } from "../perf/perfLog"; import { buildPrAiResolutionContextKey, isAdeUsageRangePreset, isAdeUsageScope } from "../../../shared/types"; import { detectCliAuthStatuses } from "../ai/authDetector"; @@ -323,8 +324,12 @@ import type { AgentChatPrepareCrossMachineHandoffResult, AgentChatValidateCrossMachineSourceArgs, AgentChatInterruptArgs, + AgentChatRecoverTurnArgs, + AgentChatRecoverTurnResult, AgentChatRecoverCodexTurnArgs, AgentChatRecoverCodexTurnResult, + AgentChatResolveUnprocessedMessageArgs, + AgentChatResolveUnprocessedMessageResult, AgentChatRecoverContinuityArgs, AgentChatContinuityRecoveryResult, AgentChatListArgs, @@ -6134,6 +6139,68 @@ export function registerIpc({ return { sessionId: record.sessionId.trim(), steerId: record.steerId.trim() }; }; + const parseAgentChatRecoveryIdentifier = ( + value: unknown, + label: "sessionId" | "turnId" | "steerId", + ): string => { + const normalized = typeof value === "string" ? value.trim() : ""; + if ( + !normalized + || normalized.length > 256 + || !/^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(normalized) + ) { + throw new Error(`Agent chat recovery ${label} is malformed`); + } + return normalized; + }; + + const parseAgentChatRecoverTurnArgs = ( + value: unknown, + ): AgentChatRecoverTurnArgs => { + const record = requireRecord(value, "Agent chat recovery request"); + if (!isAgentChatTurnRecoveryAction(record.action)) { + throw new Error("Agent chat recovery action is unsupported"); + } + return { + sessionId: parseAgentChatRecoveryIdentifier(record.sessionId, "sessionId"), + turnId: parseAgentChatRecoveryIdentifier(record.turnId, "turnId"), + action: record.action, + }; + }; + + const parseAgentChatRecoverCodexTurnArgs = ( + value: unknown, + ): AgentChatRecoverCodexTurnArgs => { + const record = requireRecord(value, "Agent chat Codex recovery request"); + if ( + record.action !== "wait" + && record.action !== "steer" + && record.action !== "interrupt_retry_same_thread" + && record.action !== "restart_resume_thread" + ) { + throw new Error("Agent chat Codex recovery action is unsupported"); + } + return { + sessionId: parseAgentChatRecoveryIdentifier(record.sessionId, "sessionId"), + turnId: parseAgentChatRecoveryIdentifier(record.turnId, "turnId"), + action: record.action, + }; + }; + + const parseAgentChatResolveUnprocessedMessageArgs = ( + value: unknown, + ): AgentChatResolveUnprocessedMessageArgs => { + const record = requireRecord(value, "Agent chat unprocessed message request"); + if (record.action !== "run_next" && record.action !== "dismiss") { + throw new Error("Agent chat unprocessed message action is unsupported"); + } + return { + sessionId: parseAgentChatRecoveryIdentifier(record.sessionId, "sessionId"), + steerId: parseAgentChatRecoveryIdentifier(record.steerId, "steerId"), + action: record.action, + }; + }; + const parseAgentChatSuggestLaneNameArgs = (value: unknown): AgentChatSuggestLaneNameArgs => { const record = requireRecord(value, "Agent chat suggest lane name request"); if (typeof record.prompt !== "string" || !record.prompt.trim()) { @@ -6755,12 +6822,43 @@ export function registerIpc({ await ctx.agentChatService.interrupt(arg); }); + ipcMain.handle(IPC.agentChatRecoverTurn, async ( + _event, + arg: unknown, + ): Promise => { + const ctx = ensureAgentChatContext(); + const request = parseAgentChatRecoverTurnArgs(arg); + await ctx.agentChatService.assertRecoveryTargetOwned({ + sessionId: request.sessionId, + turnId: request.turnId, + }); + return await ctx.agentChatService.recoverTurn(request); + }); + ipcMain.handle(IPC.agentChatRecoverCodexTurn, async ( _event, - arg: AgentChatRecoverCodexTurnArgs, + arg: unknown, ): Promise => { const ctx = ensureAgentChatContext(); - return await ctx.agentChatService.recoverCodexTurn(arg); + const request = parseAgentChatRecoverCodexTurnArgs(arg); + await ctx.agentChatService.assertRecoveryTargetOwned({ + sessionId: request.sessionId, + turnId: request.turnId, + }); + return await ctx.agentChatService.recoverCodexTurn(request); + }); + + ipcMain.handle(IPC.agentChatResolveUnprocessedMessage, async ( + _event, + arg: unknown, + ): Promise => { + const ctx = ensureAgentChatContext(); + const request = parseAgentChatResolveUnprocessedMessageArgs(arg); + await ctx.agentChatService.assertRecoveryTargetOwned({ + sessionId: request.sessionId, + steerId: request.steerId, + }); + return await ctx.agentChatService.resolveUnprocessedMessage(request); }); ipcMain.handle(IPC.agentChatApprove, async (_event, arg: AgentChatApproveArgs): Promise => { diff --git a/apps/desktop/src/main/services/ipc/runtimeBridge.test.ts b/apps/desktop/src/main/services/ipc/runtimeBridge.test.ts index 10dfba45d..fcbd3e719 100644 --- a/apps/desktop/src/main/services/ipc/runtimeBridge.test.ts +++ b/apps/desktop/src/main/services/ipc/runtimeBridge.test.ts @@ -1393,6 +1393,88 @@ describe("registerIpc sync bridge", () => { vi.useRealTimers(); }); + it("validates recovery identifiers and target ownership before mutating chat state", async () => { + const assertRecoveryTargetOwned = vi.fn(async (args: { + sessionId: string; + turnId?: string; + steerId?: string; + }) => { + if ( + args.sessionId !== "chat-1" + || (args.turnId !== undefined && args.turnId !== "turn-1") + || (args.steerId !== undefined && args.steerId !== "steer-1") + ) { + throw new Error("Recovery target does not belong to this project chat."); + } + }); + const recoverTurn = vi.fn(async (args) => ({ + action: args.action, + turnId: args.turnId, + status: "waiting", + })); + const recoverCodexTurn = vi.fn(async (args) => ({ + action: args.action, + turnId: args.turnId, + status: "waiting", + })); + const resolveUnprocessedMessage = vi.fn(async (args) => ({ + steerId: args.steerId, + action: args.action, + status: "completed", + })); + registerIpc({ + getCtx: () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn() }, + agentChatService: { + assertRecoveryTargetOwned, + recoverTurn, + recoverCodexTurn, + resolveUnprocessedMessage, + }, + }) as any, + switchProjectFromDialog: vi.fn(), + closeCurrentProject: vi.fn(), + closeProjectByPath: vi.fn(), + globalStatePath: "/tmp/ade-state.json", + }); + + await expect(ipcHandlers.get(IPC.agentChatRecoverTurn)?.(eventForSender(), { + sessionId: "chat-1", + turnId: "turn-1", + action: "wait", + })).resolves.toMatchObject({ action: "wait", turnId: "turn-1" }); + await expect(ipcHandlers.get(IPC.agentChatRecoverCodexTurn)?.(eventForSender(), { + sessionId: "chat-1", + turnId: "turn-1", + action: "steer", + })).resolves.toMatchObject({ action: "steer", turnId: "turn-1" }); + await expect(ipcHandlers.get(IPC.agentChatResolveUnprocessedMessage)?.(eventForSender(), { + sessionId: "chat-1", + steerId: "steer-1", + action: "dismiss", + })).resolves.toMatchObject({ action: "dismiss", steerId: "steer-1" }); + + await expect(ipcHandlers.get(IPC.agentChatRecoverTurn)?.(eventForSender(), { + sessionId: "../foreign-chat", + turnId: "turn-1", + action: "wait", + })).rejects.toThrow(/sessionId is malformed/i); + await expect(ipcHandlers.get(IPC.agentChatRecoverTurn)?.(eventForSender(), { + sessionId: "chat-1", + turnId: "turn-from-another-chat", + action: "wait", + })).rejects.toThrow(/does not belong/i); + await expect(ipcHandlers.get(IPC.agentChatResolveUnprocessedMessage)?.(eventForSender(), { + sessionId: "chat-1", + steerId: "steer-from-another-chat", + action: "run_next", + })).rejects.toThrow(/does not belong/i); + + expect(recoverTurn).toHaveBeenCalledTimes(1); + expect(recoverCodexTurn).toHaveBeenCalledTimes(1); + expect(resolveUnprocessedMessage).toHaveBeenCalledTimes(1); + }); + it("preserves and validates exact lookup and launch overrides across external-session IPC parsing", async () => { const list = vi.fn(async () => []); const importExternalSession = vi.fn(async () => ({ diff --git a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeBootstrap.test.ts b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeBootstrap.test.ts index c401487cf..a2b4872eb 100644 --- a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeBootstrap.test.ts +++ b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeBootstrap.test.ts @@ -27,6 +27,7 @@ import { bootstrapPairedRuntime } from "./pairedRuntimeBootstrap"; import { PairedRuntimeCompatibilityError, PairedRuntimeRelayAuthRequiredError, + PairedRuntimeTransportUnavailableError, } from "./pairedRuntimeErrors"; const credentials: DesktopPairedMachineCredentials = { @@ -267,4 +268,93 @@ describe("bootstrapPairedRuntime", () => { })); expect(result.result.route?.kind).toBe("relay"); }); + + it("never sends a different account's bearer to the paired machine Relay", async () => { + const relayCredentials = { + ...credentials, + accountOwnerUserId: "account-a", + endpoints: ["wss://relay.example/connect/host-1"], + relayUrl: "wss://relay.example/connect/host-1", + }; + const openTransport = vi.fn(async () => transport()); + + await expect(bootstrapPairedRuntime({ + target, + registry: { update: vi.fn(() => target) } as any, + pairedStore: { + getForReference: vi.fn(() => relayCredentials), + save: vi.fn(), + markEndpointSucceeded: vi.fn(), + } as any, + appVersion: "1.0.0", + options: { + openTransport, + getAccountRelayProof: vi.fn(async () => ({ + userId: "account-b", + token: "must-not-leave-this-process", + })), + }, + })).rejects.toThrow(/same ADE account/i); + + expect(openTransport).not.toHaveBeenCalled(); + }); + + it("tries direct routes before Relay and returns bounded privacy-safe diagnostics", async () => { + const routedCredentials = { + ...credentials, + accountOwnerUserId: "account-a", + endpoints: [ + "wss://relay.example/connect/private-machine-key?secret=query", + "ws://studio.example.ts.net:8787", + "ws://studio.local:8787", + ], + relayUrl: "wss://relay.example/connect/private-machine-key?secret=query", + }; + const openTransport = vi.fn(async (_args: { endpoint?: string }) => { + throw new Error("socket failed with secret diagnostic-token"); + }); + let captured: unknown; + + try { + await bootstrapPairedRuntime({ + target, + registry: { update: vi.fn(() => target) } as any, + pairedStore: { + getForReference: vi.fn(() => routedCredentials), + save: vi.fn(), + markEndpointSucceeded: vi.fn(), + } as any, + appVersion: "1.0.0", + options: { + openTransport, + getAccountRelayProof: vi.fn(async () => ({ + userId: "account-a", + token: "ephemeral-account-token", + })), + }, + }); + } catch (error) { + captured = error; + } + + expect(openTransport.mock.calls.map(([args]) => args.endpoint)).toEqual([ + "ws://studio.local:8787/", + "ws://studio.example.ts.net:8787/", + "wss://relay.example/connect/private-machine-key?secret=query", + ]); + expect(captured).toBeInstanceOf(PairedRuntimeTransportUnavailableError); + const unavailable = captured as PairedRuntimeTransportUnavailableError; + expect(unavailable.diagnostic).toMatchObject({ + correlationId: expect.any(String), + attempts: [ + { kind: "lan", host: "studio.local:8787", outcome: "failed", failure: "unreachable" }, + { kind: "tailnet", host: "studio.example.ts.net:8787", outcome: "failed", failure: "unreachable" }, + { kind: "relay", host: "relay.example", outcome: "failed", failure: "unreachable" }, + ], + }); + expect(unavailable.diagnostic?.attempts.length).toBeLessThanOrEqual(8); + expect(unavailable.message).not.toContain("private-machine-key"); + expect(unavailable.message).not.toContain("diagnostic-token"); + expect(JSON.stringify(unavailable.diagnostic)).not.toContain("ephemeral-account-token"); + }); }); diff --git a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeBootstrap.ts b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeBootstrap.ts index 25001a462..638790a6a 100644 --- a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeBootstrap.ts +++ b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeBootstrap.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import type { DesktopPairedMachineCredentials, } from "../../../shared/types/pairedRuntime"; @@ -20,7 +21,14 @@ import { openSyncRuntimeTransport, type SyncRuntimeTransport, } from "./syncRuntimeTransport"; -import { buildPairedEndpointCandidates } from "./pairedRuntimeRoutes"; +import { + buildPairedEndpointCandidates, + classifyPairedRuntimeFailure, + createRouteAttemptRecorder, + MAX_ROUTE_ATTEMPTS, + orderPairedCandidates, + pairedRuntimeRouteHost, +} from "./pairedRuntimeRoutes"; import { PairedRuntimeCompatibilityError, PairedRuntimeRelayAuthRequiredError, @@ -108,8 +116,24 @@ export async function bootstrapPairedRuntime(args: { const openTransport = args.options?.openTransport ?? openSyncRuntimeTransport; const routeErrors: string[] = []; + let omittedRouteErrorCount = 0; + const addRouteError = (message: string): void => { + if (routeErrors.length < MAX_ROUTE_ATTEMPTS) { + routeErrors.push(message); + return; + } + omittedRouteErrorCount += 1; + }; + const correlationId = randomUUID(); + const attemptRecorder = createRouteAttemptRecorder(); + const { attempts, record: recordAttempt } = attemptRecorder; let relayAuthError: PairedRuntimeRelayAuthRequiredError | null = null; - for (const candidate of candidates) { + // Keep the phases explicit even if a future candidate-builder change + // accidentally reorders endpoints. + const orderedCandidates = orderPairedCandidates(candidates); + for (const candidate of orderedCandidates) { + const attemptStartedAt = Date.now(); + const safeHost = pairedRuntimeRouteHost(candidate.endpoint); let relayAccountToken: string | null = null; let relayAccountOwnerUserId: string | null = null; if (candidate.kind === "relay") { @@ -121,6 +145,14 @@ export async function bootstrapPairedRuntime(args: { "Sign in to ADE to connect through ADE Relay.", error, ); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "skipped", + failure: "authentication", + }); continue; } if ( @@ -130,6 +162,29 @@ export async function bootstrapPairedRuntime(args: { relayAuthError = new PairedRuntimeRelayAuthRequiredError( "Sign in to ADE to connect through ADE Relay.", ); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "skipped", + failure: "authentication", + }); + continue; + } + const expectedOwnerUserId = credentials.accountOwnerUserId?.trim() ?? ""; + if (expectedOwnerUserId && proof.userId.trim() !== expectedOwnerUserId) { + relayAuthError = new PairedRuntimeRelayAuthRequiredError( + "Sign in with the same ADE account as this machine to connect through ADE Relay.", + ); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "skipped", + failure: "authentication", + }); continue; } relayAccountToken = proof.token.trim(); @@ -142,21 +197,45 @@ export async function bootstrapPairedRuntime(args: { endpoint: candidate.endpoint, appVersion: args.appVersion, relayAccountToken, + ...(candidate.kind === "relay" ? { correlationId } : {}), }); } catch (error) { if (error instanceof PairedRuntimeRelayAuthRequiredError) { relayAuthError = error; + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "failed", + failure: "authentication", + }); continue; } - routeErrors.push(`${candidate.endpoint}: ${errorMessage(error)}`); + const failure = classifyPairedRuntimeFailure(error); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "failed", + failure, + }); + addRouteError(`${candidate.kind} ${safeHost}: ${failure}`); continue; } if (transport.connection.hello.features?.portForward !== true) { transport.close(); - routeErrors.push( - `${candidate.endpoint}: the paired machine does not advertise port-forward support`, - ); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "failed", + failure: "capability", + }); + addRouteError(`${candidate.kind} ${safeHost}: capability`); continue; } @@ -171,7 +250,16 @@ export async function bootstrapPairedRuntime(args: { } catch (error) { client.close(); if (isPairedTransportFailure(error)) { - routeErrors.push(`${candidate.endpoint}: ${errorMessage(error)}`); + const failure = classifyPairedRuntimeFailure(error); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "failed", + failure, + }); + addRouteError(`${candidate.kind} ${safeHost}: ${failure}`); continue; } throw compatibilityError(error); @@ -195,7 +283,16 @@ export async function bootstrapPairedRuntime(args: { } catch (error) { client.close(); if (isPairedTransportFailure(error)) { - routeErrors.push(`${candidate.endpoint}: ${errorMessage(error)}`); + const failure = classifyPairedRuntimeFailure(error); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, Date.now() - attemptStartedAt), + outcome: "failed", + failure, + }); + addRouteError(`${candidate.kind} ${safeHost}: ${failure}`); continue; } throw compatibilityError(error); @@ -203,6 +300,13 @@ export async function bootstrapPairedRuntime(args: { try { const connectedAt = Date.now(); + recordAttempt({ + kind: candidate.kind, + host: safeHost, + startedAt: attemptStartedAt, + durationMs: Math.max(0, connectedAt - attemptStartedAt), + outcome: "connected", + }); const updated = args.registry.update(args.target.id, { lastSeenArch: pairedArch(args.target, credentials), runtimeBinaryVersion: initializeInfo.version, @@ -246,6 +350,11 @@ export async function bootstrapPairedRuntime(args: { kind: candidate.kind, endpoint: candidate.endpoint, latencyMs, + correlationId, + attempts, + ...(attemptRecorder.omittedAttemptCount > 0 + ? { omittedAttemptCount: attemptRecorder.omittedAttemptCount } + : {}), }, capabilities: initializeInfo.capabilities, compatibilityWarnings: initializeInfo.compatibilityWarnings, @@ -262,6 +371,19 @@ export async function bootstrapPairedRuntime(args: { throw new PairedRuntimeTransportUnavailableError( "Could not reach the paired ADE runtime over LAN, tailnet, or relay. " - + routeErrors.join("; "), + + routeErrors.join("; ") + + ( + omittedRouteErrorCount > 0 + ? `; ${omittedRouteErrorCount} more route attempts failed` + : "" + ), + undefined, + { + correlationId, + attempts, + ...(attemptRecorder.omittedAttemptCount > 0 + ? { omittedAttemptCount: attemptRecorder.omittedAttemptCount } + : {}), + }, ); } diff --git a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeErrors.ts b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeErrors.ts index b7b4f6d83..64c38957f 100644 --- a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeErrors.ts +++ b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeErrors.ts @@ -1,4 +1,13 @@ -import type { RemoteRuntimeSshHostKeyTrustStatus } from "../../../shared/types/remoteRuntime"; +import type { + RemoteRuntimeConnectionAttempt, + RemoteRuntimeSshHostKeyTrustStatus, +} from "../../../shared/types/remoteRuntime"; + +export type PairedRuntimeRouteDiagnostic = { + correlationId: string; + attempts: RemoteRuntimeConnectionAttempt[]; + omittedAttemptCount?: number; +}; function assignCause(target: Error, cause: unknown): void { if (cause === undefined) return; @@ -12,7 +21,11 @@ function assignCause(target: Error, cause: unknown): void { export class PairedRuntimeTransportUnavailableError extends Error { readonly code = "PAIRED_RUNTIME_TRANSPORT_UNAVAILABLE" as const; - constructor(message: string, cause?: unknown) { + constructor( + message: string, + cause?: unknown, + readonly diagnostic?: PairedRuntimeRouteDiagnostic, + ) { super(message); this.name = "PairedRuntimeTransportUnavailableError"; assignCause(this, cause); @@ -32,7 +45,11 @@ export class PairedRuntimeCompatibilityError extends Error { export class PairedRuntimeRelayAuthRequiredError extends Error { readonly code = "PAIRED_RUNTIME_RELAY_AUTH_REQUIRED" as const; - constructor(message = "Sign in to ADE to connect through ADE Relay.", cause?: unknown) { + constructor( + message = "Sign in to ADE to connect through ADE Relay.", + cause?: unknown, + readonly diagnostic?: PairedRuntimeRouteDiagnostic, + ) { super(message); this.name = "PairedRuntimeRelayAuthRequiredError"; assignCause(this, cause); diff --git a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes.test.ts b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes.test.ts index e7d5c84f1..7f5a74643 100644 --- a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes.test.ts +++ b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes.test.ts @@ -2,6 +2,9 @@ import { describe, expect, it } from "vitest"; import { buildPairedEndpointCandidates, classifyPairedRuntimeEndpoint, + classifyPairedRuntimeFailure, + createRouteAttemptRecorder, + orderPairedCandidates, } from "./pairedRuntimeRoutes"; import { isTailnetHostname } from "../../../shared/tailnet"; @@ -34,6 +37,39 @@ describe("paired runtime endpoint routes", () => { ]); }); + it("prefers a freshly discovered bound port within its route kind without jumping ahead of LAN", () => { + const candidates = buildPairedEndpointCandidates({ + endpoints: [ + "ws://studio.local:8787", + "ws://studio.local:8805", + "ws://studio.example.ts.net:8805", + ], + endpointStates: [ + { + endpoint: "ws://studio.local:8787", + lastSucceededAt: 300, + lastDiscoveredAt: 100, + }, + { + endpoint: "ws://studio.local:8805", + lastSucceededAt: null, + lastDiscoveredAt: 400, + }, + { + endpoint: "ws://studio.example.ts.net:8805", + lastSucceededAt: null, + lastDiscoveredAt: 500, + }, + ], + }); + + expect(candidates.map((candidate) => candidate.endpoint)).toEqual([ + "ws://studio.local:8805/", + "ws://studio.local:8787/", + "ws://studio.example.ts.net:8805/", + ]); + }); + it("classifies normalized CGNAT and ts.net hostnames as tailnet", () => { expect(classifyPairedRuntimeEndpoint("ws://100.127.255.254:8787")).toBe( "tailnet", @@ -50,4 +86,60 @@ describe("paired runtime endpoint routes", () => { expect(isTailnetHostname(" [100.64.0.1]. ")).toBe(true); expect(isTailnetHostname("100.128.0.1")).toBe(false); }); + + it("keeps transport failures meaningful when their sanitized text mentions a token", () => { + expect(classifyPairedRuntimeFailure( + new Error("socket failed with secret diagnostic-token"), + )).toBe("unreachable"); + expect(classifyPairedRuntimeFailure( + new Error("relay token rejected"), + )).toBe("authentication"); + expect(classifyPairedRuntimeFailure( + new Error("WebSocket closed: unauthorized"), + )).toBe("authentication"); + }); + + it("shares direct-before-relay ordering and preserves failures in bounded attempts", () => { + const candidates = buildPairedEndpointCandidates({ + endpoints: [ + "wss://relay.example/connect/one", + "ws://studio.local:8787", + ], + relayUrl: "wss://relay.example/connect/one", + }); + expect(orderPairedCandidates([...candidates].reverse()).map((candidate) => candidate.kind)) + .toEqual(["lan", "relay"]); + + const recorder = createRouteAttemptRecorder(2); + recorder.record({ + kind: "relay", + host: "relay-one.example", + startedAt: 1, + durationMs: 1, + outcome: "skipped", + failure: "authentication", + }); + recorder.record({ + kind: "relay", + host: "relay-two.example", + startedAt: 2, + durationMs: 1, + outcome: "skipped", + failure: "authentication", + }); + recorder.record({ + kind: "lan", + host: "studio.local:8787", + startedAt: 3, + durationMs: 1, + outcome: "failed", + failure: "unreachable", + }); + + expect(recorder.attempts).toEqual([ + expect.objectContaining({ host: "relay-two.example", outcome: "skipped" }), + expect.objectContaining({ host: "studio.local:8787", outcome: "failed" }), + ]); + expect(recorder.omittedAttemptCount).toBe(1); + }); }); diff --git a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes.ts b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes.ts index aebefa610..c6f510111 100644 --- a/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes.ts +++ b/apps/desktop/src/main/services/remoteRuntime/pairedRuntimeRoutes.ts @@ -1,5 +1,9 @@ import type { DesktopPairedMachineEndpointState } from "../../../shared/types/pairedRuntime"; -import type { RemoteRuntimeRouteKind } from "../../../shared/types/remoteRuntime"; +import type { + RemoteRuntimeConnectionAttempt, + RemoteRuntimeConnectionAttemptFailure, + RemoteRuntimeRouteKind, +} from "../../../shared/types/remoteRuntime"; import { isTailnetHostname } from "../../../shared/tailnet"; import { normalizeSyncEndpoint } from "./syncRuntimeTransport"; @@ -7,8 +11,54 @@ export type PairedRuntimeEndpointCandidate = { endpoint: string; kind: Exclude; lastSucceededAt: number | null; + lastDiscoveredAt?: number | null; }; +export const MAX_ROUTE_ATTEMPTS = 8; + +export type PairedRouteAttemptRecorder = { + attempts: RemoteRuntimeConnectionAttempt[]; + omittedAttemptCount: number; + record: (attempt: RemoteRuntimeConnectionAttempt) => void; +}; + +export function createRouteAttemptRecorder( + maxAttempts = MAX_ROUTE_ATTEMPTS, +): PairedRouteAttemptRecorder { + const recorder: PairedRouteAttemptRecorder = { + attempts: [], + omittedAttemptCount: 0, + record: (attempt) => { + if (recorder.attempts.length < maxAttempts) { + recorder.attempts.push(attempt); + return; + } + if (attempt.outcome !== "skipped") { + const skippedIndex = recorder.attempts.findIndex( + (recorded) => recorded.outcome === "skipped", + ); + if (skippedIndex >= 0) { + recorder.attempts.splice(skippedIndex, 1); + recorder.attempts.push(attempt); + recorder.omittedAttemptCount += 1; + return; + } + } + recorder.omittedAttemptCount += 1; + }, + }; + return recorder; +} + +export function orderPairedCandidates( + candidates: readonly PairedRuntimeEndpointCandidate[], +): PairedRuntimeEndpointCandidate[] { + return [ + ...candidates.filter((candidate) => candidate.kind !== "relay"), + ...candidates.filter((candidate) => candidate.kind === "relay"), + ]; +} + function normalizedEndpointOrNull( value: string | null | undefined, ): string | null { @@ -57,19 +107,28 @@ export function buildPairedEndpointCandidates(args: { }): PairedRuntimeEndpointCandidate[] { const relayUrl = normalizedEndpointOrNull(args.relayUrl); const successByEndpoint = new Map(); + const discoveryByEndpoint = new Map(); for (const state of args.endpointStates ?? []) { const endpoint = normalizedEndpointOrNull(state.endpoint); + if (!endpoint) continue; if ( - !endpoint || - state.lastSucceededAt == null || - !Number.isFinite(state.lastSucceededAt) + state.lastSucceededAt != null + && Number.isFinite(state.lastSucceededAt) ) { - continue; + successByEndpoint.set( + endpoint, + Math.max(successByEndpoint.get(endpoint) ?? 0, state.lastSucceededAt), + ); + } + if ( + state.lastDiscoveredAt != null + && Number.isFinite(state.lastDiscoveredAt) + ) { + discoveryByEndpoint.set( + endpoint, + Math.max(discoveryByEndpoint.get(endpoint) ?? 0, state.lastDiscoveredAt), + ); } - successByEndpoint.set( - endpoint, - Math.max(successByEndpoint.get(endpoint) ?? 0, state.lastSucceededAt), - ); } const values = [ @@ -88,6 +147,9 @@ export function buildPairedEndpointCandidates(args: { endpoint, kind: classifyPairedRuntimeEndpoint(endpoint, relayUrl), lastSucceededAt: successByEndpoint.get(endpoint) ?? null, + ...(discoveryByEndpoint.has(endpoint) + ? { lastDiscoveredAt: discoveryByEndpoint.get(endpoint)! } + : {}), order: candidates.length, }); } @@ -101,8 +163,44 @@ export function buildPairedEndpointCandidates(args: { .sort( (left, right) => rank[left.kind] - rank[right.kind] || + (right.lastDiscoveredAt ?? 0) - (left.lastDiscoveredAt ?? 0) || (right.lastSucceededAt ?? 0) - (left.lastSucceededAt ?? 0) || left.order - right.order, ) .map(({ order: _order, ...candidate }) => candidate); } + +export function pairedRuntimeRouteHost( + endpointValue: string, +): string { + try { + const url = new URL(normalizeSyncEndpoint(endpointValue)); + return `${url.hostname}${url.port ? `:${url.port}` : ""}`.slice(0, 128); + } catch { + return "unknown"; + } +} + +export function classifyPairedRuntimeFailure( + error: unknown, +): RemoteRuntimeConnectionAttemptFailure { + const message = error instanceof Error ? error.message : String(error); + if (/timed? out|timeout/i.test(message)) return "timeout"; + if (/ECONN|EHOST|ENET|\bunreach|\boffline\b|\bsocket (?:error|failed)|failed to connect/i.test(message)) { + return "unreachable"; + } + if (/\bauth(?:entication|orization)?\b|\bauthori[sz]ed\b|\bsign in\b|\btoken\b|\bcredential|\bproof\b|\bforbidden\b|\bunauthorized\b/i.test(message)) { + return "authentication"; + } + if (/identity|signature|host key|device id|certificate/i.test(message)) { + return "identity"; + } + if (/feature|capabilit|port-forward|not advertise/i.test(message)) { + return "capability"; + } + if (/protocol|initialize|version|incompatib|malformed|invalid payload/i.test(message)) { + return "protocol"; + } + if (/closed|socket|websocket/i.test(message)) return "unreachable"; + return "unknown"; +} diff --git a/apps/desktop/src/main/services/remoteRuntime/remoteConnectionService.test.ts b/apps/desktop/src/main/services/remoteRuntime/remoteConnectionService.test.ts index 26a5eae1b..4648c88a1 100644 --- a/apps/desktop/src/main/services/remoteRuntime/remoteConnectionService.test.ts +++ b/apps/desktop/src/main/services/remoteRuntime/remoteConnectionService.test.ts @@ -14,6 +14,7 @@ import type { RemoteTargetRegistry } from "./remoteTargetRegistry"; import { PairedRuntimeRelayAuthRequiredError, PairedRuntimeSshTrustRequiredError, + PairedRuntimeTransportUnavailableError, } from "./pairedRuntimeErrors"; const getSshHostKeyTrustForTargetMock = vi.hoisted(() => vi.fn()); @@ -220,7 +221,10 @@ describe("RemoteConnectionService", () => { } as unknown as RemoteConnectionPool; const pairedStore = { get: vi.fn(() => savedCredentials), - save: vi.fn((value) => value), + markEndpointsDiscovered: vi.fn((_hostId, endpoints) => ({ + ...savedCredentials, + endpoints: [...endpoints, ...savedCredentials.endpoints], + })), }; const service = new RemoteConnectionService( registry, @@ -261,14 +265,14 @@ describe("RemoteConnectionService", () => { machineKey: "machine-1", }, }); - expect(pairedStore.save).toHaveBeenCalledWith(expect.objectContaining({ - endpoints: [ + expect(pairedStore.markEndpointsDiscovered).toHaveBeenCalledWith( + "host-1", + [ "ws://192.168.1.20:8787/", "ws://100.70.0.2:8787/", "ws://studio.local:8787/", - "wss://relay.example/connect/machine-1", ], - })); + ); }); it("stores structured connect errors with bounded detail and legacy text", async () => { @@ -309,6 +313,51 @@ describe("RemoteConnectionService", () => { expect(status.lastErrorInfo?.detail).toContain("TAIL"); }); + it("publishes bounded privacy-safe route diagnostics for a failed paired connection", async () => { + const remote = target("route-failed", null); + const registry = { + list: vi.fn(() => [remote]), + get: vi.fn((id: string) => id === remote.id ? remote : null), + } as unknown as RemoteTargetRegistry; + const error = new PairedRuntimeTransportUnavailableError( + "Could not reach the paired ADE runtime.", + undefined, + { + correlationId: "route-correlation", + attempts: [{ + kind: "lan", + host: "studio.local:8805", + startedAt: 100, + durationMs: 50, + outcome: "failed", + failure: "unreachable", + }], + omittedAttemptCount: 3, + }, + ); + const pool = { + connect: vi.fn(async () => { + throw error; + }), + disconnect: vi.fn(), + onEntryEvicted: vi.fn(() => () => {}), + } as unknown as RemoteConnectionPool; + const service = new RemoteConnectionService(registry, pool); + + await expect(service.connect(remote.id, { explicit: true })).rejects.toBe(error); + expect(service.snapshot().connections[0]?.lastErrorInfo).toMatchObject({ + correlationId: "route-correlation", + attempts: [{ + kind: "lan", + host: "studio.local:8805", + outcome: "failed", + failure: "unreachable", + }], + omittedAttemptCount: 3, + }); + expect(service.snapshot().connections[0]?.lastErrorInfo?.detail).toBeUndefined(); + }); + it("publishes the transport route and connect latency in connection status", async () => { const remote = target("route-status", null); const registry = { @@ -478,7 +527,7 @@ describe("RemoteConnectionService", () => { expect(pool.connect).toHaveBeenCalledWith(previouslyConnected); }); - it("reconciles account-owned target, credentials, and live pool entry as one command", () => { + it("reconciles a different account's target, credentials, and live pool entry as one command", () => { const accountOwned = { ...target("account-owned", 1_700_000_000), transport: "paired" as const, @@ -522,7 +571,7 @@ describe("RemoteConnectionService", () => { const snapshots: unknown[] = []; service.onSnapshotChanged((snapshot) => snapshots.push(snapshot)); - expect(service.reconcileAccountOwnership(null)).toEqual({ + expect(service.reconcileAccountOwnership("account-b")).toEqual({ removedTargetIds: [accountOwned.id], removedCredentialHostIds: ["owned-host"], }); @@ -532,6 +581,37 @@ describe("RemoteConnectionService", () => { expect(snapshots).toHaveLength(1); }); + it("preserves account-created paired trust on sign-out while revoking live Relay", () => { + const accountOwned = { + ...target("account-owned", null), + transport: "paired" as const, + pairedMachine: { hostIdentity: "owned-host", machineKey: "owned-key" }, + accountOwnerUserId: "account-a", + }; + const registry = { + list: vi.fn(() => [accountOwned]), + get: vi.fn(() => accountOwned), + pruneAccountOwned: vi.fn(), + remove: vi.fn(), + } as unknown as RemoteTargetRegistry; + const pool = { + reconcileAccountRelayOwner: vi.fn(() => [accountOwned.id]), + disconnect: vi.fn(), + onEntryEvicted: vi.fn(() => () => {}), + } as unknown as RemoteConnectionPool; + const pairedStore = { pruneAccountOwned: vi.fn() }; + const service = new RemoteConnectionService(registry, pool, {}, pairedStore as any); + + expect(service.reconcileAccountOwnership(null)).toEqual({ + removedTargetIds: [], + removedCredentialHostIds: [], + }); + expect(pool.reconcileAccountRelayOwner).toHaveBeenCalledWith(null); + expect(registry.pruneAccountOwned).not.toHaveBeenCalled(); + expect(pairedStore.pruneAccountOwned).not.toHaveBeenCalled(); + expect(registry.remove).not.toHaveBeenCalled(); + }); + it("keeps snapshot pure when a target disappears outside a service command", async () => { const connectedTarget = target("externally-removed", 1_700_000_000); let targets = [connectedTarget]; @@ -589,7 +669,7 @@ describe("RemoteConnectionService", () => { const service = new RemoteConnectionService( registry, pool, - { getAuthorizedAccountOwnerId: vi.fn(async () => null) }, + { getAuthorizedAccountOwnerId: vi.fn(async () => "different-account") }, pairedStore as any, ); @@ -605,6 +685,40 @@ describe("RemoteConnectionService", () => { expect(pool.connect).toHaveBeenCalledWith(localOwned); }); + it("allows a signed-out account-created target to reconnect over device-bound trust", async () => { + const accountOwned = { + ...target("account-owned", 1_700_000_000), + transport: "paired" as const, + pairedMachine: { hostIdentity: "owned-host", machineKey: "owned-key" }, + accountOwnerUserId: "expired-account", + }; + const registry = { + list: vi.fn(() => [accountOwned]), + get: vi.fn(() => accountOwned), + } as unknown as RemoteTargetRegistry; + const pool = { + connect: vi.fn(async () => ({ + ...connectResult(accountOwned), + route: { + kind: "lan" as const, + endpoint: "ws://studio.local:8805/", + }, + })), + disconnect: vi.fn(), + onEntryEvicted: vi.fn(() => () => {}), + } as unknown as RemoteConnectionPool; + const service = new RemoteConnectionService( + registry, + pool, + { getAuthorizedAccountOwnerId: vi.fn(async () => null) }, + ); + + await expect(service.connect(accountOwned.id)).resolves.toMatchObject({ + route: { kind: "lan" }, + }); + expect(pool.connect).toHaveBeenCalledWith(accountOwned); + }); + it("does not autoconnect a manually disconnected saved target", async () => { const previouslyConnected = target("previously-connected", 1_700_000_000); const registry = { diff --git a/apps/desktop/src/main/services/remoteRuntime/remoteConnectionService.ts b/apps/desktop/src/main/services/remoteRuntime/remoteConnectionService.ts index b09999e4d..0b2bbde3c 100644 --- a/apps/desktop/src/main/services/remoteRuntime/remoteConnectionService.ts +++ b/apps/desktop/src/main/services/remoteRuntime/remoteConnectionService.ts @@ -56,6 +56,7 @@ import { runRemoteRuntimeDoctor } from "./connectionDoctor"; import { PairedRuntimeRelayAuthRequiredError, PairedRuntimeSshTrustRequiredError, + PairedRuntimeTransportUnavailableError, } from "./pairedRuntimeErrors"; import { getSshHostKeyTrustForTarget, @@ -75,8 +76,9 @@ type RemoteConnectionServiceOptions = { } | null>; /** * Returns the refresh-verified account owner allowed to use account-created - * machine trust. `null` means account-owned trust must be removed before use. - * Omit only in isolated callers that do not participate in account lifecycle. + * Relay trust. `null` disables directory/Relay access while preserving + * host-issued LAN/Tailscale paired trust. Omit only in isolated callers that + * do not participate in account lifecycle. */ getAuthorizedAccountOwnerId?: () => Promise; }; @@ -96,6 +98,7 @@ type RemoteConnectionConnectOptions = { const AUTOMATIC_RECONNECT_FAILURE_LIMIT = 10; const MAX_LAST_ERROR_CHARS = 500; +const DISCOVERED_ENDPOINT_REFRESH_INTERVAL_MS = 60_000; function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); @@ -130,6 +133,20 @@ function errorInfo( message, }; } + if ( + error instanceof PairedRuntimeTransportUnavailableError + && error.diagnostic + ) { + return { + kind: "generic", + message, + correlationId: error.diagnostic.correlationId, + attempts: error.diagnostic.attempts, + ...(error.diagnostic.omittedAttemptCount != null + ? { omittedAttemptCount: error.diagnostic.omittedAttemptCount } + : {}), + }; + } return { kind: "generic", message, @@ -232,10 +249,11 @@ export class RemoteConnectionService { } /** - * Canonical account-trust reconciliation command. Account-owned target and - * credential records are one lifecycle boundary: prune them together, close - * any live transport immediately, clear all reconnect/trust state, then emit - * one authoritative snapshot. Ownerless PIN/link/SSH trust is untouched. + * Canonical account-trust reconciliation command. Sign-out revokes Relay + * leases but preserves host-issued paired secrets so LAN/Tailscale remain + * available. Switching to a different signed-in account prunes the previous + * account's records as one lifecycle boundary. Ownerless PIN/link/SSH trust + * is always untouched. */ reconcileAccountOwnership( currentOwnerUserIdValue: string | null, @@ -244,8 +262,12 @@ export class RemoteConnectionService { const disconnectedRelayTargetIds = this.pool.reconcileAccountRelayOwner?.( currentOwnerUserId, ) ?? []; - const removedTargets = this.registry.pruneAccountOwned(currentOwnerUserId); - const removedCredentials = this.pairedStore.pruneAccountOwned(currentOwnerUserId); + const removedTargets = currentOwnerUserId + ? this.registry.pruneAccountOwned(currentOwnerUserId) + : []; + const removedCredentials = currentOwnerUserId + ? this.pairedStore.pruneAccountOwned(currentOwnerUserId) + : []; const removedCredentialIds = new Set(); for (const credentials of removedCredentials) { removedCredentialIds.add(credentials.hostIdentity.deviceId); @@ -334,12 +356,10 @@ export class RemoteConnectionService { const paired = this.pairedStore.get(discoveredMachine.hostIdentity); const discoveredEndpoints = syncEndpointsForDiscoveredRuntime(discoveredMachine); if (paired && discoveredEndpoints.length > 0) { - const saved = discoveredEndpoints.some((endpoint) => !paired.endpoints.includes(endpoint)) - ? this.pairedStore.save({ - ...paired, - endpoints: [...discoveredEndpoints, ...paired.endpoints], - }) - : paired; + const saved = this.pairedStore.markEndpointsDiscovered( + paired.hostIdentity.deviceId, + discoveredEndpoints, + ); normalizedInput = { ...input, transport: "paired", @@ -356,19 +376,27 @@ export class RemoteConnectionService { } rememberDiscoveredMachines(machines: RemoteRuntimeDiscoveredMachine[]): void { + const discoveredAt = Date.now(); for (const machine of machines) { if (!machine.hostIdentity) continue; const paired = this.pairedStore.get(machine.hostIdentity); if (!paired) continue; const endpoints = syncEndpointsForDiscoveredRuntime(machine); if (endpoints.length === 0) continue; - if (endpoints.every((endpoint) => paired.endpoints.includes(endpoint))) { - continue; - } - this.pairedStore.save({ - ...paired, - endpoints: [...endpoints, ...paired.endpoints], + const discoveryIsFresh = endpoints.every((endpoint) => { + const state = paired.endpointStates?.find( + (candidate) => candidate.endpoint === endpoint, + ); + return state?.lastDiscoveredAt != null + && discoveredAt - state.lastDiscoveredAt + < DISCOVERED_ENDPOINT_REFRESH_INTERVAL_MS; }); + if (discoveryIsFresh) continue; + this.pairedStore.markEndpointsDiscovered( + paired.hostIdentity.deviceId, + endpoints, + discoveredAt, + ); } } @@ -1110,6 +1138,10 @@ export class RemoteConnectionService { const currentOwnerUserId = await this.options.getAuthorizedAccountOwnerId() .catch(() => null); if (currentOwnerUserId?.trim() === expectedOwnerUserId) return; + // Signed-out callers may still use the persisted host-issued secret over + // LAN/Tailscale. Relay remains unavailable because it separately requires + // a fresh matching account proof. + if (!currentOwnerUserId?.trim()) return; this.reconcileAccountOwnership(currentOwnerUserId); throw new PairedRuntimeRelayAuthRequiredError( "Sign in with the same ADE account as this machine to connect.", diff --git a/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.test.ts b/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.test.ts index 233934dda..d485f5b7a 100644 --- a/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.test.ts +++ b/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.test.ts @@ -37,6 +37,16 @@ afterEach(() => { else process.env.ADE_HOME = originalAdeHome; }); +function endpointWithoutCorrelation(endpoint: string): string { + const url = new URL(endpoint); + url.searchParams.delete("cid"); + return url.toString(); +} + +function endpointCorrelationId(endpoint: string): string | null { + return new URL(endpoint).searchParams.get("cid"); +} + class FakeWebSocket extends EventEmitter { readyState = 0; bufferedAmount = 0; @@ -306,6 +316,17 @@ describe("DesktopPairedMachineStore", () => { }); expect(new DesktopPairedMachineStore().get("mac-studio-host")) .toEqual(marked); + + const discovered = store.markEndpointsDiscovered( + "mac-studio-host", + ["ws://studio.local:8805"], + 1_700_000_000_500, + ); + expect(discovered.endpointStates).toContainEqual({ + endpoint: "ws://studio.local:8805/", + lastSucceededAt: null, + lastDiscoveredAt: 1_700_000_000_500, + }); }); it("replaces stale relay connection metadata only when explicitly requested", () => { @@ -535,7 +556,7 @@ describe("DesktopPairedMachineStore", () => { }, ); - expect(openedEndpoints).toEqual([ + expect(openedEndpoints.map(endpointWithoutCorrelation)).toEqual([ "wss://relay-one.example/connect/machine-account-1", "wss://relay-two.example/connect/machine-account-1", "wss://relay-one.example/connect/machine-account-1", @@ -543,6 +564,16 @@ describe("DesktopPairedMachineStore", () => { "wss://relay-one.example/connect/machine-account-1", "wss://relay-two.example/connect/machine-account-1", ]); + const correlationIds = openedEndpoints.map(endpointCorrelationId); + expect(correlationIds).toEqual([ + expect.stringMatching(/^[0-9a-f-]{36}$/), + correlationIds[0], + expect.stringMatching(/^[0-9a-f-]{36}$/), + correlationIds[2], + expect.stringMatching(/^[0-9a-f-]{36}$/), + correlationIds[4], + ]); + expect(new Set([correlationIds[0], correlationIds[2], correlationIds[4]]).size).toBe(3); expect(accountDpopVerdicts).toEqual([ { ok: true }, { ok: true }, @@ -607,12 +638,13 @@ describe("DesktopPairedMachineStore", () => { warn.mockRestore(); } - expect(openedEndpoints).toEqual([ + expect(openedEndpoints.map(endpointWithoutCorrelation)).toEqual([ "wss://relay.example/connect/machine-legacy-relay-only", ]); + expect(endpointCorrelationId(openedEndpoints[0]!)).toMatch(/^[0-9a-f-]{36}$/); }); - it("stops after a successful sealed relay adoption without dialing direct routes", async () => { + it("stops after a successful sealed LAN adoption without dialing later routes", async () => { process.env.ADE_HOME = fs.mkdtempSync( path.join(os.tmpdir(), "ade-desktop-relay-wins-"), ); @@ -663,11 +695,11 @@ describe("DesktopPairedMachineStore", () => { secret: "sealed-paired-secret", }); expect(openedEndpoints).toEqual([ - "wss://relay.example/connect/machine-relay-wins", + "ws://relay-studio.local:8787/", ]); expect(stages).toEqual([ - { kind: "relay", phase: "connecting" }, - { kind: "relay", phase: "verifying" }, + { kind: "lan", phase: "connecting" }, + { kind: "lan", phase: "verifying" }, ]); }); @@ -805,7 +837,7 @@ describe("DesktopPairedMachineStore", () => { expect(sentTypes).toEqual(["account_challenge"]); }); - it("falls through a closed relay to sealed tailnet adoption with exact stages", async () => { + it("falls through a closed LAN route to sealed tailnet adoption with exact stages", async () => { process.env.ADE_HOME = fs.mkdtempSync( path.join(os.tmpdir(), "ade-desktop-tailnet-fallback-"), ); @@ -844,7 +876,7 @@ describe("DesktopPairedMachineStore", () => { onStage: (stage) => stages.push(stage), createWebSocket: (endpoint) => { openedEndpoints.push(endpoint); - if (endpoint.startsWith("wss://")) { + if (endpoint.includes("tailnet-studio.local")) { return new FakeWebSocket((text, ws) => { const envelope = parseSyncEnvelope(wsDataToText(text)); if (envelope.type === "account_challenge") ws.close(); @@ -863,11 +895,11 @@ describe("DesktopPairedMachineStore", () => { }); expect(openedEndpoints).toEqual([ - "wss://relay.example/connect/machine-tailnet-fallback", + "ws://tailnet-studio.local:8787/", "ws://100.75.20.63:8787/", ]); expect(stages).toEqual([ - { kind: "relay", phase: "connecting" }, + { kind: "lan", phase: "connecting" }, { kind: "tailnet", phase: "connecting" }, { kind: "tailnet", phase: "verifying" }, ]); @@ -1190,7 +1222,7 @@ describe("DesktopPairedMachineStore", () => { await expect(pairing).rejects.toMatchObject({ code: "account_host_identity_verification_failed", }); - expect(openedEndpoints).toEqual(["ws://100.75.20.63:8787/"]); + expect(openedEndpoints).toEqual(["ws://expected-direct-host.local:8787/"]); expect(sentTypes).toEqual(["account_challenge"]); }); @@ -1243,11 +1275,14 @@ describe("DesktopPairedMachineStore", () => { expect(failure?.message).toMatch(/relay relay\.example:/); expect(failure?.message).toMatch(/tailnet 100\.75\.20\.63:/); expect(failure?.message).toMatch(/lan unavailable-studio\.local:/); - expect(openedEndpoints).toEqual([ - "wss://relay.example/connect/machine-all-routes-fail", - "ws://100.75.20.63:8787/", + expect(openedEndpoints.map(endpointWithoutCorrelation)).toEqual([ "ws://unavailable-studio.local:8787/", + "ws://100.75.20.63:8787/", + "wss://relay.example/connect/machine-all-routes-fail", ]); + expect(endpointCorrelationId(openedEndpoints[0]!)).toBeNull(); + expect(endpointCorrelationId(openedEndpoints[1]!)).toBeNull(); + expect(endpointCorrelationId(openedEndpoints[2]!)).toMatch(/^[0-9a-f-]{36}$/); }); it.each([ @@ -1367,7 +1402,7 @@ describe("DesktopPairedMachineStore", () => { }, )).rejects.toThrow("cancel account authentication"); - expect(openedEndpoints).toEqual([ + expect(openedEndpoints.map(endpointWithoutCorrelation)).toEqual([ "wss://relay-one.example/connect/machine-cancel", ]); }); @@ -1409,7 +1444,7 @@ describe("DesktopPairedMachineStore", () => { controller.abort(new Error("cancel account connection")); await expect(pairing).rejects.toThrow("cancel account connection"); - expect(openedEndpoints).toEqual([ + expect(openedEndpoints.map(endpointWithoutCorrelation)).toEqual([ "wss://relay-one.example/connect/machine-connect-cancel", ]); }); diff --git a/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.ts b/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.ts index 5496337d8..7566b7f68 100644 --- a/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.ts +++ b/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.ts @@ -50,6 +50,7 @@ import { type OpenSyncEnvelopeConnectionOptions, type SyncEnvelopeConnection, } from "./syncRuntimeTransport"; +import { MAX_ROUTE_ATTEMPTS } from "./pairedRuntimeRoutes"; const STORE_FILE_NAME = "desktop-paired-machines.json"; const DEFAULT_PAIRING_TIMEOUT_MS = 15_000; @@ -150,6 +151,18 @@ function formatAccountMachineAdoptionFailure( }`; } +function classifyAccountMachineAdoptionFailure(reason: string): string { + if (/timed? out|timeout/i.test(reason)) return "timeout"; + if (/auth|token|credential|proof|forbidden|unauthorized/i.test(reason)) { + return "authentication"; + } + if (/identity|signature|device id|cipher/i.test(reason)) return "identity"; + if (/ECONN|EHOST|ENET|unreach|offline|closed|socket|websocket|refused/i.test(reason)) { + return "unreachable"; + } + return "unknown"; +} + function nowIso(): string { return new Date().toISOString(); } @@ -252,7 +265,10 @@ function coerceEndpointStates( value: unknown, endpoints: string[], ): DesktopPairedMachineEndpointState[] { - const successByEndpoint = new Map(); + const stateByEndpoint = new Map(); if (Array.isArray(value)) { for (const entry of value) { if (!isRecord(entry)) continue; @@ -268,20 +284,37 @@ function coerceEndpointStates( && Number.isFinite(entry.lastSucceededAt) ? entry.lastSucceededAt : null; - const current = successByEndpoint.get(endpoint) ?? null; - if (lastSucceededAt != null && (current == null || lastSucceededAt > current)) { - successByEndpoint.set(endpoint, lastSucceededAt); - } else if (!successByEndpoint.has(endpoint)) { - successByEndpoint.set(endpoint, null); - } + const lastDiscoveredAt = typeof entry.lastDiscoveredAt === "number" + && Number.isFinite(entry.lastDiscoveredAt) + ? entry.lastDiscoveredAt + : null; + const current = stateByEndpoint.get(endpoint); + stateByEndpoint.set(endpoint, { + lastSucceededAt: Math.max( + current?.lastSucceededAt ?? 0, + lastSucceededAt ?? 0, + ) || null, + lastDiscoveredAt: Math.max( + current?.lastDiscoveredAt ?? 0, + lastDiscoveredAt ?? 0, + ) || null, + }); } } for (const endpoint of endpoints) { - if (!successByEndpoint.has(endpoint)) successByEndpoint.set(endpoint, null); + if (!stateByEndpoint.has(endpoint)) { + stateByEndpoint.set(endpoint, { + lastSucceededAt: null, + lastDiscoveredAt: null, + }); + } } - return [...successByEndpoint].map(([endpoint, lastSucceededAt]) => ({ + return [...stateByEndpoint].map(([endpoint, state]) => ({ endpoint, - lastSucceededAt, + lastSucceededAt: state.lastSucceededAt, + ...(state.lastDiscoveredAt != null + ? { lastDiscoveredAt: state.lastDiscoveredAt } + : {}), })); } @@ -508,6 +541,30 @@ export class DesktopPairedMachineStore { }); } + markEndpointsDiscovered( + hostDeviceIdOrMachineKey: string, + endpointValues: string[], + nowMs = Date.now(), + ): DesktopPairedMachineCredentials { + const machine = this.get(hostDeviceIdOrMachineKey); + if (!machine) throw new Error("Paired machine was not found."); + const discoveredEndpoints = uniqueEndpoints(...endpointValues); + const endpoints = uniqueEndpoints(...discoveredEndpoints, ...machine.endpoints); + return this.save({ + ...machine, + endpoints, + endpointStates: mergeEndpointStates( + endpoints, + machine.endpointStates, + discoveredEndpoints.map((endpoint) => ({ + endpoint, + lastSucceededAt: null, + lastDiscoveredAt: nowMs, + })), + ), + }); + } + async pairWithMachine( endpointValue: string, pinValue: string, @@ -724,6 +781,7 @@ export class DesktopPairedMachineStore { capabilities: [], ...(options.appVersion?.trim() ? { appVersion: options.appVersion.trim() } : {}), }; + const correlationId = randomUUID(); const failures: AccountMachineAdoptionFailure[] = []; for (const route of accountAuthenticationRoutes) { throwIfAborted(options.signal); @@ -743,6 +801,7 @@ export class DesktopPairedMachineStore { connectTimeoutMs: options.connectTimeoutMs, signal: options.signal, createWebSocket: options.createWebSocket, + ...(route.kind === "relay" ? { correlationId } : {}), }); } catch (error) { throwIfAborted(options.signal); @@ -1018,16 +1077,24 @@ export class DesktopPairedMachineStore { } } console.warn("[account] Account machine adoption failed on every route.", { - machineKey: machine.machineKey, - attempts: failures.map((failure) => ({ + correlationId, + attempts: failures.slice(0, MAX_ROUTE_ATTEMPTS).map((failure) => ({ kind: failure.route.kind, - endpoint: failure.route.endpoint, - reason: failure.reason, + host: accountMachineAdoptionRouteHost(failure.route), + failure: classifyAccountMachineAdoptionFailure(failure.reason), })), + omittedAttemptCount: Math.max(0, failures.length - MAX_ROUTE_ATTEMPTS), }); + const visibleFailures = failures.slice(0, MAX_ROUTE_ATTEMPTS) + .map(formatAccountMachineAdoptionFailure); + if (failures.length > MAX_ROUTE_ATTEMPTS) { + visibleFailures.push( + `${failures.length - MAX_ROUTE_ATTEMPTS} more route attempts failed`, + ); + } throw new Error( `Could not connect to ${machine.name ?? machine.machineKey} with your ADE account. ${ - failures.map(formatAccountMachineAdoptionFailure).join("; ") + visibleFailures.join("; ") }`, ); } diff --git a/apps/desktop/src/main/services/remoteRuntime/syncRuntimeTransport.test.ts b/apps/desktop/src/main/services/remoteRuntime/syncRuntimeTransport.test.ts index 4ac3b15ac..8be31c1f9 100644 --- a/apps/desktop/src/main/services/remoteRuntime/syncRuntimeTransport.test.ts +++ b/apps/desktop/src/main/services/remoteRuntime/syncRuntimeTransport.test.ts @@ -14,6 +14,7 @@ import { buildDesktopPairedHello, openSyncEnvelopeConnection, openSyncRuntimeTransport, + withSyncRelayCorrelationId, } from "./syncRuntimeTransport"; class FakeWebSocket extends EventEmitter { @@ -79,6 +80,29 @@ function credentials(): DesktopPairedMachineCredentials { } describe("openSyncRuntimeTransport", () => { + it("adds a validated correlation id without changing the stored endpoint", async () => { + const correlationId = "123e4567-e89b-42d3-a456-426614174000"; + let openedEndpoint = ""; + const connection = await openSyncEnvelopeConnection({ + endpoint: "wss://relay.example/connect/machine?ready=2", + correlationId, + createWebSocket: (endpoint) => { + openedEndpoint = endpoint; + return new FakeWebSocket(() => {}) as unknown as WebSocket; + }, + }); + + expect(openedEndpoint).toBe( + "wss://relay.example/connect/machine?ready=2&cid=123e4567-e89b-42d3-a456-426614174000", + ); + expect(connection.endpoint).toBe("wss://relay.example/connect/machine?ready=2"); + connection.close(); + expect(() => withSyncRelayCorrelationId( + "wss://relay.example/connect/machine", + "not-a-correlation-id", + )).toThrow("canonical UUID v4"); + }); + it("adds an ephemeral account proof only to the relay hello", () => { const paired = credentials(); const hello = buildDesktopPairedHello( diff --git a/apps/desktop/src/main/services/remoteRuntime/syncRuntimeTransport.ts b/apps/desktop/src/main/services/remoteRuntime/syncRuntimeTransport.ts index 893ade796..d5bc3c359 100644 --- a/apps/desktop/src/main/services/remoteRuntime/syncRuntimeTransport.ts +++ b/apps/desktop/src/main/services/remoteRuntime/syncRuntimeTransport.ts @@ -54,11 +54,27 @@ export type AuthenticatedSyncConnection = SyncEnvelopeConnection & { export type OpenSyncEnvelopeConnectionOptions = { endpoint: string; + /** Safe operation identifier appended only to Relay URLs as `cid`. */ + correlationId?: string; connectTimeoutMs?: number; signal?: AbortSignal; createWebSocket?: (endpoint: string) => WebSocket; }; +const CORRELATION_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +export function withSyncRelayCorrelationId(endpoint: string, correlationId?: string): string { + const normalized = correlationId?.trim() ?? ""; + if (!normalized) return endpoint; + if (!CORRELATION_ID_PATTERN.test(normalized)) { + throw new Error("Sync connection correlationId must be a canonical UUID v4."); + } + const url = new URL(endpoint); + url.searchParams.set("cid", normalized.toLowerCase()); + return url.toString(); +} + export type OpenPairedSyncConnectionOptions = OpenSyncEnvelopeConnectionOptions & { credentials: DesktopPairedMachineCredentials; authTimeoutMs?: number; @@ -244,9 +260,10 @@ export async function openSyncEnvelopeConnection( options: OpenSyncEnvelopeConnectionOptions, ): Promise { const endpoint = normalizeSyncEndpoint(options.endpoint); + const socketEndpoint = withSyncRelayCorrelationId(endpoint, options.correlationId); const timeoutMs = normalizeTimeout(options.connectTimeoutMs, DEFAULT_CONNECT_TIMEOUT_MS); if (options.signal?.aborted) throw abortError(options.signal); - const ws = options.createWebSocket?.(endpoint) ?? new WebSocket(endpoint); + const ws = options.createWebSocket?.(socketEndpoint) ?? new WebSocket(socketEndpoint); const connection = createConnection(endpoint, ws); if (options.signal?.aborted) { diff --git a/apps/desktop/src/main/services/search/searchService.test.ts b/apps/desktop/src/main/services/search/searchService.test.ts index 1c0ecee5b..9d7e9cb67 100644 --- a/apps/desktop/src/main/services/search/searchService.test.ts +++ b/apps/desktop/src/main/services/search/searchService.test.ts @@ -132,6 +132,38 @@ describe("searchService", () => { expect(hit!.deepLink).toContain("event=0"); }); + it("indexes one searchable hit across accepted steer lifecycle snapshots", async () => { + const session = makeSession({ id: "chat-steer-lifecycle", title: "Steer lifecycle" }); + sessions.push(session); + for (const [sequence, deliveryState] of [ + [1, "queued"], + [2, "accepted"], + [3, "processed"], + [4, "unprocessed"], + ] as const) { + writeChatLine( + session.id, + { + type: "user_message", + text: "only one searchable follow-up", + steerId: "steer-1", + turnId: "turn-1", + deliveryState, + processed: deliveryState === "processed", + }, + `2026-07-05T10:00:0${sequence}.000Z`, + sequence, + ); + } + service.notifyChatEvent(session.id); + await service.processPendingNow(); + + const result = await service.query({ query: "searchable follow-up" }); + expect(result.results.filter((item) => + item.kind === "chat" && item.sessionId === session.id + )).toHaveLength(1); + }); + it("rebuilds searchable chat and terminal history from compressed transcripts", async () => { const chat = makeSession({ id: "chat-gz", title: "Compressed chat" }); const terminalPath = path.join(root, "transcripts", "terminal-gz.log"); @@ -1165,6 +1197,76 @@ describe("searchService owner-scoped attached terminals", () => { }); }); +describe("searchService exact session filters", () => { + it("resolves a newly-created chat before the background indexer runs", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "ade-search-session-direct-")); + const session = makeSession({ + id: "fresh-chat", + title: "Fresh reliability chat", + toolType: "codex-chat", + status: "running", + startedAt: "2026-07-05T12:00:00.000Z", + }); + const service = createSearchService({ + cacheDir: path.join(root, "cache"), + transcriptsDir: path.join(root, "transcripts"), + chatTranscriptsDir: path.join(root, "transcripts", "chat"), + sessions: { + list: async () => [session], + get: async (id) => id === session.id ? session : null, + }, + now: () => NOW, + }); + + const result = await service.query({ query: "session:fresh-chat" }); + + expect(result.results).toEqual([ + expect.objectContaining({ + kind: "chat", + sessionId: "fresh-chat", + title: "Fresh reliability chat", + }), + ]); + service.dispose(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it("prefers live session metadata without double-counting an indexed row", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "ade-search-session-live-")); + const session = makeSession({ + id: "indexed-chat", + title: "Stale indexed title", + toolType: "codex-chat", + status: "running", + startedAt: "2026-07-05T12:00:00.000Z", + }); + const service = createSearchService({ + cacheDir: path.join(root, "cache"), + transcriptsDir: path.join(root, "transcripts"), + chatTranscriptsDir: path.join(root, "transcripts", "chat"), + sessions: { + list: async () => [session], + get: async (id) => id === session.id ? session : null, + }, + now: () => NOW, + }); + service.notifyChatEvent(session.id); + await service.processPendingNow(); + session.title = "Live reliability title"; + + const result = await service.query({ query: `session:${session.id}` }); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toMatchObject({ + sessionId: session.id, + title: "Live reliability title", + }); + expect(result.totalByKind.chat).toBe(1); + service.dispose(); + fs.rmSync(root, { recursive: true, force: true }); + }); +}); + describe("searchService since: filter on delegated files", () => { it("omits file results when a since: filter is present (files carry no timestamps)", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "ade-search-test9-")); diff --git a/apps/desktop/src/main/services/search/searchService.ts b/apps/desktop/src/main/services/search/searchService.ts index c547daa64..0d68fe99e 100644 --- a/apps/desktop/src/main/services/search/searchService.ts +++ b/apps/desktop/src/main/services/search/searchService.ts @@ -203,9 +203,25 @@ function withQueryParam(link: string, key: string, value: string | number): stri } function chatEventSearchText(envelope: AgentChatEventEnvelope): string | null { - const event = envelope.event as { type?: string; text?: unknown; displayText?: unknown }; + const event = envelope.event as { + type?: string; + text?: unknown; + displayText?: unknown; + deliveryState?: unknown; + }; if (!event || typeof event !== "object") return null; if (event.type === "user_message") { + // Accepted steers are persisted again when they become processed or + // terminally unprocessed so every transcript surface can fold the latest + // delivery state. The first accepted event already owns the searchable + // message body; lifecycle snapshots must not create duplicate hits. + if ( + event.deliveryState === "queued" + || event.deliveryState === "processed" + || event.deliveryState === "unprocessed" + ) { + return null; + } const display = typeof event.displayText === "string" ? event.displayText : ""; const text = typeof event.text === "string" ? event.text : ""; return display.trim() || text.trim() || null; @@ -1418,6 +1434,37 @@ export function createSearchService(deps: SearchServiceDeps) { ); const delegated: Candidate[] = []; + // `session:` is also an exact lookup contract, not merely an FTS + // filter. Resolve the owning session directly so a newly-created chat is + // findable before the background indexer has written its metadata row. + if (parsed.sessionId && matchAll && !excludeSessionContent) { + const session = await resolveSession(parsed.sessionId); + if (session) { + const kind: "chat" | "terminal" = isChatSession(session) ? "chat" : "terminal"; + const scopeAllowsSession = !scopeChatSessionId + || session.id === scopeChatSessionId + || session.chatSessionId === scopeChatSessionId; + const laneMatches = !laneId || session.laneId === laneId; + const kindMatches = kinds.includes(kind); + const sinceMatches = !parsed.sinceIso || sessionUpdatedAt(session) >= parsed.sinceIso; + if (scopeAllowsSession && laneMatches && kindMatches && sinceMatches) { + delegated.push({ + docId: `${kind === "chat" ? "chat" : "term"}:${session.id}:meta`, + kind, + title: session.title, + rankTitle: session.title, + laneId: session.laneId || null, + laneName: session.laneName || null, + sessionId: session.id, + deepLink: sessionDeepLink(session, await envelopeForLane(session.laneId)), + updatedAt: sessionUpdatedAt(session), + bm25: 0, + snippet: (session.summary || session.lastOutputPreview || session.goal || session.title).slice(0, 240), + matchRanges: [], + }); + } + } + } if (!parsed.sessionId) { if (kinds.includes("lane") && !laneId) { delegated.push(...(await delegatedLaneCandidates(parsed, matchAll))); @@ -1433,11 +1480,19 @@ export function createSearchService(deps: SearchServiceDeps) { } } + const candidatesById = new Map(); + for (const candidate of ftsCandidates) { + if (!candidatesById.has(candidate.docId)) candidatesById.set(candidate.docId, candidate); + } for (const candidate of delegated) { - totals[candidate.kind] = (totals[candidate.kind] ?? 0) + 1; + // Exact-session lookup is live session state and should replace a stale + // background-index metadata row with the same id. + const alreadyIndexed = candidatesById.has(candidate.docId); + if (parsed.sessionId || !alreadyIndexed) candidatesById.set(candidate.docId, candidate); + if (!alreadyIndexed) totals[candidate.kind] = (totals[candidate.kind] ?? 0) + 1; } - - const tiered = rankCandidates([...ftsCandidates, ...delegated], parsed); + const candidates = Array.from(candidatesById.values()); + const tiered = rankCandidates(candidates, parsed); const page = tiered.slice(offset, offset + limit); const results: SearchResultItem[] = page.map((candidate) => ({ diff --git a/apps/desktop/src/preload/global.d.ts b/apps/desktop/src/preload/global.d.ts index 272f48e39..30cebfe41 100644 --- a/apps/desktop/src/preload/global.d.ts +++ b/apps/desktop/src/preload/global.d.ts @@ -124,8 +124,12 @@ import type { AgentChatPrepareCrossMachineHandoffResult, AgentChatValidateCrossMachineSourceArgs, AgentChatInterruptArgs, + AgentChatRecoverTurnArgs, + AgentChatRecoverTurnResult, AgentChatRecoverCodexTurnArgs, AgentChatRecoverCodexTurnResult, + AgentChatResolveUnprocessedMessageArgs, + AgentChatResolveUnprocessedMessageResult, AgentChatRecoverContinuityArgs, AgentChatContinuityRecoveryResult, AgentChatListArgs, @@ -1468,9 +1472,15 @@ declare global { args: AgentChatCancelDispatchedSteerArgs, ) => Promise; interrupt: (args: AgentChatInterruptArgs) => Promise; + recoverTurn: ( + args: AgentChatRecoverTurnArgs, + ) => Promise; recoverCodexTurn: ( args: AgentChatRecoverCodexTurnArgs, ) => Promise; + resolveUnprocessedMessage: ( + args: AgentChatResolveUnprocessedMessageArgs, + ) => Promise; recoverContinuity: ( args: AgentChatRecoverContinuityArgs, ) => Promise; diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index 779ac34dd..6d0e8d3eb 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -328,8 +328,12 @@ import type { AgentChatPrepareCrossMachineHandoffResult, AgentChatValidateCrossMachineSourceArgs, AgentChatInterruptArgs, + AgentChatRecoverTurnArgs, + AgentChatRecoverTurnResult, AgentChatRecoverCodexTurnArgs, AgentChatRecoverCodexTurnResult, + AgentChatResolveUnprocessedMessageArgs, + AgentChatResolveUnprocessedMessageResult, AgentChatRecoverContinuityArgs, AgentChatContinuityRecoveryResult, AgentChatListArgs, @@ -1254,7 +1258,9 @@ const MUTATING_CHAT_ACTIONS = new Set([ "respondToInput", "approveToolUse", "interrupt", + "recoverTurn", "recoverCodexTurn", + "resolveUnprocessedMessage", "recoverContinuity", "steer", "cancelSteer", @@ -5569,6 +5575,19 @@ contextBridge.exposeInMainWorld("ade", { await ipcRenderer.invoke(IPC.agentChatInterrupt, args); agentChatSummaryCache.clear(); }, + recoverTurn: async ( + args: AgentChatRecoverTurnArgs, + ): Promise => { + agentChatSummaryCache.clear(); + const result = await callProjectRuntimeActionOr( + "chat", + "recoverTurn", + { args }, + () => ipcRenderer.invoke(IPC.agentChatRecoverTurn, args), + ); + agentChatSummaryCache.clear(); + return result; + }, recoverCodexTurn: async ( args: AgentChatRecoverCodexTurnArgs, ): Promise => { @@ -5582,6 +5601,19 @@ contextBridge.exposeInMainWorld("ade", { agentChatSummaryCache.clear(); return result; }, + resolveUnprocessedMessage: async ( + args: AgentChatResolveUnprocessedMessageArgs, + ): Promise => { + agentChatSummaryCache.clear(); + const result = await callProjectRuntimeActionOr( + "chat", + "resolveUnprocessedMessage", + { args }, + () => ipcRenderer.invoke(IPC.agentChatResolveUnprocessedMessage, args), + ); + agentChatSummaryCache.clear(); + return result; + }, recoverContinuity: async ( args: AgentChatRecoverContinuityArgs, ): Promise => { diff --git a/apps/desktop/src/renderer/browserMock.ts b/apps/desktop/src/renderer/browserMock.ts index cdc1ee00b..e39447b37 100644 --- a/apps/desktop/src/renderer/browserMock.ts +++ b/apps/desktop/src/renderer/browserMock.ts @@ -37,7 +37,13 @@ import { deriveSmartLinkPreview } from "../shared/smartLinks"; import { isAdeUsageRangePreset, type AdeUsageRangePreset, + type AgentChatRecoverCodexTurnArgs, + type AgentChatRecoverCodexTurnResult, + type AgentChatRecoverTurnArgs, + type AgentChatRecoverTurnResult, type AgentChatPrepareCrossMachineHandoffArgs, + type AgentChatResolveUnprocessedMessageArgs, + type AgentChatResolveUnprocessedMessageResult, type RemoteRuntimeActionRequest, } from "../shared/types"; import { @@ -5082,7 +5088,22 @@ if (typeof window !== "undefined" && shouldInstallBrowserMock(window)) { }), cancelDispatchedSteer: resolvedArg({ cancelled: false }), interrupt: resolvedArg(undefined), - recoverCodexTurn: async (args: any) => ({ + recoverTurn: async ( + args: AgentChatRecoverTurnArgs, + ): Promise => ({ + action: args.action, + turnId: args.turnId, + status: args.action === "wait" + ? "waiting" + : args.action === "nudge" + ? "nudged" + : args.action === "restart_resume" + ? "resumed" + : "retrying", + }), + recoverCodexTurn: async ( + args: AgentChatRecoverCodexTurnArgs, + ): Promise => ({ action: args.action, turnId: args.turnId, status: args.action === "wait" @@ -5093,6 +5114,13 @@ if (typeof window !== "undefined" && shouldInstallBrowserMock(window)) { ? "resumed" : "retrying", }), + resolveUnprocessedMessage: async ( + args: AgentChatResolveUnprocessedMessageArgs, + ): Promise => ({ + steerId: args.steerId, + action: args.action, + status: "completed", + }), approve: resolvedArg(undefined), respondToInput: resolvedArg(undefined), models: resolvedArg([]), diff --git a/apps/desktop/src/renderer/components/chat/AgentChatMessageList.test.tsx b/apps/desktop/src/renderer/components/chat/AgentChatMessageList.test.tsx index b2585cb2c..745c5f608 100644 --- a/apps/desktop/src/renderer/components/chat/AgentChatMessageList.test.tsx +++ b/apps/desktop/src/renderer/components/chat/AgentChatMessageList.test.tsx @@ -131,6 +131,7 @@ function renderMessageList( onInsertDraft?: (text: string) => void; onApproval?: (itemId: string, decision: AgentChatApprovalDecision, responseText?: string | null, answers?: Record) => void; onCodexRecovery?: (args: AgentChatRecoverCodexTurnArgs) => Promise; + onRunUnprocessedMessage?: (event: Extract) => void | Promise; scrollToRowKeyRequest?: { key: string; requestId: number } | null; hasOlderHistory?: boolean; loadingOlderHistory?: boolean; @@ -149,6 +150,7 @@ function renderMessageList( onInsertDraft={options?.onInsertDraft} onApproval={options?.onApproval as any} onCodexRecovery={options?.onCodexRecovery} + onRunUnprocessedMessage={options?.onRunUnprocessedMessage} scrollToRowKeyRequest={options?.scrollToRowKeyRequest} hasOlderHistory={options?.hasOlderHistory} loadingOlderHistory={options?.loadingOlderHistory} @@ -894,6 +896,24 @@ describe("AgentChatMessageList transcript rendering", () => { expect(screen.getByText("interrupted")).toBeTruthy(); }); + it("labels end-of-turn wall time as ran for, not worked for", () => { + renderMessageList([ + { + sessionId: "session-1", + timestamp: "2026-03-17T10:00:00.000Z", + event: { type: "user_message", text: "Run the checks.", turnId: "turn-1" }, + }, + { + sessionId: "session-1", + timestamp: "2026-03-17T10:02:00.000Z", + event: { type: "done", turnId: "turn-1", status: "interrupted" }, + }, + ]); + + expect(screen.getByText("Ran for 2m")).toBeTruthy(); + expect(screen.queryByText(/Worked for/)).toBeNull(); + }); + it("renders provider health and thread error notices distinctly", () => { renderMessageList([ { @@ -1091,7 +1111,7 @@ describe("AgentChatMessageList transcript rendering", () => { }, ], { sessionId: "parent-session", onCodexRecovery }); - fireEvent.click(screen.getByRole("button", { name: "Wait" })); + fireEvent.click(screen.getByRole("button", { name: "Keep waiting" })); await waitFor(() => { expect(onCodexRecovery).toHaveBeenCalledWith({ sessionId: "child-session", @@ -1100,9 +1120,10 @@ describe("AgentChatMessageList transcript rendering", () => { }); }); expect(await screen.findByText("Waiting for Codex output…")).toBeTruthy(); - expect(screen.getByRole("button", { name: "Nudge" })).toBeTruthy(); - expect(screen.getByRole("button", { name: "Retry" })).toBeTruthy(); - expect(screen.getByRole("button", { name: "Resume" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Restart & resume" })).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "More" })); + expect(screen.getByRole("button", { name: "Send nudge" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Retry same server" })).toBeTruthy(); }); it("shows a Codex recovery error without making the card inert", async () => { @@ -1121,9 +1142,120 @@ describe("AgentChatMessageList transcript rendering", () => { }, ], { sessionId: "session-1", onCodexRecovery }); - fireEvent.click(screen.getByRole("button", { name: "Resume" })); + fireEvent.click(screen.getByRole("button", { name: "Restart & resume" })); expect((await screen.findByRole("alert")).textContent).toContain("no longer active"); - expect((screen.getByRole("button", { name: "Resume" }) as HTMLButtonElement).disabled).toBe(false); + expect((screen.getByRole("button", { name: "Restart & resume" }) as HTMLButtonElement).disabled).toBe(false); + }); + + it("hides raw moderation rows and keeps cumulative diagnostics behind turn details", () => { + renderMessageList([ + { + sessionId: "session-1", + timestamp: "2026-03-17T10:00:00.000Z", + event: { + type: "codex_moderation_metadata", + metadata: { turnId: "turn-1", metadata: { is_blocked: false } }, + turnId: "turn-1", + }, + }, + { + sessionId: "session-1", + timestamp: "2026-03-17T10:00:01.000Z", + event: { + type: "turn_diagnostics", + turnId: "turn-1", + moderationChecks: 1, + }, + }, + { + sessionId: "session-1", + timestamp: "2026-03-17T10:00:02.000Z", + event: { + type: "turn_diagnostics", + turnId: "turn-1", + moderationChecks: 3, + optionalIntegrationFailures: [{ integration: "unityMCP", message: "not configured" }], + }, + }, + ]); + + expect(screen.queryByText("Moderation")).toBeNull(); + expect(screen.getAllByText("Turn details")).toHaveLength(1); + expect(screen.getByText(/3 safety checks/)).toBeTruthy(); + fireEvent.click(screen.getByRole("button")); + expect(screen.getByText("unityMCP")).toBeTruthy(); + }); + + it("merges steer lifecycle updates and can run an unprocessed message next", async () => { + const onRunUnprocessedMessage = vi.fn().mockResolvedValue(undefined); + renderMessageList([ + { + sessionId: "session-1", + timestamp: "2026-03-17T10:00:00.000Z", + event: { + type: "user_message", + text: "Check the release.", + steerId: "steer-1", + deliveryState: "accepted", + turnId: "turn-1", + }, + }, + { + sessionId: "session-1", + timestamp: "2026-03-17T10:00:01.000Z", + event: { + type: "user_message", + text: "Check the release.", + steerId: "steer-1", + deliveryState: "unprocessed", + turnId: "turn-1", + }, + }, + ], { onRunUnprocessedMessage }); + + expect(screen.getAllByText("Check the release.")).toHaveLength(1); + expect(screen.getByText("not processed")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "Run next" })); + await waitFor(() => { + expect(onRunUnprocessedMessage).toHaveBeenCalledWith(expect.objectContaining({ + steerId: "steer-1", + deliveryState: "unprocessed", + })); + }); + expect(await screen.findByText("Started as the next turn")).toBeTruthy(); + }); + + it("collapses a resolved Codex recovery card into an audit receipt", () => { + renderMessageList([ + { + sessionId: "session-1", + timestamp: "2026-03-17T10:00:00.000Z", + event: { + type: "codex_turn_stalled", + turnId: "turn-stalled", + reason: "no_output", + message: "No output arrived.", + recoveryOptions: ["restart_resume_thread", "wait"], + }, + }, + { + sessionId: "session-1", + timestamp: "2026-03-17T10:00:01.000Z", + event: { + type: "codex_turn_recovery", + turnId: "turn-stalled", + action: "restart_resume_thread", + state: "recovered", + message: "ADE restarted the Codex app-server and resumed the thread.", + automatic: true, + at: "2026-03-17T10:00:01.000Z", + }, + }, + ]); + + expect(screen.queryByRole("button", { name: "Restart & resume" })).toBeNull(); + expect(screen.getByText("Recovered")).toBeTruthy(); + expect(screen.getByText(/restarted the Codex app-server/)).toBeTruthy(); }); it("keeps non-rate-limit notice details in collapsible cards", () => { diff --git a/apps/desktop/src/renderer/components/chat/AgentChatMessageList.tsx b/apps/desktop/src/renderer/components/chat/AgentChatMessageList.tsx index 57f51a7a3..37ee2e565 100644 --- a/apps/desktop/src/renderer/components/chat/AgentChatMessageList.tsx +++ b/apps/desktop/src/renderer/components/chat/AgentChatMessageList.tsx @@ -143,6 +143,11 @@ type WorkspacePathLocation = { }; type CodexTurnStalledEvent = Extract; +type CodexTurnRecoveryEvent = Extract< + AgentChatEvent, + { type: "codex_turn_recovery" | "turn_recovery" } +>; +type UserMessageEvent = Extract; function CodexTurnRecoveryCard({ event, @@ -156,12 +161,13 @@ function CodexTurnRecoveryCard({ const [pendingAction, setPendingAction] = useState(null); const [resultMessage, setResultMessage] = useState(null); const [errorMessage, setErrorMessage] = useState(null); + const [moreOpen, setMoreOpen] = useState(false); const targetSessionId = event.sourceSessionId?.trim() || sessionId?.trim() || ""; const optionLabels: Record = { - wait: "Wait", - steer: "Nudge", - interrupt_retry_same_thread: "Retry", - restart_resume_thread: "Resume", + wait: "Keep waiting", + steer: "Send nudge", + interrupt_retry_same_thread: "Retry same server", + restart_resume_thread: "Restart & resume", }; const resultLabels: Record = { waiting: "Waiting for Codex output…", @@ -169,6 +175,36 @@ function CodexTurnRecoveryCard({ retrying: "Retry started in this thread.", resumed: "Codex app-server restarted and the thread resumed.", }; + const recoveryOptions = event.recoveryOptions ?? [ + "restart_resume_thread", + "wait", + "steer", + "interrupt_retry_same_thread", + ]; + const primaryOptions = (["restart_resume_thread", "wait"] as const) + .filter((option) => recoveryOptions.includes(option)); + const secondaryOptions = (["steer", "interrupt_retry_same_thread"] as const) + .filter((option) => recoveryOptions.includes(option)); + const title = event.reason === "waiting_on_approval" + ? "Codex is waiting for approval" + : event.reason === "waiting_on_input" + ? "Codex is waiting for your input" + : event.reason === "no_progress" + ? "Codex stopped making progress" + : "Codex did not start responding"; + const timing = (() => { + const detectedAt = event.detectedAt ? Date.parse(event.detectedAt) : Number.NaN; + const turnStartedAt = event.turnStartedAt ? Date.parse(event.turnStartedAt) : Number.NaN; + const lastProgressAt = event.lastProgressAt ? Date.parse(event.lastProgressAt) : Number.NaN; + const parts: string[] = []; + if (Number.isFinite(detectedAt) && Number.isFinite(turnStartedAt) && detectedAt > turnStartedAt) { + parts.push(`Elapsed ${formatTurnDuration(detectedAt - turnStartedAt)}`); + } + if (Number.isFinite(detectedAt) && Number.isFinite(lastProgressAt) && detectedAt > lastProgressAt) { + parts.push(`inactive ${formatTurnDuration(detectedAt - lastProgressAt)}`); + } + return parts.join(" · "); + })(); const recover = useCallback(async (action: AgentChatRecoverCodexTurnArgs["action"]) => { if (!targetSessionId || !onRecover || pendingAction) return; @@ -190,19 +226,34 @@ function CodexTurnRecoveryCard({
recovery - Codex paused unexpectedly + {title}
{event.message}
- {event.recoveryOptions?.length ? ( + {event.automaticRecoveryAttempted ? ( +
+ ADE already tried one automatic restart. It will not restart again without you. +
+ ) : null} + {timing ? ( +
+ {timing} +
+ ) : null} + {primaryOptions.length ? (
- {event.recoveryOptions.slice(0, 4).map((option) => ( + {primaryOptions.map((option) => (
) : null} + {secondaryOptions.length ? ( +
+ + {moreOpen ? ( +
+ {secondaryOptions.map((option) => ( + + ))} +
+ ) : null} +
+ ) : null} {resultMessage ? (
{resultMessage} @@ -224,6 +303,73 @@ function CodexTurnRecoveryCard({ ); } +function CodexTurnRecoveryReceipt({ event }: { event: CodexTurnRecoveryEvent }) { + const tone = event.state === "failed" + ? "border-red-300/14 bg-red-500/[0.04] text-red-100/70" + : event.state === "recovered" + ? "border-emerald-300/12 bg-emerald-500/[0.035] text-emerald-100/68" + : "border-amber-300/12 bg-amber-500/[0.035] text-amber-100/68"; + const label = event.state === "recovered" + ? "Recovered" + : event.state === "failed" + ? "Recovery failed" + : "Recovering"; + return ( +
+ {event.state === "recovered" + ? + : } + {label} + {event.message} + {event.automatic ? automatic : null} +
+ ); +} + +function TurnDiagnosticsDisclosure({ + event, +}: { + event: Extract; +}) { + const moderationChecks = Math.max(0, event.moderationChecks ?? 0); + const integrations = event.optionalIntegrationFailures ?? []; + if (!moderationChecks && !integrations.length) return null; + const summaryParts = [ + moderationChecks ? `${moderationChecks} safety ${moderationChecks === 1 ? "check" : "checks"}` : null, + integrations.length ? `${integrations.length} optional ${integrations.length === 1 ? "integration warning" : "integration warnings"}` : null, + ].filter((part): part is string => Boolean(part)); + return ( + + + Turn details + {summaryParts.join(" · ")} +
+ )} + > +
+ {moderationChecks ? ( +
Safety checks recorded: {moderationChecks}.
+ ) : null} + {integrations.length ? ( +
+
Optional integrations unavailable
+
    + {integrations.map((integration) => ( +
  • + {integration.integration} + {integration.message ? · {integration.message} : null} +
    • + ))} +
+
+ ) : null} +
+ + ); +} + function formatDiffCounts(fileCount: number, additions: number, deletions: number): string { const fileLabel = fileCount === 1 ? "file" : "files"; return `${fileCount} ${fileLabel} +${additions} -${deletions}`; @@ -545,23 +691,29 @@ function describeUserDeliveryState(event: Extract void | Promise; + onEdit?: (event: UserMessageEvent) => void; + onDismiss?: (event: UserMessageEvent) => void | Promise; +}) { + const [running, setRunning] = useState(false); + const [resolved, setResolved] = useState<"run_next" | "dismiss" | null>(null); + const [error, setError] = useState(null); + const durableAction = + event.metadata?.unprocessedMessageResolution?.action ?? null; + const settledAction = durableAction ?? resolved; + if (event.deliveryState !== "unprocessed") return null; + const run = async () => { + if (!onRun || running || settledAction) return; + setRunning(true); + setError(null); + try { + await onRun(event); + setResolved("run_next"); + } catch (runError) { + setError(runError instanceof Error ? runError.message : String(runError)); + } finally { + setRunning(false); + } + }; + const dismiss = async () => { + if (!onDismiss || running || settledAction) return; + setRunning(true); + setError(null); + try { + await onDismiss(event); + setResolved("dismiss"); + } catch (dismissError) { + setError(dismissError instanceof Error ? dismissError.message : String(dismissError)); + } finally { + setRunning(false); + } + }; + if (settledAction) { + return ( +
+ {settledAction === "run_next" ? "Started as the next turn" : "Dismissed"} +
+ ); + } + return ( +
+ + {onEdit ? ( + + ) : null} + {onDismiss ? ( + + ) : null} + {error ? ( +
+ {error} +
+ ) : null} +
+ ); +} + type RenderEnvelope = { key: string; timestamp: string; @@ -2786,6 +3028,9 @@ function renderEvent( onCodexRecovery?: (args: AgentChatRecoverCodexTurnArgs) => Promise; onRetryProviderFailure?: (turnId: string | null) => Promise; onChooseProviderFailureModel?: () => void; + onRunUnprocessedMessage?: (event: UserMessageEvent) => void | Promise; + onEditUnprocessedMessage?: (event: UserMessageEvent) => void; + onDismissUnprocessedMessage?: (event: UserMessageEvent) => void | Promise; turnModel?: { label: string; modelId?: string; model?: string } | null; surfaceMode?: ChatSurfaceMode; surfaceProfile?: ChatSurfaceProfile; @@ -2987,6 +3232,12 @@ function renderEvent( /> ) : null} + ); @@ -3329,13 +3580,15 @@ function renderEvent( } if (event.type === "codex_moderation_metadata") { - return ( -
- - moderation - Checked -
- ); + return null; + } + + if (event.type === "turn_diagnostics") { + return ; + } + + if (event.type === "codex_turn_recovery" || event.type === "turn_recovery") { + return ; } if (event.type === "codex_sleep") { @@ -4402,8 +4655,8 @@ function DoneTurnDivider({ const completed = event.status === "completed"; const { label: modelLabel } = resolveModelMeta(event.modelId, event.model); const reasonLabel = completed ? null : terminalReasonLabel(event.terminalReason); - const workedFor = durationMs !== null && durationMs > 1500 - ? `Worked for ${formatTurnDuration(durationMs)}` + const ranFor = durationMs !== null && durationMs > 1500 + ? `Ran for ${formatTurnDuration(durationMs)}` : null; return (
@@ -4431,10 +4684,10 @@ function DoneTurnDivider({ {reasonLabel} ) : null} - {workedFor ? ( + {ranFor ? ( <> · - {workedFor} + {ranFor} ) : null} @@ -4538,6 +4791,9 @@ type EventRowProps = { onCodexRecovery?: (args: AgentChatRecoverCodexTurnArgs) => Promise; onRetryProviderFailure?: (turnId: string | null) => Promise; onChooseProviderFailureModel?: () => void; + onRunUnprocessedMessage?: (event: UserMessageEvent) => void | Promise; + onEditUnprocessedMessage?: (event: UserMessageEvent) => void; + onDismissUnprocessedMessage?: (event: UserMessageEvent) => void | Promise; surfaceMode?: ChatSurfaceMode; surfaceProfile?: ChatSurfaceProfile; assistantLabel?: string; @@ -4577,6 +4833,9 @@ const EventRow = React.memo(function EventRow({ onCodexRecovery, onRetryProviderFailure, onChooseProviderFailureModel, + onRunUnprocessedMessage, + onEditUnprocessedMessage, + onDismissUnprocessedMessage, surfaceMode = "standard", surfaceProfile = "standard", assistantLabel, @@ -4659,9 +4918,12 @@ const EventRow = React.memo(function EventRow({ ? : renderEvent(envelope as RenderEnvelope, { onApproval, - onCodexRecovery, - onRetryProviderFailure, - onChooseProviderFailureModel, + onCodexRecovery, + onRetryProviderFailure, + onChooseProviderFailureModel, + onRunUnprocessedMessage, + onEditUnprocessedMessage, + onDismissUnprocessedMessage, turnModel, surfaceMode, surfaceProfile, @@ -4995,6 +5257,9 @@ function AgentChatMessageListMain({ onCodexRecovery, onRetryProviderFailure, onChooseProviderFailureModel, + onRunUnprocessedMessage, + onEditUnprocessedMessage, + onDismissUnprocessedMessage, surfaceMode = "standard", surfaceProfile = "standard", assistantLabel, @@ -5025,6 +5290,9 @@ function AgentChatMessageListMain({ onCodexRecovery?: (args: AgentChatRecoverCodexTurnArgs) => Promise; onRetryProviderFailure?: (turnId: string | null) => Promise; onChooseProviderFailureModel?: () => void; + onRunUnprocessedMessage?: (event: UserMessageEvent) => void | Promise; + onEditUnprocessedMessage?: (event: UserMessageEvent) => void; + onDismissUnprocessedMessage?: (event: UserMessageEvent) => void | Promise; surfaceMode?: ChatSurfaceMode; surfaceProfile?: ChatSurfaceProfile; assistantLabel?: string; @@ -5928,6 +6196,9 @@ function AgentChatMessageListMain({ onCodexRecovery={onCodexRecovery} onRetryProviderFailure={onRetryProviderFailure} onChooseProviderFailureModel={onChooseProviderFailureModel} + onRunUnprocessedMessage={onRunUnprocessedMessage} + onEditUnprocessedMessage={onEditUnprocessedMessage} + onDismissUnprocessedMessage={onDismissUnprocessedMessage} surfaceMode={surfaceMode} surfaceProfile={surfaceProfile} assistantLabel={assistantLabel} @@ -5964,10 +6235,14 @@ function AgentChatMessageListMain({ turnDividerLabel={turnDividerLabel} showForkHistoryDivider={showForkHistoryDivider} turnModel={turnModel} + turnEndDurationMs={turnEndDurationMs} onApproval={handleApproval} onCodexRecovery={onCodexRecovery} onRetryProviderFailure={onRetryProviderFailure} onChooseProviderFailureModel={onChooseProviderFailureModel} + onRunUnprocessedMessage={onRunUnprocessedMessage} + onEditUnprocessedMessage={onEditUnprocessedMessage} + onDismissUnprocessedMessage={onDismissUnprocessedMessage} surfaceMode={surfaceMode} surfaceProfile={surfaceProfile} assistantLabel={assistantLabel} @@ -5996,7 +6271,7 @@ function AgentChatMessageListMain({ onCancelQueuedMessage={onCancelQueuedMessage} /> ); - }, [activeTurnId, anchoredRowKey, assistantLabel, assistantTurnCopyByRowKey, surfaceMode, surfaceProfile, groupedRows, latestWorkLogIndex, turnModelState, handleApproval, handleMeasure, openWorkspacePath, handleNavigateSuggestion, handleReviewChanges, onCodexRecovery, onRetryProviderFailure, onChooseProviderFailureModel, onInsertDraft, onRevealChatTerminal, onRewindFiles, turnDiffSummaries, respondingApprovalIds, pendingApprovalIds, resolvedInputStates, laneId, sessionId, sessionTurnActive, sessionEnded, runtimeName, mosaic, scrollToRowKey, forkHistoryDividerRowKey, staleInterruptReceipts, onCancelQueuedMessage]); + }, [activeTurnId, anchoredRowKey, assistantLabel, assistantTurnCopyByRowKey, surfaceMode, surfaceProfile, groupedRows, latestWorkLogIndex, turnModelState, handleApproval, handleMeasure, openWorkspacePath, handleNavigateSuggestion, handleReviewChanges, onCodexRecovery, onRetryProviderFailure, onChooseProviderFailureModel, onRunUnprocessedMessage, onEditUnprocessedMessage, onDismissUnprocessedMessage, onInsertDraft, onRevealChatTerminal, onRewindFiles, turnDiffSummaries, respondingApprovalIds, pendingApprovalIds, resolvedInputStates, laneId, sessionId, sessionTurnActive, sessionEnded, runtimeName, mosaic, scrollToRowKey, forkHistoryDividerRowKey, staleInterruptReceipts, onCancelQueuedMessage]); // Compute the bottom spacer height for virtualized mode. const bottomSpacerHeight = useMemo(() => { diff --git a/apps/desktop/src/renderer/components/chat/AgentChatPane.test.tsx b/apps/desktop/src/renderer/components/chat/AgentChatPane.test.tsx index 8073c63ba..f4dbed098 100644 --- a/apps/desktop/src/renderer/components/chat/AgentChatPane.test.tsx +++ b/apps/desktop/src/renderer/components/chat/AgentChatPane.test.tsx @@ -470,6 +470,7 @@ function installAdeMocks(options?: { aiStatus?: AiSettingsStatus; parallelLaunchState?: AgentChatParallelLaunchState | null; linkedPr?: PrSummary | null; + recoverTurnError?: Error; }) { const send = options?.sendError ? vi.fn().mockRejectedValue(options.sendError) @@ -479,6 +480,24 @@ function installAdeMocks(options?: { : vi.fn().mockResolvedValue(options?.steerResult ?? { steerId: "steer-default", queued: true }); const dispatchSteer = vi.fn().mockResolvedValue({ dispatchedAt: Date.now() }); const cancelSteer = vi.fn().mockResolvedValue(undefined); + const recoverTurn = options?.recoverTurnError + ? vi.fn().mockRejectedValue(options.recoverTurnError) + : vi.fn().mockResolvedValue({ + action: "restart_resume", + turnId: "turn-1", + status: "resumed", + }); + const recoverCodexTurn = vi.fn().mockResolvedValue({ + action: "restart_resume_thread", + turnId: "turn-1", + status: "resumed", + }); + const resolveUnprocessedMessage = vi.fn().mockResolvedValue({ + steerId: "steer-unprocessed", + action: "run_next", + status: "completed", + replacementMessageId: "message-next", + }); const list = options?.listError ? vi.fn().mockRejectedValue(options.listError) : vi.fn().mockResolvedValue(options?.sessions ?? [buildSession("session-1")]); @@ -596,6 +615,9 @@ function installAdeMocks(options?: { archive, unarchive, interrupt: vi.fn().mockResolvedValue(undefined), + recoverTurn, + recoverCodexTurn, + resolveUnprocessedMessage, approve: vi.fn().mockResolvedValue(undefined), respondToInput: vi.fn().mockResolvedValue(undefined), warmupModel: vi.fn().mockResolvedValue(undefined), @@ -685,6 +707,9 @@ function installAdeMocks(options?: { steer, dispatchSteer, cancelSteer, + recoverTurn, + recoverCodexTurn, + resolveUnprocessedMessage, list, create, createLane, @@ -1428,6 +1453,155 @@ describe("AgentChatPane companion drawers", () => { }); }); +describe("AgentChatPane durable recovery actions", () => { + it("appends an unprocessed message for editing without replacing the current draft", async () => { + const session = buildSession("session-1", { status: "idle" }); + const transcript = `${JSON.stringify({ + sessionId: session.sessionId, + timestamp: "2026-07-25T05:20:00.000Z", + event: { + type: "user_message", + text: "Original backend prompt.", + displayText: "Edit this follow-up.", + steerId: "steer-unprocessed", + deliveryState: "unprocessed", + processed: false, + turnId: "turn-1", + }, + })}\n`; + const { resolveUnprocessedMessage } = installAdeMocks({ + sessions: [session], + transcript, + }); + + renderPane(session); + + fireEvent.change(screen.getByRole("textbox"), { target: { value: "Keep this draft." } }); + fireEvent.click(await screen.findByRole("button", { name: "Edit" })); + expect((screen.getByRole("textbox") as HTMLTextAreaElement).value).toBe( + "Keep this draft.\n\nEdit this follow-up.", + ); + + fireEvent.click(screen.getByRole("button", { name: "Run next" })); + await waitFor(() => { + expect(resolveUnprocessedMessage).toHaveBeenCalledWith({ + sessionId: session.sessionId, + steerId: "steer-unprocessed", + action: "run_next", + }); + }); + }); + + it("dismisses an unprocessed message durably", async () => { + const session = buildSession("session-1", { status: "idle" }); + const transcript = `${JSON.stringify({ + sessionId: session.sessionId, + timestamp: "2026-07-25T05:20:00.000Z", + event: { + type: "user_message", + text: "Dismiss this follow-up.", + steerId: "steer-unprocessed", + deliveryState: "unprocessed", + processed: false, + turnId: "turn-1", + }, + })}\n`; + const { resolveUnprocessedMessage } = installAdeMocks({ + sessions: [session], + transcript, + }); + + renderPane(session); + + fireEvent.click(await screen.findByRole("button", { name: "Dismiss" })); + await waitFor(() => { + expect(resolveUnprocessedMessage).toHaveBeenCalledWith({ + sessionId: session.sessionId, + steerId: "steer-unprocessed", + action: "dismiss", + }); + }); + }); + + it("prefers provider-neutral recovery and falls back for older hosts", async () => { + const session = buildSession("session-1", { status: "active" }); + const transcript = `${JSON.stringify({ + sessionId: session.sessionId, + timestamp: "2026-07-25T05:20:00.000Z", + event: { + type: "turn_health", + provider: "codex", + turnId: "turn-1", + state: "stalled", + reason: "no_output", + message: "Codex accepted the turn but has not streamed output.", + turnStartedAt: "2026-07-25T05:18:00.000Z", + lastProgressAt: "2026-07-25T05:18:00.000Z", + detectedAt: "2026-07-25T05:20:00.000Z", + recoveryCount: 0, + supportedActions: ["restart_resume"], + automaticRecoveryAttempted: false, + }, + })}\n`; + const { recoverTurn, recoverCodexTurn } = installAdeMocks({ + sessions: [session], + transcript, + recoverTurnError: new Error("Action not supported by runtime"), + }); + + renderPane(session); + + fireEvent.click(await screen.findByRole("button", { name: "Restart & resume" })); + await waitFor(() => { + expect(recoverTurn).toHaveBeenCalledWith({ + sessionId: session.sessionId, + turnId: "turn-1", + action: "restart_resume", + }); + expect(recoverCodexTurn).toHaveBeenCalledWith({ + sessionId: session.sessionId, + turnId: "turn-1", + action: "restart_resume_thread", + }); + }); + }); + + it("does not hide a chat-service outage behind the legacy recovery fallback", async () => { + const session = buildSession("session-1", { status: "active" }); + const transcript = `${JSON.stringify({ + sessionId: session.sessionId, + timestamp: "2026-07-25T05:20:00.000Z", + event: { + type: "turn_health", + provider: "codex", + turnId: "turn-1", + state: "stalled", + reason: "no_output", + message: "Codex accepted the turn but has not streamed output.", + turnStartedAt: "2026-07-25T05:18:00.000Z", + lastProgressAt: "2026-07-25T05:18:00.000Z", + detectedAt: "2026-07-25T05:20:00.000Z", + recoveryCount: 0, + supportedActions: ["restart_resume"], + automaticRecoveryAttempted: false, + }, + })}\n`; + const { recoverTurn, recoverCodexTurn } = installAdeMocks({ + sessions: [session], + transcript, + recoverTurnError: new Error("Agent chat service not available."), + }); + + renderPane(session); + + fireEvent.click(await screen.findByRole("button", { name: "Restart & resume" })); + await waitFor(() => { + expect(recoverTurn).toHaveBeenCalledOnce(); + expect(recoverCodexTurn).not.toHaveBeenCalled(); + }); + }); +}); + describe("AgentChatPane submit recovery", () => { it("resends the latest user message for the selected session after auth retry", async () => { const session = buildSession("session-1", { diff --git a/apps/desktop/src/renderer/components/chat/AgentChatPane.tsx b/apps/desktop/src/renderer/components/chat/AgentChatPane.tsx index dffda5d17..2ae338f42 100644 --- a/apps/desktop/src/renderer/components/chat/AgentChatPane.tsx +++ b/apps/desktop/src/renderer/components/chat/AgentChatPane.tsx @@ -49,7 +49,10 @@ import { type TerminalSessionDetail, type TerminalToolType, } from "../../../shared/types"; -import { providerSupportsHandoffFork } from "../../../shared/types/chat"; +import { + isUnsupportedAgentChatRecoveryActionError, + providerSupportsHandoffFork, +} from "../../../shared/types/chat"; import { providerDisplayLabel } from "../../../shared/pendingInputLabels"; import { resolveSubagentCapability } from "../../../shared/subagentCapabilities"; import { @@ -3583,6 +3586,7 @@ export function AgentChatPane({ const pendingFastModeUpdateRef = useRef<{ sessionId: string; updateId: number; promise: Promise } | null>(null); const pendingEventQueueRef = useRef([]); const eventsBySessionRef = useRef>({}); + const turnActiveBySessionRef = useRef>({}); const detachedHistorySessionsRef = useRef>(new Set()); const detachedLiveEventsBySessionRef = useRef>({}); const olderHistoryCursorRef = useRef>({}); @@ -3604,6 +3608,7 @@ export function AgentChatPane({ () => (selectedSessionId ? sessions.find((session) => session.sessionId === selectedSessionId) ?? null : null), [sessions, selectedSessionId] ); + turnActiveBySessionRef.current = turnActiveBySession; const promptSuggestion = selectedSessionId ? promptSuggestionsBySession[selectedSessionId] ?? null : null; const clearPromptSuggestionForSession = useCallback((sessionId: string | null) => { if (!sessionId) return; @@ -9586,9 +9591,86 @@ export function AgentChatPane({ [handleApproval], ); const handleListCodexRecovery = useCallback( - (args: AgentChatRecoverCodexTurnArgs) => window.ade.agentChat.recoverCodexTurn(args), + async (args: AgentChatRecoverCodexTurnArgs) => { + const action = args.action === "steer" + ? "nudge" + : args.action === "interrupt_retry_same_thread" + ? "retry_same_runtime" + : args.action === "restart_resume_thread" + ? "restart_resume" + : "wait"; + try { + const result = await window.ade.agentChat.recoverTurn({ + sessionId: args.sessionId, + turnId: args.turnId, + action, + }); + return { + ...result, + action: args.action, + }; + } catch (error) { + if (!isUnsupportedAgentChatRecoveryActionError(error)) throw error; + return window.ade.agentChat.recoverCodexTurn(args); + } + }, [], ); + const handleRunUnprocessedMessage = useCallback( + async (event: Extract) => { + const sessionId = selectedSessionIdRef.current; + if (!sessionId) throw new Error("This chat is no longer selected."); + if (turnActiveBySessionRef.current[sessionId]) { + throw new Error("A turn is already active. Wait for it to finish, then run this message."); + } + if (submitInFlightRef.current) { + throw new Error("Another message is already being sent."); + } + try { + submitInFlightRef.current = true; + setBusy(true); + setError(null); + touchSession(sessionId); + const steerId = event.steerId?.trim(); + if (!steerId) throw new Error("This message is missing its durable delivery identifier."); + await window.ade.agentChat.resolveUnprocessedMessage({ + sessionId, + steerId, + action: "run_next", + }); + void refreshSessions().catch(() => {}); + } catch (runError) { + const message = runError instanceof Error ? runError.message : String(runError); + setError(message); + throw runError; + } finally { + submitInFlightRef.current = false; + setBusy(false); + } + }, + [refreshSessions, touchSession], + ); + const handleEditUnprocessedMessage = useCallback( + (event: Extract) => { + insertComposerDraft(event.displayText?.trim() || event.text); + }, + [insertComposerDraft], + ); + const handleDismissUnprocessedMessage = useCallback( + async (event: Extract) => { + const sessionId = selectedSessionIdRef.current; + const steerId = event.steerId?.trim(); + if (!sessionId) throw new Error("This chat is no longer selected."); + if (!steerId) throw new Error("This message is missing its durable delivery identifier."); + await window.ade.agentChat.resolveUnprocessedMessage({ + sessionId, + steerId, + action: "dismiss", + }); + void refreshSessions().catch(() => {}); + }, + [refreshSessions], + ); const handleListRetryProviderFailure = useCallback( async (failedTurnId: string | null) => { if (!selectedSessionId) return "This chat is no longer selected."; @@ -11789,6 +11871,9 @@ export function AgentChatPane({ onCancelQueuedMessage={!subagentView && selectedSessionId ? cancelQueuedMessageFromReceipt : undefined} onApproval={handleListApproval} onCodexRecovery={handleListCodexRecovery} + onRunUnprocessedMessage={handleRunUnprocessedMessage} + onEditUnprocessedMessage={handleEditUnprocessedMessage} + onDismissUnprocessedMessage={handleDismissUnprocessedMessage} onRetryProviderFailure={handleListRetryProviderFailure} onChooseProviderFailureModel={handleListChooseProviderFailureModel} mosaic={subagentView ? undefined : mosaicContext} diff --git a/apps/desktop/src/renderer/components/chat/chatTranscriptRows.test.ts b/apps/desktop/src/renderer/components/chat/chatTranscriptRows.test.ts index 7b54dde18..0d1c54a57 100644 --- a/apps/desktop/src/renderer/components/chat/chatTranscriptRows.test.ts +++ b/apps/desktop/src/renderer/components/chat/chatTranscriptRows.test.ts @@ -2613,6 +2613,161 @@ describe("subagent two-row rendering", () => { }); expect(rows[1]?.event.type).toBe("user_message"); }); + + it("folds durable steer and diagnostic lifecycle snapshots into stable rows", () => { + const rows = collapseChatTranscriptEvents([ + env("2026-06-01T09:00:00.000Z", { + type: "user_message", + text: "Check the release.", + steerId: "steer-1", + deliveryState: "accepted", + turnId: "turn-1", + }), + env("2026-06-01T09:00:01.000Z", { + type: "user_message", + text: "Check the release.", + steerId: "steer-1", + deliveryState: "processed", + processed: true, + turnId: "turn-1", + }), + env("2026-06-01T09:00:02.000Z", { + type: "turn_diagnostics", + turnId: "turn-1", + moderationChecks: 1, + }), + env("2026-06-01T09:00:03.000Z", { + type: "turn_diagnostics", + turnId: "turn-1", + moderationChecks: 2, + optionalIntegrationFailures: [{ integration: "unityMCP" }], + }), + ]); + + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ + event: { + type: "user_message", + steerId: "steer-1", + deliveryState: "processed", + processed: true, + }, + }); + expect(rows[1]).toMatchObject({ + event: { + type: "turn_diagnostics", + moderationChecks: 2, + optionalIntegrationFailures: [{ integration: "unityMCP" }], + }, + }); + }); + + it("keeps durable message resolution across out-of-order hydration and later metadata", () => { + const rows = collapseChatTranscriptEvents([ + env("2026-06-01T09:00:00.000Z", { + type: "user_message_resolution", + steerId: "steer-early-resolution", + action: "run_next", + state: "completed", + resolvedAt: "2026-06-01T09:00:00.000Z", + replacementMessageId: "message-2", + }), + env("2026-06-01T09:00:01.000Z", { + type: "user_message", + text: "Run this next.", + steerId: "steer-early-resolution", + deliveryState: "unprocessed", + processed: false, + metadata: { + scheduledWake: { + scheduleId: "wake-1", + kind: "wakeup", + firedAt: "2026-06-01T09:00:01.000Z", + }, + }, + }), + env("2026-06-01T09:00:02.000Z", { + type: "user_message", + text: "Run this next.", + steerId: "steer-early-resolution", + deliveryState: "unprocessed", + processed: false, + metadata: { + spawnCompletion: { + childSessionId: "child-1", + childTitle: "Child", + spawnKind: "subagent", + status: "completed", + }, + }, + }), + ]); + + const userMessage = rows.find((row) => row.event.type === "user_message"); + expect(userMessage?.event).toMatchObject({ + type: "user_message", + metadata: { + scheduledWake: { scheduleId: "wake-1" }, + spawnCompletion: { childSessionId: "child-1" }, + unprocessedMessageResolution: { + action: "run_next", + state: "completed", + replacementMessageId: "message-2", + }, + }, + }); + }); + + it("preserves the actual provider-neutral recovery action", () => { + const rows = collapseChatTranscriptEvents([ + env("2026-06-01T09:00:00.000Z", { + type: "turn_recovery", + provider: "claude", + turnId: "turn-1", + action: "nudge", + state: "recovered", + message: "The provider resumed.", + automatic: false, + at: "2026-06-01T09:00:00.000Z", + recoveryCount: 1, + }), + ]); + + expect(rows).toHaveLength(1); + expect(rows[0]?.event).toMatchObject({ + type: "turn_recovery", + action: "nudge", + state: "recovered", + }); + }); + + it("preserves the child session when adapting provider-neutral turn health for recovery UI", () => { + const rows = collapseChatTranscriptEvents([ + env("2026-06-01T09:00:00.000Z", { + type: "turn_health", + provider: "codex", + turnId: "turn-child", + state: "stalled", + reason: "no_output", + message: "The child turn accepted the request but has not produced output.", + detectedAt: "2026-06-01T09:00:00.000Z", + turnStartedAt: "2026-06-01T08:58:00.000Z", + lastProgressAt: "2026-06-01T08:58:00.000Z", + recoveryCount: 1, + supportedActions: ["wait", "nudge", "retry_same_runtime", "restart_resume"], + automaticRecoveryAttempted: true, + sourceSessionId: "child-session", + }), + ]); + + expect(rows).toHaveLength(1); + expect(rows[0]?.event).toMatchObject({ + type: "codex_turn_stalled", + turnId: "turn-child", + sourceSessionId: "child-session", + recoveryOptions: ["wait", "steer", "interrupt_retry_same_thread", "restart_resume_thread"], + }); + }); }); describe("interrupt-stopped subagent grouping", () => { diff --git a/apps/desktop/src/renderer/components/chat/chatTranscriptRows.ts b/apps/desktop/src/renderer/components/chat/chatTranscriptRows.ts index 663a2126e..bed7b04b4 100644 --- a/apps/desktop/src/renderer/components/chat/chatTranscriptRows.ts +++ b/apps/desktop/src/renderer/components/chat/chatTranscriptRows.ts @@ -325,10 +325,32 @@ type CollapseTranscriptContext = { subagentAnchors: Map; /** Semantic provider failures already rendered for a specific turn. */ errorKeysByTurn: Set; + /** Durable user-message lifecycle rows keyed by ADE steer id. */ + userMessageRowIndexBySteer: Map; + /** Resolution events that arrived before their durable user-message row. */ + unmatchedUserMessageResolutionsBySteer: Map< + string, + Extract + >; + /** Latest cumulative diagnostic snapshot keyed by turn id. */ + diagnosticsRowIndexByTurn: Map; + /** Latest recovery receipt keyed by turn id. */ + recoveryRowIndexByTurn: Map; + /** Actionable stalled-turn card keyed by turn id. */ + stalledRowIndexByTurn: Map; }; export function createCollapseTranscriptContext(): CollapseTranscriptContext { - return { latestTodoItemsByTurn: new Map(), subagentAnchors: new Map(), errorKeysByTurn: new Set() }; + return { + latestTodoItemsByTurn: new Map(), + subagentAnchors: new Map(), + errorKeysByTurn: new Set(), + userMessageRowIndexBySteer: new Map(), + unmatchedUserMessageResolutionsBySteer: new Map(), + diagnosticsRowIndexByTurn: new Map(), + recoveryRowIndexByTurn: new Map(), + stalledRowIndexByTurn: new Map(), + }; } function todoSnapshotKey(turnId: string | null): string { @@ -946,6 +968,33 @@ function repairSubagentRowPositionsAfterSplice( } } +function repairIndexedTranscriptRowsAfterSplice( + context: CollapseTranscriptContext, + removedIndex: number, +): void { + repairSubagentRowPositionsAfterSplice(context, removedIndex); + for (const rowIndexes of [ + context.userMessageRowIndexBySteer, + context.diagnosticsRowIndexByTurn, + context.recoveryRowIndexByTurn, + context.stalledRowIndexByTurn, + ]) { + for (const [key, storedIndex] of rowIndexes) { + if (storedIndex === removedIndex) rowIndexes.delete(key); + else if (storedIndex > removedIndex) rowIndexes.set(key, storedIndex - 1); + } + } +} + +function removeCollapsedTranscriptRow( + rows: ChatTranscriptRenderEnvelope[], + context: CollapseTranscriptContext, + index: number, +): void { + rows.splice(index, 1); + repairIndexedTranscriptRowsAfterSplice(context, index); +} + function durationMsBetween(startedAt: string | null, endedAt: string): number | null { if (!startedAt) return null; const start = Date.parse(startedAt); @@ -1240,6 +1289,48 @@ export function appendCollapsedChatTranscriptEvent( const { event } = envelope; if (event.type === "user_message") { + const steerId = event.steerId?.trim(); + const pendingResolution = steerId + ? context?.unmatchedUserMessageResolutionsBySteer.get(steerId) + : undefined; + const existingIndex = steerId ? context?.userMessageRowIndexBySteer.get(steerId) : undefined; + const existing = existingIndex != null ? rows[existingIndex] : null; + if (existingIndex != null && existing?.event.type === "user_message") { + rows[existingIndex] = { + ...existing, + event: { + ...existing.event, + ...event, + text: event.text || existing.event.text, + displayText: event.displayText ?? existing.event.displayText, + attachments: event.attachments ?? existing.event.attachments, + contextAttachments: event.contextAttachments ?? existing.event.contextAttachments, + metadata: event.metadata || existing.event.metadata || pendingResolution + ? { + ...(existing.event.metadata ?? {}), + ...(event.metadata ?? {}), + ...(pendingResolution + ? { + unprocessedMessageResolution: { + action: pendingResolution.action, + state: pendingResolution.state, + resolvedAt: pendingResolution.resolvedAt, + ...(pendingResolution.replacementMessageId + ? { replacementMessageId: pendingResolution.replacementMessageId } + : {}), + }, + } + : {}), + } + : existing.event.metadata, + turnId: event.turnId ?? existing.event.turnId, + }, + }; + if (steerId && pendingResolution) { + context?.unmatchedUserMessageResolutionsBySteer.delete(steerId); + } + return; + } const wake = event.metadata?.scheduledWake; if ( wake @@ -1292,13 +1383,190 @@ export function appendCollapsedChatTranscriptEvent( } } + if (event.type === "user_message_resolution") { + const steerId = event.steerId.trim(); + const existingIndex = context?.userMessageRowIndexBySteer.get(steerId); + const existing = existingIndex != null ? rows[existingIndex] : null; + if (existingIndex != null && existing?.event.type === "user_message") { + rows[existingIndex] = { + ...existing, + timestamp: envelope.timestamp, + event: { + ...existing.event, + metadata: { + ...(existing.event.metadata ?? {}), + unprocessedMessageResolution: { + action: event.action, + state: event.state, + resolvedAt: event.resolvedAt, + ...(event.replacementMessageId ? { replacementMessageId: event.replacementMessageId } : {}), + }, + }, + }, + }; + } else if (steerId) { + context?.unmatchedUserMessageResolutionsBySteer.set(steerId, event); + } + return; + } + if (event.type === "step_boundary" || event.type === "activity" || event.type === "pending_input_resolved") { return; } // Codex token usage drives the chat-bottom token footer; inline transcript // rows would be duplicate noise. - if (event.type === "codex_token_usage") { + if (event.type === "codex_token_usage" || event.type === "codex_moderation_metadata") { + return; + } + + if (event.type === "turn_diagnostics") { + const turnKey = event.turnId?.trim() || "__session_startup__"; + const existingIndex = context?.diagnosticsRowIndexByTurn.get(turnKey); + if (existingIndex != null && rows[existingIndex]?.event.type === "turn_diagnostics") { + rows[existingIndex] = { + ...rows[existingIndex]!, + timestamp: envelope.timestamp, + event, + }; + return; + } + const rowIndex = rows.length; + rows.push({ + key: `turn-diagnostics:${turnKey}`, + timestamp: envelope.timestamp, + event, + }); + context?.diagnosticsRowIndexByTurn.set(turnKey, rowIndex); + return; + } + + if (event.type === "turn_recovery") { + const turnKey = event.turnId.trim(); + if (event.state === "recovered" && context) { + const stalledIndex = context.stalledRowIndexByTurn.get(turnKey); + if (stalledIndex != null && rows[stalledIndex]?.event.type === "codex_turn_stalled") { + removeCollapsedTranscriptRow(rows, context, stalledIndex); + } + } + const existingIndex = context?.recoveryRowIndexByTurn.get(turnKey); + if ( + existingIndex != null + && ( + rows[existingIndex]?.event.type === "turn_recovery" + || rows[existingIndex]?.event.type === "codex_turn_recovery" + ) + ) { + rows[existingIndex] = { + ...rows[existingIndex]!, + timestamp: envelope.timestamp, + event, + }; + return; + } + const rowIndex = rows.length; + rows.push({ + key: `turn-recovery:${turnKey}`, + timestamp: envelope.timestamp, + event, + }); + context?.recoveryRowIndexByTurn.set(turnKey, rowIndex); + return; + } + + if (event.type === "codex_turn_recovery") { + const turnKey = event.turnId.trim(); + if (event.state === "recovered" && context) { + const stalledIndex = context.stalledRowIndexByTurn.get(turnKey); + if (stalledIndex != null && rows[stalledIndex]?.event.type === "codex_turn_stalled") { + removeCollapsedTranscriptRow(rows, context, stalledIndex); + } + } + const existingIndex = context?.recoveryRowIndexByTurn.get(turnKey); + if (existingIndex != null && rows[existingIndex]?.event.type === "turn_recovery") { + // Provider-neutral receipts are canonical when a legacy alias arrives too. + return; + } + if (existingIndex != null && rows[existingIndex]?.event.type === "codex_turn_recovery") { + rows[existingIndex] = { + ...rows[existingIndex]!, + timestamp: envelope.timestamp, + event, + }; + return; + } + const rowIndex = rows.length; + rows.push({ + key: `codex-turn-recovery:${turnKey}`, + timestamp: envelope.timestamp, + event, + }); + context?.recoveryRowIndexByTurn.set(turnKey, rowIndex); + return; + } + + if (event.type === "codex_turn_stalled") { + const turnKey = event.turnId.trim(); + const recoveryIndex = context?.recoveryRowIndexByTurn.get(turnKey); + const recoveryEvent = recoveryIndex != null ? rows[recoveryIndex]?.event : null; + if ( + (recoveryEvent?.type === "codex_turn_recovery" || recoveryEvent?.type === "turn_recovery") + && recoveryEvent.state === "recovered" + ) { + return; + } + const existingIndex = context?.stalledRowIndexByTurn.get(turnKey); + if (existingIndex != null && rows[existingIndex]?.event.type === "codex_turn_stalled") { + rows[existingIndex] = { + ...rows[existingIndex]!, + timestamp: envelope.timestamp, + event, + }; + return; + } + const rowIndex = rows.length; + rows.push({ + key: `codex-turn-stalled:${turnKey}`, + timestamp: envelope.timestamp, + event, + }); + context?.stalledRowIndexByTurn.set(turnKey, rowIndex); + return; + } + + if (event.type === "turn_health") { + const recoveryOptions = event.supportedActions.flatMap((action) => { + switch (action) { + case "wait": + return ["wait" as const]; + case "nudge": + return ["steer" as const]; + case "retry_same_runtime": + return ["interrupt_retry_same_thread" as const]; + case "restart_resume": + return ["restart_resume_thread" as const]; + } + }); + const legacyStall: Extract = { + type: "codex_turn_stalled", + turnId: event.turnId, + reason: event.reason === "runtime_state_unknown" + ? "app_server_state_unknown" + : event.reason, + message: event.message, + recoveryOptions, + detectedAt: event.detectedAt, + turnStartedAt: event.turnStartedAt, + lastProgressAt: event.lastProgressAt, + automaticRecoveryAttempted: event.automaticRecoveryAttempted, + sourceSessionId: event.sourceSessionId, + }; + appendCollapsedChatTranscriptEvent( + rows, + { ...envelope, event: legacyStall }, + sequence, + context, + ); return; } @@ -1408,7 +1676,7 @@ export function appendCollapsedChatTranscriptEvent( const row = rows[index]; if (row?.event.type === "text" && row.event.messageId && retractedIds.has(row.event.messageId)) { rows.splice(index, 1); - if (context) repairSubagentRowPositionsAfterSplice(context, index); + if (context) repairIndexedTranscriptRowsAfterSplice(context, index); } } return; @@ -1596,11 +1864,38 @@ export function appendCollapsedChatTranscriptEvent( return; } + const pendingResolution = event.type === "user_message" && event.steerId?.trim() + ? context?.unmatchedUserMessageResolutionsBySteer.get(event.steerId.trim()) + : undefined; + const renderEvent: ChatTranscriptVisibleEvent = event.type === "user_message" && pendingResolution + ? { + ...event, + metadata: { + ...(event.metadata ?? {}), + unprocessedMessageResolution: { + action: pendingResolution.action, + state: pendingResolution.state, + resolvedAt: pendingResolution.resolvedAt, + ...(pendingResolution.replacementMessageId + ? { replacementMessageId: pendingResolution.replacementMessageId } + : {}), + }, + }, + } + : event as ChatTranscriptVisibleEvent; + const rowIndex = rows.length; rows.push({ key: event.type === "text" ? buildTextRenderKey(event, envelope, sequence) : buildRenderKey(envelope, sequence), timestamp: envelope.timestamp, - event, + event: renderEvent, }); + if (event.type === "user_message" && event.steerId?.trim()) { + const steerId = event.steerId.trim(); + context?.userMessageRowIndexBySteer.set(steerId, rowIndex); + if (pendingResolution) { + context?.unmatchedUserMessageResolutionsBySteer.delete(steerId); + } + } } /** diff --git a/apps/desktop/src/renderer/components/remoteTargets/RemoteTargetList.test.tsx b/apps/desktop/src/renderer/components/remoteTargets/RemoteTargetList.test.tsx index c7f5fbdec..c98646537 100644 --- a/apps/desktop/src/renderer/components/remoteTargets/RemoteTargetList.test.tsx +++ b/apps/desktop/src/renderer/components/remoteTargets/RemoteTargetList.test.tsx @@ -1040,6 +1040,77 @@ describe("RemoteTargetList", () => { expect(screen.getByText(/df: \/: 100% used \(0 bytes free\)/)).toBeTruthy(); }); + it("shows bounded privacy-safe route diagnostics without endpoint paths", async () => { + const target = { + id: "target-1", + name: "Mac Studio", + hostname: "studio.local", + sshUser: null, + port: 22, + sshKeyPath: null, + lastSeenArch: null, + runtimeBinaryVersion: null, + lastConnectedAt: null, + transport: "paired" as const, + }; + Object.defineProperty(remoteRuntimeMock, "getConnectionSnapshot", { + configurable: true, + value: vi.fn().mockResolvedValue({ + connections: [ + { + target, + state: "error", + arch: null, + version: null, + projects: [], + lastError: "Connection failed", + lastErrorInfo: { + kind: "generic", + message: "ADE could not reach this machine.", + correlationId: "123e4567-e89b-42d3-a456-426614174000", + attempts: [ + { + kind: "lan", + host: "studio.local:8787", + startedAt: 1, + durationMs: 250.4, + outcome: "failed", + failure: "timeout", + }, + { + kind: "relay", + host: "wss://relay.example/connect/machine?token=secret", + startedAt: 2, + durationMs: 81.2, + outcome: "failed", + failure: "unreachable", + }, + ], + }, + lastAttemptedAt: 1, + connectedAt: null, + }, + ], + connectedCount: 0, + updatedAt: 1, + }), + }); + remoteRuntimeMock.listDiscoveredMachines.mockResolvedValue({ + machines: [], + diagnostics: [], + }); + installAdeMock(); + + render(); + + fireEvent.click(await screen.findByText("Route details")); + expect(screen.getByText("studio.local:8787 · 250ms · Failed (timeout)")).toBeTruthy(); + expect(screen.getByText("relay.example · 81ms · Failed (unreachable)")).toBeTruthy(); + expect(screen.getByText("123e4567-e89b-42d3-a456-426614174000")).toBeTruthy(); + expect(screen.queryByText(/connect\/machine/)).toBeNull(); + expect(screen.queryByText(/token=secret/)).toBeNull(); + }); + it("surfaces Tailscale discovery diagnostics separately from empty results", async () => { remoteRuntimeMock.listTargets.mockResolvedValue([]); remoteRuntimeMock.listDiscoveredMachines.mockResolvedValue({ diff --git a/apps/desktop/src/renderer/components/remoteTargets/SavedMachineRow.tsx b/apps/desktop/src/renderer/components/remoteTargets/SavedMachineRow.tsx index df0648a4e..ac16567d5 100644 --- a/apps/desktop/src/renderer/components/remoteTargets/SavedMachineRow.tsx +++ b/apps/desktop/src/renderer/components/remoteTargets/SavedMachineRow.tsx @@ -7,6 +7,7 @@ import { Warning, } from "@phosphor-icons/react"; import type { + RemoteRuntimeConnectionAttempt, RemoteRuntimeConnectResult, RemoteRuntimeSshHostKeyTrustStatus, RemoteRuntimeTarget, @@ -41,6 +42,95 @@ import { subTextStyle, } from "./remoteTargetListStyles"; +const ROUTE_LABELS: Record = { + lan: "LAN", + tailnet: "Tailscale", + relay: "ADE relay", + ssh: "SSH", +}; + +function safeDiagnosticHost(value: string): string { + const normalized = value + .replace(/[\u0000-\u001f\u007f]/g, " ") + .replace(/\s+/g, " ") + .trim() + .slice(0, 160); + if (!normalized) return "Unknown host"; + try { + const url = new URL(normalized.includes("://") ? normalized : `ws://${normalized}`); + return url.host || "Unknown host"; + } catch { + return normalized.split(/[/?#]/, 1)[0] || "Unknown host"; + } +} + +function ConnectionRouteDetails({ + attempts, + correlationId, +}: { + attempts: RemoteRuntimeConnectionAttempt[]; + correlationId: string | null; +}) { + if (attempts.length === 0 && !correlationId) return null; + return ( +
+ + Route details + +
+ {attempts.slice(0, 8).map((attempt, index) => { + const outcome = attempt.outcome === "connected" + ? "Connected" + : attempt.outcome === "skipped" + ? "Skipped" + : `Failed${attempt.failure ? ` (${attempt.failure})` : ""}`; + return ( +
+ + {ROUTE_LABELS[attempt.kind]} + + + {`${safeDiagnosticHost(attempt.host)} · ${Math.max(0, Math.round(attempt.durationMs))}ms · ${outcome}`} + +
+ ); + })} + {correlationId ? ( +
+ + Diagnostic ID + + {" "} + {correlationId} +
+ ) : null} +
+
+ ); +} + type SavedMachineRowProps = { row: SavedMachineRowModel; section: MachineSection; @@ -109,6 +199,12 @@ export function SavedMachineRow({ rawError: status?.state === "error" ? status.lastError : null, overrideMessage: selected ? error : null, }); + const activeRoute = connected?.target.id === target.id + ? connected.route + : status?.route; + const errorInfo = status?.state === "error" ? status.lastErrorInfo : null; + const routeAttempts = (activeRoute?.attempts ?? errorInfo?.attempts ?? []).slice(0, 8); + const routeCorrelationId = activeRoute?.correlationId ?? errorInfo?.correlationId ?? null; const formOpen = formPrefill?.targetId === target.id; return ( @@ -265,6 +361,11 @@ export function SavedMachineRow({ /> ) : null} + + {warnings.length > 0 ? (
controller.abort(), DIRECTORY_MUTATION_TIMEOUT_MS); + const correlationId = createAccountDirectoryCorrelationId(); try { const response = await (this.options.fetchImpl ?? fetch)( `${config.directoryBaseUrl}/account/machines/${encodeURIComponent(machineKey)}`, @@ -680,6 +682,7 @@ export class BrowserAccountClient { headers: { accept: "application/json", authorization: `Bearer ${accessToken}`, + "x-ade-correlation-id": correlationId, }, credentials: "omit", referrerPolicy: "no-referrer", diff --git a/apps/desktop/src/renderer/webclient/account/leaseMonitor.ts b/apps/desktop/src/renderer/webclient/account/leaseMonitor.ts index dc3446717..ce18cc35e 100644 --- a/apps/desktop/src/renderer/webclient/account/leaseMonitor.ts +++ b/apps/desktop/src/renderer/webclient/account/leaseMonitor.ts @@ -27,15 +27,18 @@ export function accountLeaseOwnerForActiveConnection(args: { relayAccess: WebRelayAccess; }): string | null { const environmentOwnerUserId = args.environment.accountOwnerUserId?.trim() ?? ""; - if (environmentOwnerUserId) return environmentOwnerUserId; - if (args.relayAccess.kind !== "signed_in" || !args.endpoint) return null; + if (!args.endpoint) return null; const requiresRelayLease = deriveBrowserSyncEndpoints({ environment: args.environment, }).some((candidate) => ( candidate.url === args.endpoint && browserEndpointRequiresRelayAccess(candidate) )); - return requiresRelayLease ? args.relayAccess.userId.trim() || null : null; + if (!requiresRelayLease) return null; + if (environmentOwnerUserId) return environmentOwnerUserId; + return args.relayAccess.kind === "signed_in" + ? args.relayAccess.userId.trim() || null + : null; } export async function reconcileActiveAccountLease(args: { diff --git a/apps/desktop/src/renderer/webclient/shell/MachinePicker.tsx b/apps/desktop/src/renderer/webclient/shell/MachinePicker.tsx index fe413c370..993317edc 100644 --- a/apps/desktop/src/renderer/webclient/shell/MachinePicker.tsx +++ b/apps/desktop/src/renderer/webclient/shell/MachinePicker.tsx @@ -130,7 +130,7 @@ export function MachinePicker({ }) { const signedIn = browserAccountIsSignedIn(account.state); const directEnvironments = environments.filter((environment) => ( - environment.accountOwnerUserId == null && hasDirectRoute(environment) + hasDirectRoute(environment) )); const savedRelayEnvironments = environments.filter((environment) => ( canUseRelayForEnvironment(environment, relayAccess) diff --git a/apps/desktop/src/renderer/webclient/shell/WebClientRoot.tsx b/apps/desktop/src/renderer/webclient/shell/WebClientRoot.tsx index ed0f39e36..6569c4023 100644 --- a/apps/desktop/src/renderer/webclient/shell/WebClientRoot.tsx +++ b/apps/desktop/src/renderer/webclient/shell/WebClientRoot.tsx @@ -128,6 +128,7 @@ function environmentsVisibleToOwner( environments: WebClientEnvironmentRecord[], ownerUserId: string | null, ): WebClientEnvironmentRecord[] { + if (!ownerUserId) return environments; return environments.filter((environment) => ( environment.accountOwnerUserId == null || environment.accountOwnerUserId === ownerUserId diff --git a/apps/desktop/src/renderer/webclient/shell/__tests__/MachinePicker.test.tsx b/apps/desktop/src/renderer/webclient/shell/__tests__/MachinePicker.test.tsx index cafab2907..f76baff13 100644 --- a/apps/desktop/src/renderer/webclient/shell/__tests__/MachinePicker.test.tsx +++ b/apps/desktop/src/renderer/webclient/shell/__tests__/MachinePicker.test.tsx @@ -100,6 +100,18 @@ describe("MachinePicker account states", () => { expect(screen.queryByText(/pairing link/i)).toBeNull(); }); + it("keeps account-created device trust available over a direct route after sign-out", () => { + const environment = { + ...savedEnvironment(), + accountOwnerUserId: "signed-out-account", + }; + const { onSelect } = renderPicker(account(), [environment]); + + const savedButton = screen.getByRole("button", { name: /Saved Studio/i }); + fireEvent.click(savedButton); + expect(onSelect).toHaveBeenCalledWith(environment); + }); + it("treats directory presence as a hint when a secure route is still available", () => { const available = { machineKey: "mk-online", diff --git a/apps/desktop/src/renderer/webclient/sync/__tests__/sync.test.ts b/apps/desktop/src/renderer/webclient/sync/__tests__/sync.test.ts index 5c216f3cf..799f5438f 100644 --- a/apps/desktop/src/renderer/webclient/sync/__tests__/sync.test.ts +++ b/apps/desktop/src/renderer/webclient/sync/__tests__/sync.test.ts @@ -221,11 +221,22 @@ class ScriptedSocket implements WebSocketLike { readonly closeHistory: Array<{ code: number; reason: string }> = []; closedWith: { code: number; reason: string } | null = null; + readonly url: string; + readonly rawUrl: string; + constructor( - readonly url: string, + rawUrl: string, private readonly onClientEnvelope: (socket: ScriptedSocket, envelope: SyncEnvelope) => void | Promise, openDelayMs = 0, ) { + this.rawUrl = rawUrl; + const normalized = new URL(rawUrl); + if (normalized.searchParams.has("cid")) { + normalized.searchParams.delete("cid"); + this.url = normalized.toString().replace(/\?$/, ""); + } else { + this.url = rawUrl; + } setTimeout(() => { if (this.readyState !== 0) return; this.readyState = 1; @@ -587,7 +598,7 @@ describe("browser sync envelope codec", () => { }); describe("browser sync endpoints and storage", () => { - it("orders relay and browser-safe endpoints while marking HTTPS plain ws routes undialable", () => { + it("orders LAN then Tailscale then Relay while marking HTTPS plain ws routes undialable", () => { const endpoints = deriveBrowserSyncEndpoints({ payload: pairingPayload, location: { protocol: "https:", hostname: "app.ade-app.dev" }, @@ -600,6 +611,41 @@ describe("browser sync endpoints and storage", () => { expect(endpoints.find((candidate) => candidate.url === "ws://192.168.1.10:8787")?.reason).toBe("plain_ws_blocked_from_https"); }); + it("keeps direct phases ahead of a cached Relay last-good endpoint", () => { + const endpoints = deriveBrowserSyncEndpoints({ + environment: { + envId: "ordered", + machineName: "Studio", + hostDeviceId: "host", + relayUrl: "wss://relay.example/connect/machine-key", + machineKeyUrl: null, + addressCandidates: [ + { host: "100.64.0.2", kind: "tailscale" }, + { host: "192.168.1.10", kind: "lan" }, + ], + explicitWssEndpoints: [], + port: 8787, + pairedDeviceId: "browser", + secret: "secret", + dpopKeys: {} as CryptoKeyPair, + siteId: "site", + localDeviceId: "local", + localDeviceName: "Browser", + createdAt: "2026-07-01T00:00:00.000Z", + lastGoodEndpoint: "wss://relay.example/connect/machine-key", + }, + location: { protocol: "http:", hostname: "localhost" }, + }); + + expect(endpoints.filter((candidate) => candidate.dialable).map((candidate) => candidate.url)).toEqual([ + "ws://127.0.0.1:8787", + "ws://localhost:8787", + "ws://192.168.1.10:8787", + "ws://100.64.0.2:8787", + "wss://relay.example/connect/machine-key", + ]); + }); + it("allows loopback candidates from local browser pages", () => { const endpoints = deriveBrowserSyncEndpoints({ payload: pairingPayload, @@ -893,6 +939,12 @@ describe("browser sync connection and client", () => { expect(script.sockets[0]?.closedWith?.reason).toBe("Relay readiness negotiation timeout"); expect(script.sockets[1]?.url).toBe(pairingPayload.relayUrl); expect(script.sockets[1]?.sent.map((envelope) => envelope.type)).toEqual(["hello"]); + const readyCorrelation = new URL(script.sockets[0]!.rawUrl).searchParams.get("cid"); + const legacyCorrelation = new URL(script.sockets[1]!.rawUrl).searchParams.get("cid"); + expect(readyCorrelation).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i, + ); + expect(legacyCorrelation).toBe(readyCorrelation); connection.dispose(); }); @@ -2305,7 +2357,7 @@ describe("browser sync connection and client", () => { client.dispose(); }); - it("disconnects and prunes revoked account trust while connected without deleting local pairings", async () => { + it("disconnects a revoked Relay lease while preserving direct reconnect trust", async () => { const storage = new MemoryStorage(); await makeEnvironment(storage, { envId: "local-env", @@ -2348,7 +2400,10 @@ describe("browser sync connection and client", () => { expect(result.state).toBe("revoked"); expect(client.getStatus().state).toBe("disconnected"); - expect((await client.listEnvironments()).map((environment) => environment.envId)).toEqual(["local-env"]); + expect((await client.listEnvironments()).map((environment) => environment.envId).sort()).toEqual([ + "account-env", + "local-env", + ]); client.dispose(); }); diff --git a/apps/desktop/src/renderer/webclient/sync/connection.ts b/apps/desktop/src/renderer/webclient/sync/connection.ts index 5dcf26f19..5400d904e 100644 --- a/apps/desktop/src/renderer/webclient/sync/connection.ts +++ b/apps/desktop/src/renderer/webclient/sync/connection.ts @@ -28,6 +28,7 @@ import { import { resolveAccountHelloPairing } from "../../../shared/accountDirectory"; import { browserEndpointRequiresRelayAccess, + browserDialCandidateRouteKind, type BrowserDialCandidate, } from "./endpoints"; import type { WebClientEnvironmentRecord } from "./envStore"; @@ -239,6 +240,20 @@ function withRelayReadyNegotiation(endpoint: string): string { return url.toString(); } +function browserConnectionCorrelationId(): string { + if (typeof globalThis.crypto?.randomUUID === "function") { + return globalThis.crypto.randomUUID(); + } + const raw = randomHex(16).toLowerCase(); + return `${raw.slice(0, 8)}-${raw.slice(8, 12)}-4${raw.slice(13, 16)}-8${raw.slice(17, 20)}-${raw.slice(20, 32)}`; +} + +function withRelayCorrelationId(endpoint: string, correlationId: string): string { + const url = new URL(endpoint); + url.searchParams.set("cid", correlationId); + return url.toString(); +} + function relayTransportFrame(data: unknown): SyncRelayClientAccepted | SyncRelayClientReady | null { if (typeof data !== "string") return null; try { @@ -369,12 +384,13 @@ export class SyncConnection { this.consecutiveAuthFailures = 0; const dialable = args.endpoints.filter((candidate) => candidate.dialable); if (dialable.length === 0) throw new Error("That machine has no secure account connection route."); + const correlationId = browserConnectionCorrelationId(); let lastError: Error | null = null; for (const candidate of dialable) { try { const dpop = await args.createDpop(); if (operationGeneration !== this.operationGeneration) throw new StaleSocketAttemptError(); - const result = await this.pairWithAccountOnEndpoint(candidate, args, dpop); + const result = await this.pairWithAccountOnEndpoint(candidate, args, dpop, correlationId); if (operationGeneration !== this.operationGeneration) throw new StaleSocketAttemptError(); return result; } catch (error) { @@ -451,10 +467,11 @@ export class SyncConnection { hostName: environment.machineName, error: null, }); + const correlationId = browserConnectionCorrelationId(); let lastError: Error | null = null; for (const candidate of dialable) { try { - await this.connectEndpoint(environment, candidate); + await this.connectEndpoint(environment, candidate, correlationId); return; } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); @@ -476,13 +493,17 @@ export class SyncConnection { throw lastError ?? new Error(message); } - private async connectEndpoint(environment: WebClientEnvironmentRecord, candidate: BrowserDialCandidate): Promise { + private async connectEndpoint( + environment: WebClientEnvironmentRecord, + candidate: BrowserDialCandidate, + correlationId: string, + ): Promise { const throughRelay = browserEndpointRequiresRelayAccess(candidate); try { - await this.connectEndpointAttempt(environment, candidate, throughRelay); + await this.connectEndpointAttempt(environment, candidate, throughRelay, correlationId); } catch (error) { if (!throughRelay || !(error instanceof RelayReadyNegotiationTimeoutError)) throw error; - await this.connectEndpointAttempt(environment, candidate, false); + await this.connectEndpointAttempt(environment, candidate, false, correlationId); } } @@ -490,6 +511,7 @@ export class SyncConnection { environment: WebClientEnvironmentRecord, candidate: BrowserDialCandidate, useRelayReadyV2: boolean, + correlationId: string, ): Promise { const generation = ++this.connectionGeneration; const endpoint = candidate.url; @@ -502,7 +524,12 @@ export class SyncConnection { // rejection until onopen awaits the same promise so it cannot be reported // as unhandled while the transport is still opening. void preparedAuth.catch(() => undefined); - const socket = this.socketFactory(useRelayReadyV2 ? withRelayReadyNegotiation(endpoint) : endpoint); + const relayEndpoint = throughRelay + ? withRelayCorrelationId(endpoint, correlationId) + : endpoint; + const socket = this.socketFactory( + useRelayReadyV2 ? withRelayReadyNegotiation(relayEndpoint) : relayEndpoint, + ); this.ws = socket; this.status.endpoint = endpoint; this.emit("statusChanged", this.getStatus()); @@ -643,13 +670,14 @@ export class SyncConnection { candidate: BrowserDialCandidate, args: AccountPairAndConnectArgs, dpop: SyncDpopProof, + correlationId: string, ): Promise<{ environment: WebClientEnvironmentRecord; helloOk: SyncHelloOkPayload; endpoint: string }> { const throughRelay = browserEndpointRequiresRelayAccess(candidate); try { - return await this.pairWithAccountOnEndpointAttempt(candidate, args, dpop, throughRelay); + return await this.pairWithAccountOnEndpointAttempt(candidate, args, dpop, throughRelay, correlationId); } catch (error) { if (!throughRelay || !(error instanceof RelayReadyNegotiationTimeoutError)) throw error; - return this.pairWithAccountOnEndpointAttempt(candidate, args, dpop, false); + return this.pairWithAccountOnEndpointAttempt(candidate, args, dpop, false, correlationId); } } @@ -658,11 +686,17 @@ export class SyncConnection { args: AccountPairAndConnectArgs, dpop: SyncDpopProof, useRelayReadyV2: boolean, + correlationId: string, ): Promise<{ environment: WebClientEnvironmentRecord; helloOk: SyncHelloOkPayload; endpoint: string }> { const generation = ++this.connectionGeneration; const endpoint = candidate.url; const throughRelay = browserEndpointRequiresRelayAccess(candidate); - const socket = this.socketFactory(useRelayReadyV2 ? withRelayReadyNegotiation(endpoint) : endpoint); + const relayEndpoint = throughRelay + ? withRelayCorrelationId(endpoint, correlationId) + : endpoint; + const socket = this.socketFactory( + useRelayReadyV2 ? withRelayReadyNegotiation(relayEndpoint) : relayEndpoint, + ); this.ws = socket; this.shouldReconnect = false; this.setStatus({ state: "connecting", endpoint, error: null }); @@ -971,10 +1005,31 @@ export class SyncConnection { ): boolean { if (!this.isCurrentSocket(socket, generation)) return false; this.environment = environment; - this.endpoints = [ - ...this.endpoints.filter((candidate) => candidate.url === endpoint), - ...this.endpoints.filter((candidate) => candidate.url !== endpoint), - ]; + const winningCandidate = this.endpoints.find( + (candidate) => candidate.url === endpoint, + ); + if (winningCandidate) { + const winningKind = browserDialCandidateRouteKind(winningCandidate); + this.endpoints = this.endpoints + .map((candidate, order) => ({ + candidate, + order, + winning: candidate.url === endpoint, + })) + .sort((left, right) => { + const leftKind = browserDialCandidateRouteKind(left.candidate); + const rightKind = browserDialCandidateRouteKind(right.candidate); + const rank = { lan: 0, tailnet: 1, relay: 2 } as const; + return rank[leftKind] - rank[rightKind] + || ( + leftKind === winningKind && rightKind === winningKind + ? Number(right.winning) - Number(left.winning) + : 0 + ) + || left.order - right.order; + }) + .map(({ candidate }) => candidate); + } this.latestHello = helloOk; this.consecutiveAuthFailures = 0; this.relayAuthorizationTerminalError = null; diff --git a/apps/desktop/src/renderer/webclient/sync/endpoints.ts b/apps/desktop/src/renderer/webclient/sync/endpoints.ts index 29b34938f..14c1b0198 100644 --- a/apps/desktop/src/renderer/webclient/sync/endpoints.ts +++ b/apps/desktop/src/renderer/webclient/sync/endpoints.ts @@ -3,6 +3,7 @@ import type { SyncAddressCandidate, SyncPairingQrPayload, } from "../../../shared/types/sync"; +import { isTailnetHostname } from "../../../shared/tailnet"; import type { WebClientEnvironmentRecord } from "./envStore"; export type BrowserDialCandidate = { @@ -22,6 +23,23 @@ export function browserEndpointRequiresRelayAccess(candidate: BrowserDialCandida return candidate.kind === "relay" || candidate.kind === "lastGood"; } +export function browserDialCandidateRouteKind( + candidate: BrowserDialCandidate, +): "lan" | "tailnet" | "relay" { + if (browserEndpointRequiresRelayAccess(candidate)) return "relay"; + if ( + candidate.source?.kind === "tailscale" + || (() => { + try { + return isTailnetHostname(new URL(candidate.url).hostname); + } catch { + return false; + } + })() + ) return "tailnet"; + return "lan"; +} + export type EndpointDerivationInput = { payload?: SyncPairingQrPayload | string | null; environment?: WebClientEnvironmentRecord | null; @@ -149,5 +167,13 @@ export function deriveBrowserSyncEndpoints(input: EndpointDerivationInput): Brow } } - return candidates; + const rank = { lan: 0, tailnet: 1, relay: 2 } as const; + return candidates + .map((candidate, order) => ({ candidate, order })) + .sort((left, right) => ( + rank[browserDialCandidateRouteKind(left.candidate)] + - rank[browserDialCandidateRouteKind(right.candidate)] + || left.order - right.order + )) + .map(({ candidate }) => candidate); } diff --git a/apps/desktop/src/renderer/webclient/sync/envStore.test.ts b/apps/desktop/src/renderer/webclient/sync/envStore.test.ts index 82c382c4f..91a032cb7 100644 --- a/apps/desktop/src/renderer/webclient/sync/envStore.test.ts +++ b/apps/desktop/src/renderer/webclient/sync/envStore.test.ts @@ -132,7 +132,7 @@ describe("web-client trust reset migration", () => { await expect(store.listEnvironments()).resolves.toHaveLength(2); }); - it("prunes signed-out and wrong-account environments before launch", async () => { + it("preserves signed-out direct trust but prunes a different signed-in account", async () => { const storage = new MemoryStorage(); const store = new WebClientEnvStore(storage); await store.saveEnvironment(legacyEnvironment("manual")); @@ -159,11 +159,12 @@ describe("web-client trust reset migration", () => { ]), ); await expect(store.pruneAccountOwnedEnvironments(null)).resolves.toEqual({ - removedIds: ["current"], - environments: [expect.objectContaining({ envId: "manual" })], + removedIds: [], + environments: expect.arrayContaining([ + expect.objectContaining({ envId: "manual" }), + expect.objectContaining({ envId: "current" }), + ]), }); - await expect(store.listEnvironments()).resolves.toEqual([ - expect.objectContaining({ envId: "manual" }), - ]); + await expect(store.listEnvironments()).resolves.toHaveLength(2); }); }); diff --git a/apps/desktop/src/renderer/webclient/sync/envStore.ts b/apps/desktop/src/renderer/webclient/sync/envStore.ts index 379c9eada..d9e1381b3 100644 --- a/apps/desktop/src/renderer/webclient/sync/envStore.ts +++ b/apps/desktop/src/renderer/webclient/sync/envStore.ts @@ -412,7 +412,8 @@ export class WebClientEnvStore { .filter((environment) => environment.accountOwnerUserId != null) .filter((environment) => options.removeCurrent ? environment.accountOwnerUserId === currentOwnerUserId - : environment.accountOwnerUserId !== currentOwnerUserId) + : currentOwnerUserId != null + && environment.accountOwnerUserId !== currentOwnerUserId) .map((environment) => environment.envId); const removedIdSet = new Set(removedIds); for (const envId of removedIds) transaction.delete("environments", envId); diff --git a/apps/desktop/src/shared/accountDirectory.test.ts b/apps/desktop/src/shared/accountDirectory.test.ts index 866f8584d..83d4e87ba 100644 --- a/apps/desktop/src/shared/accountDirectory.test.ts +++ b/apps/desktop/src/shared/accountDirectory.test.ts @@ -5,7 +5,7 @@ import { import type { AdeAccountMachine } from "./types/account"; describe("accountMachineAdoptionRoutes", () => { - it("orders validated relay, tailnet, and LAN routes", () => { + it("orders validated LAN, tailnet, and relay routes", () => { const machine: AdeAccountMachine = { machineKey: "machine-studio", deviceId: "device-studio", @@ -29,16 +29,16 @@ describe("accountMachineAdoptionRoutes", () => { expect(accountMachineAdoptionRoutes(machine, ["https://relay.example"])) .toEqual([ { - endpoint: "wss://relay.example/connect/machine-studio", - kind: "relay", + endpoint: "ws://studio.local:8787/", + kind: "lan", }, { endpoint: "ws://100.75.20.63:8787/", kind: "tailnet", }, { - endpoint: "ws://studio.local:8787/", - kind: "lan", + endpoint: "wss://relay.example/connect/machine-studio", + kind: "relay", }, ]); }); diff --git a/apps/desktop/src/shared/accountDirectory.ts b/apps/desktop/src/shared/accountDirectory.ts index c1f57ecf0..3581750ce 100644 --- a/apps/desktop/src/shared/accountDirectory.ts +++ b/apps/desktop/src/shared/accountDirectory.ts @@ -362,6 +362,24 @@ export async function readAccountDirectoryHttpReason(response: Response): Promis return shortHttpReason(text); } +export function createAccountDirectoryCorrelationId(): string { + if (typeof globalThis.crypto?.randomUUID === "function") { + return globalThis.crypto.randomUUID(); + } + const bytes = new Uint8Array(16); + globalThis.crypto?.getRandomValues?.(bytes); + if (bytes.every((value) => value === 0)) { + const seed = `${Date.now()}-${Math.random()}`; + for (let index = 0; index < bytes.length; index += 1) { + bytes[index] = seed.charCodeAt(index % seed.length) & 0xff; + } + } + bytes[6] = (bytes[6]! & 0x0f) | 0x40; + bytes[8] = (bytes[8]! & 0x3f) | 0x80; + const hex = Array.from(bytes, (value) => value.toString(16).padStart(2, "0")).join(""); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + export async function fetchAccountMachines(args: { baseUrl: string | null | undefined; accessToken: string; @@ -380,6 +398,7 @@ export async function fetchAccountMachines(args: { } const token = args.accessToken.trim(); if (!token) return { state: "signed_out", machines: [], message: null }; + const correlationId = createAccountDirectoryCorrelationId(); const controller = new AbortController(); const onAbort = () => controller.abort(args.signal?.reason); @@ -398,6 +417,7 @@ export async function fetchAccountMachines(args: { headers: { accept: "application/json", authorization: `Bearer ${accessToken}`, + "x-ade-correlation-id": correlationId, }, credentials: "omit", referrerPolicy: "no-referrer", @@ -638,12 +658,12 @@ export function accountMachineAdoptionRoutes( } return [ + ...lan, + ...tailnet, ...relayEndpoints.map((endpoint) => ({ endpoint, kind: "relay" as const, })), - ...tailnet, - ...lan, ]; } diff --git a/apps/desktop/src/shared/ipc.ts b/apps/desktop/src/shared/ipc.ts index 0dac35aa1..e03a2f27a 100644 --- a/apps/desktop/src/shared/ipc.ts +++ b/apps/desktop/src/shared/ipc.ts @@ -227,7 +227,9 @@ export const IPC = { agentChatDispatchSteer: "ade.agentChat.dispatchSteer", agentChatCancelDispatchedSteer: "ade.agentChat.cancelDispatchedSteer", agentChatInterrupt: "ade.agentChat.interrupt", + agentChatRecoverTurn: "ade.agentChat.recoverTurn", agentChatRecoverCodexTurn: "ade.agentChat.recoverCodexTurn", + agentChatResolveUnprocessedMessage: "ade.agentChat.resolveUnprocessedMessage", agentChatRecoverContinuity: "ade.agentChat.recoverContinuity", agentChatApprove: "ade.agentChat.approve", agentChatRespondToInput: "ade.agentChat.respondToInput", diff --git a/apps/desktop/src/shared/types/chat.ts b/apps/desktop/src/shared/types/chat.ts index 8e6071306..c4df82ef4 100644 --- a/apps/desktop/src/shared/types/chat.ts +++ b/apps/desktop/src/shared/types/chat.ts @@ -462,12 +462,29 @@ export type AgentChatScheduledWakeMetadata = { late?: boolean; }; +export type AgentChatUnprocessedReplayMetadata = { + sourceSteerId: string; + action: "run_next"; + replacementMessageId: string; +}; + +export type AgentChatUnprocessedMessageResolutionMetadata = { + action: "run_next" | "dismiss"; + state: "completed"; + resolvedAt: string; + replacementMessageId?: string; +}; + export type AgentChatEventMetadata = Record & { /** Marks a synthetic unattended turn started by ADE's durable scheduler. */ scheduledWake?: AgentChatScheduledWakeMetadata; /** Rides the `subagent` completion wake so the renderer can render the * "ADE woke this chat" divider off a typed shape (mirrors `scheduledWake`). */ spawnCompletion?: AgentChatSpawnCompletion; + /** Provenance on the replacement message created by Run next. */ + replayedFromUnprocessedSteer?: AgentChatUnprocessedReplayMetadata; + /** Renderer-folded terminal state for the original unprocessed bubble. */ + unprocessedMessageResolution?: AgentChatUnprocessedMessageResolutionMetadata; }; export type AgentChatScheduledWorkKind = @@ -508,10 +525,29 @@ export type AgentChatEvent = contextAttachments?: AgentChatContextAttachment[]; turnId?: string; steerId?: string; - deliveryState?: "queued" | "delivered" | "inline" | "failed"; + /** + * Durable user-message lifecycle. `delivered` and `inline` remain + * accepted legacy values for older transcripts and clients. + */ + deliveryState?: "queued" | "accepted" | "processed" | "unprocessed" | "delivered" | "inline" | "failed"; processed?: boolean; runtime?: AgentChatRuntime; } + | { + /** + * Durable resolution for a terminal accepted-but-unprocessed steer. + * Keeping this separate from the replacement user message makes + * Run next and Dismiss idempotent across retries, app restarts, and + * different ADE surfaces. + */ + type: "user_message_resolution"; + steerId: string; + action: "run_next" | "dismiss"; + state: "completed"; + resolvedAt: string; + replacementMessageId?: string; + turnId?: string; + } | { type: "text"; text: string; @@ -941,6 +977,15 @@ export type AgentChatEvent = metadata: CodexModerationMetadata; turnId?: string; } + | { + type: "turn_diagnostics"; + turnId?: string; + moderationChecks?: number; + optionalIntegrationFailures?: Array<{ + integration: string; + message?: string | null; + }>; + } | { type: "codex_sleep"; itemId: string; @@ -1065,11 +1110,56 @@ export type AgentChatEvent = type: "codex_turn_stalled"; turnId: string; threadId?: string; - reason: "no_output" | "waiting_on_input" | "waiting_on_approval" | "app_server_state_unknown"; + reason: "no_output" | "no_progress" | "waiting_on_input" | "waiting_on_approval" | "app_server_state_unknown"; message: string; recoveryOptions?: Array<"wait" | "steer" | "interrupt_retry_same_thread" | "restart_resume_thread">; sourceSessionId?: string; parentSessionId?: string; + detectedAt?: string; + turnStartedAt?: string; + lastProgressAt?: string; + automaticRecoveryAttempted?: boolean; + } + | { + /** + * Provider-neutral turn-health contract. Codex-specific events remain + * readable for backwards compatibility, while new surfaces should prefer + * this event when both are present. + */ + type: "turn_health"; + provider: string; + turnId: string; + state: "stalled"; + reason: "no_output" | "no_progress" | "waiting_on_input" | "waiting_on_approval" | "runtime_state_unknown"; + message: string; + turnStartedAt: string; + lastProgressAt: string; + detectedAt: string; + recoveryCount: number; + supportedActions: AgentChatTurnRecoveryAction[]; + automaticRecoveryAttempted: boolean; + /** Owning child chat when this health event is mirrored into a parent. */ + sourceSessionId?: string; + } + | { + type: "codex_turn_recovery"; + turnId: string; + action: "restart_resume_thread"; + state: "recovering" | "recovered" | "failed"; + message: string; + automatic: boolean; + at: string; + } + | { + type: "turn_recovery"; + provider: string; + turnId: string; + action: AgentChatTurnRecoveryAction; + state: "recovering" | "recovered" | "failed"; + message: string; + automatic: boolean; + at: string; + recoveryCount: number; } | { type: "codex_thread_deleted"; @@ -2317,6 +2407,45 @@ export type AgentChatCodexRecoveryAction = | "interrupt_retry_same_thread" | "restart_resume_thread"; +export const AGENT_CHAT_TURN_RECOVERY_ACTIONS = [ + "wait", + "nudge", + "retry_same_runtime", + "restart_resume", +] as const; + +export type AgentChatTurnRecoveryAction = + (typeof AGENT_CHAT_TURN_RECOVERY_ACTIONS)[number]; + +export function isAgentChatTurnRecoveryAction( + value: unknown, +): value is AgentChatTurnRecoveryAction { + return typeof value === "string" + && (AGENT_CHAT_TURN_RECOVERY_ACTIONS as readonly string[]).includes(value); +} + +export function isUnsupportedAgentChatRecoveryActionError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + /\b(?:unsupported|unknown)\s+(?:chat\s+)?(?:method|action|command)\b/i.test(message) + || /\b(?:method|action|command)\s+(?:is\s+)?not supported\b/i.test(message) + || /\b(?:recoverTurn|chat\.recoverTurn)\b.*\b(?:not supported|not available|not found)\b/i.test(message) + || /\b(?:not supported|not available|not found)\b.*\b(?:recoverTurn|chat\.recoverTurn)\b/i.test(message) + ); +} + +export type AgentChatRecoverTurnArgs = { + sessionId: string; + turnId: string; + action: AgentChatTurnRecoveryAction; +}; + +export type AgentChatRecoverTurnResult = { + action: AgentChatTurnRecoveryAction; + turnId: string; + status: "waiting" | "nudged" | "retrying" | "resumed"; +}; + export type AgentChatRecoverCodexTurnArgs = { sessionId: string; turnId: string; @@ -2329,6 +2458,19 @@ export type AgentChatRecoverCodexTurnResult = { status: "waiting" | "nudged" | "retrying" | "resumed"; }; +export type AgentChatResolveUnprocessedMessageArgs = { + sessionId: string; + steerId: string; + action: "run_next" | "dismiss"; +}; + +export type AgentChatResolveUnprocessedMessageResult = { + steerId: string; + action: "run_next" | "dismiss"; + status: "completed" | "already_completed"; + replacementMessageId?: string; +}; + export type AgentChatCodexGetGoalArgs = { sessionId: string; }; diff --git a/apps/desktop/src/shared/types/pairedRuntime.ts b/apps/desktop/src/shared/types/pairedRuntime.ts index 390dbe6d3..7a9046aa3 100644 --- a/apps/desktop/src/shared/types/pairedRuntime.ts +++ b/apps/desktop/src/shared/types/pairedRuntime.ts @@ -132,6 +132,8 @@ export type DesktopPairedMachineCredentials = { export type DesktopPairedMachineEndpointState = { endpoint: string; lastSucceededAt: number | null; + /** Fresh discovery wins within a route kind before historical success. */ + lastDiscoveredAt?: number | null; }; export type DesktopPairedMachinesFile = { diff --git a/apps/desktop/src/shared/types/personalChats.ts b/apps/desktop/src/shared/types/personalChats.ts index e9c54afed..0cc0b9293 100644 --- a/apps/desktop/src/shared/types/personalChats.ts +++ b/apps/desktop/src/shared/types/personalChats.ts @@ -14,6 +14,10 @@ import type { AgentChatEditSteerArgs, AgentChatModelCatalog, AgentChatModelCatalogArgs, + AgentChatRecoverTurnArgs, + AgentChatRecoverTurnResult, + AgentChatResolveUnprocessedMessageArgs, + AgentChatResolveUnprocessedMessageResult, AgentChatRespondToInputArgs, AgentChatSendArgs, AgentChatSession, @@ -38,6 +42,8 @@ export const PERSONAL_CHAT_ACTIONS = [ "dispatchSteer", "cancelDispatchedSteer", "interrupt", + "recoverTurn", + "resolveUnprocessedMessage", "respondToInput", "approve", "createScheduledWork", @@ -106,6 +112,8 @@ export type PersonalChatCallArgs = | { action: "dispatchSteer"; args: AgentChatDispatchSteerArgs } | { action: "cancelDispatchedSteer"; args: AgentChatCancelDispatchedSteerArgs } | { action: "interrupt"; args: AgentChatInterruptArgs } + | { action: "recoverTurn"; args: AgentChatRecoverTurnArgs } + | { action: "resolveUnprocessedMessage"; args: AgentChatResolveUnprocessedMessageArgs } | { action: "respondToInput"; args: AgentChatRespondToInputArgs } | { action: "approve"; args: AgentChatApproveArgs } | { action: "createScheduledWork"; args: AgentChatCreateScheduledWorkArgs } @@ -143,6 +151,8 @@ export type PersonalChatCallResult = | AgentChatCreateScheduledWorkResult | AgentChatCancelScheduledWorkResult | AgentChatSetScheduledWorkPausedResult + | AgentChatRecoverTurnResult + | AgentChatResolveUnprocessedMessageResult | AgentChatModelCatalog | PtyCreateResult | PtyDisposeResult diff --git a/apps/desktop/src/shared/types/remoteRuntime.ts b/apps/desktop/src/shared/types/remoteRuntime.ts index e35ac2289..bce545ddb 100644 --- a/apps/desktop/src/shared/types/remoteRuntime.ts +++ b/apps/desktop/src/shared/types/remoteRuntime.ts @@ -170,10 +170,35 @@ export type RemoteRuntimeConnectionState = export type RemoteRuntimeRouteKind = "lan" | "tailnet" | "relay" | "ssh"; +export type RemoteRuntimeConnectionAttemptFailure = + | "unreachable" + | "timeout" + | "authentication" + | "identity" + | "capability" + | "protocol" + | "unknown"; + +export type RemoteRuntimeConnectionAttempt = { + kind: RemoteRuntimeRouteKind; + /** Host and optional port only. Paths, query strings, and credentials are excluded. */ + host: string; + startedAt: number; + durationMs: number; + outcome: "connected" | "failed" | "skipped"; + failure?: RemoteRuntimeConnectionAttemptFailure; +}; + export type RemoteRuntimeConnectionRoute = { kind: RemoteRuntimeRouteKind; endpoint: string; latencyMs?: number; + /** Correlates the bounded route attempts for this connection without exposing secrets. */ + correlationId?: string; + /** At most eight privacy-safe attempts, ordered exactly as they were tried. */ + attempts?: RemoteRuntimeConnectionAttempt[]; + /** Number of route attempts omitted from the bounded diagnostic list. */ + omittedAttemptCount?: number; }; export type RemoteRuntimeConnectErrorInfo = { @@ -182,6 +207,9 @@ export type RemoteRuntimeConnectErrorInfo = { detail?: string; freeBytes?: number; requiredBytes?: number; + correlationId?: string; + attempts?: RemoteRuntimeConnectionAttempt[]; + omittedAttemptCount?: number; }; const REMOTE_RUNTIME_ERROR_DETAIL_MAX_CHARS = 4_000; diff --git a/apps/desktop/src/shared/types/sync.ts b/apps/desktop/src/shared/types/sync.ts index dad3d14aa..4effcb5ed 100644 --- a/apps/desktop/src/shared/types/sync.ts +++ b/apps/desktop/src/shared/types/sync.ts @@ -1496,6 +1496,8 @@ export type SyncRemoteCommandAction = | "chat.send" | "chat.interrupt" | "chat.recoverCodexTurn" + | "chat.recoverTurn" + | "chat.resolveUnprocessedMessage" | "chat.steer" | "chat.cancelSteer" | "chat.editSteer" diff --git a/apps/ios/ADE/Models/RemoteModels.swift b/apps/ios/ADE/Models/RemoteModels.swift index 8ccf1af2d..7a4c40307 100644 --- a/apps/ios/ADE/Models/RemoteModels.swift +++ b/apps/ios/ADE/Models/RemoteModels.swift @@ -1911,6 +1911,11 @@ struct CodexModerationMetadata: Codable, Equatable { var metadata: RemoteJSONValue? } +struct AgentChatOptionalIntegrationFailure: Codable, Equatable, Hashable { + var integration: String + var message: String? +} + struct AgentChatEventProvenance: Decodable, Equatable { var messageId: String? var threadId: String? @@ -2054,6 +2059,14 @@ struct AgentChatFileRef: Codable, Equatable, Hashable { enum AgentChatEvent: Decodable, Equatable { case userMessage(text: String, attachments: [AgentChatFileRef]?, turnId: String?, steerId: String?, deliveryState: String?, processed: Bool?) + case userMessageResolution( + steerId: String, + action: String, + state: String, + resolvedAt: String, + replacementMessageId: String?, + turnId: String? + ) case text(text: String, messageId: String?, turnId: String?, itemId: String?) case toolCall(tool: String, args: RemoteJSONValue, itemId: String, logicalItemId: String?, parentItemId: String?, turnId: String?) case toolResult(tool: String, result: RemoteJSONValue, itemId: String, logicalItemId: String?, parentItemId: String?, turnId: String?, status: String?) @@ -2104,8 +2117,56 @@ enum AgentChatEvent: Decodable, Equatable { ) case codexSafetyBuffering(state: CodexSafetyBufferingState, turnId: String?) case codexModerationMetadata(metadata: CodexModerationMetadata, turnId: String?) + case turnDiagnostics( + turnId: String?, + moderationChecks: Int?, + optionalIntegrationFailures: [AgentChatOptionalIntegrationFailure]? + ) case codexSleep(itemId: String, turnId: String?, durationMs: Int?, status: String) - case codexTurnStalled(turnId: String, threadId: String?, reason: String, message: String, recoveryOptions: [String]?, sourceSessionId: String?) + case codexTurnStalled( + turnId: String, + threadId: String?, + reason: String, + message: String, + recoveryOptions: [String]?, + sourceSessionId: String?, + detectedAt: String?, + turnStartedAt: String?, + lastProgressAt: String?, + automaticRecoveryAttempted: Bool? + ) + case turnHealth( + provider: String, + turnId: String, + state: String, + reason: String, + message: String, + turnStartedAt: String, + lastProgressAt: String, + detectedAt: String, + recoveryCount: Int, + supportedActions: [String], + automaticRecoveryAttempted: Bool, + sourceSessionId: String? + ) + case codexTurnRecovery( + turnId: String, + action: String, + state: String, + message: String, + automatic: Bool, + at: String + ) + case turnRecovery( + provider: String, + turnId: String, + action: String, + state: String, + message: String, + automatic: Bool, + at: String, + recoveryCount: Int + ) case codexThreadDeleted(threadId: String, turnId: String?) case systemNotice(noticeKind: AgentChatNoticeKind, message: String, detail: RemoteJSONValue?, turnId: String?, steerId: String?) case completionReport(report: ChatCompletionReport, turnId: String?) @@ -2128,6 +2189,8 @@ extension AgentChatEvent { case steerId case deliveryState case processed + case resolvedAt + case replacementMessageId case messageId case itemId case logicalItemId @@ -2201,7 +2264,6 @@ extension AgentChatEvent { case sourceTaskId case error case messageIds - case replacementMessageId case toolUseIds case trigger case preTokens @@ -2211,9 +2273,19 @@ extension AgentChatEvent { case compactionId case state case reasons + case moderationChecks + case optionalIntegrationFailures case recoveryOptions case sourceSessionId case threadId + case detectedAt + case turnStartedAt + case lastProgressAt + case automaticRecoveryAttempted + case automatic + case at + case recoveryCount + case supportedActions case metadata case noticeKind case report @@ -2260,6 +2332,15 @@ extension AgentChatEvent { deliveryState: try container.decodeIfPresent(String.self, forKey: .deliveryState), processed: try container.decodeIfPresent(Bool.self, forKey: .processed) ) + case "user_message_resolution": + self = .userMessageResolution( + steerId: try container.decode(String.self, forKey: .steerId), + action: try container.decode(String.self, forKey: .action), + state: try container.decode(String.self, forKey: .state), + resolvedAt: try container.decode(String.self, forKey: .resolvedAt), + replacementMessageId: try container.decodeIfPresent(String.self, forKey: .replacementMessageId), + turnId: try container.decodeIfPresent(String.self, forKey: .turnId) + ) case "text": self = .text( text: try container.decode(String.self, forKey: .text), @@ -2599,6 +2680,15 @@ extension AgentChatEvent { metadata: try container.decode(CodexModerationMetadata.self, forKey: .metadata), turnId: try container.decodeIfPresent(String.self, forKey: .turnId) ) + case "turn_diagnostics": + self = .turnDiagnostics( + turnId: try container.decodeIfPresent(String.self, forKey: .turnId), + moderationChecks: try container.decodeIfPresent(Int.self, forKey: .moderationChecks), + optionalIntegrationFailures: try container.decodeIfPresent( + [AgentChatOptionalIntegrationFailure].self, + forKey: .optionalIntegrationFailures + ) + ) case "codex_sleep": self = .codexSleep( itemId: try container.decode(String.self, forKey: .itemId), @@ -2618,8 +2708,47 @@ extension AgentChatEvent { reason: reason, message: message, recoveryOptions: recoveryOptions, + sourceSessionId: try container.decodeIfPresent(String.self, forKey: .sourceSessionId), + detectedAt: try container.decodeIfPresent(String.self, forKey: .detectedAt), + turnStartedAt: try container.decodeIfPresent(String.self, forKey: .turnStartedAt), + lastProgressAt: try container.decodeIfPresent(String.self, forKey: .lastProgressAt), + automaticRecoveryAttempted: try container.decodeIfPresent(Bool.self, forKey: .automaticRecoveryAttempted) + ) + case "turn_health": + self = .turnHealth( + provider: try container.decode(String.self, forKey: .provider), + turnId: try container.decode(String.self, forKey: .turnId), + state: try container.decode(String.self, forKey: .state), + reason: try container.decode(String.self, forKey: .reason), + message: try container.decode(String.self, forKey: .message), + turnStartedAt: try container.decode(String.self, forKey: .turnStartedAt), + lastProgressAt: try container.decode(String.self, forKey: .lastProgressAt), + detectedAt: try container.decode(String.self, forKey: .detectedAt), + recoveryCount: try container.decodeIfPresent(Int.self, forKey: .recoveryCount) ?? 0, + supportedActions: try container.decodeIfPresent([String].self, forKey: .supportedActions) ?? [], + automaticRecoveryAttempted: try container.decodeIfPresent(Bool.self, forKey: .automaticRecoveryAttempted) ?? false, sourceSessionId: try container.decodeIfPresent(String.self, forKey: .sourceSessionId) ) + case "codex_turn_recovery": + self = .codexTurnRecovery( + turnId: try container.decode(String.self, forKey: .turnId), + action: try container.decode(String.self, forKey: .action), + state: try container.decode(String.self, forKey: .state), + message: try container.decode(String.self, forKey: .message), + automatic: try container.decode(Bool.self, forKey: .automatic), + at: try container.decode(String.self, forKey: .at) + ) + case "turn_recovery": + self = .turnRecovery( + provider: try container.decode(String.self, forKey: .provider), + turnId: try container.decode(String.self, forKey: .turnId), + action: try container.decode(String.self, forKey: .action), + state: try container.decode(String.self, forKey: .state), + message: try container.decode(String.self, forKey: .message), + automatic: try container.decodeIfPresent(Bool.self, forKey: .automatic) ?? false, + at: try container.decode(String.self, forKey: .at), + recoveryCount: try container.decodeIfPresent(Int.self, forKey: .recoveryCount) ?? 0 + ) case "codex_thread_deleted": self = .codexThreadDeleted( threadId: try container.decode(String.self, forKey: .threadId), @@ -2701,6 +2830,7 @@ extension AgentChatEvent { var typeName: String { switch self { case .userMessage: return "user_message" + case .userMessageResolution: return "user_message_resolution" case .text: return "text" case .toolCall: return "tool_call" case .toolResult: return "tool_result" @@ -2736,8 +2866,12 @@ extension AgentChatEvent { case .codexContextCompaction: return "codex_context_compaction" case .codexSafetyBuffering: return "codex_safety_buffering" case .codexModerationMetadata: return "codex_moderation_metadata" + case .turnDiagnostics: return "turn_diagnostics" case .codexSleep: return "codex_sleep" case .codexTurnStalled: return "codex_turn_stalled" + case .turnHealth: return "turn_health" + case .codexTurnRecovery: return "codex_turn_recovery" + case .turnRecovery: return "turn_recovery" case .codexThreadDeleted: return "codex_thread_deleted" case .systemNotice: return "system_notice" case .completionReport: return "completion_report" @@ -2820,6 +2954,31 @@ struct AgentChatRecoverCodexTurnResult: Codable, Equatable { var status: String } +struct AgentChatRecoverTurnRequest: Codable, Equatable { + var sessionId: String + var turnId: String + var action: String +} + +struct AgentChatRecoverTurnResult: Codable, Equatable { + var action: String + var turnId: String + var status: String +} + +struct AgentChatResolveUnprocessedMessageRequest: Codable, Equatable { + var sessionId: String + var steerId: String + var action: String +} + +struct AgentChatResolveUnprocessedMessageResult: Codable, Equatable { + var steerId: String + var action: String + var status: String + var replacementMessageId: String? +} + struct AgentChatSessionIdRequest: Codable, Equatable { var sessionId: String } diff --git a/apps/ios/ADE/Services/AccountDirectory.swift b/apps/ios/ADE/Services/AccountDirectory.swift index 8b963fa22..662c49994 100644 --- a/apps/ios/ADE/Services/AccountDirectory.swift +++ b/apps/ios/ADE/Services/AccountDirectory.swift @@ -153,11 +153,13 @@ struct AccountDirectoryClient { token: String, refreshToken: (() async -> String?)? = nil ) async throws -> [AccountMachine] { + let correlationID = UUID().uuidString.lowercased() func request(using accessToken: String) async throws -> (Data, HTTPURLResponse) { var request = URLRequest(url: baseURL.appendingPathComponent("account/machines")) request.httpMethod = "GET" request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") request.setValue("application/json", forHTTPHeaderField: "Accept") + request.setValue(correlationID, forHTTPHeaderField: "X-ADE-Correlation-ID") request.timeoutInterval = 12 request.cachePolicy = .reloadIgnoringLocalCacheData diff --git a/apps/ios/ADE/Services/SyncConnectionRace.swift b/apps/ios/ADE/Services/SyncConnectionRace.swift index 47442eec6..7b22b8854 100644 --- a/apps/ios/ADE/Services/SyncConnectionRace.swift +++ b/apps/ios/ADE/Services/SyncConnectionRace.swift @@ -49,6 +49,15 @@ func syncRelayLegacyURL(_ rawValue: String) -> String { return components.string ?? rawValue } +func syncRelayCorrelatedURL(_ rawValue: String, correlationID: String) -> String { + guard var components = URLComponents(string: rawValue) else { return rawValue } + var queryItems = components.queryItems ?? [] + queryItems.removeAll(where: { $0.name == "cid" }) + queryItems.append(URLQueryItem(name: "cid", value: correlationID.lowercased())) + components.queryItems = queryItems + return components.string ?? rawValue +} + enum SyncRelayTransportControl: Equatable { case accepted case ready diff --git a/apps/ios/ADE/Services/SyncService.swift b/apps/ios/ADE/Services/SyncService.swift index 31721af3c..4f85e8f83 100644 --- a/apps/ios/ADE/Services/SyncService.swift +++ b/apps/ios/ADE/Services/SyncService.swift @@ -457,7 +457,7 @@ enum SyncRequestTimeout { private struct LaneDeletionFailure: Equatable { let projectId: String - let projectRootPath: String + let projectRootPath: String? let message: String } @@ -857,6 +857,27 @@ func syncRelayAuthorizationState( return relayOwner == currentOwner ? .eligible : .requires(.sameAccountRequired) } +func syncRelayReconnectAuthorizationRequirement( + usedAuthorization: AccountPairingAuthorization?, + currentAuthorization: AccountPairingAuthorization?, + relayAccountOwnerId: String? +) -> SyncRelayAuthorizationRequirement? { + guard let usedAuthorization, let currentAuthorization else { + return .signInRequired + } + let storedOwner = relayAccountOwnerId? + .trimmingCharacters(in: .whitespacesAndNewlines) + let expectedOwner = (storedOwner?.isEmpty == false ? storedOwner : nil) + ?? usedAuthorization.ownerId + if currentAuthorization.ownerId != expectedOwner { + return .sameAccountRequired + } + guard currentAuthorization == usedAuthorization else { + return .signInRequired + } + return nil +} + func syncEligibleRelayCandidates( for profile: HostConnectionProfile, currentAccountOwnerId: String? @@ -2403,6 +2424,27 @@ private struct SyncDomainHydrationAttempt { let domains: [SyncDomain] } +func syncProviderNeutralRecoveryAction(_ legacyAction: String) -> String? { + switch legacyAction { + case "wait": return "wait" + case "steer": return "nudge" + case "interrupt_retry_same_thread": return "retry_same_runtime" + case "restart_resume_thread": return "restart_resume" + default: return nil + } +} + +func syncPreferredRecoveryActionName( + supportsProviderNeutral: Bool, + supportsLegacyCodex: Bool, + providerNeutralActionName: String = "chat.recoverTurn", + legacyCodexActionName: String = "chat.recoverCodexTurn" +) -> String? { + if supportsProviderNeutral { return providerNeutralActionName } + if supportsLegacyCodex { return legacyCodexActionName } + return nil +} + @MainActor final class SyncService: ObservableObject { @Published private(set) var connectionState: RemoteConnectionState = .disconnected @@ -4713,7 +4755,10 @@ final class SyncService: ObservableObject { for machine: AccountMachine, hasSigningKey: Bool ) -> [AccountAdoptionRoute] { - let orderedKinds: [AccountMachineEndpoint.Kind] = [.relay, .tailnet, .lan] + // Match desktop paired-runtime routing. Eligible device-bound routes are + // attempted before the authenticated relay; an unsigned legacy host stays + // relay-only so account credentials never reach an unverified endpoint. + let orderedKinds: [AccountMachineEndpoint.Kind] = [.lan, .tailnet, .relay] var seen = Set() var routes: [AccountAdoptionRoute] = [] for kind in orderedKinds { @@ -6162,11 +6207,9 @@ final class SyncService: ObservableObject { objectWillChange.send() } - /// Removes only pairings created through a specific ADE account. Directly - /// paired machines intentionally have no owner id and remain available after - /// sign-out. This is also called when Clerk reports a session switch or - /// expiry, so another person cannot reopen this ADE install and see the - /// previous account's machines. + /// Explicitly revokes pairings created through one ADE account. Ordinary + /// sign-out uses `removeAccountOwnedPairings(exceptOwnerId:)` and preserves + /// the host-issued device credential for direct routes. func removeAccountOwnedPairings(ownerId: String) { let owner = ownerId.trimmingCharacters(in: .whitespacesAndNewlines) guard !owner.isEmpty else { return } @@ -6174,21 +6217,24 @@ final class SyncService: ObservableObject { removeAccountOwnedPairings(matching: { $0 == owner }) } - /// On cold launch, Clerk may already be signed out and there is no in-memory - /// "previous" identity to compare. Prune every account-owned profile except - /// the currently authenticated owner so stale sessions never leak machines. + /// Reconcile saved account-adopted pairings with the current account. + /// Signing out preserves the host-issued, device-bound paired secret so + /// LAN/Tailscale remain available; Relay still requires a fresh matching + /// account proof. Switching directly to another account removes pairings + /// owned by the previous account, matching desktop. func removeAccountOwnedPairings(exceptOwnerId ownerId: String?) { let allowedOwner = ownerId?.trimmingCharacters(in: .whitespacesAndNewlines) - removeAccountOwnedPairings { owner in - !owner.isEmpty && owner != allowedOwner + if let allowedOwner, !allowedOwner.isEmpty { + removeAccountOwnedPairings { owner in + !owner.isEmpty && owner != allowedOwner + } } enforceRelayAuthorization(currentOwnerId: allowedOwner) } - /// A direct-owned pairing may have a relay learned through an account without - /// becoming account-owned itself. When that account signs out or changes, end - /// an active relay session immediately, keep the local credential, and try the - /// direct LAN/Tailscale routes instead. + /// Any paired profile may have Relay metadata learned through an account. + /// When that account signs out or changes, end an active Relay session + /// immediately, keep the device credential, and try LAN/Tailscale instead. private func enforceRelayAuthorization(currentOwnerId: String?) { guard currentAddress.map(syncIsFullWebSocketRoute) == true, let profile = activeHostProfile ?? loadProfile() else { return } @@ -9603,10 +9649,17 @@ final class SyncService: ObservableObject { throw NSError( domain: "ADE", code: 24, - userInfo: [NSLocalizedDescriptionKey: "This Codex recovery action is not supported."] + userInfo: [NSLocalizedDescriptionKey: "This recovery action is not supported."] ) } - guard supportsRemoteAction("chat.recoverCodexTurn") else { + let neutralActionName = chatActionName("chat.recoverTurn", sessionId: sessionId) + let legacyActionName = chatActionName("chat.recoverCodexTurn", sessionId: sessionId) + guard let selectedActionName = syncPreferredRecoveryActionName( + supportsProviderNeutral: supportsRemoteAction(neutralActionName), + supportsLegacyCodex: supportsRemoteAction(legacyActionName), + providerNeutralActionName: neutralActionName, + legacyCodexActionName: legacyActionName + ) else { throw NSError( domain: "ADE", code: 17, @@ -9614,8 +9667,33 @@ final class SyncService: ObservableObject { ) } let scope = chatCommandScope(for: sessionId) + if selectedActionName == neutralActionName { + guard let neutralAction = syncProviderNeutralRecoveryAction(action) else { + throw NSError( + domain: "ADE", + code: 24, + userInfo: [NSLocalizedDescriptionKey: "This recovery action is not supported."] + ) + } + let result = try await sendDecodableChatCommand( + action: neutralActionName, + payload: AgentChatRecoverTurnRequest( + sessionId: sessionId, + turnId: turnId, + action: neutralAction + ), + targetProjectId: scope.projectId, + targetProjectRootPath: scope.rootPath, + as: AgentChatRecoverTurnResult.self + ) + return AgentChatRecoverCodexTurnResult( + action: action, + turnId: result.turnId, + status: result.status + ) + } return try await sendDecodableChatCommand( - action: "chat.recoverCodexTurn", + action: legacyActionName, payload: AgentChatRecoverCodexTurnRequest( sessionId: sessionId, turnId: turnId, @@ -9627,6 +9705,48 @@ final class SyncService: ObservableObject { ) } + func resolveUnprocessedMessage( + sessionId: String, + steerId: String, + action: String + ) async throws -> AgentChatResolveUnprocessedMessageResult { + guard action == "run_next" || action == "dismiss" else { + throw NSError( + domain: "ADE", + code: 24, + userInfo: [NSLocalizedDescriptionKey: "This unprocessed-message action is not supported."] + ) + } + let normalizedSteerId = steerId.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalizedSteerId.isEmpty else { + throw NSError( + domain: "ADE", + code: 25, + userInfo: [NSLocalizedDescriptionKey: "This message is missing its durable delivery identifier."] + ) + } + let actionName = chatActionName("chat.resolveUnprocessedMessage", sessionId: sessionId) + guard supportsRemoteAction(actionName) else { + throw NSError( + domain: "ADE", + code: 17, + userInfo: [NSLocalizedDescriptionKey: "Message recovery is not available on this machine version. Update ADE on the machine and reconnect."] + ) + } + let scope = chatCommandScope(for: sessionId) + return try await sendDecodableChatCommand( + action: actionName, + payload: AgentChatResolveUnprocessedMessageRequest( + sessionId: sessionId, + steerId: normalizedSteerId, + action: action + ), + targetProjectId: scope.projectId, + targetProjectRootPath: scope.rootPath, + as: AgentChatResolveUnprocessedMessageResult.self + ) + } + @discardableResult func steerChatSession( sessionId: String, @@ -11278,6 +11398,7 @@ final class SyncService: ObservableObject { var scheduled: SyncConnectionRaceScheduledCandidate var task: URLSessionWebSocketTask var helloPayload: [String: Any] + var relayAuthorization: AccountPairingAuthorization? } private enum AuthenticatedConnectionRaceEvent: @unchecked Sendable { @@ -11310,8 +11431,7 @@ final class SyncService: ObservableObject { ? automaticReconnectAddresses(for: profile) : prioritizedAddresses(for: profile) // Relay routes (full wss:// URLs) carry their own path and port. Keep them - // out of the direct port sweep, but race them as first-class authenticated - // candidates rather than waiting for stale direct timeouts. + // out of the direct port sweep and reserve them for the fallback phase. let relayRoutes = rawAddresses.filter(syncIsFullWebSocketRoute) let usableRelayRoutes = AccountService.shared.currentPairingAuthorization == nil ? [] : relayRoutes let addresses = connectableAddresses(from: rawAddresses.filter { !syncIsFullWebSocketRoute($0) }) @@ -11360,18 +11480,53 @@ final class SyncService: ObservableObject { } guard !orderedEndpointAttempts.isEmpty else { throw noConnectableAddressError() } - let racePlan = syncConnectionRaceCandidatePlan(rankedAttempts: orderedEndpointAttempts) - if publishConnecting, let first = racePlan.first { + let directRacePlan = syncConnectionRaceCandidatePlan( + rankedAttempts: orderedEndpointAttempts.filter { + !syncIsFullWebSocketRoute($0.address) + } + ) + let relayRacePlan = syncConnectionRaceCandidatePlan( + rankedAttempts: orderedEndpointAttempts.filter { + syncIsFullWebSocketRoute($0.address) + } + ) + if publishConnecting, let first = directRacePlan.first ?? relayRacePlan.first { publishSocketConnecting(to: first.endpoint.address) } let connectedCandidate: AuthenticatedConnectionCandidate do { - connectedCandidate = try await raceAuthenticatedConnectionCandidates( - racePlan, - profile: profile, - pairedSecret: token, - connectAttemptGeneration: connectAttemptGeneration - ) + if directRacePlan.isEmpty { + connectedCandidate = try await raceAuthenticatedConnectionCandidates( + relayRacePlan, + profile: profile, + pairedSecret: token, + connectAttemptGeneration: connectAttemptGeneration + ) + } else { + do { + connectedCandidate = try await raceAuthenticatedConnectionCandidates( + directRacePlan, + profile: profile, + pairedSecret: token, + connectAttemptGeneration: connectAttemptGeneration + ) + } catch { + guard !relayRacePlan.isEmpty, + isCurrentConnectAttempt(connectAttemptGeneration), + !Task.isCancelled else { + throw error + } + syncConnectLog.notice( + "ADE_SYNC_TRACE direct routes exhausted; beginning authenticated relay fallback" + ) + connectedCandidate = try await raceAuthenticatedConnectionCandidates( + relayRacePlan, + profile: profile, + pairedSecret: token, + connectAttemptGeneration: connectAttemptGeneration + ) + } + } } catch { if !shouldInvalidateSavedPairing(for: error), case .requires(let requirement) = relayAuthorizationState(for: profile) { @@ -11383,6 +11538,15 @@ final class SyncService: ObservableObject { connectedCandidate.task.cancel(with: .goingAway, reason: nil) throw CancellationError() } + if syncIsFullWebSocketRoute(connectedCandidate.scheduled.endpoint.address), + let requirement = syncRelayReconnectAuthorizationRequirement( + usedAuthorization: connectedCandidate.relayAuthorization, + currentAuthorization: AccountService.shared.currentPairingAuthorization, + relayAccountOwnerId: profile.relayAccountOwnerId + ) { + connectedCandidate.task.cancel(with: .goingAway, reason: nil) + throw requirement + } if socket != nil { failPendingRequests(with: NSError( domain: "ADE", @@ -11563,7 +11727,12 @@ final class SyncService: ObservableObject { guard let rawURLString = syncWebSocketURLString(host: urlHost, port: socketPort) else { throw NSError(domain: "ADE", code: 1, userInfo: [NSLocalizedDescriptionKey: "Invalid machine address."]) } - let legacyURLString = isRelay ? syncRelayLegacyURL(rawURLString) : rawURLString + let legacyURLString = isRelay + ? syncRelayCorrelatedURL( + syncRelayLegacyURL(rawURLString), + correlationID: connectionAttempt.id + ) + : rawURLString guard let legacyURL = URL(string: legacyURLString) else { throw NSError(domain: "ADE", code: 1, userInfo: [NSLocalizedDescriptionKey: "Invalid machine address."]) } @@ -11652,6 +11821,7 @@ final class SyncService: ObservableObject { } var auth: [String: Any] + var relayAuthorization: AccountPairingAuthorization? if profile.authKind == "paired", let pairedDeviceId = profile.pairedDeviceId { let proof = DpopKeyService.shared.buildProof(deviceId: pairedDeviceId, secret: pairedSecret) if isRelay, proof == nil { @@ -11669,6 +11839,7 @@ final class SyncService: ObservableObject { if let proof { auth["dpop"] = proof } if isRelay { let relaySession = try await AccountService.shared.freshRelaySession() + relayAuthorization = relaySession.authorization guard profile.relayAccountOwnerId == nil || profile.relayAccountOwnerId == relaySession.authorization.ownerId else { throw SyncRelayAuthorizationRequirement.sameAccountRequired @@ -11709,7 +11880,8 @@ final class SyncService: ObservableObject { return AuthenticatedConnectionCandidate( scheduled: candidate, task: candidateTask, - helloPayload: payload + helloPayload: payload, + relayAuthorization: relayAuthorization ) case "hello_error": throw candidateHelloError(preprocessed.payload, profile: profile) @@ -12344,7 +12516,13 @@ final class SyncService: ObservableObject { throw NSError(domain: "ADE", code: 1, userInfo: [NSLocalizedDescriptionKey: "Invalid machine address."]) } let isRelay = syncIsFullWebSocketRoute(host) - let legacyURLString = isRelay ? syncRelayLegacyURL(rawURLString) : rawURLString + let correlationID = UUID().uuidString + let legacyURLString = isRelay + ? syncRelayCorrelatedURL( + syncRelayLegacyURL(rawURLString), + correlationID: correlationID + ) + : rawURLString let socketAttempts: [(urlString: String, awaitsRelayReadyV2: Bool)] = isRelay ? [(syncRelayReadyV2URL(legacyURLString), true), (legacyURLString, false)] : [(legacyURLString, false)] diff --git a/apps/ios/ADE/Views/Work/WorkActivityIndicator.swift b/apps/ios/ADE/Views/Work/WorkActivityIndicator.swift index f16733850..b8fce0175 100644 --- a/apps/ios/ADE/Views/Work/WorkActivityIndicator.swift +++ b/apps/ios/ADE/Views/Work/WorkActivityIndicator.swift @@ -111,24 +111,45 @@ struct WorkActivityIndicator: View { } } - /// Timestamp of the last user message / turn-start boundary, i.e. when the - /// currently-streaming turn began. + /// Timestamp of the active turn's start boundary. Follow-up steers are user + /// messages too, but they must not reset this anchor — doing so made a + /// four-hour stalled turn appear to have started seconds ago after a nudge. static func activeTurnStartTimestamp(from transcript: [WorkChatEnvelope]) -> String? { - var latestActiveStatus: String? - for envelope in transcript.reversed() { + let ordered = sortedWorkChatEnvelopes(transcript) + let endedTurnIds = Set(ordered.compactMap(Self.endedTurnId(from:))) + + if let activeBoundary = ordered.reversed().first(where: { envelope in + guard case .status(let turnStatus, _, let turnId) = envelope.event, + Self.isActiveStatus(turnStatus) + else { + return false + } + guard let turnId = normalizedActivityTurnId(turnId) else { return true } + return !endedTurnIds.contains(turnId) + }) { + return activeBoundary.timestamp + } + + let lastTerminalIndex = ordered.lastIndex(where: { envelope in switch envelope.event { - case .userMessage: - return envelope.timestamp + case .done: + return true case .status(let turnStatus, _, _): - if latestActiveStatus == nil, - ["started", "active", "running"].contains(turnStatus.lowercased()) { - latestActiveStatus = envelope.timestamp - } + return Self.isTerminalStatus(turnStatus) default: - continue + return false } + }) + let activeStartIndex = lastTerminalIndex.map { ordered.index(after: $0) } ?? ordered.startIndex + if activeStartIndex < ordered.endIndex, + let primaryMessage = ordered[activeStartIndex...].first(where: { envelope in + if case .userMessage = envelope.event { return true } + return false + }) { + return primaryMessage.timestamp } - return latestActiveStatus ?? transcript.last?.timestamp + + return ordered.last?.timestamp } struct Presentation: Equatable { @@ -153,6 +174,9 @@ struct WorkActivityIndicator: View { case .done: return nil + case .userMessageResolution: + continue + case .userMessage: return workingFallback @@ -257,7 +281,7 @@ struct WorkActivityIndicator: View { .todoUpdate, .approvalRequest, .structuredQuestion, .toolUseSummary, .systemNotice, .error, .promptSuggestion, .contextCompact, .autoApprovalReview, .pendingInputResolved, .subagentResult, - .codexState, .codexTurnStalled, + .codexState, .turnDiagnostics, .codexTurnStalled, .codexTurnRecovery, .scheduledWorkUpdate, .transcriptRetraction, .completionReport, .tokens, .claudeGoalUpdated, .claudeGoalCleared, .unknown: @@ -339,7 +363,7 @@ func workChatShouldShowInterruptControl(isStreamingTurn: Bool, transcript: [Work /// Parse an ISO-8601 timestamp (with or without fractional seconds) into a /// `Date` for elapsed-time math. Returns nil on host quirks so callers can fall /// back gracefully. -private func workActivityTimestamp(_ iso: String) -> Date? { +func workActivityTimestamp(_ iso: String) -> Date? { if let date = workActivityIsoFormatter.date(from: iso) { return date } return workActivityIsoFallbackFormatter.date(from: iso) } diff --git a/apps/ios/ADE/Views/Work/WorkChatHeaderAndMessageViews.swift b/apps/ios/ADE/Views/Work/WorkChatHeaderAndMessageViews.swift index b37ab0193..ac29b2526 100644 --- a/apps/ios/ADE/Views/Work/WorkChatHeaderAndMessageViews.swift +++ b/apps/ios/ADE/Views/Work/WorkChatHeaderAndMessageViews.swift @@ -293,6 +293,9 @@ struct WorkChatMessageBubble: View { /// Computed once by the parent transcript view. Avoids installing one /// GeometryReader per user row while preserving the desktop-style max width. var maxUserBubbleWidth: CGFloat? = nil + var onRunUnprocessed: (@MainActor (WorkChatMessage) async throws -> Void)? = nil + var onEditUnprocessed: (@MainActor (WorkChatMessage) async throws -> Void)? = nil + var onDismissUnprocessed: (@MainActor (WorkChatMessage) async throws -> Void)? = nil @State private var assistantLineBudget = workAssistantMessageInitialLineBudget /// Provider string for the current chat session (e.g. "claude", "codex", "cursor"). @@ -455,8 +458,9 @@ struct WorkChatMessageBubble: View { Spacer(minLength: 0) VStack(alignment: .trailing, spacing: 6) { if let deliveryBadge { - // Delivery badges only render when a non-default state applies - // (queued/sending/failed). Successful deliveries stay silent. + // Follow-up delivery is a durable lifecycle. Showing the precise + // state prevents "accepted by the server" from being mistaken for + // "the agent actually processed this message." WorkDeliveryBadge(state: deliveryBadge) } if hasText || hasAttachments { @@ -487,6 +491,17 @@ struct WorkChatMessageBubble: View { ) .frame(maxWidth: maxBubbleWidth, alignment: .trailing) .fixedSize(horizontal: false, vertical: true) + .accessibilityElement(children: .combine) + .accessibilityLabel(userMessageAccessibilityLabel) + } + if message.deliveryState == "unprocessed" { + WorkUnprocessedMessageActions( + message: message, + onRun: onRunUnprocessed, + onEdit: onEditUnprocessed, + onDismiss: onDismissUnprocessed + ) + .frame(maxWidth: maxBubbleWidth, alignment: .trailing) } } } @@ -498,8 +513,7 @@ struct WorkChatMessageBubble: View { Label("Copy message", systemImage: "doc.on.doc") } } - .accessibilityElement(children: .combine) - .accessibilityLabel(userMessageAccessibilityLabel) + .accessibilityElement(children: .contain) .adeInspectable( "Work.Chat.MessageBubble.User", metadata: [ @@ -539,18 +553,10 @@ struct WorkChatMessageBubble: View { var deliveryBadge: WorkDeliveryBadge.State? { guard message.role == "user" else { return nil } - if let state = message.deliveryState { - switch state { - case "queued": return .queued - case "delivered": - return message.processed == true ? nil : .delivered - case "inline": return .inline - case "failed": return .failed - case "sending": return .sending - default: return nil - } - } - return nil + return workDeliveryBadgeState( + deliveryState: message.deliveryState, + processed: message.processed + ) } @ViewBuilder @@ -1044,13 +1050,13 @@ struct WorkTurnEndMarkerView: View { private var markerAccessibilityLabel: String { if completed { - return "Turn ended at \(workTurnSeparatorTimeLabel(marker.time)). Worked for \(marker.workedDurationLabel)" + return "Turn ended at \(workTurnSeparatorTimeLabel(marker.time)). Ran for \(marker.workedDurationLabel)" } return [ "Turn \(status)", marker.terminalReasonLabel, marker.modelLabel.isEmpty ? nil : marker.modelLabel, - "Worked for \(marker.workedDurationLabel)", + "Elapsed \(marker.workedDurationLabel)", ].compactMap { $0 }.joined(separator: ". ") } @@ -1069,7 +1075,7 @@ struct WorkTurnEndMarkerView: View { @ViewBuilder private var content: some View { if completed { - Text("\(workTurnSeparatorTimeLabel(marker.time)) · Worked for \(marker.workedDurationLabel)") + Text("\(workTurnSeparatorTimeLabel(marker.time)) · Ran for \(marker.workedDurationLabel)") .font(.caption2) .foregroundStyle(ADEColor.textMuted) .lineLimit(1) @@ -1094,7 +1100,7 @@ struct WorkTurnEndMarkerView: View { } Text("·") .opacity(0.42) - Text("Worked for \(marker.workedDurationLabel)") + Text("Elapsed \(marker.workedDurationLabel)") .font(.caption2) } .foregroundStyle(statusTint.opacity(0.9)) @@ -1226,15 +1232,16 @@ extension EnvironmentValues { } struct WorkDeliveryBadge: View { - enum State { - case queued, sending, delivered, inline, failed + enum State: Equatable { + case queued, sending, accepted, processed, unprocessed, failed var label: String { switch self { case .queued: return "Queued" case .sending: return "Sending" - case .delivered: return "Delivered" - case .inline: return "During turn" + case .accepted: return "Accepted" + case .processed: return "Processed" + case .unprocessed: return "Not processed" case .failed: return "Failed" } } @@ -1243,8 +1250,9 @@ struct WorkDeliveryBadge: View { switch self { case .queued: return "clock" case .sending: return "arrow.up.circle" - case .delivered: return "checkmark.circle" - case .inline: return "arrow.turn.down.right" + case .accepted: return "tray.and.arrow.down" + case .processed: return "checkmark.circle" + case .unprocessed: return "arrow.clockwise.circle" case .failed: return "exclamationmark.triangle" } } @@ -1253,8 +1261,9 @@ struct WorkDeliveryBadge: View { switch self { case .queued: return ADEColor.accent case .sending: return ADEColor.accent - case .delivered: return ADEColor.success - case .inline: return ADEColor.accent + case .accepted: return ADEColor.accent + case .processed: return ADEColor.success + case .unprocessed: return ADEColor.warning case .failed: return ADEColor.danger } } @@ -1275,3 +1284,156 @@ struct WorkDeliveryBadge: View { .accessibilityLabel("Delivery state: \(state.label)") } } + +private struct WorkUnprocessedMessageActions: View { + let message: WorkChatMessage + let onRun: (@MainActor (WorkChatMessage) async throws -> Void)? + let onEdit: (@MainActor (WorkChatMessage) async throws -> Void)? + let onDismiss: (@MainActor (WorkChatMessage) async throws -> Void)? + + @State private var pendingAction: String? + @State private var optimisticResolution: String? + @State private var errorMessage: String? + + private var settledAction: String? { + message.unprocessedResolution?.action ?? optimisticResolution + } + + private var hasDurableSteerId: Bool { + !(message.steerId?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true) + } + + var body: some View { + if let settledAction { + Text(settledAction == "run_next" ? "Started as the next turn" : "Dismissed") + .font(.caption2) + .foregroundStyle(ADEColor.textMuted) + .accessibilityLabel( + settledAction == "run_next" + ? "This message started as the next turn." + : "This message was dismissed." + ) + } else { + VStack(alignment: .trailing, spacing: 6) { + ViewThatFits(in: .horizontal) { + HStack(spacing: 6) { actionButtons } + VStack(alignment: .trailing, spacing: 6) { actionButtons } + } + if let errorMessage { + Text(errorMessage) + .font(.caption2) + .foregroundStyle(ADEColor.danger) + .fixedSize(horizontal: false, vertical: true) + .accessibilityAddTraits(.isStaticText) + } + } + } + } + + @ViewBuilder + private var actionButtons: some View { + if hasDurableSteerId, onRun != nil { + actionButton( + title: pendingAction == "run_next" ? "Starting…" : "Run next", + systemImage: "play.fill", + action: "run_next", + primary: true, + accessibilityHint: "Starts this message as a new turn when the current turn is idle." + ) + } + if onEdit != nil { + actionButton( + title: "Edit", + systemImage: "pencil", + action: "edit", + primary: false, + accessibilityHint: "Replaces the composer draft with this message for editing." + ) + } + if hasDurableSteerId, onDismiss != nil { + actionButton( + title: "Dismiss", + systemImage: "xmark", + action: "dismiss", + primary: false, + accessibilityHint: "Marks this unprocessed message as dismissed." + ) + } + } + + private func actionButton( + title: String, + systemImage: String, + action: String, + primary: Bool, + accessibilityHint: String + ) -> some View { + Button { + Task { await perform(action) } + } label: { + Label(title, systemImage: systemImage) + .font(.caption.weight(.semibold)) + .foregroundStyle(primary ? Color.white : ADEColor.textSecondary) + .frame(minWidth: 44, minHeight: 44) + .padding(.horizontal, 10) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .background( + primary ? ADEColor.warning : ADEColor.cardBackground.opacity(0.42), + in: RoundedRectangle(cornerRadius: 10, style: .continuous) + ) + .overlay( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .stroke(primary ? ADEColor.warning.opacity(0.4) : ADEColor.glassBorder, lineWidth: 0.8) + ) + .disabled(pendingAction != nil) + .accessibilityLabel(title) + .accessibilityHint(accessibilityHint) + } + + @MainActor + private func perform(_ action: String) async { + guard pendingAction == nil else { return } + let handler: (@MainActor (WorkChatMessage) async throws -> Void)? + switch action { + case "run_next": handler = onRun + case "edit": handler = onEdit + case "dismiss": handler = onDismiss + default: return + } + guard let handler else { return } + pendingAction = action + errorMessage = nil + defer { pendingAction = nil } + do { + try await handler(message) + if action == "run_next" || action == "dismiss" { + optimisticResolution = action + } + } catch { + ADEHaptics.error() + errorMessage = error.localizedDescription + } + } +} + +func workDeliveryBadgeState( + deliveryState: String?, + processed: Bool? +) -> WorkDeliveryBadge.State? { + switch deliveryState { + case "queued": return .queued + case "accepted": return .accepted + case "processed": return .processed + case "unprocessed": return .unprocessed + case "delivered": + return processed == true ? .processed : .accepted + case "inline": + return .processed + case "failed": return .failed + case "sending": return .sending + default: + return processed == true ? .processed : nil + } +} diff --git a/apps/ios/ADE/Views/Work/WorkChatRichCardViews.swift b/apps/ios/ADE/Views/Work/WorkChatRichCardViews.swift index 849e48aff..f06682a7e 100644 --- a/apps/ios/ADE/Views/Work/WorkChatRichCardViews.swift +++ b/apps/ios/ADE/Views/Work/WorkChatRichCardViews.swift @@ -1254,16 +1254,20 @@ struct WorkCodexRecoveryCardView: View { @State private var feedbackMessage: String? @State private var errorMessage: String? - private let labels: [String: String] = [ - "wait": "Wait", - "steer": "Nudge", - "interrupt_retry_same_thread": "Retry", - "restart_resume_thread": "Resume", - ] - - private var visibleOptions: [String] { + private var availableOptions: [String] { var seen = Set() - return card.recoveryOptions.filter { labels[$0] != nil && seen.insert($0).inserted }.prefix(4).map { $0 } + return card.recoveryOptions + .filter { workCodexRecoveryActionLabel(for: $0) != nil && seen.insert($0).inserted } + .prefix(4) + .map { $0 } + } + + private var primaryOptions: [String] { + workCodexRecoveryPrimaryOptions(availableOptions) + } + + private var moreOptions: [String] { + workCodexRecoveryMoreOptions(availableOptions) } private var canRecover: Bool { @@ -1273,6 +1277,28 @@ struct WorkCodexRecoveryCardView: View { && !(card.recoveryTurnId?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true) } + private var recoveryProviderName: String { + guard card.recoveryContext?.providerNeutral == true else { return "Codex" } + return workChatSurfaceProviderName(card.recoveryContext?.provider) + } + + private var timingSummary: String? { + guard let context = card.recoveryContext, + let detectedAt = context.detectedAt.flatMap(workActivityTimestamp) else { + return nil + } + var parts: [String] = [] + if let turnStartedAt = context.turnStartedAt.flatMap(workActivityTimestamp) { + let elapsed = max(0, Int(detectedAt.timeIntervalSince(turnStartedAt))) + parts.append("Elapsed \(WorkActivityIndicator.formatElapsedSeconds(elapsed))") + } + if let lastProgressAt = context.lastProgressAt.flatMap(workActivityTimestamp) { + let inactive = max(0, Int(detectedAt.timeIntervalSince(lastProgressAt))) + parts.append("No progress for \(WorkActivityIndicator.formatElapsedSeconds(inactive))") + } + return parts.isEmpty ? nil : parts.joined(separator: " · ") + } + var body: some View { VStack(alignment: .leading, spacing: 9) { HStack(spacing: 8) { @@ -1283,7 +1309,7 @@ struct WorkCodexRecoveryCardView: View { .font(.caption2.monospaced().weight(.bold)) .tracking(1.2) .foregroundStyle(ADEColor.warning.opacity(0.8)) - Text("Codex paused unexpectedly") + Text("\(recoveryProviderName) stopped responding") .font(.caption.weight(.semibold)) .foregroundStyle(ADEColor.textPrimary) .lineLimit(1) @@ -1296,14 +1322,50 @@ struct WorkCodexRecoveryCardView: View { .fixedSize(horizontal: false, vertical: true) } - if !visibleOptions.isEmpty { + if let timingSummary { + Label(timingSummary, systemImage: "clock") + .font(.caption2.monospacedDigit()) + .foregroundStyle(ADEColor.textMuted) + } + + if card.recoveryContext?.automaticRecoveryAttempted == true { + Label("Automatic restart was already attempted once.", systemImage: "arrow.clockwise") + .font(.caption2) + .foregroundStyle(ADEColor.textMuted) + .fixedSize(horizontal: false, vertical: true) + } + + if !primaryOptions.isEmpty { ViewThatFits(in: .horizontal) { - HStack(spacing: 7) { recoveryButtons } - LazyVGrid(columns: [GridItem(.flexible()), GridItem(.flexible())], spacing: 7) { recoveryButtons } + HStack(spacing: 7) { + ForEach(primaryOptions, id: \.self) { recoveryButton(for: $0) } + } + VStack(spacing: 7) { + ForEach(primaryOptions, id: \.self) { recoveryButton(for: $0) } + } + } + } + + if !moreOptions.isEmpty { + Menu { + ForEach(moreOptions, id: \.self) { option in + Button(workCodexRecoveryActionLabel(for: option) ?? option) { + Task { await recover(option) } + } + .disabled(!canRecover || pendingAction != nil) + } + } label: { + Label("More recovery options", systemImage: "ellipsis") + .font(.caption.weight(.semibold)) + .foregroundStyle(ADEColor.textSecondary) + .frame(minHeight: 44) + .contentShape(Rectangle()) } + .disabled(!canRecover || pendingAction != nil) + .accessibilityHint("Shows nudge and same-server retry actions.") } - if !canRecover, !visibleOptions.isEmpty { + if !canRecover, !availableOptions.isEmpty { Text(enabled ? "Update or reconnect the paired machine to use recovery." : "Reconnect to the paired machine to use recovery.") .font(.caption2) .foregroundStyle(ADEColor.textMuted) @@ -1331,29 +1393,35 @@ struct WorkCodexRecoveryCardView: View { } @ViewBuilder - private var recoveryButtons: some View { - ForEach(visibleOptions, id: \.self) { option in - let label = labels[option] ?? option - Button { - Task { await recover(option) } - } label: { - Text(pendingAction == option ? "\(label)…" : label) - .font(.caption.monospaced().weight(.semibold)) - .foregroundStyle(ADEColor.warning) - .frame(maxWidth: .infinity, minHeight: 44) - .padding(.horizontal, 10) - .contentShape(Rectangle()) - } - .buttonStyle(.plain) - .background(ADEColor.warning.opacity(0.07), in: RoundedRectangle(cornerRadius: 10, style: .continuous)) - .overlay( - RoundedRectangle(cornerRadius: 10, style: .continuous) - .stroke(ADEColor.warning.opacity(0.18), lineWidth: 1) - ) - .disabled(!canRecover || pendingAction != nil) - .accessibilityLabel("\(label) Codex recovery") - .accessibilityHint("Runs this recovery action for the stalled Codex turn.") + private func recoveryButton(for option: String) -> some View { + let label = workCodexRecoveryActionLabel(for: option) ?? option + let isPrimary = option == "restart_resume_thread" + Button { + Task { await recover(option) } + } label: { + Text(pendingAction == option ? "\(label)…" : label) + .font(.caption.weight(.semibold)) + .foregroundStyle(isPrimary ? Color.white : ADEColor.warning) + .frame(maxWidth: .infinity, minHeight: 44) + .padding(.horizontal, 10) + .contentShape(Rectangle()) } + .buttonStyle(.plain) + .background( + isPrimary ? ADEColor.warning : ADEColor.warning.opacity(0.07), + in: RoundedRectangle(cornerRadius: 10, style: .continuous) + ) + .overlay( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .stroke(ADEColor.warning.opacity(isPrimary ? 0.36 : 0.18), lineWidth: 1) + ) + .disabled(!canRecover || pendingAction != nil) + .accessibilityLabel("\(label) \(recoveryProviderName) recovery") + .accessibilityHint( + isPrimary + ? "Restarts the \(recoveryProviderName) runtime once, resumes this thread, and keeps its context." + : "Keeps the current \(recoveryProviderName) runtime running while you wait for more output." + ) } @MainActor @@ -1375,6 +1443,101 @@ struct WorkCodexRecoveryCardView: View { } } +func workCodexRecoveryActionLabel(for action: String) -> String? { + switch action { + case "wait": return "Keep waiting" + case "steer": return "Send nudge" + case "interrupt_retry_same_thread": return "Retry same server" + case "restart_resume_thread": return "Restart & resume" + default: return nil + } +} + +func workCodexRecoveryPrimaryOptions(_ options: [String]) -> [String] { + ["restart_resume_thread", "wait"].filter(options.contains) +} + +func workCodexRecoveryMoreOptions(_ options: [String]) -> [String] { + ["steer", "interrupt_retry_same_thread"].filter(options.contains) +} + +struct WorkTurnDiagnosticsDisclosureView: View { + let card: WorkEventCardModel + + @State private var isExpanded = false + + private var summary: String { + var parts: [String] = [] + if card.diagnosticModerationChecks > 0 { + parts.append("Safety checked") + } + if !card.diagnosticIntegrationFailures.isEmpty { + let count = card.diagnosticIntegrationFailures.count + parts.append("\(count) optional integration\(count == 1 ? "" : "s") unavailable") + } + return parts.isEmpty ? "No notable diagnostics" : parts.joined(separator: " · ") + } + + var body: some View { + DisclosureGroup(isExpanded: $isExpanded) { + VStack(alignment: .leading, spacing: 8) { + if card.diagnosticModerationChecks > 0 { + Label( + card.diagnosticModerationChecks == 1 + ? "Safety check completed" + : "\(card.diagnosticModerationChecks) safety checks completed", + systemImage: "checkmark.shield" + ) + .font(.caption) + .foregroundStyle(ADEColor.textSecondary) + } + + ForEach(card.diagnosticIntegrationFailures, id: \.self) { failure in + VStack(alignment: .leading, spacing: 2) { + Label("\(failure.integration) unavailable for this turn", systemImage: "puzzlepiece.extension") + .font(.caption) + .foregroundStyle(ADEColor.textSecondary) + if let message = failure.message?.trimmingCharacters(in: .whitespacesAndNewlines), + !message.isEmpty { + Text(message) + .font(.caption2) + .foregroundStyle(ADEColor.textMuted) + } + } + } + } + .padding(.top, 4) + .padding(.leading, 2) + } label: { + HStack(spacing: 8) { + Image(systemName: "info.circle") + .foregroundStyle(ADEColor.textMuted) + VStack(alignment: .leading, spacing: 1) { + Text("Turn details") + .font(.caption.weight(.semibold)) + .foregroundStyle(ADEColor.textSecondary) + Text(summary) + .font(.caption2) + .foregroundStyle(ADEColor.textMuted) + .lineLimit(1) + } + } + .frame(maxWidth: .infinity, minHeight: 44, alignment: .leading) + .contentShape(Rectangle()) + } + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background(ADEColor.cardBackground.opacity(0.28), in: RoundedRectangle(cornerRadius: 12, style: .continuous)) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(ADEColor.glassBorder.opacity(0.75), lineWidth: 0.7) + ) + .accessibilityElement(children: .contain) + .accessibilityLabel("Turn details. \(summary)") + .accessibilityHint(isExpanded ? "Double tap to collapse details." : "Double tap to show safety and integration details.") + } +} + /// Resolved / historical structured-question card shown in the transcript once /// a question is no longer pending. Collapses to a single compact row — status /// icon plus a one-line summary ("Answered · {choice}", a quoted typed answer, diff --git a/apps/ios/ADE/Views/Work/WorkChatSessionView+Timeline.swift b/apps/ios/ADE/Views/Work/WorkChatSessionView+Timeline.swift index 209ae7b76..ccc3f6ee6 100644 --- a/apps/ios/ADE/Views/Work/WorkChatSessionView+Timeline.swift +++ b/apps/ios/ADE/Views/Work/WorkChatSessionView+Timeline.swift @@ -89,7 +89,10 @@ extension WorkChatSessionView { WorkChatMessageBubble( message: message, isStreaming: message.id == streamingAssistantMessageId, - maxUserBubbleWidth: maxUserBubbleWidth + maxUserBubbleWidth: maxUserBubbleWidth, + onRunUnprocessed: onRunUnprocessedMessage, + onEditUnprocessed: onEditUnprocessedMessage, + onDismissUnprocessed: onDismissUnprocessedMessage ) case .toolCard(let toolCard): timelineToolCard(toolCard) @@ -254,6 +257,8 @@ extension WorkChatSessionView { enabled: isLive, onRecover: onRecoverCodexTurn ) + } else if card.kind == "turnDiagnostics" { + WorkTurnDiagnosticsDisclosureView(card: card) } else { WorkEventCardView( card: card, diff --git a/apps/ios/ADE/Views/Work/WorkChatSessionView.swift b/apps/ios/ADE/Views/Work/WorkChatSessionView.swift index caf180fd4..ec021323a 100644 --- a/apps/ios/ADE/Views/Work/WorkChatSessionView.swift +++ b/apps/ios/ADE/Views/Work/WorkChatSessionView.swift @@ -222,10 +222,14 @@ struct WorkChatSessionView: View { var attachmentsAvailable: Bool = true var personalModelCatalogAvailable: Bool = true var personalSessionUpdatesAvailable: Bool = true - /// Present only when the paired host advertises `chat.recoverCodexTurn`. + /// Present only when the paired host advertises provider-neutral + /// `chat.recoverTurn` or the legacy Codex-specific recovery action. /// Keeping this optional prevents a new phone build from offering controls /// that an older ADE brain cannot execute. var onRecoverCodexTurn: (@MainActor (String, String, String) async throws -> String)? = nil + var onRunUnprocessedMessage: (@MainActor (WorkChatMessage) async throws -> Void)? = nil + var onEditUnprocessedMessage: (@MainActor (WorkChatMessage) async throws -> Void)? = nil + var onDismissUnprocessedMessage: (@MainActor (WorkChatMessage) async throws -> Void)? = nil @State var steerEditDrafts: [String: String] = [:] @State var modelPickerPresented = false @@ -1440,6 +1444,12 @@ private func workTimelinePresentationSignature( if case .message(let message) = timelineEntry.payload { hasher.combine(message.id) hasher.combine(message.role) + hasher.combine(message.steerId) + hasher.combine(message.deliveryState) + hasher.combine(message.processed) + hasher.combine(message.unprocessedResolution?.action) + hasher.combine(message.unprocessedResolution?.state) + hasher.combine(message.unprocessedResolution?.resolvedAt) workTimelineCombineTextSignature(message.markdown, into: &hasher) if let preview = message.assistantPreview { workTimelineCombineTextSignature(preview.text, into: &hasher) @@ -2153,10 +2163,16 @@ private struct WorkContextUsagePopover: View { struct WorkChatComposerDraftRestore: Equatable, Identifiable { let id: UUID let text: String + let replacesExistingDraft: Bool - init(text: String, id: UUID = UUID()) { + init( + text: String, + id: UUID = UUID(), + replacesExistingDraft: Bool = false + ) { self.id = id self.text = text + self.replacesExistingDraft = replacesExistingDraft } } @@ -2195,7 +2211,12 @@ final class WorkChatComposerDraftState: ObservableObject { func applyRestore(_ restore: WorkChatComposerDraftRestore?) { guard let restore, appliedRestoreId != restore.id else { return } appliedRestoreId = restore.id - restoreUnsentText(restore.text) + if restore.replacesExistingDraft { + text = restore.text + isFocused = true + } else { + restoreUnsentText(restore.text) + } } } diff --git a/apps/ios/ADE/Views/Work/WorkErrorAndMessageHelpers.swift b/apps/ios/ADE/Views/Work/WorkErrorAndMessageHelpers.swift index 6833710d7..ac730bcbc 100644 --- a/apps/ios/ADE/Views/Work/WorkErrorAndMessageHelpers.swift +++ b/apps/ios/ADE/Views/Work/WorkErrorAndMessageHelpers.swift @@ -35,6 +35,35 @@ func errorPresentation(for category: String) -> WorkErrorPresentation { func buildWorkChatMessages(from transcript: [WorkChatEnvelope]) -> [WorkChatMessage] { var messages: [WorkChatMessage] = [] let metadataByTurn = workTurnModelMetadataByTurn(from: transcript) + var resolutionBySteerId: [String: WorkUserMessageResolution] = [:] + var resolutionOrderBySteerId: [String: (timestamp: String, sequence: Int)] = [:] + for envelope in transcript { + guard case .userMessageResolution( + let steerId, + let action, + let state, + let resolvedAt, + let replacementMessageId, + _ + ) = envelope.event else { + continue + } + let normalizedSteerId = steerId.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalizedSteerId.isEmpty else { continue } + let sequence = envelope.sequence ?? 0 + if let existing = resolutionOrderBySteerId[normalizedSteerId], + envelope.timestamp < existing.timestamp + || (envelope.timestamp == existing.timestamp && sequence <= existing.sequence) { + continue + } + resolutionBySteerId[normalizedSteerId] = WorkUserMessageResolution( + action: action, + state: state, + resolvedAt: resolvedAt, + replacementMessageId: replacementMessageId + ) + resolutionOrderBySteerId[normalizedSteerId] = (envelope.timestamp, sequence) + } // Tracks whether the previous envelope was assistantText so nil-itemId // streaming fragments can merge into it. MUST be reset to false on every // non-assistantText branch below — otherwise a subsequent nil-itemId @@ -138,6 +167,9 @@ func buildWorkChatMessages(from transcript: [WorkChatEnvelope]) -> [WorkChatMess )) } previousEnvelopeWasAssistantText = true + case .userMessageResolution: + previousEnvelopeWasAssistantText = false + continue case .transcriptRetraction(let messageIds, _, _, _): previousEnvelopeWasAssistantText = false let retractedIds = Set( @@ -160,6 +192,15 @@ func buildWorkChatMessages(from transcript: [WorkChatEnvelope]) -> [WorkChatMess } } + for index in messages.indices { + guard let steerId = messages[index].steerId?.trimmingCharacters(in: .whitespacesAndNewlines), + !steerId.isEmpty, + let resolution = resolutionBySteerId[steerId] else { + continue + } + messages[index].unprocessedResolution = resolution + } + return messages } @@ -1740,6 +1781,16 @@ func workChatEventMergeKey(_ event: WorkChatEvent) -> String { } let attachmentDigest = (attachments ?? []).map { "\($0.type):\($0.path)" }.joined(separator: ",") return ["user_message", turnId ?? "", steerId ?? "", deliveryState ?? "", processed.map { $0 ? "1" : "0" } ?? "", attachmentDigest, text].joined(separator: "|") + case .userMessageResolution(let steerId, let action, let state, let resolvedAt, let replacementMessageId, let turnId): + return [ + "user_message_resolution", + turnId ?? "", + steerId, + action, + state, + resolvedAt, + replacementMessageId ?? "", + ].joined(separator: "|") case .assistantText(let text, let turnId, let itemId): let normalizedItemId = itemId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" if !normalizedItemId.isEmpty { @@ -1819,8 +1870,22 @@ func workChatEventMergeKey(_ event: WorkChatEvent) -> String { return ["web_search", turnId ?? "", itemId, query, actionKey, status.rawValue].joined(separator: "|") case .codexState(let title, let message, _, let turnId): return ["codex_state", turnId ?? "", title, message].joined(separator: "|") - case .codexTurnStalled(let message, _, let turnId, let sourceSessionId): - return ["codex_turn_stalled", sourceSessionId ?? "", turnId ?? "", message].joined(separator: "|") + case .turnDiagnostics(_, _, let turnId): + return ["turn_diagnostics", turnId ?? ""].joined(separator: "|") + case .codexTurnStalled(let message, _, let turnId, let sourceSessionId, let context): + return [ + "codex_turn_stalled", + context.providerNeutral ? "provider_neutral" : "legacy", + sourceSessionId ?? "", + turnId ?? "", + message, + ].joined(separator: "|") + case .codexTurnRecovery(_, let receipt, let turnId): + return [ + "codex_turn_recovery", + receipt.providerNeutral ? "provider_neutral" : "legacy", + turnId ?? "", + ].joined(separator: "|") case .planText(let text, let turnId): return ["plan_text", turnId ?? "", text].joined(separator: "|") case .toolUseSummary(let text, let turnId): diff --git a/apps/ios/ADE/Views/Work/WorkEventMapping.swift b/apps/ios/ADE/Views/Work/WorkEventMapping.swift index 3598d6cb4..8483902d3 100644 --- a/apps/ios/ADE/Views/Work/WorkEventMapping.swift +++ b/apps/ios/ADE/Views/Work/WorkEventMapping.swift @@ -180,6 +180,15 @@ func makeWorkChatEvent(from event: AgentChatEvent) -> WorkChatEvent { deliveryState: deliveryState, processed: processed ) + case .userMessageResolution(let steerId, let action, let state, let resolvedAt, let replacementMessageId, let turnId): + return .userMessageResolution( + steerId: steerId, + action: action, + state: state, + resolvedAt: resolvedAt, + replacementMessageId: replacementMessageId, + turnId: turnId + ) case .text(let text, let messageId, let turnId, let itemId): return .assistantText( text: text, @@ -529,19 +538,100 @@ func makeWorkChatEvent(from event: AgentChatEvent) -> WorkChatEvent { case .codexSafetyBuffering(let state, let turnId): let detail = state.fasterModel.map { "Buffering, \($0) ready" } ?? "Buffering" return .codexState(title: "Safety", message: detail, icon: "shield.checkered", turnId: turnId ?? state.turnId) - case .codexModerationMetadata(_, let turnId): - return .codexState(title: "Moderation", message: "Checked", icon: "checkmark.shield", turnId: turnId) + case .codexModerationMetadata: + // Routine per-check rows are intentionally absent from the main timeline. + // New hosts publish a privacy-safe aggregate `turn_diagnostics` event. + return .unknown(type: "codex_moderation_metadata") + case .turnDiagnostics(let turnId, let moderationChecks, let optionalIntegrationFailures): + return .turnDiagnostics( + moderationChecks: max(0, moderationChecks ?? 0), + optionalIntegrationFailures: optionalIntegrationFailures ?? [], + turnId: turnId + ) case .codexSleep(_, let turnId, let durationMs, _): let duration = durationMs.map { $0 < 1000 ? "\($0)ms" : "\(($0 + 500) / 1000)s" } return .codexState(title: "Wait", message: duration.map { "Sleeping \($0)" } ?? "Sleeping", icon: "hourglass", turnId: turnId) case .codexThreadDeleted(_, let turnId): return .codexState(title: "Thread", message: "Deleted upstream. Next message starts fresh.", icon: "exclamationmark.triangle", turnId: turnId) - case .codexTurnStalled(let turnId, _, _, let message, let recoveryOptions, let sourceSessionId): + case .codexTurnStalled( + let turnId, + _, + let reason, + let message, + let recoveryOptions, + let sourceSessionId, + let detectedAt, + let turnStartedAt, + let lastProgressAt, + let automaticRecoveryAttempted + ): return .codexTurnStalled( message: message, recoveryOptions: recoveryOptions ?? [], turnId: turnId, - sourceSessionId: sourceSessionId + sourceSessionId: sourceSessionId, + context: WorkCodexStallContext( + reason: reason, + detectedAt: detectedAt, + turnStartedAt: turnStartedAt, + lastProgressAt: lastProgressAt, + automaticRecoveryAttempted: automaticRecoveryAttempted ?? false + ) + ) + case .turnHealth( + let provider, + let turnId, + _, + let reason, + let message, + let turnStartedAt, + let lastProgressAt, + let detectedAt, + let recoveryCount, + let supportedActions, + let automaticRecoveryAttempted, + let sourceSessionId + ): + return .codexTurnStalled( + message: message, + recoveryOptions: supportedActions.compactMap(workLegacyRecoveryAction), + turnId: turnId, + sourceSessionId: sourceSessionId, + context: WorkCodexStallContext( + reason: reason, + detectedAt: detectedAt, + turnStartedAt: turnStartedAt, + lastProgressAt: lastProgressAt, + automaticRecoveryAttempted: automaticRecoveryAttempted, + provider: provider, + recoveryCount: recoveryCount, + providerNeutral: true + ) + ) + case .codexTurnRecovery(let turnId, let action, let state, let message, let automatic, let at): + return .codexTurnRecovery( + message: message, + receipt: WorkCodexRecoveryReceipt( + action: action, + state: state, + automatic: automatic, + at: at + ), + turnId: turnId + ) + case .turnRecovery(let provider, let turnId, let action, let state, let message, let automatic, let at, let recoveryCount): + return .codexTurnRecovery( + message: message, + receipt: WorkCodexRecoveryReceipt( + action: action, + state: state, + automatic: automatic, + at: at, + provider: provider, + recoveryCount: recoveryCount, + providerNeutral: true + ), + turnId: turnId ) case .planText(let text, let turnId, _): return .planText(text: text, turnId: turnId) @@ -587,6 +677,16 @@ func makeWorkChatEvent(from event: AgentChatEvent) -> WorkChatEvent { } } +func workLegacyRecoveryAction(_ action: String) -> String? { + switch action { + case "wait": return "wait" + case "nudge": return "steer" + case "retry_same_runtime": return "interrupt_retry_same_thread" + case "restart_resume": return "restart_resume_thread" + default: return nil + } +} + private func workStructuredErrorCategory(from errorInfo: RemoteJSONValue?) -> String? { guard case .object(let fields)? = errorInfo, case .string(let rawCategory)? = fields["category"] diff --git a/apps/ios/ADE/Views/Work/WorkModels.swift b/apps/ios/ADE/Views/Work/WorkModels.swift index 2d9e7a495..305e42f50 100644 --- a/apps/ios/ADE/Views/Work/WorkModels.swift +++ b/apps/ios/ADE/Views/Work/WorkModels.swift @@ -123,6 +123,7 @@ struct WorkChatMessage: Identifiable, Equatable { var deliveryState: String? = nil var processed: Bool? = nil var attachments: [AgentChatFileRef]? = nil + var unprocessedResolution: WorkUserMessageResolution? = nil } struct WorkLocalEchoMessage: Identifiable, Equatable { @@ -358,6 +359,34 @@ struct WorkCompletionArtifactModel: Equatable { let reference: String? } +struct WorkCodexStallContext: Equatable { + let reason: String + let detectedAt: String? + let turnStartedAt: String? + let lastProgressAt: String? + let automaticRecoveryAttempted: Bool + var provider: String? = nil + var recoveryCount: Int = 0 + var providerNeutral: Bool = false +} + +struct WorkUserMessageResolution: Equatable { + let action: String + let state: String + let resolvedAt: String + let replacementMessageId: String? +} + +struct WorkCodexRecoveryReceipt: Equatable { + let action: String + let state: String + let automatic: Bool + let at: String + var provider: String? = nil + var recoveryCount: Int = 0 + var providerNeutral: Bool = false +} + struct WorkCommandCardModel: Identifiable, Equatable { let id: String let command: String @@ -408,7 +437,8 @@ enum WorkTimelinePayload: Equatable { /// transcript's turn separators. case turnSeparator(WorkTurnSeparator) /// Centered end-of-turn completion row rendered after a terminal `done` - /// event, matching desktop's "time · Worked for ..." divider. + /// event. Completed turns say "Ran for"; interrupted/failed turns say + /// "Elapsed" so wall time is never presented as continuous agent work. case turnEndMarker(WorkTurnEndMarker) case pendingQuestion(WorkPendingQuestionModel) case pendingPermission(WorkPendingPermissionModel) @@ -751,6 +781,12 @@ struct WorkEventCardModel: Identifiable, Equatable { let recoveryOptions: [String] let recoveryTurnId: String? let recoverySessionId: String? + let recoveryContext: WorkCodexStallContext? + let recoveryReceipt: WorkCodexRecoveryReceipt? + /// Aggregated, low-noise diagnostics disclosed from "Turn details" instead + /// of rendering each routine moderation or optional integration event. + let diagnosticModerationChecks: Int + let diagnosticIntegrationFailures: [AgentChatOptionalIntegrationFailure] init( id: String, @@ -769,7 +805,11 @@ struct WorkEventCardModel: Identifiable, Equatable { resolution: String? = nil, recoveryOptions: [String] = [], recoveryTurnId: String? = nil, - recoverySessionId: String? = nil + recoverySessionId: String? = nil, + recoveryContext: WorkCodexStallContext? = nil, + recoveryReceipt: WorkCodexRecoveryReceipt? = nil, + diagnosticModerationChecks: Int = 0, + diagnosticIntegrationFailures: [AgentChatOptionalIntegrationFailure] = [] ) { self.id = id self.kind = kind @@ -788,6 +828,10 @@ struct WorkEventCardModel: Identifiable, Equatable { self.recoveryOptions = recoveryOptions self.recoveryTurnId = recoveryTurnId self.recoverySessionId = recoverySessionId + self.recoveryContext = recoveryContext + self.recoveryReceipt = recoveryReceipt + self.diagnosticModerationChecks = diagnosticModerationChecks + self.diagnosticIntegrationFailures = diagnosticIntegrationFailures } } @@ -896,6 +940,14 @@ struct WorkChatEnvelope: Identifiable, Equatable { enum WorkChatEvent: Equatable { case userMessage(text: String, attachments: [AgentChatFileRef]?, turnId: String?, steerId: String?, deliveryState: String?, processed: Bool?) + case userMessageResolution( + steerId: String, + action: String, + state: String, + resolvedAt: String, + replacementMessageId: String?, + turnId: String? + ) case assistantText(text: String, turnId: String?, itemId: String?) case toolCall(tool: String, argsText: String, itemId: String, parentItemId: String?, turnId: String?) case toolResult(tool: String, resultText: String, itemId: String, parentItemId: String?, turnId: String?, status: WorkToolCardStatus) @@ -921,7 +973,23 @@ enum WorkChatEvent: Equatable { case autoApprovalReview(summary: String, turnId: String?) case webSearch(query: String, action: String?, actions: [CodexWebSearchAction]?, results: [CodexWebSearchResult]?, status: WorkToolCardStatus, itemId: String, turnId: String?) case codexState(title: String, message: String, icon: String, turnId: String?) - case codexTurnStalled(message: String, recoveryOptions: [String], turnId: String?, sourceSessionId: String?) + case turnDiagnostics( + moderationChecks: Int, + optionalIntegrationFailures: [AgentChatOptionalIntegrationFailure], + turnId: String? + ) + case codexTurnStalled( + message: String, + recoveryOptions: [String], + turnId: String?, + sourceSessionId: String?, + context: WorkCodexStallContext + ) + case codexTurnRecovery( + message: String, + receipt: WorkCodexRecoveryReceipt, + turnId: String? + ) case planText(text: String, turnId: String?) case toolUseSummary(text: String, turnId: String?) case status(turnStatus: String, message: String?, turnId: String?) @@ -934,6 +1002,7 @@ enum WorkChatEvent: Equatable { var typeKey: String { switch self { case .userMessage: return "user_message" + case .userMessageResolution: return "user_message_resolution" case .assistantText: return "text" case .toolCall: return "tool_call" case .toolResult: return "tool_result" @@ -959,7 +1028,9 @@ enum WorkChatEvent: Equatable { case .autoApprovalReview: return "auto_approval_review" case .webSearch: return "web_search" case .codexState: return "codex_state" + case .turnDiagnostics: return "turn_diagnostics" case .codexTurnStalled: return "codex_turn_stalled" + case .codexTurnRecovery: return "codex_turn_recovery" case .planText: return "plan_text" case .toolUseSummary: return "tool_use_summary" case .status: return "status" diff --git a/apps/ios/ADE/Views/Work/WorkRootScreen.swift b/apps/ios/ADE/Views/Work/WorkRootScreen.swift index 0879ad8e1..fa41e2dd0 100644 --- a/apps/ios/ADE/Views/Work/WorkRootScreen.swift +++ b/apps/ios/ADE/Views/Work/WorkRootScreen.swift @@ -311,7 +311,13 @@ struct WorkRootScreen: View { var body: some View { NavigationStack(path: $path) { ScrollViewReader { proxy in - List { + workList(proxy: proxy) + } + } + } + + private func workList(proxy: ScrollViewProxy) -> some View { + List { if isLoadingSkeleton && sessions.isEmpty && optimisticSessions.isEmpty { ForEach(0..<3, id: \.self) { _ in ADECardSkeleton(rows: 3) @@ -396,115 +402,8 @@ struct WorkRootScreen: View { .listRowBackground(Color.clear) .listRowSeparator(.hidden) } else { - let rowLaneById = laneById - let rowPrTagsByLaneId = lanePrTagsByLaneId - let rowArchivedSessionIds = archivedSessionIds - let rowCollapsedSectionIds = collapsedSectionIds - let rowTopLevelDisplaySessionIds = sessionPresentation.topLevelDisplaySessionIds - let rowChildGroupsByParentId = sessionPresentation.childGroupsByParentId - let rowDeletingLaneIds = syncService.pendingLaneDeletionIds - ForEach(sessionGroups) { group in - let isLaneDeleting = group.laneId.map(rowDeletingLaneIds.contains) ?? false - WorkSidebarSectionHeader( - group: group, - collapsed: rowCollapsedSectionIds.contains(group.id), - onToggle: { - withAnimation(ADEMotion.quick(reduceMotion: reduceMotion)) { - toggleCollapsed(group.id) - } - }, - pullRequest: group.laneId.flatMap { rowPrTagsByLaneId[$0] }, - onOpenPullRequest: { tag in - openLanePullRequest(tag: tag, laneId: group.laneId) - } - ) - .disabled(isLaneDeleting) - .redacted(reason: isLaneDeleting ? .placeholder : []) - .id(group.id) - .listRowBackground(Color.clear) - .listRowSeparator(.hidden) - .listRowInsets(EdgeInsets(top: 8, leading: 16, bottom: 2, trailing: 16)) - - if !rowCollapsedSectionIds.contains(group.id) { - ForEach(group.sessions.filter { rowTopLevelDisplaySessionIds.contains($0.id) }) { session in - WorkSessionListRow( - session: session, - lane: rowLaneById[session.laneId], - // Fall back to the resolved lane (name/branch match) so - // legacy sessions with a stale laneId still surface their - // PR shortcut — same resolution goToLane/openPullRequest use. - pullRequest: rowPrTagsByLaneId[session.laneId] - ?? rowPrTagsByLaneId[resolvedWorkNavigationLaneId(for: session, lanes: lanes)], - chatSummary: chatSummaries[session.id], - isArchived: rowArchivedSessionIds.contains(session.id), - isLaneDeleting: rowDeletingLaneIds.contains(session.laneId), - transitionNamespace: ADEMotion.allowsMatchedGeometry(reduceMotion: reduceMotion) ? sessionTransitionNamespace : nil, - selectedSessionId: $selectedSessionTransitionId, - isSelecting: isSelecting, - isChecked: selectedSessionIds.contains(session.id), - onLongPressSelect: startSelection, - onToggleSelect: toggleSelection, - onOpen: openSession, - onPin: togglePin, - onRename: beginRename, - onStopRuntime: { session in stopRuntimeTarget = session }, - onDelete: deleteChatSession, - onCopyId: copySessionId, - onCopyDeepLink: copySessionDeepLink, - onGoToLane: goToLane, - onOpenPullRequest: openPullRequest - ) - .id(session.id) - .listRowInsets(EdgeInsets(top: 4, leading: 16, bottom: 4, trailing: 16)) - .listRowBackground(Color.clear) - .listRowSeparator(.hidden) - - if let childGroup = rowChildGroupsByParentId[session.id] { - WorkChildShellSection( - group: childGroup, - collapsed: rowCollapsedSectionIds.contains(childGroup.collapsedSectionId), - onToggle: { - withAnimation(ADEMotion.quick(reduceMotion: reduceMotion)) { - toggleCollapsed(childGroup.collapsedSectionId) - } - } - ) { - ForEach(childGroup.children) { child in - WorkSessionListRow( - session: child, - lane: rowLaneById[child.laneId], - pullRequest: rowPrTagsByLaneId[child.laneId] - ?? rowPrTagsByLaneId[resolvedWorkNavigationLaneId(for: child, lanes: lanes)], - chatSummary: chatSummaries[child.id], - isArchived: rowArchivedSessionIds.contains(child.id), - isLaneDeleting: rowDeletingLaneIds.contains(child.laneId), - transitionNamespace: nil, - compact: true, - selectedSessionId: $selectedSessionTransitionId, - isSelecting: isSelecting, - isChecked: selectedSessionIds.contains(child.id), - onLongPressSelect: startSelection, - onToggleSelect: toggleSelection, - onOpen: openSession, - onPin: togglePin, - onRename: beginRename, - onStopRuntime: { session in stopRuntimeTarget = session }, - onDelete: deleteChatSession, - onCopyId: copySessionId, - onCopyDeepLink: copySessionDeepLink, - onGoToLane: goToLane, - onOpenPullRequest: openPullRequest - ) - .id(child.id) - } - } - .listRowInsets(EdgeInsets(top: 0, leading: 30, bottom: 6, trailing: 16)) - .listRowBackground(Color.clear) - .listRowSeparator(.hidden) - } - } - } + workSessionGroupRows(group) } } } @@ -789,7 +688,115 @@ struct WorkRootScreen: View { } message: { session in Text("ADE will stop the running process. The saved session stays available unless you delete it.") } + } + + @ViewBuilder + private func workSessionGroupRows(_ group: WorkSessionGroup) -> some View { + let isLaneDeleting = group.laneId.map(syncService.pendingLaneDeletionIds.contains) ?? false + WorkSidebarSectionHeader( + group: group, + collapsed: collapsedSectionIds.contains(group.id), + onToggle: { + withAnimation(ADEMotion.quick(reduceMotion: reduceMotion)) { + toggleCollapsed(group.id) + } + }, + pullRequest: group.laneId.flatMap { lanePrTagsByLaneId[$0] }, + onOpenPullRequest: { tag in + openLanePullRequest(tag: tag, laneId: group.laneId) + } + ) + .disabled(isLaneDeleting) + .redacted(reason: isLaneDeleting ? .placeholder : []) + .id(group.id) + .listRowBackground(Color.clear) + .listRowSeparator(.hidden) + .listRowInsets(EdgeInsets(top: 8, leading: 16, bottom: 2, trailing: 16)) + + if !collapsedSectionIds.contains(group.id) { + ForEach(group.sessions.filter { sessionPresentation.topLevelDisplaySessionIds.contains($0.id) }) { session in + workSessionRows(session) + } + } + } + + @ViewBuilder + private func workSessionRows(_ session: TerminalSessionSummary) -> some View { + WorkSessionListRow( + session: session, + lane: laneById[session.laneId], + // Fall back to the resolved lane (name/branch match) so legacy sessions + // with a stale laneId still surface their PR shortcut. + pullRequest: lanePrTagsByLaneId[session.laneId] + ?? lanePrTagsByLaneId[resolvedWorkNavigationLaneId(for: session, lanes: lanes)], + chatSummary: chatSummaries[session.id], + isArchived: archivedSessionIds.contains(session.id), + transitionNamespace: ADEMotion.allowsMatchedGeometry(reduceMotion: reduceMotion) + ? sessionTransitionNamespace + : nil, + isLaneDeleting: syncService.pendingLaneDeletionIds.contains(session.laneId), + selectedSessionId: $selectedSessionTransitionId, + isSelecting: isSelecting, + isChecked: selectedSessionIds.contains(session.id), + onLongPressSelect: startSelection, + onToggleSelect: toggleSelection, + onOpen: openSession, + onPin: togglePin, + onRename: beginRename, + onStopRuntime: { session in stopRuntimeTarget = session }, + onDelete: deleteChatSession, + onCopyId: copySessionId, + onCopyDeepLink: copySessionDeepLink, + onGoToLane: goToLane, + onOpenPullRequest: openPullRequest + ) + .id(session.id) + .listRowInsets(EdgeInsets(top: 4, leading: 16, bottom: 4, trailing: 16)) + .listRowBackground(Color.clear) + .listRowSeparator(.hidden) + + if let childGroup = sessionPresentation.childGroupsByParentId[session.id] { + WorkChildShellSection( + group: childGroup, + collapsed: collapsedSectionIds.contains(childGroup.collapsedSectionId), + onToggle: { + withAnimation(ADEMotion.quick(reduceMotion: reduceMotion)) { + toggleCollapsed(childGroup.collapsedSectionId) + } + } + ) { + ForEach(childGroup.children) { child in + WorkSessionListRow( + session: child, + lane: laneById[child.laneId], + pullRequest: lanePrTagsByLaneId[child.laneId] + ?? lanePrTagsByLaneId[resolvedWorkNavigationLaneId(for: child, lanes: lanes)], + chatSummary: chatSummaries[child.id], + isArchived: archivedSessionIds.contains(child.id), + transitionNamespace: nil, + compact: true, + isLaneDeleting: syncService.pendingLaneDeletionIds.contains(child.laneId), + selectedSessionId: $selectedSessionTransitionId, + isSelecting: isSelecting, + isChecked: selectedSessionIds.contains(child.id), + onLongPressSelect: startSelection, + onToggleSelect: toggleSelection, + onOpen: openSession, + onPin: togglePin, + onRename: beginRename, + onStopRuntime: { session in stopRuntimeTarget = session }, + onDelete: deleteChatSession, + onCopyId: copySessionId, + onCopyDeepLink: copySessionDeepLink, + onGoToLane: goToLane, + onOpenPullRequest: openPullRequest + ) + .id(child.id) + } } + .listRowInsets(EdgeInsets(top: 0, leading: 30, bottom: 6, trailing: 16)) + .listRowBackground(Color.clear) + .listRowSeparator(.hidden) } } diff --git a/apps/ios/ADE/Views/Work/WorkSessionDestinationView+Actions.swift b/apps/ios/ADE/Views/Work/WorkSessionDestinationView+Actions.swift index 0967ed13e..f0e7edb09 100644 --- a/apps/ios/ADE/Views/Work/WorkSessionDestinationView+Actions.swift +++ b/apps/ios/ADE/Views/Work/WorkSessionDestinationView+Actions.swift @@ -134,13 +134,78 @@ extension WorkSessionDestinationView { ) await refreshChatStateAfterAction(forceRemote: true) errorMessage = nil - switch result.status { - case "waiting": return "Waiting for Codex output…" - case "nudged": return "Status nudge sent." - case "retrying": return "Retry started in this thread." - case "resumed": return "Codex app server restarted and the thread resumed." - default: return "Recovery action sent." + return workTurnRecoveryFeedback(status: result.status) + } + + @MainActor + func runUnprocessedMessage(_ message: WorkChatMessage) async throws { + guard liveTurnActiveHint != true && !shouldSteerActiveTurn else { + throw NSError( + domain: "ADE", + code: 28, + userInfo: [NSLocalizedDescriptionKey: "A turn is already active. Wait for it to finish, then run this message."] + ) + } + guard !sending else { + throw NSError( + domain: "ADE", + code: 27, + userInfo: [NSLocalizedDescriptionKey: "Another message is already being sent."] + ) + } + let steerId = message.steerId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !steerId.isEmpty else { + throw NSError( + domain: "ADE", + code: 25, + userInfo: [NSLocalizedDescriptionKey: "This message is missing its durable delivery identifier."] + ) + } + sending = true + defer { sending = false } + _ = try await syncService.resolveUnprocessedMessage( + sessionId: sessionId, + steerId: steerId, + action: "run_next" + ) + await refreshChatStateAfterAction(forceRemote: true) + errorMessage = nil + } + + @MainActor + func editUnprocessedMessage(_ message: WorkChatMessage) async throws { + let text = message.markdown.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { + throw NSError( + domain: "ADE", + code: 26, + userInfo: [NSLocalizedDescriptionKey: "This message has no editable text."] + ) } + composerDraftRestore = WorkChatComposerDraftRestore( + text: text, + replacesExistingDraft: true + ) + errorMessage = nil + } + + @MainActor + func dismissUnprocessedMessage(_ message: WorkChatMessage) async throws { + let steerId = message.steerId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard !steerId.isEmpty else { + throw NSError( + domain: "ADE", + code: 25, + userInfo: [NSLocalizedDescriptionKey: "This message is missing its durable delivery identifier."] + ) + } + _ = try await syncService.resolveUnprocessedMessage( + sessionId: sessionId, + steerId: steerId, + action: "dismiss" + ) + await refreshChatStateAfterAction(forceRemote: true) + errorMessage = nil } @MainActor @@ -911,6 +976,16 @@ extension WorkSessionDestinationView { } } +func workTurnRecoveryFeedback(status: String) -> String { + switch status { + case "waiting": return "Waiting for runtime output…" + case "nudged": return "Status nudge sent." + case "retrying": return "Retry started in this thread." + case "resumed": return "Runtime restarted and the thread resumed." + default: return "Recovery action sent." + } +} + struct WorkChatPrResolution { var tag: LanePrTag? var mappedPr: PullRequestListItem? diff --git a/apps/ios/ADE/Views/Work/WorkSessionDestinationView.swift b/apps/ios/ADE/Views/Work/WorkSessionDestinationView.swift index d9e9e04b9..5732c9d87 100644 --- a/apps/ios/ADE/Views/Work/WorkSessionDestinationView.swift +++ b/apps/ios/ADE/Views/Work/WorkSessionDestinationView.swift @@ -1200,6 +1200,17 @@ struct WorkSessionDestinationView: View { } else { loadOlderTranscriptAction = { await loadOlderTranscriptEntries() } } + let supportsRecovery = syncService.supportsChatRemoteAction( + "chat.recoverTurn", + sessionId: session.id + ) || syncService.supportsChatRemoteAction( + "chat.recoverCodexTurn", + sessionId: session.id + ) + let supportsUnprocessedResolution = syncService.supportsChatRemoteAction( + "chat.resolveUnprocessedMessage", + sessionId: session.id + ) return WorkChatSessionView( session: WorkChatSessionRenderContext(session), chatSummaryContext: WorkChatSummaryRenderContext(composerChatSummary), @@ -1288,9 +1299,16 @@ struct WorkSessionDestinationView: View { personalSessionUpdatesAvailable: !personalChat || syncService.canInvokeRemoteAction("personalChats.updateSession"), onRecoverCodexTurn: workChatCodexRecoveryAvailable( - hostSupportsRecovery: syncService.supportsRemoteAction("chat.recoverCodexTurn"), + hostSupportsRecovery: supportsRecovery, viewingSubagent: viewingSubagent - ) ? recoverCodexTurn : nil + ) ? recoverCodexTurn : nil, + onRunUnprocessedMessage: !viewingSubagent && supportsUnprocessedResolution + ? runUnprocessedMessage + : nil, + onEditUnprocessedMessage: !viewingSubagent ? editUnprocessedMessage : nil, + onDismissUnprocessedMessage: !viewingSubagent && supportsUnprocessedResolution + ? dismissUnprocessedMessage + : nil ) } diff --git a/apps/ios/ADE/Views/Work/WorkTimelineHelpers.swift b/apps/ios/ADE/Views/Work/WorkTimelineHelpers.swift index a053290fc..99cc67b56 100644 --- a/apps/ios/ADE/Views/Work/WorkTimelineHelpers.swift +++ b/apps/ios/ADE/Views/Work/WorkTimelineHelpers.swift @@ -147,6 +147,13 @@ private func combineWorkChatEventSignature(_ event: WorkChatEvent, into hasher: combineOptional(steerId, into: &hasher) combineOptional(deliveryState, into: &hasher) combineOptional(processed, into: &hasher) + case .userMessageResolution(let steerId, let action, let state, let resolvedAt, let replacementMessageId, let turnId): + hasher.combine(steerId) + hasher.combine(action) + hasher.combine(state) + hasher.combine(resolvedAt) + combineOptional(replacementMessageId, into: &hasher) + combineOptional(turnId, into: &hasher) case .assistantText(let text, let turnId, let itemId): combineLongTextSignature(text, into: &hasher) combineOptional(turnId, into: &hasher) @@ -311,11 +318,36 @@ private func combineWorkChatEventSignature(_ event: WorkChatEvent, into hasher: combineLongTextSignature(message, into: &hasher) hasher.combine(icon) combineOptional(turnId, into: &hasher) - case .codexTurnStalled(let message, let recoveryOptions, let turnId, let sourceSessionId): + case .turnDiagnostics(let moderationChecks, let integrationFailures, let turnId): + hasher.combine(moderationChecks) + for failure in integrationFailures { + hasher.combine(failure.integration) + combineOptionalText(failure.message, into: &hasher) + } + combineOptional(turnId, into: &hasher) + case .codexTurnStalled(let message, let recoveryOptions, let turnId, let sourceSessionId, let context): combineLongTextSignature(message, into: &hasher) recoveryOptions.forEach { hasher.combine($0) } hasher.combine(turnId) hasher.combine(sourceSessionId) + hasher.combine(context.reason) + combineOptional(context.detectedAt, into: &hasher) + combineOptional(context.turnStartedAt, into: &hasher) + combineOptional(context.lastProgressAt, into: &hasher) + hasher.combine(context.automaticRecoveryAttempted) + combineOptional(context.provider, into: &hasher) + hasher.combine(context.recoveryCount) + hasher.combine(context.providerNeutral) + case .codexTurnRecovery(let message, let receipt, let turnId): + combineLongTextSignature(message, into: &hasher) + hasher.combine(receipt.action) + hasher.combine(receipt.state) + hasher.combine(receipt.automatic) + hasher.combine(receipt.at) + combineOptional(receipt.provider, into: &hasher) + hasher.combine(receipt.recoveryCount) + hasher.combine(receipt.providerNeutral) + combineOptional(turnId, into: &hasher) case .planText(let text, let turnId): combineLongTextSignature(text, into: &hasher) combineOptional(turnId, into: &hasher) @@ -2241,6 +2273,14 @@ func buildWorkEventCards( var byId: [String: WorkEventCardModel] = [:] var order: [String] = [] let terminalDoneTurnIds = workTerminalDoneTurnIds(from: transcript) + let recoveredCodexTurnIds = Set(transcript.compactMap { envelope -> String? in + guard case .codexTurnRecovery(_, let receipt, let turnId) = envelope.event, + receipt.state.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "recovered" + else { + return nil + } + return normalizedWorkTurnId(turnId) + }) // Join `pending_input_resolved` events onto the question / plan / approval // card they resolve so those cards can show the outcome inline. Any resolved // itemId that lands on such a card gets its standalone "Input resolved" ribbon @@ -2265,6 +2305,11 @@ func buildWorkEventCards( if redundantWorkTerminalStatus(envelope.event, terminalDoneTurnIds: terminalDoneTurnIds) { continue } + if case .codexTurnStalled(_, _, let turnId, _, _) = envelope.event, + let turnId = normalizedWorkTurnId(turnId), + recoveredCodexTurnIds.contains(turnId) { + continue + } guard let card = eventCard(for: envelope, resolutionByItemId: resolutionByItemId) else { continue } if let existing = byId[card.id], let merged = mergedWorkEventCard(existing, with: card) { byId[card.id] = merged @@ -2497,8 +2542,58 @@ private func truncatedWorkTimelineText(_ value: String, limit: Int) -> String { return "\(value.prefix(limit - 3))..." } +private func normalizedWorkIntegrationFailures( + _ failures: [AgentChatOptionalIntegrationFailure] +) -> [AgentChatOptionalIntegrationFailure] { + var byIntegration: [String: AgentChatOptionalIntegrationFailure] = [:] + for failure in failures { + let integration = failure.integration.trimmingCharacters(in: .whitespacesAndNewlines) + guard !integration.isEmpty else { continue } + let message = failure.message?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedMessage = message?.isEmpty == false ? message : byIntegration[integration]?.message + byIntegration[integration] = AgentChatOptionalIntegrationFailure( + integration: integration, + message: resolvedMessage + ) + } + return byIntegration.values.sorted { + $0.integration.localizedCaseInsensitiveCompare($1.integration) == .orderedAscending + } +} + private func mergedWorkEventCard(_ existing: WorkEventCardModel, with incoming: WorkEventCardModel) -> WorkEventCardModel? { guard existing.kind == incoming.kind else { return nil } + if existing.kind == "turnDiagnostics" { + let normalizedFailures = normalizedWorkIntegrationFailures( + existing.diagnosticIntegrationFailures + incoming.diagnosticIntegrationFailures + ) + return WorkEventCardModel( + id: incoming.id, + kind: incoming.kind, + title: incoming.title, + icon: incoming.icon, + tint: incoming.tint, + timestamp: laterWorkTimestamp(existing.timestamp, incoming.timestamp), + body: nil, + bullets: [], + metadata: [], + diagnosticModerationChecks: max( + existing.diagnosticModerationChecks, + incoming.diagnosticModerationChecks + ), + diagnosticIntegrationFailures: normalizedFailures + ) + } + if existing.kind == "codexRecovery", + existing.recoveryContext?.providerNeutral == true, + incoming.recoveryContext?.providerNeutral != true { + return existing + } + if existing.kind == "codexRecoveryReceipt", + existing.recoveryReceipt?.providerNeutral == true, + incoming.recoveryReceipt?.providerNeutral != true { + return existing + } if existing.kind == "reasoning" { return WorkEventCardModel( id: incoming.id, @@ -2556,6 +2651,9 @@ private func eventCard( // redundant rows under each tool group. Live streaming hints come from // WorkActivityIndicator, not the persisted timeline. return nil + case .userMessageResolution: + // Folded into the originating user bubble by `buildWorkChatMessages`. + return nil case .plan(let steps, let explanation, let turnId): guard !steps.isEmpty || nonEmptyWorkTimelineText(explanation) != nil else { return nil @@ -2771,9 +2869,25 @@ private func eventCard( bullets: [], metadata: [] ) - case .codexTurnStalled(let message, let recoveryOptions, let turnId, let sourceSessionId): + case .turnDiagnostics(let moderationChecks, let integrationFailures, let turnId): + guard moderationChecks > 0 || !integrationFailures.isEmpty else { return nil } + let normalizedTurnId = normalizedWorkTurnId(turnId) ?? "session" return WorkEventCardModel( - id: envelope.id, + id: "turn-diagnostics:\(envelope.sessionId):\(normalizedTurnId)", + kind: "turnDiagnostics", + title: "Turn details", + icon: "checkmark.shield", + tint: .secondary, + timestamp: envelope.timestamp, + body: nil, + bullets: [], + metadata: [], + diagnosticModerationChecks: moderationChecks, + diagnosticIntegrationFailures: normalizedWorkIntegrationFailures(integrationFailures) + ) + case .codexTurnStalled(let message, let recoveryOptions, let turnId, let sourceSessionId, let context): + return WorkEventCardModel( + id: "turn-health:\(envelope.sessionId):\(turnId ?? "unknown")", kind: "codexRecovery", title: "Recovery", icon: "exclamationmark.triangle", @@ -2784,7 +2898,38 @@ private func eventCard( metadata: [], recoveryOptions: recoveryOptions, recoveryTurnId: turnId, - recoverySessionId: sourceSessionId + recoverySessionId: sourceSessionId, + recoveryContext: context + ) + case .codexTurnRecovery(let message, let receipt, let turnId): + let normalizedState = receipt.state.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + let providerName = receipt.providerNeutral + ? workChatSurfaceProviderName(receipt.provider) + : "Codex" + return WorkEventCardModel( + id: "codex-recovery-receipt:\(envelope.sessionId):\(turnId ?? "unknown")", + kind: "codexRecoveryReceipt", + title: normalizedState == "recovered" + ? "\(providerName) connection recovered" + : normalizedState == "failed" + ? "\(providerName) recovery failed" + : "Recovering \(providerName) connection", + icon: normalizedState == "recovered" + ? "checkmark.circle.fill" + : normalizedState == "failed" + ? "exclamationmark.triangle.fill" + : "arrow.clockwise.circle", + tint: normalizedState == "recovered" + ? .success + : normalizedState == "failed" + ? .danger + : .secondary, + timestamp: receipt.at, + body: message, + bullets: [], + metadata: [receipt.automatic ? "Automatic recovery" : "Manual recovery"], + recoveryTurnId: turnId, + recoveryReceipt: receipt ) case .planText(let text, let turnId): return WorkEventCardModel( @@ -3067,6 +3212,7 @@ private func normalizedWorkTurnId(_ turnId: String?) -> String? { private func workTurnId(for event: WorkChatEvent) -> String? { switch event { case .userMessage(_, _, let turnId, _, _, _), + .userMessageResolution(_, _, _, _, _, let turnId), .assistantText(_, let turnId, _), .toolCall(_, _, _, _, let turnId), .toolResult(_, _, _, _, let turnId, _), @@ -3088,7 +3234,9 @@ private func workTurnId(for event: WorkChatEvent) -> String? { .autoApprovalReview(_, let turnId), .webSearch(_, _, _, _, _, _, let turnId), .codexState(_, _, _, let turnId), - .codexTurnStalled(_, _, let turnId, _), + .turnDiagnostics(_, _, let turnId), + .codexTurnStalled(_, _, let turnId, _, _), + .codexTurnRecovery(_, _, let turnId), .planText(_, let turnId), .toolUseSummary(_, let turnId), .status(_, _, let turnId), diff --git a/apps/ios/ADE/Views/Work/WorkTranscriptParser.swift b/apps/ios/ADE/Views/Work/WorkTranscriptParser.swift index 252a2e63c..0f5c27012 100644 --- a/apps/ios/ADE/Views/Work/WorkTranscriptParser.swift +++ b/apps/ios/ADE/Views/Work/WorkTranscriptParser.swift @@ -148,6 +148,15 @@ func parseWorkChatTranscript(_ raw: String) -> [WorkChatEnvelope] { deliveryState: optionalString(eventDict["deliveryState"]), processed: eventDict["processed"] as? Bool ) + case "user_message_resolution": + event = .userMessageResolution( + steerId: stringValue(eventDict["steerId"]), + action: stringValue(eventDict["action"]), + state: optionalString(eventDict["state"]) ?? "completed", + resolvedAt: optionalString(eventDict["resolvedAt"]) ?? timestamp, + replacementMessageId: optionalString(eventDict["replacementMessageId"]), + turnId: turnId + ) case "text": event = .assistantText( text: stringValue(eventDict["text"]), @@ -475,12 +484,78 @@ func parseWorkChatTranscript(_ raw: String) -> [WorkChatEnvelope] { turnId: turnId ?? "", itemId: nil ) + case "codex_moderation_metadata": + event = .unknown(type: "codex_moderation_metadata") + case "turn_diagnostics": + let integrationFailures = (eventDict["optionalIntegrationFailures"] as? [[String: Any]] ?? []) + .compactMap { failure -> AgentChatOptionalIntegrationFailure? in + guard let integration = optionalString(failure["integration"]) else { return nil } + return AgentChatOptionalIntegrationFailure( + integration: integration, + message: optionalString(failure["message"]) + ) + } + event = .turnDiagnostics( + moderationChecks: max(0, optionalWorkInt(eventDict["moderationChecks"]) ?? 0), + optionalIntegrationFailures: integrationFailures, + turnId: turnId + ) case "codex_turn_stalled": event = .codexTurnStalled( message: stringValue(eventDict["message"]), recoveryOptions: eventDict["recoveryOptions"] as? [String] ?? [], turnId: turnId, - sourceSessionId: optionalString(eventDict["sourceSessionId"]) + sourceSessionId: optionalString(eventDict["sourceSessionId"]), + context: WorkCodexStallContext( + reason: optionalString(eventDict["reason"]) ?? "no_output", + detectedAt: optionalString(eventDict["detectedAt"]), + turnStartedAt: optionalString(eventDict["turnStartedAt"]), + lastProgressAt: optionalString(eventDict["lastProgressAt"]), + automaticRecoveryAttempted: eventDict["automaticRecoveryAttempted"] as? Bool ?? false + ) + ) + case "turn_health": + let supportedActions = eventDict["supportedActions"] as? [String] ?? [] + event = .codexTurnStalled( + message: stringValue(eventDict["message"]), + recoveryOptions: supportedActions.compactMap(workLegacyRecoveryAction), + turnId: turnId, + sourceSessionId: optionalString(eventDict["sourceSessionId"]), + context: WorkCodexStallContext( + reason: optionalString(eventDict["reason"]) ?? "runtime_state_unknown", + detectedAt: optionalString(eventDict["detectedAt"]), + turnStartedAt: optionalString(eventDict["turnStartedAt"]), + lastProgressAt: optionalString(eventDict["lastProgressAt"]), + automaticRecoveryAttempted: eventDict["automaticRecoveryAttempted"] as? Bool ?? false, + provider: optionalString(eventDict["provider"]), + recoveryCount: max(0, optionalWorkInt(eventDict["recoveryCount"]) ?? 0), + providerNeutral: true + ) + ) + case "codex_turn_recovery": + event = .codexTurnRecovery( + message: stringValue(eventDict["message"]), + receipt: WorkCodexRecoveryReceipt( + action: optionalString(eventDict["action"]) ?? "restart_resume_thread", + state: optionalString(eventDict["state"]) ?? "recovering", + automatic: eventDict["automatic"] as? Bool ?? false, + at: optionalString(eventDict["at"]) ?? timestamp + ), + turnId: turnId + ) + case "turn_recovery": + event = .codexTurnRecovery( + message: stringValue(eventDict["message"]), + receipt: WorkCodexRecoveryReceipt( + action: optionalString(eventDict["action"]) ?? "restart_resume_thread", + state: optionalString(eventDict["state"]) ?? "recovering", + automatic: eventDict["automatic"] as? Bool ?? false, + at: optionalString(eventDict["at"]) ?? timestamp, + provider: optionalString(eventDict["provider"]), + recoveryCount: max(0, optionalWorkInt(eventDict["recoveryCount"]) ?? 0), + providerNeutral: true + ), + turnId: turnId ) case "completion_report": let report = eventDict["report"] as? [String: Any] ?? [:] diff --git a/apps/ios/ADETests/ADETests.swift b/apps/ios/ADETests/ADETests.swift index 8dfb7c6cb..4b022bc5e 100644 --- a/apps/ios/ADETests/ADETests.swift +++ b/apps/ios/ADETests/ADETests.swift @@ -2249,6 +2249,42 @@ final class ADETests: XCTestCase { ) } + func testRelayWinnerIsRejectedAfterSignOutOrAccountSwitch() { + let used = AccountPairingAuthorization(ownerId: "user_123", generation: 7) + + XCTAssertNil( + syncRelayReconnectAuthorizationRequirement( + usedAuthorization: used, + currentAuthorization: used, + relayAccountOwnerId: "user_123" + ) + ) + XCTAssertEqual( + syncRelayReconnectAuthorizationRequirement( + usedAuthorization: used, + currentAuthorization: nil, + relayAccountOwnerId: "user_123" + ), + .signInRequired + ) + XCTAssertEqual( + syncRelayReconnectAuthorizationRequirement( + usedAuthorization: used, + currentAuthorization: AccountPairingAuthorization(ownerId: "user_123", generation: 8), + relayAccountOwnerId: "user_123" + ), + .signInRequired + ) + XCTAssertEqual( + syncRelayReconnectAuthorizationRequirement( + usedAuthorization: used, + currentAuthorization: AccountPairingAuthorization(ownerId: "user_other", generation: 8), + relayAccountOwnerId: "user_123" + ), + .sameAccountRequired + ) + } + func testRelayHostRejectionBecomesTypedAccountRequirement() { XCTAssertEqual( syncRelayAuthorizationRequirementForHostRejection( @@ -2665,7 +2701,7 @@ final class ADETests: XCTestCase { } @MainActor - func testAccountSignOutRemovesOnlyAccountOwnedMachineCredentials() { + func testAccountSignOutPreservesDeviceBoundDirectMachineCredentials() { let service = SyncService(database: makeDatabase(baseURL: makeTemporaryDirectory())) service.clearSavedProfilesForTesting() defer { service.clearSavedProfilesForTesting() } @@ -2692,20 +2728,32 @@ final class ADETests: XCTestCase { lastRemoteDbVersion: 0, lastHostDeviceId: "account-host", lastSuccessfulAddress: "wss://relay.ade.app/connect/account-host", - savedAddressCandidates: [], - discoveredLanAddresses: [], - tailscaleAddress: nil, + savedAddressCandidates: ["192.168.1.9", "100.75.20.64"], + discoveredLanAddresses: ["192.168.1.9"], + tailscaleAddress: "100.75.20.64", savedRelayCandidates: ["wss://relay.ade.app/connect/account-host"], - accountOwnerId: "user_123" + accountOwnerId: "user_123", + relayAccountOwnerId: "user_123" ) service.installSavedProfileForTesting(manual, token: "manual-secret", makeActive: true) service.installSavedProfileForTesting(account, token: "account-secret") service.removeAccountOwnedPairings(exceptOwnerId: nil) - XCTAssertEqual(service.savedProfilesForTesting().map(\.hostIdentity), ["manual-host"]) + XCTAssertEqual( + Set(service.savedProfilesForTesting().map(\.hostIdentity)), + Set(["manual-host", "account-host"]) + ) XCTAssertTrue(service.hasCredentialForTesting(manual)) - XCTAssertFalse(service.hasCredentialForTesting(account)) + XCTAssertTrue(service.hasCredentialForTesting(account)) + XCTAssertEqual( + syncAddressesAllowedByRelayPolicy( + ["192.168.1.9", "100.75.20.64", "wss://relay.ade.app/connect/account-host"], + profile: account, + currentAccountOwnerId: nil + ), + ["192.168.1.9", "100.75.20.64"] + ) } @MainActor @@ -2786,7 +2834,7 @@ final class ADETests: XCTestCase { } @MainActor - func testAccountSignOutClearsActiveAccountMachine() { + func testAccountSignOutKeepsActiveAccountMachineDeviceTrust() { let service = SyncService(database: makeDatabase(baseURL: makeTemporaryDirectory())) service.clearSavedProfilesForTesting() defer { service.clearSavedProfilesForTesting() } @@ -2799,19 +2847,28 @@ final class ADETests: XCTestCase { lastRemoteDbVersion: 0, lastHostDeviceId: "account-host", lastSuccessfulAddress: "wss://relay.ade.app/connect/account-host", - savedAddressCandidates: [], - discoveredLanAddresses: [], + savedAddressCandidates: ["192.168.1.9"], + discoveredLanAddresses: ["192.168.1.9"], tailscaleAddress: nil, savedRelayCandidates: ["wss://relay.ade.app/connect/account-host"], - accountOwnerId: "user_123" + accountOwnerId: "user_123", + relayAccountOwnerId: "user_123" ) service.installSavedProfileForTesting(account, token: "account-secret", makeActive: true) - service.removeAccountOwnedPairings(ownerId: "user_123") + service.removeAccountOwnedPairings(exceptOwnerId: nil) - XCTAssertTrue(service.savedProfilesForTesting().isEmpty) - XCTAssertNil(service.activeHostProfile) - XCTAssertFalse(service.hasCredentialForTesting(account)) + XCTAssertEqual(service.savedProfilesForTesting().map(\.hostIdentity), ["account-host"]) + XCTAssertEqual(service.activeHostProfile?.hostIdentity, "account-host") + XCTAssertTrue(service.hasCredentialForTesting(account)) + XCTAssertEqual( + syncAddressesAllowedByRelayPolicy( + ["wss://relay.ade.app/connect/account-host", "192.168.1.9"], + profile: account, + currentAccountOwnerId: nil + ), + ["192.168.1.9"] + ) } func testSyncRoamDecisionUsesSavedTailnetWhenWifiDrops() { @@ -5485,7 +5542,10 @@ final class ADETests: XCTestCase { @MainActor func testSyncServiceAcceptsLegacyHelloInLimitedCompatibilityMode() async throws { - let service = SyncService(database: makeDatabase(baseURL: makeTemporaryDirectory())) + let pendingOperationsKey = "ade.sync.pendingOperations" + UserDefaults.standard.removeObject(forKey: pendingOperationsKey) + defer { UserDefaults.standard.removeObject(forKey: pendingOperationsKey) } + let service = SyncService(database: makeControllerHydrationDatabase(baseURL: makeTemporaryDirectory())) try service.applyHelloPayloadForTesting([ "brain": [ @@ -5687,7 +5747,10 @@ final class ADETests: XCTestCase { @MainActor func testScheduledWorkCancellationStaysGatedWhenAdvertisedHostOmitsAction() async throws { - let service = SyncService(database: makeDatabase(baseURL: makeTemporaryDirectory())) + let pendingOperationsKey = "ade.sync.pendingOperations" + UserDefaults.standard.removeObject(forKey: pendingOperationsKey) + defer { UserDefaults.standard.removeObject(forKey: pendingOperationsKey) } + let service = SyncService(database: makeControllerHydrationDatabase(baseURL: makeTemporaryDirectory())) try service.applyHelloPayloadForTesting([ "brain": [ "deviceId": "host-1", @@ -8911,6 +8974,7 @@ final class ADETests: XCTestCase { } let service = SyncService(database: database) + service.setActiveProjectForTesting(projectId: "project-1", rootPath: "/tmp/project-one") try await service.archiveLane("lane-child") XCTAssertEqual(service.pendingOperationCount, 1) @@ -9189,10 +9253,78 @@ final class ADETests: XCTestCase { } catch { XCTAssertTrue(error.localizedDescription.contains("not available on this machine version")) } + XCTAssertFalse(service.supportsRemoteAction("chat.recoverTurn")) XCTAssertFalse(service.supportsRemoteAction("chat.recoverCodexTurn")) XCTAssertEqual(service.pendingOperationCount, 0) } + func testRecoveryActionSelectionPrefersProviderNeutralContract() { + XCTAssertEqual(syncProviderNeutralRecoveryAction("wait"), "wait") + XCTAssertEqual(syncProviderNeutralRecoveryAction("steer"), "nudge") + XCTAssertEqual( + syncProviderNeutralRecoveryAction("interrupt_retry_same_thread"), + "retry_same_runtime" + ) + XCTAssertEqual( + syncProviderNeutralRecoveryAction("restart_resume_thread"), + "restart_resume" + ) + XCTAssertNil(syncProviderNeutralRecoveryAction("unknown")) + + XCTAssertEqual( + syncPreferredRecoveryActionName( + supportsProviderNeutral: true, + supportsLegacyCodex: true + ), + "chat.recoverTurn" + ) + XCTAssertEqual( + syncPreferredRecoveryActionName( + supportsProviderNeutral: false, + supportsLegacyCodex: true + ), + "chat.recoverCodexTurn" + ) + XCTAssertNil(syncPreferredRecoveryActionName( + supportsProviderNeutral: false, + supportsLegacyCodex: false + )) + XCTAssertEqual( + syncPreferredRecoveryActionName( + supportsProviderNeutral: true, + supportsLegacyCodex: true, + providerNeutralActionName: "personalChats.recoverTurn", + legacyCodexActionName: "personalChats.recoverCodexTurn" + ), + "personalChats.recoverTurn" + ) + } + + @MainActor + func testUnprocessedMessageResolutionIsGatedWithoutAdvertisedCapability() async throws { + let remoteCommandDescriptorsKey = "ade.sync.remoteCommandDescriptors" + UserDefaults.standard.removeObject(forKey: remoteCommandDescriptorsKey) + defer { UserDefaults.standard.removeObject(forKey: remoteCommandDescriptorsKey) } + + let database = makeDatabase(baseURL: makeTemporaryDirectory()) + defer { database.close() } + let service = SyncService(database: database) + service.disconnect() + + do { + _ = try await service.resolveUnprocessedMessage( + sessionId: "chat-legacy", + steerId: "steer-1", + action: "run_next" + ) + XCTFail("Expected resolution to be rejected before an unsupported command is sent.") + } catch { + XCTAssertTrue(error.localizedDescription.contains("not available on this machine version")) + } + XCTAssertFalse(service.supportsRemoteAction("chat.resolveUnprocessedMessage")) + XCTAssertEqual(service.pendingOperationCount, 0) + } + @MainActor func testIntentCommandRegistryQueuesCommandsUntilBridgeRegisters() async { ADEIntentCommandRegistry.resetForTesting() @@ -13730,6 +13862,22 @@ final class ADETests: XCTestCase { ) } + @MainActor + func testEditingUnprocessedMessageReplacesComposerDraftAndFocusesIt() { + let state = WorkChatComposerDraftState() + state.text = "A newer local draft" + state.isFocused = false + + state.applyRestore(WorkChatComposerDraftRestore( + text: "The original unprocessed message", + id: UUID(uuidString: "4D31F415-1FA2-44BC-87C5-2AB71DA11CB5")!, + replacesExistingDraft: true + )) + + XCTAssertEqual(state.text, "The original unprocessed message") + XCTAssertTrue(state.isFocused) + } + func testWorkChatComposerPlaceholderUsesPlanReviewCopyForPlanApprovalOnly() { let planInput = WorkPendingInputItem.planApproval(WorkPendingPlanApprovalModel( id: "plan-1", @@ -14952,9 +15100,24 @@ final class ADETests: XCTestCase { "message": "Codex accepted the turn but has not streamed output yet.", "recoveryOptions": ["wait", "steer", "interrupt_retry_same_thread", "restart_resume_thread"], "sourceSessionId": "chat-child", + "detectedAt": "2026-07-09T00:02:01.000Z", + "turnStartedAt": "2026-07-09T00:00:01.000Z", + "lastProgressAt": "2026-07-09T00:00:31.000Z", + "automaticRecoveryAttempted": true, ] let decoded = try AgentChatEvent.decode(from: eventObject) - guard case .codexTurnStalled(let turnId, let threadId, let reason, let message, let options, let sourceSessionId) = decoded else { + guard case .codexTurnStalled( + let turnId, + let threadId, + let reason, + let message, + let options, + let sourceSessionId, + let detectedAt, + let turnStartedAt, + let lastProgressAt, + let automaticRecoveryAttempted + ) = decoded else { return XCTFail("Expected a Codex stalled-turn event.") } XCTAssertEqual(turnId, "turn-child") @@ -14962,6 +15125,10 @@ final class ADETests: XCTestCase { XCTAssertEqual(reason, "no_output") XCTAssertEqual(sourceSessionId, "chat-child") XCTAssertEqual(options, ["wait", "steer", "interrupt_retry_same_thread", "restart_resume_thread"]) + XCTAssertEqual(detectedAt, "2026-07-09T00:02:01.000Z") + XCTAssertEqual(turnStartedAt, "2026-07-09T00:00:01.000Z") + XCTAssertEqual(lastProgressAt, "2026-07-09T00:00:31.000Z") + XCTAssertEqual(automaticRecoveryAttempted, true) let mapped = makeWorkChatEvent(from: decoded) let envelope = WorkChatEnvelope( @@ -14976,14 +15143,355 @@ final class ADETests: XCTestCase { XCTAssertEqual(card.recoverySessionId, "chat-child") XCTAssertEqual(card.recoveryTurnId, "turn-child") XCTAssertEqual(card.recoveryOptions, options) + XCTAssertEqual(card.recoveryContext?.reason, "no_output") + XCTAssertEqual(card.recoveryContext?.automaticRecoveryAttempted, true) + XCTAssertEqual( + workCodexRecoveryPrimaryOptions(options ?? []), + ["restart_resume_thread", "wait"] + ) + XCTAssertEqual( + workCodexRecoveryMoreOptions(options ?? []), + ["steer", "interrupt_retry_same_thread"] + ) + XCTAssertEqual( + workCodexRecoveryActionLabel(for: "restart_resume_thread"), + "Restart & resume" + ) + XCTAssertEqual(workCodexRecoveryActionLabel(for: "wait"), "Keep waiting") - let raw = #"{"sessionId":"chat-parent","timestamp":"2026-07-09T00:00:01.000Z","sequence":1,"event":{"type":"codex_turn_stalled","turnId":"turn-child","threadId":"thread-child","reason":"no_output","message":"Codex accepted the turn but has not streamed output yet.","recoveryOptions":["wait","steer","interrupt_retry_same_thread","restart_resume_thread"],"sourceSessionId":"chat-child"}}"# - guard case .codexTurnStalled(_, let parsedOptions, let parsedTurnId, let parsedSourceSessionId) = parseWorkChatTranscript(raw).first?.event else { + let raw = #"{"sessionId":"chat-parent","timestamp":"2026-07-09T00:00:01.000Z","sequence":1,"event":{"type":"codex_turn_stalled","turnId":"turn-child","threadId":"thread-child","reason":"no_output","message":"Codex accepted the turn but has not streamed output yet.","recoveryOptions":["wait","steer","interrupt_retry_same_thread","restart_resume_thread"],"sourceSessionId":"chat-child","automaticRecoveryAttempted":true}}"# + guard case .codexTurnStalled(_, let parsedOptions, let parsedTurnId, let parsedSourceSessionId, let parsedContext) = parseWorkChatTranscript(raw).first?.event else { return XCTFail("Expected transcript fallback to preserve the recovery event.") } XCTAssertEqual(parsedOptions, options) XCTAssertEqual(parsedTurnId, "turn-child") XCTAssertEqual(parsedSourceSessionId, "chat-child") + XCTAssertTrue(parsedContext.automaticRecoveryAttempted) + } + + func testUnprocessedMessageResolutionDecodesAndFoldsIntoOriginalBubble() throws { + let decoded = try AgentChatEvent.decode(from: [ + "type": "user_message_resolution", + "steerId": "steer-1", + "action": "run_next", + "state": "completed", + "resolvedAt": "2026-07-09T00:01:00.000Z", + "replacementMessageId": "message-2", + "turnId": "turn-2", + ]) + guard case .userMessageResolution( + let steerId, + let action, + let state, + let resolvedAt, + let replacementMessageId, + let turnId + ) = decoded else { + return XCTFail("Expected a durable user-message resolution event.") + } + XCTAssertEqual(steerId, "steer-1") + XCTAssertEqual(action, "run_next") + XCTAssertEqual(state, "completed") + XCTAssertEqual(resolvedAt, "2026-07-09T00:01:00.000Z") + XCTAssertEqual(replacementMessageId, "message-2") + XCTAssertEqual(turnId, "turn-2") + + let raw = """ + {"sessionId":"chat-1","timestamp":"2026-07-09T00:00:01.000Z","sequence":1,"event":{"type":"user_message","text":"Please keep going","turnId":"turn-1","steerId":"steer-1","deliveryState":"unprocessed","processed":false}} + {"sessionId":"chat-1","timestamp":"2026-07-09T00:01:00.000Z","sequence":2,"event":{"type":"user_message_resolution","steerId":"steer-1","action":"run_next","state":"completed","resolvedAt":"2026-07-09T00:01:00.000Z","replacementMessageId":"message-2","turnId":"turn-2"}} + """ + let transcript = parseWorkChatTranscript(raw) + XCTAssertEqual(transcript.count, 2) + guard case .userMessageResolution = transcript[1].event else { + return XCTFail("Expected fallback parsing to preserve the resolution event.") + } + + let reloadedTranscript = mergeWorkChatTranscripts( + base: [transcript[0]], + live: [transcript[1]] + ) + XCTAssertEqual(reloadedTranscript.count, 2) + let messages = buildWorkChatMessages(from: reloadedTranscript) + XCTAssertEqual(messages.count, 1) + XCTAssertEqual(messages[0].steerId, "steer-1") + XCTAssertEqual(messages[0].deliveryState, "unprocessed") + XCTAssertEqual(messages[0].processed, false) + XCTAssertEqual(messages[0].unprocessedResolution?.action, "run_next") + XCTAssertEqual(messages[0].unprocessedResolution?.state, "completed") + XCTAssertEqual(messages[0].unprocessedResolution?.replacementMessageId, "message-2") + XCTAssertTrue(buildWorkEventCards(from: reloadedTranscript).isEmpty) + } + + func testDismissedUnprocessedMessageFoldsWithoutStandaloneTimelineCard() { + let raw = """ + {"sessionId":"chat-1","timestamp":"2026-07-09T00:00:01.000Z","sequence":1,"event":{"type":"user_message","text":"Never mind","turnId":"turn-1","steerId":"steer-dismiss","deliveryState":"unprocessed","processed":false}} + {"sessionId":"chat-1","timestamp":"2026-07-09T00:00:02.000Z","sequence":2,"event":{"type":"user_message_resolution","steerId":"steer-dismiss","action":"dismiss","state":"completed","resolvedAt":"2026-07-09T00:00:02.000Z","turnId":"turn-1"}} + """ + let transcript = parseWorkChatTranscript(raw) + let messages = buildWorkChatMessages(from: transcript) + + XCTAssertEqual(messages.count, 1) + XCTAssertEqual(messages[0].unprocessedResolution?.action, "dismiss") + XCTAssertNil(messages[0].unprocessedResolution?.replacementMessageId) + XCTAssertTrue(buildWorkEventCards(from: transcript).isEmpty) + } + + func testProviderNeutralTurnHealthWinsWhenLegacyAliasAlsoArrives() throws { + let decoded = try AgentChatEvent.decode(from: [ + "type": "turn_health", + "provider": "claude", + "turnId": "turn-1", + "state": "stalled", + "reason": "no_output", + "message": "The runtime accepted this turn but has not streamed output yet.", + "turnStartedAt": "2026-07-09T00:00:00.000Z", + "lastProgressAt": "2026-07-09T00:00:10.000Z", + "detectedAt": "2026-07-09T00:01:00.000Z", + "recoveryCount": 2, + "supportedActions": ["wait", "nudge", "retry_same_runtime", "restart_resume"], + "automaticRecoveryAttempted": true, + "sourceSessionId": "chat-child", + ]) + guard case .codexTurnStalled( + _, + let decodedOptions, + _, + let decodedSourceSessionId, + let decodedContext + ) = makeWorkChatEvent(from: decoded) else { + return XCTFail("Expected provider-neutral health to map to the recovery UI.") + } + XCTAssertEqual( + decodedOptions, + ["wait", "steer", "interrupt_retry_same_thread", "restart_resume_thread"] + ) + XCTAssertEqual(decodedSourceSessionId, "chat-child") + XCTAssertEqual(decodedContext.provider, "claude") + XCTAssertEqual(decodedContext.recoveryCount, 2) + XCTAssertTrue(decodedContext.providerNeutral) + + let raw = """ + {"sessionId":"chat-parent","timestamp":"2026-07-09T00:01:00.000Z","sequence":1,"event":{"type":"turn_health","provider":"claude","turnId":"turn-1","state":"stalled","reason":"no_output","message":"Provider-neutral health","turnStartedAt":"2026-07-09T00:00:00.000Z","lastProgressAt":"2026-07-09T00:00:10.000Z","detectedAt":"2026-07-09T00:01:00.000Z","recoveryCount":2,"supportedActions":["wait","nudge","retry_same_runtime","restart_resume"],"automaticRecoveryAttempted":true,"sourceSessionId":"chat-child"}} + {"sessionId":"chat-parent","timestamp":"2026-07-09T00:01:00.100Z","sequence":2,"event":{"type":"codex_turn_stalled","turnId":"turn-1","reason":"no_output","message":"Legacy alias","recoveryOptions":["wait","steer","interrupt_retry_same_thread","restart_resume_thread"],"sourceSessionId":"chat-child","automaticRecoveryAttempted":true}} + """ + let cards = buildWorkEventCards(from: parseWorkChatTranscript(raw)) + XCTAssertEqual(cards.count, 1) + XCTAssertEqual(cards[0].kind, "codexRecovery") + XCTAssertEqual(cards[0].body, "Provider-neutral health") + XCTAssertEqual(cards[0].recoveryOptions, decodedOptions) + XCTAssertEqual(cards[0].recoverySessionId, "chat-child") + XCTAssertEqual(cards[0].recoveryContext?.provider, "claude") + XCTAssertEqual(cards[0].recoveryContext?.recoveryCount, 2) + XCTAssertTrue(cards[0].recoveryContext?.providerNeutral == true) + } + + func testProviderNeutralRecoveryEventsDefaultOmittedCompatibilityFields() throws { + let health = try AgentChatEvent.decode(from: [ + "type": "turn_health", + "provider": "claude", + "turnId": "turn-compat", + "state": "stalled", + "reason": "no_output", + "message": "Waiting for output.", + "turnStartedAt": "2026-07-09T00:00:00.000Z", + "lastProgressAt": "2026-07-09T00:00:10.000Z", + "detectedAt": "2026-07-09T00:01:00.000Z", + ]) + guard case .turnHealth( + _, + _, + _, + _, + _, + _, + _, + _, + let recoveryCount, + let supportedActions, + let automaticRecoveryAttempted, + _ + ) = health else { + return XCTFail("Expected provider-neutral turn health.") + } + XCTAssertEqual(recoveryCount, 0) + XCTAssertEqual(supportedActions, []) + XCTAssertFalse(automaticRecoveryAttempted) + + let recovery = try AgentChatEvent.decode(from: [ + "type": "turn_recovery", + "provider": "claude", + "turnId": "turn-compat", + "action": "wait", + "state": "recovered", + "message": "Output resumed.", + "at": "2026-07-09T00:02:00.000Z", + ]) + guard case .turnRecovery(_, _, _, _, _, let automatic, _, let count) = recovery else { + return XCTFail("Expected provider-neutral turn recovery.") + } + XCTAssertFalse(automatic) + XCTAssertEqual(count, 0) + } + + func testUnprocessedResolutionUsesNewestTimestampInsteadOfArrayOrder() { + let transcript = [ + WorkChatEnvelope( + sessionId: "chat-1", + timestamp: "2026-07-09T00:00:00.000Z", + sequence: 1, + event: .userMessage( + text: "Run this next.", + attachments: nil, + turnId: "turn-1", + steerId: "steer-1", + deliveryState: "unprocessed", + processed: false + ) + ), + WorkChatEnvelope( + sessionId: "chat-1", + timestamp: "2026-07-09T00:03:00.000Z", + sequence: 3, + event: .userMessageResolution( + steerId: "steer-1", + action: "run_next", + state: "completed", + resolvedAt: "2026-07-09T00:03:00.000Z", + replacementMessageId: "message-2", + turnId: "turn-2" + ) + ), + WorkChatEnvelope( + sessionId: "chat-1", + timestamp: "2026-07-09T00:02:00.000Z", + sequence: 2, + event: .userMessageResolution( + steerId: "steer-1", + action: "dismiss", + state: "completed", + resolvedAt: "2026-07-09T00:02:00.000Z", + replacementMessageId: nil, + turnId: "turn-1" + ) + ), + ] + + let message = buildWorkChatMessages(from: transcript).first + XCTAssertEqual(message?.unprocessedResolution?.action, "run_next") + XCTAssertEqual(message?.unprocessedResolution?.replacementMessageId, "message-2") + } + + func testProviderNeutralRecoveryReceiptWinsWhenLegacyAliasAlsoArrives() throws { + let decoded = try AgentChatEvent.decode(from: [ + "type": "turn_recovery", + "provider": "claude", + "turnId": "turn-1", + "action": "restart_resume", + "state": "recovered", + "message": "Provider-neutral recovery receipt", + "automatic": false, + "at": "2026-07-09T00:02:00.000Z", + "recoveryCount": 3, + ]) + guard case .codexTurnRecovery(_, let decodedReceipt, _) = makeWorkChatEvent(from: decoded) else { + return XCTFail("Expected provider-neutral recovery to map to the recovery receipt UI.") + } + XCTAssertTrue(decodedReceipt.providerNeutral) + XCTAssertEqual(decodedReceipt.provider, "claude") + XCTAssertEqual(decodedReceipt.recoveryCount, 3) + + let raw = """ + {"sessionId":"chat-1","timestamp":"2026-07-09T00:02:00.000Z","sequence":1,"event":{"type":"turn_recovery","provider":"claude","turnId":"turn-1","action":"restart_resume","state":"recovered","message":"Provider-neutral recovery receipt","automatic":false,"at":"2026-07-09T00:02:00.000Z","recoveryCount":3}} + {"sessionId":"chat-1","timestamp":"2026-07-09T00:02:00.100Z","sequence":2,"event":{"type":"codex_turn_recovery","turnId":"turn-1","action":"restart_resume_thread","state":"recovered","message":"Legacy recovery receipt","automatic":false,"at":"2026-07-09T00:02:00.100Z"}} + """ + let transcript = parseWorkChatTranscript(raw) + guard case .codexTurnRecovery(_, let fallbackReceipt, _) = transcript[0].event else { + return XCTFail("Expected fallback parsing to preserve provider-neutral recovery.") + } + XCTAssertTrue(fallbackReceipt.providerNeutral) + XCTAssertEqual(fallbackReceipt.provider, "claude") + XCTAssertEqual(fallbackReceipt.recoveryCount, 3) + + let cards = buildWorkEventCards(from: transcript) + XCTAssertEqual(cards.count, 1) + XCTAssertEqual(cards[0].kind, "codexRecoveryReceipt") + XCTAssertEqual(cards[0].title, "Claude connection recovered") + XCTAssertEqual(cards[0].body, "Provider-neutral recovery receipt") + XCTAssertEqual(cards[0].recoveryReceipt?.action, "restart_resume") + XCTAssertEqual(cards[0].recoveryReceipt?.provider, "claude") + XCTAssertEqual(cards[0].recoveryReceipt?.recoveryCount, 3) + XCTAssertTrue(cards[0].recoveryReceipt?.providerNeutral == true) + } + + func testProviderNeutralRecoveryFallbackUsesNormalizedDefaultAction() { + let raw = #"{"sessionId":"chat-1","timestamp":"2026-07-09T00:02:00.000Z","sequence":1,"event":{"type":"turn_recovery","provider":"claude","turnId":"turn-1","state":"recovered","message":"Recovered.","automatic":false,"at":"2026-07-09T00:02:00.000Z"}}"# + guard case .codexTurnRecovery(_, let receipt, _) = parseWorkChatTranscript(raw).first?.event else { + return XCTFail("Expected fallback recovery receipt.") + } + XCTAssertEqual(receipt.action, "restart_resume_thread") + } + + func testCodexTurnDiagnosticsCollapseRoutineModerationAndIntegrationNoise() throws { + let moderation = try AgentChatEvent.decode(from: [ + "type": "codex_moderation_metadata", + "turnId": "turn-1", + "metadata": [ + "threadId": "thread-1", + "turnId": "turn-1", + "metadata": [:], + ], + ]) + let diagnostics = try AgentChatEvent.decode(from: [ + "type": "turn_diagnostics", + "turnId": "turn-1", + "moderationChecks": 3, + "optionalIntegrationFailures": [ + ["integration": "unityMCP", "message": "MCP client unavailable"], + ], + ]) + let transcript = [ + WorkChatEnvelope( + sessionId: "chat-1", + timestamp: "2026-07-09T00:00:01.000Z", + sequence: 1, + event: makeWorkChatEvent(from: moderation) + ), + WorkChatEnvelope( + sessionId: "chat-1", + timestamp: "2026-07-09T00:00:02.000Z", + sequence: 2, + event: makeWorkChatEvent(from: diagnostics) + ), + ] + + let cards = buildWorkEventCards(from: transcript) + XCTAssertEqual(cards.count, 1) + XCTAssertEqual(cards[0].kind, "turnDiagnostics") + XCTAssertEqual(cards[0].title, "Turn details") + XCTAssertEqual(cards[0].diagnosticModerationChecks, 3) + XCTAssertEqual(cards[0].diagnosticIntegrationFailures.map(\.integration), ["unityMCP"]) + if case .unknown(let type) = makeWorkChatEvent(from: moderation) { + XCTAssertEqual(type, "codex_moderation_metadata") + } else { + XCTFail("Routine moderation checks must stay out of the main timeline.") + } + + let raw = #"{"sessionId":"chat-1","timestamp":"2026-07-09T00:00:02.000Z","sequence":2,"event":{"type":"turn_diagnostics","turnId":"turn-1","moderationChecks":3,"optionalIntegrationFailures":[{"integration":"unityMCP","message":"MCP client unavailable"}]}}"# + let fallbackCard = try XCTUnwrap(buildWorkEventCards(from: parseWorkChatTranscript(raw)).first) + XCTAssertEqual(fallbackCard.kind, "turnDiagnostics") + XCTAssertEqual(fallbackCard.diagnosticModerationChecks, 3) + XCTAssertEqual(fallbackCard.diagnosticIntegrationFailures.first?.integration, "unityMCP") + } + + func testRecoveredCodexTurnReplacesStallActionsWithAuditReceipt() throws { + let raw = """ + {"sessionId":"chat-1","timestamp":"2026-07-09T00:00:01.000Z","sequence":1,"event":{"type":"codex_turn_stalled","turnId":"turn-1","reason":"no_output","message":"No output.","recoveryOptions":["wait","restart_resume_thread"]}} + {"sessionId":"chat-1","timestamp":"2026-07-09T00:00:03.000Z","sequence":2,"event":{"type":"codex_turn_recovery","turnId":"turn-1","action":"restart_resume_thread","state":"recovered","message":"Restarted and resumed the thread.","automatic":true,"at":"2026-07-09T00:00:03.000Z"}} + """ + let cards = buildWorkEventCards(from: parseWorkChatTranscript(raw)) + XCTAssertEqual(cards.map(\.kind), ["codexRecoveryReceipt"]) + XCTAssertEqual(cards.first?.recoveryReceipt?.state, "recovered") + XCTAssertEqual(cards.first?.metadata, ["Automatic recovery"]) } func testCodexRecoveryRemainsAvailableInSubagentTranscriptWhenHostSupportsIt() { @@ -16277,7 +16785,7 @@ final class ADETests: XCTestCase { laneId: "lane-2", laneName: "release", toolType: "shell", - runtimeState: "idle", + runtimeState: "running", title: "Deploy logs", lastOutputPreview: "Tail the deploy terminal output" ) @@ -16308,7 +16816,7 @@ final class ADETests: XCTestCase { laneId: "lane-1", laneName: "release", toolType: "codex-chat", - runtimeState: "idle", + runtimeState: "running", title: "Phone terminal" ) let outputSearch = workSessionOutputSearchIndexBySessionId(buffers: [ @@ -16944,6 +17452,77 @@ final class ADETests: XCTestCase { XCTAssertEqual(WorkActivityIndicator.formatElapsedSeconds(2610), "43m 30s") } + func testWorkActivityIndicatorFollowUpDoesNotResetTurnStart() { + let transcript = [ + WorkChatEnvelope( + sessionId: "chat-1", + timestamp: "2026-07-09T00:00:00.000Z", + sequence: 1, + event: .userMessage( + text: "Start the work", + attachments: nil, + turnId: "turn-1", + steerId: nil, + deliveryState: nil, + processed: nil + ) + ), + WorkChatEnvelope( + sessionId: "chat-1", + timestamp: "2026-07-09T00:00:01.000Z", + sequence: 2, + event: .status(turnStatus: "started", message: nil, turnId: "turn-1") + ), + WorkChatEnvelope( + sessionId: "chat-1", + timestamp: "2026-07-09T00:45:00.000Z", + sequence: 3, + event: .userMessage( + text: "Any update?", + attachments: nil, + turnId: "turn-1", + steerId: "steer-1", + deliveryState: "accepted", + processed: nil + ) + ), + ] + + XCTAssertEqual( + WorkActivityIndicator.activeTurnStartTimestamp(from: transcript), + "2026-07-09T00:00:01.000Z" + ) + } + + func testWorkDeliveryBadgeDistinguishesAcceptedFromProcessed() { + XCTAssertEqual( + workDeliveryBadgeState(deliveryState: "accepted", processed: nil), + .accepted + ) + XCTAssertEqual( + workDeliveryBadgeState(deliveryState: "delivered", processed: nil), + .accepted + ) + XCTAssertEqual( + workDeliveryBadgeState(deliveryState: "processed", processed: true), + .processed + ) + XCTAssertEqual( + workDeliveryBadgeState(deliveryState: "unprocessed", processed: false), + .unprocessed + ) + XCTAssertEqual(WorkDeliveryBadge.State.accepted.label, "Accepted") + XCTAssertEqual(WorkDeliveryBadge.State.processed.label, "Processed") + XCTAssertEqual(WorkDeliveryBadge.State.unprocessed.label, "Not processed") + } + + func testProviderNeutralRecoveryFeedbackDoesNotAssumeCodex() { + XCTAssertEqual(workTurnRecoveryFeedback(status: "waiting"), "Waiting for runtime output…") + XCTAssertEqual(workTurnRecoveryFeedback(status: "nudged"), "Status nudge sent.") + XCTAssertEqual(workTurnRecoveryFeedback(status: "retrying"), "Retry started in this thread.") + XCTAssertEqual(workTurnRecoveryFeedback(status: "resumed"), "Runtime restarted and the thread resumed.") + } + func testWorkActivityIndicatorUsesToolSpecificVerbAndArgPreview() { let transcript = [ WorkChatEnvelope( diff --git a/apps/tunnel-relay/src/tunnelDo.ts b/apps/tunnel-relay/src/tunnelDo.ts index 1929e1adf..5c0a4aa7d 100644 --- a/apps/tunnel-relay/src/tunnelDo.ts +++ b/apps/tunnel-relay/src/tunnelDo.ts @@ -42,6 +42,15 @@ const MAX_CLOSE_REASON_BYTES = 123; const WS_OPEN = 1; const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); +const CORRELATION_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +function relayCorrelationId(url: URL): string { + const provided = url.searchParams.get("cid")?.trim() ?? ""; + return CORRELATION_ID_PATTERN.test(provided) + ? provided.toLowerCase() + : crypto.randomUUID(); +} function applicationCloseCode(code: unknown, fallback: number): number { return typeof code === "number" && Number.isInteger(code) && code >= 4000 && code <= 4999 @@ -88,6 +97,8 @@ type SocketAttachment = { legacyBuffered?: true; /** Pre-deploy data socket whose volatile pre-ready state cannot be proven. */ legacyStateUnknown?: true; + /** Safe client-generated operation id; never contains endpoint/auth data. */ + correlationId?: string; ts: number; }; @@ -263,8 +274,16 @@ export class TunnelDurableObject implements DurableObject { } }); } + const clientAttachment = this.normalizedAttachment(client); if (this.pipeForId(id, epoch)) { - return this.acceptSocket({ role: "pipe", id, epoch }, (server) => { + return this.acceptSocket({ + role: "pipe", + id, + epoch, + ...(clientAttachment?.correlationId + ? { correlationId: clientAttachment.correlationId } + : {}), + }, (server) => { try { server.close(CLOSE_STALE_PIPE, "duplicate pipe"); } catch { @@ -275,7 +294,14 @@ export class TunnelDurableObject implements DurableObject { const control = this.currentControl(); const controlAttachment = control ? this.normalizedAttachment(control) : null; if (this.epochOf(controlAttachment) !== epoch) { - return this.acceptSocket({ role: "pipe", id, epoch }, (server) => { + return this.acceptSocket({ + role: "pipe", + id, + epoch, + ...(clientAttachment?.correlationId + ? { correlationId: clientAttachment.correlationId } + : {}), + }, (server) => { try { server.close(CLOSE_STALE_PIPE, "stale control epoch"); } catch { @@ -287,7 +313,14 @@ export class TunnelDurableObject implements DurableObject { // also OPEN with an epoch-matched control {t:"ready"} message. A legacy // pipe arrival already proves its old Node bridge is OPEN, so that branch // becomes ready immediately for both old and ready-v2 clients. - const response = await this.acceptSocket({ role: "pipe", id, epoch }); + const response = await this.acceptSocket({ + role: "pipe", + id, + epoch, + ...(clientAttachment?.correlationId + ? { correlationId: clientAttachment.correlationId } + : {}), + }); if (!requestedEpoch) this.markPairReady(id, epoch); return response; } @@ -295,12 +328,13 @@ export class TunnelDurableObject implements DurableObject { private async handleConnect(request: Request, url: URL): Promise { const notWs = await this.requireWebSocket(request); if (notWs) return notWs; + const correlationId = relayCorrelationId(url); const control = this.currentControl(); if (!control) { // No host is registered — accept then close so the phone gets a clean, // distinguishable code rather than a bare handshake failure. - logTunnel("connect_rejected", { reason: "host_offline" }); - return this.acceptSocket({ role: "client", epoch: "offline" }, (server) => { + logTunnel("connect_rejected", { reason: "host_offline", correlationId }); + return this.acceptSocket({ role: "client", epoch: "offline", correlationId }, (server) => { try { server.close(CLOSE_HOST_OFFLINE, "host offline"); } catch { @@ -312,11 +346,17 @@ export class TunnelDurableObject implements DurableObject { const activeClients = this.openSocketsForRole("client").length; const maxTunnels = this.maxTunnels(); if (activeClients >= maxTunnels) { - logTunnel("connect_rejected", { reason: "too_many", activeClients, maxTunnels }); + logTunnel("connect_rejected", { + reason: "too_many", + activeClients, + maxTunnels, + correlationId, + }); const controlAttachment = this.normalizedAttachment(control); return this.acceptSocket({ role: "client", epoch: this.epochOf(controlAttachment) ?? LEGACY_CONTROL_EPOCH, + correlationId, }, (server) => { try { server.close(CLOSE_TOO_MANY, "too many tunnels"); @@ -329,7 +369,7 @@ export class TunnelDurableObject implements DurableObject { const controlAttachment = this.normalizedAttachment(control); const controlEpoch = this.epochOf(controlAttachment); if (!controlAttachment || !controlEpoch) { - return this.acceptSocket({ role: "client", epoch: "offline" }, (server) => { + return this.acceptSocket({ role: "client", epoch: "offline", correlationId }, (server) => { try { server.close(CLOSE_HOST_OFFLINE, "host offline"); } catch { @@ -346,6 +386,7 @@ export class TunnelDurableObject implements DurableObject { role: "client", id, epoch: controlEpoch, + correlationId, ...(readyVersion ? { readyVersion } : {}), }, (server) => { if (!readyVersion) return; @@ -374,7 +415,7 @@ export class TunnelDurableObject implements DurableObject { } catch { // The control socket died between lookup and signaling. Do not leave the // phone occupying a tunnel slot while it waits for a pipe that cannot come. - logTunnel("connect_rejected", { reason: "host_offline" }); + logTunnel("connect_rejected", { reason: "host_offline", correlationId }); const client = this.clientForId(id, controlEpoch); try { client?.close(CLOSE_HOST_OFFLINE, "host offline"); @@ -538,7 +579,11 @@ export class TunnelDurableObject implements DurableObject { const code = applicationCloseCode(parsed.code, CLOSE_BRIDGE_REJECTED); const reason = sanitizedCloseReason(parsed.reason, "bridge rejected"); this.pendingClientFrames.delete(parsed.id); - logTunnel("connect_rejected", { reason: "bridge_rejected", code }); + logTunnel("connect_rejected", { + reason: "bridge_rejected", + code, + correlationId: clientAttachment?.correlationId ?? null, + }); this.closePair(client, clientAttachment, code, reason); } else if ( parsed?.t === "ready" @@ -651,6 +696,7 @@ export class TunnelDurableObject implements DurableObject { epochMode: this.epochOf(attachment) === LEGACY_CONTROL_EPOCH ? "legacy" : "epoch", code, established: attachment?.established === true, + correlationId: attachment?.correlationId ?? null, }); } @@ -737,6 +783,7 @@ export class TunnelDurableObject implements DurableObject { connectionId: attachment?.id ?? null, sourceRole: attachment?.role ?? null, reason: closeReason, + correlationId: attachment?.correlationId ?? null, }); } if (attachment?.id) this.pendingClientFrames.delete(attachment.id); diff --git a/apps/tunnel-relay/test/relay.test.ts b/apps/tunnel-relay/test/relay.test.ts index 4ce185624..cef633e43 100644 --- a/apps/tunnel-relay/test/relay.test.ts +++ b/apps/tunnel-relay/test/relay.test.ts @@ -67,6 +67,7 @@ class FakeSocket { established?: boolean; legacyBuffered?: true; legacyStateUnknown?: true; + correlationId?: string; ts: number; }, ) {} @@ -104,6 +105,7 @@ class FakeState { readyVersion?: typeof RELAY_READY_VERSION; established?: boolean; legacyBuffered?: boolean; + correlationId?: string; } = {}, ): FakeSocket { const epoch = options.epoch ?? CONTROL_EPOCH; @@ -115,6 +117,7 @@ class FakeState { ...(options.readyVersion ? { readyVersion: options.readyVersion } : {}), ...(options.established ? { established: true } : {}), ...(options.legacyBuffered ? { legacyBuffered: true as const } : {}), + ...(options.correlationId ? { correlationId: options.correlationId } : {}), ts, }); this.sockets.push(socket); @@ -148,6 +151,7 @@ function installAcceptSocketStub(durable: TunnelDurableObject, state: FakeState) id?: string; epoch?: string; readyVersion?: typeof RELAY_READY_VERSION; + correlationId?: string; }, afterAccept?: (socket: WebSocket) => void, ) => Promise; @@ -156,6 +160,7 @@ function installAcceptSocketStub(durable: TunnelDurableObject, state: FakeState) const socket = state.addSocket(attachment.role, attachment.id, Date.now(), { epoch: attachment.epoch, readyVersion: attachment.readyVersion, + correlationId: attachment.correlationId, }); afterAccept?.(socket as unknown as WebSocket); return new Response(null, { status: 200 }); @@ -500,6 +505,50 @@ describe("durable socket lifecycle", () => { log.mockRestore(); }); + it("propagates a safe client correlation id to pipe lifecycle logs", async () => { + const { durable, state, storage } = makeDoHarness(); + await storage.put("secret", SECRET); + installAcceptSocketStub(durable, state); + const controlEpoch = CONTROL_EPOCH; + const correlationId = "123e4567-e89b-42d3-a456-426614174000"; + const control = state.addSocket("control", undefined, Date.now(), { + epoch: controlEpoch, + }); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + await durable.fetch(new Request( + `https://relay.test/connect/${MACHINE_KEY}?ready=2&cid=${correlationId}`, + { headers: { Upgrade: "websocket" } }, + )); + const open = JSON.parse(String(control.sent.at(-1))) as { + id: string; + epoch: string; + }; + const client = state.sockets.find((socket) => socket.tags.includes("client"))!; + expect(client.deserializeAttachment()).toMatchObject({ correlationId }); + + await durable.fetch(await signedPipeRequest({ + id: open.id, + epoch: controlEpoch, + })); + const pipe = state.sockets.find((socket) => socket.tags.includes("pipe"))!; + expect(pipe.deserializeAttachment()).toMatchObject({ correlationId }); + + await durable.webSocketClose( + client as unknown as WebSocket, + 1006, + "abnormal close", + false, + ); + const terminalLog = log.mock.calls + .flatMap(([entry]) => [String(entry)]) + .find((entry) => entry.includes('"kind":"socket_closed"')); + expect(terminalLog).toContain(`"correlationId":"${correlationId}"`); + expect(terminalLog).not.toContain("ready=2"); + expect(terminalLog).not.toContain(MACHINE_KEY); + log.mockRestore(); + }); + it("migrates a proven pre-deploy client/pipe pair after hibernation", async () => { const { state } = makeDoHarness(); const client = state.addSocket("client", "abcdef01", Date.now(), { omitEpoch: true }); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5fbd8428b..8c7b4d8c7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -232,7 +232,7 @@ Build outputs (configured in `apps/desktop/tsup.config.ts`): Terminal-native **Work** chat client (Ink 7 + React 19) for agents and power users who live in a shell, built into `apps/ade-cli/src/tuiClient/`. Its UI dependencies live under `apps/ade-cli` and are intentionally independent of the desktop renderer's React stack. It is a peer of the desktop client, not a wrapper around it: it speaks the same multi-project JSON-RPC surface and binds to an ADE runtime the same way. - **Attached mode** (default): connects to `$ADE_HOME/sock/ade.sock`, or to an explicit endpoint passed on the parent `ade` invocation. Starts the brain if the endpoint is missing. -- **Remote mode**: `ade code remote` reads the desktop's saved remote-target registry, picks a target/project/session, and uses the target's declared transport. Interactive launches always show the machine chooser, even with one saved target; non-interactive launches auto-select only when that choice is unambiguous. Paired targets use the DPoP-bound sync runtime bridge and try LAN → tailnet → Relay unless `--route lan|tailscale|relay` pins one class; SSH targets start `ade rpc --stdio` over a validated route, and an explicit paired-route choice never falls back to SSH. `remoteLauncher.ts` coordinates selection, `pairedRemoteConnector.ts` owns paired route/auth/health policy, `remoteLaunchBudget.ts` bounds connection work, and `remoteBridge.ts` exposes the local socket consumed by the normal TUI. Before mounting the TUI, the launcher closes the discovery client and verifies the long-lived paired connection; the bridge consumes that connection for its first local client instead of redialing during handoff. Later TUI retries leave the bridge listener alive, reload the saved target, and redial the eligible paths. Account-created targets are paired-only: account authentication is accepted only for initial adoption over an exact allowlisted WSS relay. Later direct LAN/tailnet connections use the stored paired secret plus pinned DPoP; every later Relay connection additionally fetches and attaches a fresh in-memory account proof that the host accepts only for its currently signed-in owner. The launcher recognizes the exact legacy account-machine shape that older desktop builds saved as credentialless SSH, adopts it into the paired store, and fails closed if account verification or pairing is unavailable; an explicitly configured SSH user/key continues to mean SSH. For true SSH targets, the saved hostname remains the OpenSSH `Host` selector while `-o HostName=` dials each concrete route, preserving alias-scoped credentials, agent/proxy settings, and strict host-key verification. Account resolution, paired dialing, and the SSH route × channel-home × binary probe matrix share one 45-second cancellable startup budget and return aggregated attempt diagnostics. Compatibility checks that only make sense for local processes (entrypoint build hash and project-root equality) are skipped. +- **Remote mode**: `ade code remote` reads the desktop's saved remote-target registry, picks a target/project/session, and uses the target's declared transport. Interactive launches always show the machine chooser, even with one saved target; non-interactive launches auto-select only when that choice is unambiguous. Paired targets use the DPoP-bound sync runtime bridge and try LAN → tailnet → Relay unless `--route lan|tailscale|relay` pins one class; SSH targets start `ade rpc --stdio` over a validated route, and an explicit paired-route choice never falls back to SSH. `remoteLauncher.ts` coordinates selection, `pairedRemoteConnector.ts` owns paired route/auth/health policy, `remoteLaunchBudget.ts` bounds connection work, and `remoteBridge.ts` exposes the local socket consumed by the normal TUI. Before mounting the TUI, the launcher closes the discovery client and verifies the long-lived paired connection; the bridge consumes that connection for its first local client instead of redialing during handoff. Later TUI retries leave the bridge listener alive, reload the saved target, and redial the eligible paths. Account-created targets are paired-only: a directory row with a signed host identity can be adopted over LAN or tailnet through the sealed `ade-adopt-v1` handshake; unsigned legacy rows remain Relay-only. Later direct LAN/tailnet connections use the stored paired secret plus pinned DPoP and survive account sign-out; every Relay connection additionally fetches and attaches a fresh in-memory account proof that the host accepts only for its currently signed-in owner. The launcher recognizes the exact legacy account-machine shape that older desktop builds saved as credentialless SSH, adopts it into the paired store, and fails closed if account verification or pairing is unavailable; an explicitly configured SSH user/key continues to mean SSH. For true SSH targets, the saved hostname remains the OpenSSH `Host` selector while `-o HostName=` dials each concrete route, preserving alias-scoped credentials, agent/proxy settings, and strict host-key verification. Account resolution, paired dialing, and the SSH route × channel-home × binary probe matrix share one 45-second cancellable startup budget and return aggregated attempt diagnostics with one correlation id, bounded endpoint metadata, and coarse failure classes. Compatibility checks that only make sense for local processes (entrypoint build hash and project-root equality) are skipped. - **Embedded mode**: `--embedded` / `--headless` runs the shared `apps/ade-cli` services in-process without going through a machine brain. Used when no brain endpoint or manual runtime endpoint is reachable. Shared DTOs and cross-client policies are imported from `apps/desktop/src/shared/*` (never the renderer barrel) so `npm run typecheck` in `apps/ade-cli` covers both typed commands and the TUI. This includes `externalSessionAffordances.ts`, which keeps ADE Code's provider-native Continue/Copy choices aligned with desktop while `externalSessionBrowser.ts` owns TUI-only navigation and the Open-existing action. Entry: `apps/ade-cli/src/tuiClient/cli.tsx` → `apps/ade-cli/dist/tuiClient/cli.mjs`, loaded by `ade code`. The built TUI bundle is intended to run in isolation: tsup bundles its Ink/xterm/highlight dependencies and injects ESM shims for `__dirname` / `__filename`; both `apps/ade-cli/scripts/verify-built-cli.mjs` and the desktop artifact validators smoke-import it and run `runAdeCodeCli(["--help"])`. Provider/model/interface setup is kept in pure helpers (`modelState.ts`, `providerMetadata.ts`, `modelPickerController.ts`) so Chat-vs-CLI availability, Cursor SDK-vs-CLI model filtering, permission presets, Fast Mode, and setup rows stay testable outside the Ink root. Chat Info uses shared derivations for subagents, tasks, and scheduled work, so Claude wakeups/cron/background activity rendered in desktop also appears in ADE Code; `AgentChatSessionSummary.nextWakeAt` adds the runtime scheduler's earliest armed fire as an alarm countdown in the Schedule block. The TUI can hand off to a desktop window via the `app/navigate` JSON-RPC method when a desktop client is attached to the same runtime. @@ -255,7 +255,7 @@ Native SwiftUI app acting as a controller. It pairs with an ADE machine over Web - Shipped project tabs: Lanes, Files, Work, PRs, CTO, Settings (including a Push delivery panel). The projectless Chats surface is entered only from the Hub, outside the project tab bar. It uses runtime-scoped commands and the same chat event union/Work transcript renderer while suppressing lane/project actions. The Work chat decodes the same chat event union as desktop for live transcripts, including scheduled-work updates and transcript retractions; scheduled work appears in a native Chat Info popup/sheet while the phone remains a controller only. Durable active rows expose Cancel and the Schedule header exposes per-chat Pause/Resume when the host advertises those actions. Project chats use `chat.cancelScheduledWork` / `chat.setScheduledWorkPaused`; Hub personal chats map the same UI to runtime-scoped `personalChats.*` actions. The host also advertises non-queueable schedule creation, but iOS does not render a create control. Native clients gate every implemented control on its descriptor, so transport availability does not make an older brain accept unsupported mutations. - Shipped widgets: a Lock Screen widget for prioritized agent/PR/sync/offline/idle status, plus an ActivityKit Live Activity + Dynamic Island for active agent runs (`ADEWidgets/ADEAgentActivityWidget.swift`). - Push: APNs alert pushes (deep-linked) and Live Activity updates arrive via the Cloudflare push relay (§2.7); the phone hands tokens/prefs to the brain over the paired sync WebSocket. -- Connection: ADE account sign-in is the primary PIN-less path; direct pairing uses a user-set 6-digit PIN after scanning the v3 smart-URL QR or choosing a Nearby machine. Pairing is hardened with device-bound DPoP proofs. +- Connection: ADE account sign-in is the primary PIN-less path; direct pairing uses a user-set 6-digit PIN after scanning the v3 smart-URL QR or choosing a Nearby machine. Both paths produce device-bound DPoP trust and reconnect in LAN → Tailscale → Relay order. Sign-out disables account discovery and Relay but retains direct machine trust until the user explicitly forgets that machine. - Planned: Automations, Graph, History tabs; iPad layout; Spotlight. - Target: iOS 26+, iPhone + iPad. @@ -263,7 +263,10 @@ Native SwiftUI app acting as a controller. It pairs with an ADE machine over Web Static Cloudflare Pages controller built from the desktop renderer package. New connections start with ADE account sign-in and adopt a machine from the -account directory over Relay; the resulting credential is DPoP-bound. The +account directory; the resulting credential is DPoP-bound. Endpoint ranking is +still LAN → Tailscale → Relay, but a production HTTPS page cannot dial insecure +`ws://` LAN/tailnet endpoints, so Relay is normally the only browser-eligible +route. The client keeps no local ADE DB and installs a sync-backed subset of `window.ade`. Its static HTML paints the loading shell before React, while account bootstrap, directory loading, and transactional IndexedDB privacy cleanup do not serialize @@ -295,7 +298,7 @@ Four independent Cloudflare Workers, each its own npm package / lockfile / `wran - **`apps/push-relay/`** — fans ADE agent-state transitions out to iPhones as APNs alert pushes and Live Activity updates (Worker + a single D1 database; free-plan compatible, no Durable Objects). The brain is the only publisher: it claims an unguessable 32–64-hex `machineKey` with a relay secret (`POST /machines/:key/claim`, first-writer-wins) and HMAC-signs every later call (`x-ade-push-signature: sha256=HMAC(secret, "...")`). It stores only device tokens and in-flight notification payloads — no chat/PR content. APNs auth is an ES256 provider JWT from the `.p8` (wrangler secrets `APNS_KEY` / `APNS_KEY_ID` / `APNS_TEAM_ID`). Brain-side publisher lives at `apps/ade-cli/src/services/push/`. See [features/sync-and-multi-device/push-notifications.md](./features/sync-and-multi-device/push-notifications.md). - **`apps/tunnel-relay/`** — pipes ADE **sync** WebSocket frames between a controller and a brain when there is no direct LAN/Tailscale path (Worker + Durable Object with SQLite storage, one instance per `machineKey`, WebSocket Hibernation API). The brain holds a persistent HMAC-signed outbound control socket while the machine has a valid ADE account session; a controller dials `/connect/:machineKey`; the DO pairs it with a dedicated brain-side pipe socket and passes bytes through 1:1 with no frame wrapping, so the normal ADE hello / pairing / DPoP handshake is unchanged. Native 30-second ping / 10-second pong transport liveness is the primary keepalive; because a hibernated or wedged DO can leave the edge answering those transport pings after the machine's control registration is dead, the brain adds a low-frequency application-level `{t:"ping"}`/`{t:"pong"}` keepalive (180 s interval, 30 s deadline) to catch such "zombie" controls, and verifies the path end-to-end with a self-probe (`syncRelaySelfProbe`) that dials `/connect/:machineKey?ready=2` like a real controller. The account directory advertises a `relay` endpoint only after that self-probe round-trips (honest relay publication); an at-capacity `4503` close is treated as liveness proof, not failure. Failed bridge opens are rejected explicitly; application close codes and bounded sanitized reasons survive the phone/pipe/local boundaries. Early controller frames are bounded by both 64 frames and 256 KiB, and idle-sweep alarms run only while a client or pipe exists. Brain-side client is `apps/ade-cli/src/services/sync/syncTunnelClientService.ts`. There is no user relay toggle: sign-in starts and advertises Relay, while sign-out closes it. It remains the lowest-priority `relay` address candidate after LAN and Tailscale. TLS terminates at the Worker, so this is a trusted-operator plaintext path rather than end-to-end encryption; relay payload E2E encryption is planned security work. -- **`apps/account-directory/`** — Clerk-authenticated machine directory and OAuth device-authorization bridge (Worker + D1). The machine brain publishes a health-filtered registration through `accountMachinePublisherService.ts`: a 30-second heartbeat keeps the row inside the Worker's 90-second online window, while sign-in and publish-relevant relay-route changes trigger coalesced immediate writes and reset the heartbeat deadline. The Worker scopes rows by Clerk `sub`, selects at most the 500 most recently seen machines, then returns online-first order. Machine-list responses expose separate auth and D1 durations through `Server-Timing`, including auth failures. Authentication failures return only fixed classifications such as `token expired`, `invalid issuer`, and `invalid audience`; directory clients consume at most 512 response bytes before exposing the short reason in machine-list results and publisher health. Desktop, ADE Code, hosted web, and iOS use the compiled HTTPS Worker origin by default. Headless login binds each short-lived device code to a daemon secret, uses Clerk OAuth + PKCE in any browser, and atomically burns the approved token pair on redemption. Each published row also carries the machine's long-lived Ed25519 identity as `pubkey`; a same-account desktop/iOS client verifies that key during the sealed `ade-adopt-v1` handshake to adopt a machine over a direct LAN/Tailscale route (relay → tailnet → LAN fallback) without exposing the account bearer in plaintext — see [features/sync-and-multi-device/README.md](./features/sync-and-multi-device/README.md). +- **`apps/account-directory/`** — Clerk-authenticated machine directory and OAuth device-authorization bridge (Worker + D1). The machine brain publishes a health-filtered registration through `accountMachinePublisherService.ts`: a 30-second heartbeat keeps the row inside the Worker's 90-second online window, while sign-in and publish-relevant relay-route changes trigger coalesced immediate writes and reset the heartbeat deadline. The Worker scopes rows by Clerk `sub`, selects at most the 500 most recently seen machines, then returns online-first order. Machine-list responses expose separate auth and D1 durations through `Server-Timing`, including auth failures. Authentication failures return only fixed classifications such as `token expired`, `invalid issuer`, and `invalid audience`; directory clients consume at most 512 response bytes before exposing the short reason in machine-list results and publisher health. Clients attach `X-ADE-Correlation-ID`; the Worker reflects and CORS-exposes it and logs it with route, method, status, and duration so a connection attempt can be followed without recording account tokens or full endpoint URLs. Desktop, ADE Code, hosted web, and iOS use the compiled HTTPS Worker origin by default. Headless login binds each short-lived device code to a daemon secret, uses Clerk OAuth + PKCE in any browser, and atomically burns the approved token pair on redemption. Each published row also carries the machine's long-lived Ed25519 identity as `pubkey`; a same-account desktop/iOS client verifies that key during the sealed `ade-adopt-v1` handshake to adopt a machine over a direct LAN/Tailscale route (LAN → Tailscale → Relay fallback) without exposing the account bearer in plaintext — see [features/sync-and-multi-device/README.md](./features/sync-and-multi-device/README.md). - **`apps/webhook-relay/`** — the pre-existing GitHub webhook relay (different trust model and lifecycle again). See its own docs. --- @@ -478,6 +481,14 @@ Service entry points live under `apps/desktop/src/main/services/ai/`. The subsys `ai.getOpenCodeRuntimeDiagnostics` expose the same provider readiness, stored-key, and OpenCode runtime health data to renderer settings and `ade code` model setup through the shared ADE action registry. + +Agent-chat adapters publish a provider-neutral delivery and health contract: +accepted user messages retain processed/unprocessed resolution state; +`turn_health`, `turn_recovery`, and `turn_diagnostics` describe stalls, +recovery, and aggregated diagnostics; `chat.recoverTurn` and +`chat.resolveUnprocessedMessage` are the shared action surfaces. Provider-native +events remain compatibility inputs, not client UI contracts. Repeated raw +moderation checks are collapsed into one quiet per-turn diagnostics summary. - **Fallback**: if no usable provider is present, ADE runs in **guest mode** — deterministic features (packs, diffs, conflicts) continue; AI surfaces are disabled with explanatory UI. ### 4.2 Permission modes (provider-native + ADE) @@ -595,8 +606,9 @@ ade.agentChat.* # agent chat sessions, model inventory, parallel la # `{ mode: "cached"|"refresh-stale"|"force", refreshProvider?: "opencode"|"cursor"|"droid"|"lmstudio"|"ollama" }`) # and ade.agentChat.codex.* goal controls backed by # Codex app-server thread/goal RPCs, plus - # recoverCodexTurn for guarded Wait/Nudge/Retry/Resume - # handling of the currently active stalled turn, and + # provider-neutral recoverTurn and + # resolveUnprocessedMessage controls (plus the + # legacy recoverCodexTurn compatibility action), and # recoverContinuity (retry_original / recover_from_history / # start_new_chat) for a chat whose provider thread could not be # resumed — see features/storage-and-recovery/README.md. Also includes the typed @@ -692,7 +704,7 @@ Most services described here live under `apps/desktop/src/main/services/ | `appControl/` | `appControlService.ts`, `appControlLaunchCommand.ts` | Chrome DevTools Protocol bridge for developer-owned Electron apps. Launches a chat-owned PTY running the user's dev command (or connects to an existing `--remote-debugging-port`), polls `/json` for ready CDP targets, attaches a long-lived `CdpClient` WebSocket, and exposes screenshot / DOM snapshot / hit-test / click / type / scroll / key dispatch / screencast frames. `appControlLaunchCommand.ts` owns the shell-command detection and debug-flag injection helpers for direct Electron and package-script launches. `inspectPoint` and `selectPoint` produce `AppControlContextItem`s for the chat composer (DOM packet + screenshot + source-file candidates resolved by `findSourceMatches` over an indexed tree of project source files). See [features/computer-use/app-control.md](./features/computer-use/app-control.md). | | `builtInBrowser/` | `builtInBrowserService.ts`, `builtInBrowserAgentAccess.ts`, `builtInBrowserActorCapabilities.ts`, `builtInBrowserAuthentication.ts`, `builtInBrowserProfileMigration.ts`, `builtInBrowserStateStore.ts`, `builtInBrowserNavigation.ts`, `builtInBrowserPermissions.ts`, `builtInBrowserWebAuthn.ts`, `desktopBridgeServer.ts` | In-app web browser owned by the main process. Every remote-content `WebContentsView` uses the single persistent `persist:ade-browser` storage profile (`storageProfileKey: "global"`), while service keys combine the ADE window id with a project/window/personal tab-collection key so visible tabs stay independent. Project roots route project commands and scratch observations; validated personal commands retain the personal tab collection and use the channel-specific machine-local browser-observation scratch root. Neither route partitions cookies or site storage. On first use, a bounded, idempotent migration copies unexpired persistent cookies from this channel's legacy project-derived partitions into the global profile without overwriting global cookies or copying session cookies; it preserves the old partition directories because Chromium DOM storage, IndexedDB, service-worker state, and WebAuthn credentials cannot be safely merged across partitions. The bounded machine-local state store restores HTTP(S)/blank tab URLs and the active tab for each collection, but never restores agent leases, lightweight browser sessions, or synthetic session cookies. The service caps each collection at 10 tabs, routes global-session network events back to their owning collection, drives OAuth popups and downloads, and emits targeted events. HTTP/proxy authentication uses a sandboxed, local credential prompt and passes values directly to Chromium without persisting or logging them; client-certificate requests use an explicit native chooser and only accept a certificate Electron offered. Permission requests are deny-by-default, limited to managed browser web contents and secure origins, and use persisted per-origin/embedding-origin decisions with a native human prompt; only Google's `storage-access` and `top-level-storage-access` requests retain a narrow accounts-domain compatibility exception. The Browser toolbar's trusted-renderer Profile panel exposes non-secret cookie/cache/flush diagnostics and list/remove/clear controls for remembered permission decisions; these operations are not bridged to agents or unbound CLI callers. A separate non-persistent agent-access controller requires a per-chat/lane native human grant for every non-local origin and for local origins with allowed privileged permissions; cross-origin navigations and redirects are intercepted, and sensitive popups are blocked until explicitly approved. The grant follows the agent-owned tab without a timer and clears only when an explicit trusted-renderer navigation reclaims the tab. Tabs carry owner/lease metadata. ADE-launched chats receive opaque in-memory browser actor capabilities bound to their trusted chat/lane/project or personal collection. The runtime requires the token and strips caller routing; Electron validates it in the issuing process, restores only the bound scope, forces `force: false`, and separately authenticates the bridge with the desktop launch's rotating token. Agents cannot force or impersonate a takeover, read another agent's tab status, inspect global cookie-domain diagnostics, or administer permissions. Browser sessions bind one workflow to one tab. Project observations live under `.ade/cache/browser-observations/`; personal observations live under the channel user-data `browser-observations/personal/` root, which is narrowly allowlisted for proof promotion. The issuer-restored scope selects the matching independent tab collection. Navigation/protocol policy lives in `builtInBrowserNavigation.ts`; WebAuthn account selection lives in `builtInBrowserWebAuthn.ts`. | | `automations/` | `automationService.ts`, `automationPlannerService.ts`, `automationIngressService.ts`, `automationSecretService.ts` | Rule lifecycle, NL → rule planner, inbound triggers, per-rule secrets. | -| `chat/` | `agentChatService.ts`, `chatScheduledWorkScheduler.ts`, `runtimeEvents.ts`, `claudeStructuredActivity.ts`, `openCodeStructuredActivity.ts`, `codexMcpElicitation.ts`, `buildClaudeV2Message.ts`, `markdownSlashCommandDiscovery.ts`, `claudeSlashCommandDiscovery.ts`, `codexSlashCommandDiscovery.ts`, `cursorSlashCommandDiscovery.ts`, `projectSlashCommandDiscovery.ts`, `slashCommandPromptExpansion.ts`, `cursorSdk*` (`cursorSdkPool.ts`, `cursorSdkWorker.ts`, `cursorSdkProtocol.ts`, `cursorSdkPolicy.ts`, `cursorSdkSystemPrompt.ts`, `cursorSdkEventMapper.ts`, `cursorSdkErrors.ts`), `droidSdkEventMapper.ts`, `sessionRecovery.ts` | Agent chat sessions (lane-scoped + orchestration worker/coordinator). Builds Claude messages, hosts the Cursor SDK in a Node worker pool with official local-store persistence, formalizes the cross-runtime event vocabulary, normalizes provider-native web/MCP/image activity into compact shared events, handles Codex app-server MCP elicitations and stalled-turn recovery, recovers sessions on restart, derives prompt-based lane names for parallel model launches, keeps Claude Agent SDK streams alive for scheduled wake/cron/background work after visible turns, emits transcript retractions for provider-superseded assistant rows, and manages Codex app-server goals with persisted, unlimited-budget session state. `chat.createScheduledWork` validates a five-field cron plus a bounded prompt and writes an ADE-owned recurring or one-shot row for any chat provider runtime or ADE-tracked provider CLI. Claude remains authoritative for provider schedule tool success and canonical ids, while ADE's store is authoritative for delivery: successful `PostToolUse` mirrors `ScheduleWakeup`, every successful `CronCreate`, and `/loop` records in `kv`, and scopes Stop/SubagentStop reconciliation to the exact provider-session owner so a new session's empty snapshot preserves prior-owner rows. `durable: true` persists Claude's provider copy, but the SDK's schedule view remains advisory; ADE state wins. The SDK gets the native fire opportunity at `fireAt`; ADE's timer waits through a 90-second grace window before backstopping a skipped or unavailable provider. A native claim requires an explicit SDK cron-task start; an exact provider id wins when present, while older ambiguous task events may claim only the earliest due CronCreate-owned row and can never consume a `ScheduleWakeup` or loop. Every managed chat row that becomes due during a foreground Claude, Codex, Cursor, Droid, or OpenCode turn stays armed and retries after 20 seconds rather than entering that turn's disposable input queue; only an actual delivery advances a cron, and expiry still wins. At an idle chat boundary the scheduler sends `messageSession(kind: "wake")`. Tracked CLI rows wait for a provider-specific visible composer boundary, resume ended sessions, and retry proven pre-delivery failures without consuming the occurrence. The scheduler restores timers, coalesces missed occurrences to one late fire, applies session/global pause state, cold-starts idle chats when necessary, expires recurring crons after seven days, and emits lifecycle rows while summaries and `chat.getScheduledWorkState` expose management state. Cancellation of Claude-owned jobs routes through `CronDelete` and remains visible until provider confirmation; ADE-owned rows cancel directly. There is no scheduled-work-specific spend cap. | +| `chat/` | `agentChatService.ts`, `chatScheduledWorkScheduler.ts`, `runtimeEvents.ts`, `claudeStructuredActivity.ts`, `openCodeStructuredActivity.ts`, `codexMcpElicitation.ts`, `buildClaudeV2Message.ts`, `markdownSlashCommandDiscovery.ts`, `claudeSlashCommandDiscovery.ts`, `codexSlashCommandDiscovery.ts`, `cursorSlashCommandDiscovery.ts`, `projectSlashCommandDiscovery.ts`, `slashCommandPromptExpansion.ts`, `cursorSdk*` (`cursorSdkPool.ts`, `cursorSdkWorker.ts`, `cursorSdkProtocol.ts`, `cursorSdkPolicy.ts`, `cursorSdkSystemPrompt.ts`, `cursorSdkEventMapper.ts`, `cursorSdkErrors.ts`), `droidSdkEventMapper.ts`, `sessionRecovery.ts` | Agent chat sessions (lane-scoped + orchestration worker/coordinator). Builds Claude messages, hosts the Cursor SDK in a Node worker pool with official local-store persistence, formalizes the cross-runtime event vocabulary, normalizes provider-native web/MCP/image activity into compact shared events, persists accepted/processed/unprocessed message delivery, emits provider-neutral turn health/recovery/diagnostics, aggregates moderation checks quietly, handles Codex app-server MCP elicitations, recovers sessions on restart, derives prompt-based lane names for parallel model launches, keeps Claude Agent SDK streams alive for scheduled wake/cron/background work after visible turns, emits transcript retractions for provider-superseded assistant rows, and manages Codex app-server goals with persisted, unlimited-budget session state. `chat.createScheduledWork` validates a five-field cron plus a bounded prompt and writes an ADE-owned recurring or one-shot row for any chat provider runtime or ADE-tracked provider CLI. Claude remains authoritative for provider schedule tool success and canonical ids, while ADE's store is authoritative for delivery: successful `PostToolUse` mirrors `ScheduleWakeup`, every successful `CronCreate`, and `/loop` records in `kv`, and scopes Stop/SubagentStop reconciliation to the exact provider-session owner so a new session's empty snapshot preserves prior-owner rows. `durable: true` persists Claude's provider copy, but the SDK's schedule view remains advisory; ADE state wins. The SDK gets the native fire opportunity at `fireAt`; ADE's timer waits through a 90-second grace window before backstopping a skipped or unavailable provider. A native claim requires an explicit SDK cron-task start; an exact provider id wins when present, while older ambiguous task events may claim only the earliest due CronCreate-owned row and can never consume a `ScheduleWakeup` or loop. Every managed chat row that becomes due during a foreground Claude, Codex, Cursor, Droid, or OpenCode turn stays armed and retries after 20 seconds rather than entering that turn's disposable input queue; only an actual delivery advances a cron, and expiry still wins. At an idle chat boundary the scheduler sends `messageSession(kind: "wake")`. Tracked CLI rows wait for a provider-specific visible composer boundary, resume ended sessions, and retry proven pre-delivery failures without consuming the occurrence. The scheduler restores timers, coalesces missed occurrences to one late fire, applies session/global pause state, cold-starts idle chats when necessary, expires recurring crons after seven days, and emits lifecycle rows while summaries and `chat.getScheduledWorkState` expose management state. Cancellation of Claude-owned jobs routes through `CronDelete` and remains visible until provider confirmation; ADE-owned rows cancel directly. There is no scheduled-work-specific spend cap. | | `computerUse/` | `computerUseArtifactBrokerService.ts`, `controlPlane.ts`, `localComputerUse.ts`, `syntheticToolResult.ts` | Proof-artifact broker (ingests, owner links, review state, routing), control-plane snapshot helpers, macOS capture capability descriptor, and the synthetic-tool-result helper used by the Claude compaction path. `proofObserver.ts` was removed in the rebuild — there is no passive auto-ingest. Direct Codex Computer Use executable resolution lives outside this folder in `main/utils/codexComputerUse.ts` because it configures provider runtimes rather than ingesting proof. | | `proof/` | `agentBrowserArtifactAdapter.ts` | Parses agent-browser payloads into broker inputs. | | `config/` | `projectConfigService.ts`, `laneOverlayMatcher.ts` | Load/save `.ade/ade.yaml` + `local.yaml`; trust enforcement; lane overlays. | @@ -722,7 +734,7 @@ Most services described here live under `apps/desktop/src/main/services/ | `pty/` | `ptyService.ts` | `node-pty` spawn, PTY I/O bridging, transcript writing. | | `remoteRuntime/` | `remoteTargetRegistry.ts`, `sshTransport.ts`, `remoteBootstrap.ts`, `remoteConnectionPool.ts`, `remoteConnectionService.ts`, `runtimeRpcClient.ts`, `runtimeDiscovery.ts` | Saved SSH machines (manual host + alternate `routes[]` with `lastSucceededAt` and manual-disconnect state), ssh-agent/key transport with bounded connect/exec timeouts and multi-route fallback, first-connect runtime upload/version/SHA verification with channel-home fallback (`.ade` / `.ade-alpha` / `.ade-beta`) and capability/version skew demoted from fatal errors to `RemoteRuntimeConnectResult.compatibilityWarnings`, remote project catalog, action dispatch (with a `projects.*` capability gate against `RemoteRuntimeCapabilities.machineProjects`), handoff storage/Git preflight, route-pinned sensitive calls, local TCP forwards for remote preview ports, reconnect/eviction with pool eviction listeners and implicit reconnect backoff, `powerMonitor` resume probe, and LAN + Tailscale discovery that returns diagnostics alongside machines. The JSON-RPC client formats remote errors with the original method name plus the JSON-RPC `code` / `message` / `data` for clearer diagnostics. See [Cross-machine session handoff](./features/sync-and-multi-device/cross-machine-session-handoff.md). | | `runtime/` | `tempCleanupService.ts`, `processRegistryService.ts`, `machineStateMigration.ts`, `packagedNodePath.ts`, `lastFailureStore.ts`, `projectRecoveryService.ts` | Runtime temp cleanup. `processRegistryService` is the per-process heartbeat registrar against machine-local `runtime_processes` (see §3.4); reconcile/dispose paths in `sessionService` and `ptyService` consult live and known owner sets before sweeping `terminal_sessions` rows so sibling processes and synced remote-machine owners are preserved. `machineStateMigration` carries one-shot migrations of the per-machine state files under `~/.ade/`. `packagedNodePath.ts` centralizes the `Resources/app*.asar(.unpacked)/node_modules` search path used by packaged runtime children. `lastFailureStore` records bounded typed project/machine failure reports and crash-loop backoff; `projectRecoveryService` runs the brain-independent diagnose/repair sequence behind `ade.recovery.*` (see [features/storage-and-recovery/README.md](./features/storage-and-recovery/README.md)). | -| `search/` | `searchService.ts`, `searchIndexDb.ts`, `searchQueryParser.ts`, `searchRanking.ts`, `terminalChunking.ts`, `searchServiceWiring.ts` | Universal search over chat/terminal/PR/commit/branch text via a disposable FTS5 index (`.ade/cache/search-index.db`, never inside `ade.db`, never synced), unioned at query time with delegated lanes/files/artifacts/Linear. Debounced off-hot-path ingestion with cursor-based incremental reads, deterministic ranking tiers, and the `search` ADE action domain (`query`/`indexStatus`/`rebuildIndex`). `searchServiceWiring.ts` is shared with the `ade` runtime so wiring can't drift. See [features/search/README.md](./features/search/README.md). | +| `search/` | `searchService.ts`, `searchIndexDb.ts`, `searchQueryParser.ts`, `searchRanking.ts`, `terminalChunking.ts`, `searchServiceWiring.ts` | Universal search over chat/terminal/PR/commit/branch text via a disposable FTS5 index (`.ade/cache/search-index.db`, never inside `ade.db`, never synced), unioned at query time with delegated lanes/files/artifacts/Linear. Accepted chat messages own the searchable document while processed/unprocessed events remain lifecycle-only; an exact `session:` query reads live ownership state, overrides stale same-document FTS metadata, and deduplicates totals. Debounced off-hot-path ingestion with cursor-based incremental reads, deterministic ranking tiers, and the `search` ADE action domain (`query`/`indexStatus`/`rebuildIndex`). `searchServiceWiring.ts` is shared with the `ade` runtime so wiring can't drift. See [features/search/README.md](./features/search/README.md). | | `sessions/` | `sessionService.ts`, `sessionDeltaService.ts`, `chatSessionProjection.ts`, `settleTerminalSession.ts` | Terminal session CRUD, post-session delta computation, provider-chat runtime projection onto resumable terminal rows, and the atomic settle/dismiss-pending-input boundary shared by IPC and ADE actions. | | `shared/` | `utils.ts`, `imageDimensions.ts`, `queueRebase.ts`, `packLegacyUtils.ts`, `transcriptInsights.ts` | Cross-domain utilities, including shared record guards and PNG/JPEG dimension parsing used by App Control and iOS Simulator capture paths. | | `state/` | `kvDb.ts`, `crsqliteExtension.ts`, `dbMaintenanceApi.ts`, `globalState.ts`, `projectState.ts`, `onConflictAudit.ts` | SQLite schema + open (WAL + `synchronous = NORMAL`), CRR extension loader, global state file, per-project state init. `kvDb` also attaches the optional `maintenance` (`DbMaintenanceApi`) handle — retention prunes, zero-peers-only cr-sqlite compaction, and fragmentation-gated vacuum — whose interface and shared retention constants live in `dbMaintenanceApi.ts` and are invoked by the storage doctor. `globalState.upsertRecentProject` accepts `preserveRecentOrder` so reactivating an already-known project (by app focus, deep link, etc.) refreshes its `lastOpenedAt` in place instead of jumping it to the front of the recents list. Recent projects use stable keys: local rows are keyed by absolute root path, remote rows by `remote::`, so a remote path string never collides with a local project. Pinned rows are retained above normal recency ordering and survive beyond the cap. `model_picker_favorites` and `model_picker_recents` are per-project CRR tables shared by desktop, TUI, and iOS; they are primary-key-only so CRR can convert them, with the recents cap enforced in `modelPickerStore.ts`. `AdeDb.sync.discardUnpublishedChangesForTables(tableNames)` lets a service clear local CRR state for specific tables without leaking those clears to sync peers — it records the cleared tables and `through_db_version` in the local-only `local_crr_change_suppressions` table, and `exportChangesSince` filters local-site rows for those tables at or below that version on the way out. The local-only excluded set (still kept out of replication) includes that suppression table itself, the snapshot caches, `local_worktree_residual_cleanups`, `pr_auto_link_ignores`, `pull_request_ai_summaries`, and `runtime_processes`. `crsql_changes` DELETE statements run through a helper that swallows the read-only-table error the cr-sqlite extension raises when a CRR-managed table is wiped, with a `db.crr_changes_cleanup_skipped` warn log instead of failing the migration. | @@ -1124,10 +1136,24 @@ The sync subsystem is **owned by the ADE runtime** (`apps/ade-cli/src/services/s advanced SSH bootstrap available. There is no pairing-link paste or manual address + PIN surface. Signed-in launches enter directly. - App launch reads pairing secret from iOS Keychain after that choice. -- Opens WebSocket candidates after direct TCP ranking, then races the authenticated hellos (250 ms stagger, at most three candidates, 10-second budget) across route diversity — a dead LAN IP no longer delays the live Tailscale/Relay route, and only `hello_ok` can win. Sends local `db_version` plus the per-host-DB cursor map (`remoteDbVersionBySite`); host replies with its `serverDbSiteId` and sends incremental catch-up changesets or, for an eligible gap over 5,000 versions, one compact ACK-gated catch-up batch through the existing chunked envelope transport. +- Opens authenticated WebSocket candidates in two phases: direct LAN then + Tailscale routes first, followed by Relay only when direct routes do not win. + Each phase uses a 250 ms stagger, bounded candidate count and connection + budget, and only `hello_ok` can win. Attempts share one correlation id and + retain only bounded host/port plus coarse failure classes. Sends local + `db_version` plus the per-host-DB cursor map (`remoteDbVersionBySite`); host + replies with its `serverDbSiteId` and sends incremental catch-up changesets + or, for an eligible gap over 5,000 versions, one compact ACK-gated catch-up + batch through the existing chunked envelope transport. - `hello_ok` can include the host's mobile project catalog and project-action feature flag. The iOS app shows a native project home until an active project is selected, can browse/open/create/clone projects on the paired machine when project actions are available, then drives `project_switch_request` / `project_switch_result`; the port stays stable across switches. - Bidirectional sync continues; inbound processing (envelope parse, gunzip, chunk reassembly, changeset decode + apply) runs off the main actor. On disconnect: a fast exponential-backoff burst, then an indefinite ~30 s slow-heartbeat retry — the phone never permanently gives up. `reconnectIfPossible` is guarded against overlapping runs. - Chat streaming resumes by sequence: each `chat_event` carries a host-assigned per-session `seq` backed by a replay buffer; `chat_subscribe` passes `sinceSeq` so reconnects replay only the missed events. A per-session hydration barrier holds the live broadcaster and transcript pump until the snapshot ack, then resumes from the pre-capture logical byte offset so concurrent appends cannot overtake or fall between snapshot and stream. The subscribe ack also carries `turnActive` (live turn state from the agent chat service) so a phone subscribing mid-turn renders streaming/stop affordances immediately even when the byte-capped snapshot tail dropped the turn's start event. `chat.getTranscript` pages older history via an opaque cursor; full runtimes advertise append-stable `cursorKind: "byte"` offsets and the minimal headless fallback advertises `cursorKind: "index"`. When the host advertises the `crossProjectChat` feature flag, `chat_subscribe` can also name a foreign (non-active) project via `projectId`/`projectRootPath`; the host streams that project's transcript read-only straight off its `.ade` transcript files, so the all-projects Hub can open any project's chat without a project switch or runtime boot. Personal subscriptions instead send `chatScope: "personal"`; the host resolves the durable transcript and active-turn state through `PersonalChatScope` with no project id. +- User-message delivery is durable across that stream: accepted messages retain + processed/unprocessed state, and unprocessed rows expose Run next / Edit / + Dismiss through idempotent `chat.resolveUnprocessedMessage`. Turn stalls and + recovery use provider-neutral `turn_health`, `turn_recovery`, and + `chat.recoverTurn`; raw moderation activity is summarized once in + `turn_diagnostics` instead of rendered as repeated cards. - Session lifecycle columns (`settled_at`, `status_note`, `attention_requested_at`, `attention_message`, `last_turn_failed_at`) replicate with `terminal_sessions`. The all-project roster carries the same additive @@ -1158,7 +1184,7 @@ The sync subsystem is **owned by the ADE runtime** (`apps/ade-cli/src/services/s input is one-at-a-time and ACK/dedupe protected by stable input ids; timeout and reconnect reuse the same id, while legacy hosts get one-shot input. - Pairing is a **user-set 6-digit PIN** stored at `.ade/secrets/sync-pin.json` on the host. The phone sends the PIN once after scanning the QR or choosing a Nearby machine; the host returns a durable per-device secret. The QR payload is a **v3 smart URL** (`https://ade-app.dev/pair#` — host identity + port + address candidates + optional cloud-relay URL, no pairing code) used only as the internal system-camera/App Clip wire encoding; there is no user-facing pairing link. Pairing is hardened with **device-bound DPoP**: iOS keeps a Secure Enclave P-256 key and every paired hello carries a signed proof (`requireDpop` / `ADE_SYNC_REQUIRE_DPOP` on the host, enforced on both the project host and the brain ingress path). -- Off-LAN transport: an account-gated **cloud tunnel relay** (`apps/tunnel-relay`, §2.7) advertised as the lowest-priority `relay` address candidate. Direct LAN/Tailscale routes remain preferred and work without an account. Relay is active whenever the host is signed in and has a current account lease; there is no separate user toggle or CLI kill-switch. Control connection/reconnection is single-flight, and a transient account-token refresh exception keeps the control route only through the last known lease expiry. Every Relay connection carries a fresh in-memory account token, and the host accepts it only when both clients are signed in to the same account; sign-out closes Relay immediately. Capable paired peers renew that authorization in place with DPoP-bound `relayReauthorizeV1`; terminal identity/proof failures close, while token-expired/verifier-unavailable failures may retry within a short grace. A claim endpoint response with exact status `409` permits one serialized machine-key/secret rotation and re-claim; other failures never rotate identity. Account-created pairing records are owner-scoped and deleted with that account, while direct QR/Nearby/SSH pairings remain local. Control-route health preserves open and bridge-validation timestamps independently and exposes its specific skip/error reason; structured lifecycle logs retain claim status, HTTP upgrade status plus a bounded response body, and close code/reason/opened state. Relay TLS terminates at the operator, so payloads are not end-to-end encrypted; adding relay E2E encryption remains planned security work. +- Off-LAN transport: an account-gated **cloud tunnel relay** (`apps/tunnel-relay`, §2.7) advertised as the lowest-priority `relay` address candidate. Direct LAN/Tailscale routes remain preferred and work without an account. Relay is active whenever the host is signed in and has a current account lease; there is no separate user toggle or CLI kill-switch. Control connection/reconnection is single-flight, and a transient account-token refresh exception keeps the control route only through the last known lease expiry. Every Relay connection carries a fresh in-memory account token, and the host accepts it only when both clients are signed in to the same account; sign-out closes Relay immediately. Capable paired peers renew that authorization in place with DPoP-bound `relayReauthorizeV1`; terminal identity/proof failures close, while token-expired/verifier-unavailable failures may retry within a short grace. A claim endpoint response with exact status `409` permits one serialized machine-key/secret rotation and re-claim; other failures never rotate identity. Signed account-directory adoption creates device-bound direct trust equivalent to QR/Nearby/SSH pairing: sign-out removes directory and Relay access but keeps that direct trust until explicit Forget. Control-route health preserves open and bridge-validation timestamps independently and exposes its specific skip/error reason; structured lifecycle logs retain claim status, HTTP upgrade status plus a bounded response body, and close code/reason/opened state. Relay TLS terminates at the operator, so payloads are not end-to-end encrypted; adding relay E2E encryption remains planned security work. - Push: APNs alert pushes (deep-linked) and Live Activity updates via `apps/push-relay` (§2.7); the phone hands tokens/prefs to the brain over the paired sync WebSocket (`push.*` runtime-scoped commands) and never talks to the relay directly. - Widgets: `ADELockScreenWidget` reads from a shared `WorkspaceSnapshot` in the App Group container. `ADEAgentActivityWidget` registers an ActivityKit Live Activity + Dynamic Island for active agent runs. Home Screen and Control Center surfaces are not registered. - Tabs: Lanes, Files, Work, PRs, CTO, Settings. @@ -1173,6 +1199,9 @@ The sync subsystem is **owned by the ADE runtime** (`apps/ade-cli/src/services/s - `.ade/local.secret.yaml` (API keys, ADE CLI configs), sync site ID, sync device ID, sync bootstrap token: **never sync**. - Each device stores its own pairing secret in OS Keychain. +- Device-bound direct pairing secrets survive account sign-out and are removed + only by explicit machine forget or the versioned trust-reset policy; Relay + authorization always requires a fresh matching account proof. - Linear creds, GitHub tokens, provider API keys stay on the host. - Commands from non-host devices validated and executed by the host only. - The release's versioned trust reset clears only saved connection grants: diff --git a/docs/features/chat/README.md b/docs/features/chat/README.md index 7c38f3918..ad4f41bca 100644 --- a/docs/features/chat/README.md +++ b/docs/features/chat/README.md @@ -755,13 +755,12 @@ happen to begin with `User request:`. 4. The runtime streams events through the main-process event emitter and into the renderer via `ade.agentChat.event` (a push channel owned by `registerIpc.ts`). - Codex turns also run a narrow no-first-output watchdog: if `turn/start` - succeeds but no useful model/tool event arrives, ADE reconciles the same - app-server thread with `thread/read` and `thread/turns/list` before - surfacing a `codex_turn_stalled` event plus one visible `system_notice`. - The watchdog never auto-handoffs or interrupts; parent/orchestrator - sessions receive the structured stall event and decide whether to wait, - steer, interrupt, or retry the same thread. + Codex turns also run the watchdog described below: if `turn/start` succeeds + but no useful model/tool event arrives, ADE reconciles the same app-server + thread with `thread/read` and `thread/turns/list`, attempts at most one + restart + thread resume, then publishes provider-neutral `turn_health` when + manual recovery is still needed. Parent/orchestrator sessions receive the + structured event with `sourceSessionId` and target the owning child. 5. On completion the service emits `status: "completed" | "failed" | "interrupted"`, optionally emits a `turn_diff_summary`, flushes buffered text, marks the session idle, and pulls the next queued steer. @@ -881,6 +880,59 @@ cold-start. Project/window close probes still fail closed: if the chat workload probe throws, ADE keeps the project alive instead of closing over a possibly running agent. +### Message delivery, turn health, and quiet diagnostics + +The transcript is the durable truth for whether ADE merely accepted a message +or the provider actually processed it. A user message may move through +`accepted` to `processed`; if the active turn ends without consuming an +accepted steer, the service persists `unprocessed`. Desktop, ADE Code, hosted +web, and iOS fold those lifecycle snapshots onto one user bubble rather than +showing duplicate messages. An unprocessed bubble offers three explicit +outcomes: + +- **Run next** sends the original text and attachments as a new turn only after + the current turn is idle. +- **Edit** restores the original content to the composer without changing the + durable message. +- **Dismiss** records that no turn should be created. + +Run next and Dismiss converge through +`chat.resolveUnprocessedMessage` / `resolveUnprocessedMessage()` and a durable +`user_message_resolution` event. The replacement message carries +`metadata.replayedFromUnprocessedSteer`; that replacement is the dispatch +commit point. If the runtime stops after dispatch but before writing the +resolution receipt, a later retry reconstructs the missing receipt and still +resolves to Run next. Concurrent clients are serialized by session + steer id, +so Dismiss cannot race a replacement turn and repeated calls return the +already-completed outcome. + +Turn stalls use the provider-neutral `turn_health` event and +`chat.recoverTurn`; `codex_turn_stalled` / `recoverCodexTurn` remain readable +for older clients. The event names the provider, owning turn, timestamps, +recovery count, whether automatic recovery was attempted, and the supported +actions (`wait`, `nudge`, `retry_same_runtime`, `restart_resume`). When a +child's health event is mirrored into a parent, `sourceSessionId` keeps every +surface's recovery action targeted at the child. Recovery progress is recorded +as `turn_recovery`, so a successful recovery replaces the stale warning instead +of leaving contradictory cards. + +Codex is the first provider with an active watchdog behind that shared +contract. A turn that produces no useful model/tool event for 120 seconds is +reconciled against app-server thread state. ADE backfills any missed items or +terminal state it finds; otherwise it attempts one app-server restart + thread +resume per session before exposing manual recovery. After useful progress, +10 minutes without further model/tool activity produces a non-destructive +stall warning. Pending approvals and user input suspend reconciliation, and +answering them re-arms the 10-minute progress window. + +Moderation metadata is operational evidence, not a conversational response. +Raw `codex_moderation_metadata` events are retained for compatibility but do +not render as individual cards. The service counts checks per turn and merges +them with deduplicated optional-integration startup failures in one +`turn_diagnostics` snapshot. Transcript folds keep only the latest snapshot per +turn; graphical clients render a quiet **Turn details** disclosure and ADE Code +renders one compact details line. + ## Spawn types and completion reporting A chat can spawn another chat. The relationship is captured by @@ -1077,19 +1129,20 @@ Provider connection management lives on the `ade.ai.*` surface (handled in `regi summary cannot put the UI back into running/Stop state. Keep every renderer rehydration path on this helper so failures always restore a sendable composer. -- **Codex runtime recovery events.** MCP startup status notifications are - warnings, not model progress. Do not let them clear the no-first-output - watchdog. If app-server state can be read, recovered turn items are - backfilled into the transcript and terminal turn state is finalized; only - a genuinely silent or unreadable turn emits `codex_turn_stalled`. The - desktop transcript, `ade code`, and iOS should render that structured - event as the recovery surface; do not hide it as a generic activity row. - Desktop recovery buttons call `recoverCodexTurn` and must remain scoped to - the active turn: Wait re-arms reconciliation, Nudge sends a progress steer, - Retry interrupts/finalizes the turn before retrying in the same thread, and - Resume additionally tears down app-server and resumes the persisted thread. - Only one action may run for a session/turn, and a late card must fail rather - than mutating a newer turn. +- **Turn-health events are provider-neutral; the Codex watchdog is the first + producer.** MCP startup status and moderation notifications are diagnostics, + not model progress. Do not let them clear the no-first-output watchdog. If + app-server state can be read, recovered turn items are backfilled into the + transcript and terminal turn state is finalized; only a genuinely silent or + unreadable turn emits the neutral `turn_health` plus its legacy + `codex_turn_stalled` twin. Desktop, ADE Code, hosted web, and iOS prefer the + neutral event, preserve a mirrored child event's `sourceSessionId`, and avoid + rendering both forms. Recovery calls `recoverTurn` when advertised and falls + back to `recoverCodexTurn` for older hosts. Wait re-arms reconciliation, + Nudge sends a progress steer, Retry interrupts/finalizes the turn before + retrying on the same runtime, and Resume additionally tears down app-server + and resumes the persisted thread. Only one action may run for a session/turn, + and a late card must fail rather than mutating a newer turn. App-server `thread/deleted` notifications must clear live turn state, stop active Codex subagents, and remove the persisted thread id so the next user message starts from a fresh thread. diff --git a/docs/features/search/README.md b/docs/features/search/README.md index 1f9b38549..fd9ae246a 100644 --- a/docs/features/search/README.md +++ b/docs/features/search/README.md @@ -29,7 +29,8 @@ Main-process service (`apps/desktop/src/main/services/search/`): `sources` table, the deferred `startBackfill` reconcile pass, and `query` (FTS candidate fetch + query-time delegation + deterministic ranking + cursor pagination). Public surface: `query`, `indexStatus`, `rebuildIndex`, - `startBackfill`, the `notify*` hooks (`notifyChatEvent`, + `startBackfill`, exact-session live lookup (which overrides stale indexed + metadata for the same document), the `notify*` hooks (`notifyChatEvent`, `notifyTerminalData`, `notifySessionChanged`, `notifyPrChanged`, `notifyLaneActivity`), `processPendingNow` (tests/rebuild), `dispose`. - `searchIndexDb.ts` — opens/creates the disposable index DB. Owns the DDL @@ -159,6 +160,11 @@ re-derive their docs wholesale each run. `startBackfill` runs once, well past the host boot window, enqueues every session/PR/lane, and reconciles docs whose sessions were deleted while the service was down. +For chat, the accepted user-message event owns the searchable message body. +Later processed/unprocessed lifecycle snapshots update delivery state but do +not create another searchable document, so reconnect replay and resolution +events cannot duplicate one message in search. + ### Deterministic ranking tiers Ranking is exactly specifiable and stable — the same query over the same corpus @@ -182,13 +188,22 @@ validated form and can be mixed with inline filters. Linear is opt-in: it can hit the network, so it is excluded from the default kind set and only consulted when a caller explicitly asks for `kind:linear`. +An exact `session:` filter is also a freshness contract. The service reads +the owning session directly before relying on the disposable index, replaces +stale FTS metadata for that same document with the live result, and deduplicates +the merged candidate set and totals. A just-accepted or just-resolved message +therefore appears once even when the background ingestion debounce has not run. + ### Query-time delegation vs. FTS Only chat, terminal, PR, commit, and branch text is FTS-indexed. Lanes, files, artifacts, and Linear issues are **delegated at query time** to their owning service so results are always fresh and nothing duplicates an authoritative -store. FTS candidates and delegated candidates are ranked together through the -one comparator, then paginated with an opaque base64 cursor. +store. Exact session lookup is a narrow live-source exception for indexed chat +and terminal ownership: it supplements the FTS cache, then replaces stale +same-document metadata rather than appending a duplicate. FTS candidates and +delegated candidates are ranked together through the one comparator, then +paginated with an opaque base64 cursor. ### Caller scoping policy diff --git a/docs/features/sync-and-multi-device/README.md b/docs/features/sync-and-multi-device/README.md index 13c000fe2..63983ee53 100644 --- a/docs/features/sync-and-multi-device/README.md +++ b/docs/features/sync-and-multi-device/README.md @@ -3,9 +3,10 @@ ADE syncs live runtime state across an ADE machine runtime and any connected controllers (other Macs, iPhones) using **cr-sqlite** as a CRDT-backed replication layer over a **WebSocket** transport. The design is local-first: -direct LAN/Tailscale routes are preferred, while the account-gated cloud tunnel -relay is a byte-transport fallback whenever the machine is signed in to an ADE -account. There is no separate relay toggle. Two +eligible routes are ordered **LAN → Tailscale → Relay** for desktop-to-runtime, +ADE Code, and iOS connections. The account-gated cloud tunnel relay is a +byte-transport fallback whenever the machine is signed in to an ADE account; +there is no separate relay toggle. Two machines on the same LAN (or Tailscale tailnet) converge their application state directly. @@ -88,6 +89,42 @@ The older terms "brain" and "host" still appear in code, schema, and protocol types. In the current product vocabulary, they refer to the same thing: the runtime that is the current **sync authority**. +## Connection route policy + +Every native paired controller uses the same phase order: + +1. current LAN endpoints; +2. current Tailscale/tailnet endpoints; +3. ADE Relay, only with a fresh matching account proof. + +Within the LAN and tailnet phases, current discovery outranks stale saved +metadata and recent successful endpoints break ties. iOS may race several +authenticated candidates within a direct phase, but Relay does not race ahead +of an eligible direct phase; it starts only after direct routes exhaust. ADE +Code and the desktop paired-runtime pool use the same +`buildPairedEndpointCandidates` ordering. First-time same-account adoption also +uses LAN → tailnet → Relay when the directory row contains a verifiable host +signing key, because the `ade-adopt-v1` sealed challenge protects the account +credential on direct routes. An unsigned legacy directory host is Relay-only; +ADE never sends a plaintext account bearer to an unverified LAN/tailnet peer. + +The hosted HTTPS web client applies the same ranking to routes the browser may +legally dial, but browsers cannot open insecure `ws://` LAN/Tailscale sockets +from `https://app.ade-app.dev`. In production that eligibility filter normally +leaves Relay only; local HTTP development and previously verified secure +direct endpoints can exercise the direct phases. + +Successful and failed native paired-runtime connects retain one random +correlation id plus at most eight privacy-safe ordered attempts. An attempt +contains route kind, host + optional port, start time, duration, outcome, and a +coarse failure class; paths, query strings, tokens, pairing secrets, and raw +error text are excluded. The same correlation id is forwarded on Relay dials +and appears in tunnel lifecycle logs. Account-directory list/register/delete +requests send `X-ADE-Correlation-ID`; the Worker validates or replaces it, +returns it on every response, exposes it to the trusted web origin, and writes +one structured completion record with route, method, status class, and +duration. + ## Mobile compatibility contract Mobile clients must be able to connect to older and newer ADE brains long enough @@ -225,7 +262,10 @@ Runtime support files outside `services/sync/`: written account-owned credential if sign-out or an account switch wins the race. The directory's `online` field is a short presence lease, not a transport verdict: a machine with a verified secure Relay endpoint remains - connectable after that presence bit expires. + connectable after that presence bit expires. Every HTTP operation carries + one bounded correlation id across the initial request and its one auth-refresh + retry, so a user-visible failure can be joined to the Worker's structured + lifecycle record without logging an account token or response body. - `apps/ade-cli/src/services/account/accountMachinePublisherService.ts` — the single machine-brain publisher for the account directory. It derives the stable machine key from the cloud-relay store, publishes only currently @@ -303,7 +343,10 @@ Runtime support files outside `services/sync/`: register/list/delete Worker routes. Machine listing selects the owner's 500 most recently seen rows before computing online-first order and exposes separate authentication and D1 durations through `Server-Timing`; the - trusted web-client CORS response exposes that header. + trusted web-client CORS response exposes that header. Every request also + receives a validated/generated `X-ADE-Correlation-ID`, echoed on the + response and included in one privacy-safe structured completion log; trusted + web CORS exposes the id and allows the request header. - `apps/desktop/src/shared/accountDirectory.ts` — canonical account-directory origin, bounded success/error response decoding, route allowlisting, machine selection, and paired endpoint validation shared by desktop, the brain, ADE @@ -951,8 +994,8 @@ iOS service files (`apps/ios/ADE/Services/`): - `SyncService.swift` — WebSocket client, envelope encoding (zlib), command routing, keychain integration, PIN-based pairing, the sealed `ade-adopt-v1` account-adoption client (challenge/verify against the - directory `pubkey`, sealed `account_sealed` hello, and relay → Tailscale → - LAN route fallback with per-stage progress and a PIN-pairing fallback), lane + directory `pubkey`, sealed `account_sealed` hello, and LAN → Tailscale → + Relay route fallback with per-stage progress and a PIN-pairing fallback), lane presence announcements, terminal subscribe/unsubscribe tracking, terminal input/resize senders, mobile CLI launch/continuation, external-session list/import commands for Work, @@ -1115,7 +1158,12 @@ access-token refresh. Only a repeated 401/403 is classified as failures remain retryable and do not erase pairing trust. Account adoption captures the account owner/session generation and rechecks it before and after credential persistence so a late result cannot recreate trust after sign-out -or an account switch. +or an account switch. Once the host has minted a device-bound paired secret, +that host-issued direct trust is distinct from the account session that found +the machine: sign-out removes directory visibility and Relay authorization but +does not delete the secret needed for LAN/Tailscale reconnect. Forgetting the +machine is the explicit trust-deletion boundary. Signing into a different +account cannot use the previous account's directory or Relay lease. Relay has two related but distinct leases. The machine's control tunnel may survive a transient refresh failure only until its last known account-token @@ -1189,9 +1237,9 @@ grace. Older peers close exactly when their initial token expires. Tailscale IP or DNS name, and only falls back to the opaque `saved` kind for a host that no longer matches the live address set. This is what lets the account directory publish a LAN-backed saved host as a real LAN endpoint. iOS treats - relay as an - automatic fallback after direct LAN/Tailscale routes and promotes it when the phone has no Tailscale - tunnel (see the transport race in `ios-companion.md`). Already-paired + Relay as a separate authenticated fallback phase after direct + LAN/Tailscale routes exhaust (see the transport race in + `ios-companion.md`). Already-paired phones also learn the relay URL from `hello_ok` / `brain_status` (`cloudRelayWssUrl`) and persist it with the host profile for reconnects. @@ -1566,9 +1614,9 @@ feature is merged or because a deliberately isolated-port host is running. outbound HMAC-authenticated tunnel to the `apps/tunnel-relay` Cloudflare Worker so a phone off the LAN/tailnet can dial the machine over TLS with zero configuration. - Phones prefer direct routes when they are currently usable; when an - iPhone has no Tailscale tunnel and holds a saved relay URL, reconnect - dials the relay ahead of stale saved LAN/Tailscale sweeps. The relay + Phones attempt authenticated direct routes in LAN → Tailscale order and use + Relay only as the separate fallback phase. Candidate work is bounded, so a + stale saved LAN/Tailscale endpoint cannot hold Relay off indefinitely. The relay pipes WebSocket bytes after terminating TLS. The normal ADE hello / PIN / paired-secret / DPoP handshake still runs inside that pipe, but it is not end-to-end encrypted: the relay can read paired secrets and runtime/sync @@ -1580,9 +1628,10 @@ feature is merged or because a deliberately isolated-port host is running. signed in on the host; the proof is never persisted. Direct LAN/Tailscale hellos do not need an account token. Sign-out, account switch, expiry, or a refresh failure after the last known lease has expired closes Relay peers and - revokes account-owned pairing records; a transient refresh exception while - that lease is current leaves the route intact. Direct QR/Nearby/SSH records - carry local provenance and survive. A verified + removes directory access; a transient refresh exception while that lease is + current leaves the route intact. The device-bound paired secret remains + available for direct LAN/Tailscale reconnect regardless of whether it was + minted after PIN pairing or sealed same-account adoption. A verified same-owner account hello with the pinned DPoP key may rotate the paired secret so a lost credential-delivery response can be retried safely. The `machineKey` is an unguessable 32-hex identifier and the tunnel @@ -1661,7 +1710,7 @@ feature is merged or because a deliberately isolated-port host is running. | Device-bound pairing (DPoP, Secure Enclave P-256) | Implemented (host + brain ingress; `requireDpop` / `ADE_SYNC_REQUIRE_DPOP`) | | Cloud tunnel relay (off-LAN transport, `relay` candidate) | Implemented whenever the host is signed in, with no separate toggle and with same-account per-connection proof (`syncTunnelClientService` + `apps/tunnel-relay`) | | Relay end-to-end self-probe + zombie-control detection (honest relay publication) | Implemented (`syncRelaySelfProbe`, JSON control keepalive, `sync.runSelfProbe`, `ade doctor` relay check) | -| Sealed account adoption over direct routes (`ade-adopt-v1`, host `pubkey` identity, relay → tailnet → LAN fallback, negotiated ChaCha20-Poly1305 / AES-256-GCM AEAD) | Implemented (`machineIdentitySigningStore` + `adoptChannelCrypto`; desktop + iOS clients) | +| Sealed account adoption over direct routes (`ade-adopt-v1`, host `pubkey` identity, LAN → tailnet → Relay fallback, negotiated ChaCha20-Poly1305 / AES-256-GCM AEAD) | Implemented (`machineIdentitySigningStore` + `adoptChannelCrypto`; desktop + iOS clients) | | Push notifications + Live Activities (APNs relay) | Implemented (see `push-notifications.md`; on-device E2E needs a physical iPhone) | | Tailscale integration | Implemented (address candidate + mDNS TXT + per-node `tailscale serve` publication on the live sync port) | | Clean, published lane + Work chat handoff between connected desktops | Implemented ([contract](./cross-machine-session-handoff.md)) | diff --git a/docs/features/sync-and-multi-device/ios-companion.md b/docs/features/sync-and-multi-device/ios-companion.md index cf57e0e71..f2e158ade 100644 --- a/docs/features/sync-and-multi-device/ios-companion.md +++ b/docs/features/sync-and-multi-device/ios-companion.md @@ -41,21 +41,22 @@ reconnect without asking for the PIN again. A signed-in launch enters the app directly. Choosing a signed-in account machine performs first-time adoption through the -directory-verified WSS relay. The phone stores the returned per-device secret -and DPoP key for direct reconnects, and adds a fresh in-memory account token to -every later Relay hello. The token is never saved with the machine. Those -profiles are tagged with the Clerk user id that created them. Signing out, -switching accounts, or confirmed session loss removes only that account's -profiles and Keychain pairing secrets. Machines paired directly by QR, -Nearby, or SSH have no account owner and remain saved after -sign-out. - -Pairing ownership and Relay eligibility are intentionally separate. A direct -profile can learn verified Relay metadata for the current account without -becoming account-owned. On sign-out or account switch, an active Relay socket -is closed immediately and ADE tries the saved LAN/Tailscale routes; the direct -profile and its pairing secret remain. A transient Clerk or directory outage is -not treated as logout and does not erase saved machines. +directory using LAN, Tailscale, then Relay. LAN/Tailscale adoption is allowed +only when the directory row contains the host's Ed25519 signing key and the +phone verifies a sealed `ade-adopt-v1` challenge before releasing the account +credential; an unsigned legacy host remains Relay-only. The phone stores the +returned per-device secret and DPoP key for direct reconnects, and adds a fresh +in-memory account token to every later Relay hello. The account token is never +saved with the machine. + +Device-bound machine trust and account transport authorization are separate. +Signing out, switching accounts, or confirmed session loss closes an active +Relay socket, removes account-directory visibility, and blocks every saved +Relay route. It does not delete the host-issued paired secret, DPoP key, or +machine profile needed for a LAN/Tailscale reconnect. ADE immediately tries +those direct routes, and **Forget machine** remains the explicit trust-deletion +boundary. A transient Clerk or directory outage is not treated as logout and +does not erase saved machines. ### Pair with SSH @@ -420,8 +421,10 @@ The Work model/activity parity path is concentrated in these files: `WorkUsageActivityCarousel`. - `ADE/Services/SyncService.swift`, `ADE/Views/Work/WorkSessionDestinationView.swift`, and `WorkSessionDestinationView+Actions.swift` — host-advertised chat action - dispatch, including `chat.recoverCodexTurn` for stalled-turn buttons and the - non-queueable `chat.cancelScheduledWork` wrapper used by Chat Info. + dispatch, including provider-neutral `chat.recoverTurn` (with the legacy + Codex action as a compatibility fallback), durable + `chat.resolveUnprocessedMessage`, and the non-queueable + `chat.cancelScheduledWork` wrapper used by Chat Info. Deployment target: iOS 26+. iPhone and iPad (adaptive layouts planned for Phase 7). @@ -644,14 +647,14 @@ Sources: `apps/ios/ADE/Services/SyncService.swift` and connect path's recovery sweep. Cloud-relay candidates (full `wss://…/connect/` URLs) are zero-config and carry their own path/port, so they are never mixed into the host:port TCP probe - or fallback-port sweep. When the phone has a Tailscale tunnel, - direct routes stay preferred. When the phone has no Tailscale tunnel - and a saved relay route exists, reconnect probes only currently live - same-LAN routes, then dials the relay before stale saved LAN/Tailscale - routes; this prevents "Tailscale off" from waiting behind a dead - tailnet sweep. Pairing uses the same endpoint ordering, so a phone - without Tailscale can pair through relay without extra user setup. - Ordered endpoints then enter an **authenticated** race: candidates start + or fallback-port sweep. Connection proceeds in the same explicit phases as + desktop and ADE Code: LAN, then Tailscale, then authenticated Relay. + Direct candidates enter an **authenticated** race first; only after that + phase exhausts does a separate Relay race begin. A phone with Tailscale + disabled skips ineligible tailnet routes but does not promote Relay ahead of + a currently usable LAN route. Pairing and ordinary reconnect share this + policy, so the route order does not change after a pairing code succeeds. + Within each race, candidates start 250 ms apart, at most three are active, the whole wave has a 10-second budget, and only a completed `hello_ok` can win. The initial wave covers the best candidate plus transport/route diversity, and failures admit the @@ -670,8 +673,9 @@ Sources: `apps/ios/ADE/Services/SyncService.swift` and fresh account token in memory for every Relay attempt and adds it only to the paired hello. Missing/different account state reports a clear sign-in or same-account requirement without consuming the reconnect retry budget; LAN - and Tailscale attempts remain available. `accountOwnerId` separately marks - profiles created by account adoption and therefore deleted on owner loss. + and Tailscale attempts remain available. A host-issued paired secret remains + direct-route trust after account loss; forgetting the machine is the action + that removes the profile and Keychain secret. The primary Settings status stays route-neutral (for example, "Connected in 0.3s"); the host-observed LAN/Tailscale/relay route remains available in the diagnostics row for troubleshooting. `reconnectIfPossible` is the single @@ -1711,11 +1715,11 @@ different machine's cached limits. | PIN pairing flow | Implemented | | QR pairing payload (v3 smart URL) + camera scanner (`SettingsPairingScannerSheet`) | Implemented | | Account launch gate + account machine directory | Implemented; sign-in is the primary PIN-less path, while signed-out launches can continue with QR + PIN, Nearby + PIN, or advanced SSH pairing | -| Account-owned pairing and same-account Relay lifetime | Implemented; exact session-generation commit checks, owner-scoped deletion, fresh Relay token per connection | +| Account discovery + device-bound direct trust | Implemented; signed directory adoption, exact session-generation commit checks, direct trust retained across sign-out, fresh Relay proof per connection | | SSH one-time pairing bootstrap | Implemented; explicit host fingerprint trust, JSON-stdin device grant, optional Keychain recovery credentials | | One-time mobile machine-trust reset | Implemented; clears connection tokens/profiles after update while preserving account and stable device/DPoP identity | | Device-bound pairing (DPoP, Secure Enclave P-256) | Implemented (`DpopKeyService`; signed proof on every paired hello) | -| Cloud relay (same-account `relay` transport, promoted when the phone has no Tailscale tunnel) | Implemented; fresh in-memory account proof on every Relay connection | +| Cloud relay (same-account `relay` transport after LAN and Tailscale direct routes) | Implemented; fresh in-memory account proof on every Relay connection | | Project home + machine project switching | Implemented, including Add project actions for browsing/opening existing Git repos, creating local projects, cloning GitHub repos on the paired machine, and removing projects from the list | | Hub personal chats | Implemented; runtime-scoped list/create/read/send/interactive actions, owner-only scheduled-work creation capability, controller Cancel/Pause actions, per-host offline summary cache, explicit personal transcript subscriptions, native new-chat/model flow, Chat Info Cancel/Pause controls, and project/lane actions suppressed | | Lanes tab | Implemented to live machine parity (with `devicesOpen`, multi-attach, stack canvas, stack-position/base-branch editing in Manage Lane, and template environment progress) | @@ -1763,12 +1767,14 @@ different machine's cached limits. account keys, the stable device id and DPoP identity, analytics preferences, and pairing-PIN state. The completion marker is written only after Keychain token clearing succeeds, so a failed Keychain operation retries next launch. -- **Account ownership and Relay ownership are different fields.** - `accountOwnerId` decides whether sign-out deletes a saved profile; - `relayAccountOwnerId` decides whether its Relay route may be used. Never make - a QR/Nearby/SSH profile account-owned just because the same Mac - later appears in the account directory. On account loss, disconnect Relay - and retry direct routes; delete only account-owned profiles. Treat transient +- **Device-bound direct trust and account-bound Relay access have different + lifetimes.** A successful signed directory adoption creates the same + machine-scoped direct credential as QR, Nearby, PIN, or SSH pairing. Sign-out + removes directory and Relay authorization but retains that direct credential, + so LAN and Tailscale reconnects keep working; "Forget machine" is the user + boundary that deletes it. Never make a QR/Nearby/SSH profile Relay-dependent + just because the same Mac later appears in the account directory. On account + loss, disconnect Relay and retry direct routes. Treat transient account/directory failures as retryable, not as logout. The directory's `online` flag is only its 90-second publisher lease: if a row still contains a verified secure endpoint, the phone may dial it and let the authenticated @@ -1875,14 +1881,19 @@ different machine's cached limits. new event or card type, keep the phone's decoders tolerant — a foreign event should degrade to "that one event is missing", never "the whole transcript is gone". -- **Codex app-server chat events are first-class on Work.** Mobile - mirrors the desktop/TUI Codex runtime rows by decoding - `codex_safety_buffering`, `codex_moderation_metadata`, `codex_sleep`, - `codex_thread_deleted`, and `codex_turn_stalled` in - `RemoteModels.swift`, mapping them through `WorkEventMapping.swift`, - and rendering compact timeline cards. Stalled rows preserve the - `sourceSessionId` from child chats and expose Wait / Nudge / Retry / Resume - only when the host advertises `chat.recoverCodexTurn`. Web-search events also carry +- **Turn delivery and recovery are provider-neutral on Work.** Mobile mirrors + desktop and ADE Code by decoding the durable message-delivery lifecycle, + `turn_health`, `turn_recovery`, `turn_diagnostics`, and legacy provider + events in `RemoteModels.swift`, then mapping them through + `WorkEventMapping.swift`. Accepted-but-unprocessed user messages remain + visible and expose Run next / Edit / Dismiss only when the host advertises + `chat.resolveUnprocessedMessage`; those resolutions are durable and + idempotent across reconnects. Stalled rows preserve the `sourceSessionId` + from child chats and expose Wait / Nudge / Retry / Resume through + provider-neutral `chat.recoverTurn`, with the Codex-specific action retained + only as a compatibility fallback. Raw moderation checks are not rendered as + repeated cards: counts and any integration failures are summarized in one + quiet turn-diagnostics disclosure. Web-search events also carry provider action metadata (`query` / `queries`, `title`, `url`, `snippet`); Work keeps those in the enriched web-search tool card so URLs are visible without duplicating the same event as a second row. diff --git a/docs/features/sync-and-multi-device/remote-commands.md b/docs/features/sync-and-multi-device/remote-commands.md index 77e873905..ca0eed95e 100644 --- a/docs/features/sync-and-multi-device/remote-commands.md +++ b/docs/features/sync-and-multi-device/remote-commands.md @@ -250,9 +250,26 @@ the cursor opaque and use `cursorKind` only to select their local merge strategy. - `create`, `send`, `interrupt`, `steer`, `cancelSteer`, `editSteer`, `dispatchSteer`, `cancelDispatchedSteer`, `approve`, `respondToInput` +- `recoverTurn`, legacy `recoverCodexTurn`, `resolveUnprocessedMessage` - `restart`, `updateSession`, `archive`, `unarchive`, `delete`, `models`, `modelCatalog` +`chat.recoverTurn` is the provider-neutral stall-recovery action. It takes +`{ sessionId, turnId, action }`, where `action` is `wait`, `nudge`, +`retry_same_runtime`, or `restart_resume`. It is viewer-allowed and +non-queueable: recovery must be applied to the currently active turn, never +replayed after reconnect. Older hosts may advertise only +`chat.recoverCodexTurn`; clients map the same four controls to its legacy +action names without rendering duplicate recovery cards. + +`chat.resolveUnprocessedMessage` takes +`{ sessionId, steerId, action: "run_next" | "dismiss" }`. It is also +viewer-allowed and non-queueable. The runtime persists a terminal +`user_message_resolution` receipt and makes both actions idempotent across +client retries and runtime restarts. `run_next` is accepted only while the +session is idle and commits only after the replacement turn is dispatched; +`dismiss` never sends a turn. + `chat.resolveSmartLinkPreview` is a viewer-allowed, non-mutating enrichment read. Its `{ url }` payload returns the shared deterministic provider/kind/label shape and may add a title or bounded favicon data URL. GitHub and Linear titles diff --git a/docs/features/web-client/README.md b/docs/features/web-client/README.md index 40438c37c..bd3643b9b 100644 --- a/docs/features/web-client/README.md +++ b/docs/features/web-client/README.md @@ -15,6 +15,14 @@ their saved direct routes, but the hosted client no longer creates non-account pairings. Its retired `/pair` route discards the payload and opens the normal account sign-in flow. +The browser uses the same abstract route order as native controllers: +LAN → Tailscale → Relay. Eligibility is surface-specific, however: production +HTTPS blocks insecure `ws://` LAN and tailnet addresses, so they are filtered +before dialing and Relay is normally the only eligible route. Local HTTP +development can exercise the direct candidates. Each connection cycle uses one +correlation id, bounded endpoint metadata, and coarse failure classes so browser, +directory, and relay logs can be joined without exposing credentials or URLs. + Production hosting is Cloudflare Pages. The Pages URL is `https://ade-web-client.pages.dev`; the canonical product URL in source is `https://app.ade-app.dev` (`WEB_CLIENT_BASE_URL`). The canonical domain is live @@ -83,7 +91,8 @@ Browser sync client: - `apps/desktop/src/renderer/webclient/sync/connection.ts` - WebSocket lifecycle, account adoption, paired hello, DPoP proof on reconnect, heartbeat, reconnect/backoff, project catalog chunks, and auth-failure - attribution. DPoP/token preparation begins in parallel with the socket dial; + attribution, with one correlation id spanning directory lookup and route + attempts. DPoP/token preparation begins in parallel with the socket dial; transport open and authenticated hello have separate 8-second and 12-second deadlines. Relay authorization is renewed in place ahead of expiry even while the tab is hidden, so background timer throttling does not turn a @@ -100,8 +109,9 @@ Browser sync client: account ownership, filters Relay routes while signed out, and surfaces the sign-in-required state before a socket is opened. - `apps/desktop/src/renderer/webclient/sync/endpoints.ts` - derives the - browser-safe endpoint list. Relay and explicit `wss://` are dialable from the - hosted page; plain `ws://` is dialable only from local/http pages. + ordered browser-safe endpoint list (LAN, Tailscale, Relay). Relay and explicit + `wss://` are dialable from the hosted page; plain `ws://` is dialable only + from local/http pages. - `apps/desktop/src/renderer/webclient/sync/envStore.ts` - IndexedDB storage for paired machine environments, per-device secret, host/candidate metadata, the WebCrypto `CryptoKeyPair`, and the separate account-session object store. @@ -315,7 +325,9 @@ Account connection and entry points: - `apps/account-directory/src/directory.ts` - Clerk-scoped machine register/list/delete routes. The list query reads the 500 most recently seen rows, computes online-first order, and exposes auth/D1 durations through the - CORS-visible `Server-Timing` header. + CORS-visible `Server-Timing` header. It accepts, reflects, and CORS-exposes + `X-ADE-Correlation-ID`, then records it with route, method, status, and + duration for credential-free connection tracing. - `apps/desktop/src/renderer/components/settings/SyncDevicesSection.tsx` - the focused **Connections > Phone** and **Connections > Web** tab bodies plus the shared **This Mac** card. The Web variant is account-sign-in only; the