From 7d12adda14d40168a01aba2e678094117e8bdb85 Mon Sep 17 00:00:00 2001 From: Pedram Amini Date: Fri, 14 Aug 2026 22:28:03 -0500 Subject: [PATCH 01/66] MAESTRO: add A Cappella architecture docs (system overview, protocol, 2 ADRs) --- docs/.mintignore | 5 + .../decisions/adr-001-webrtc-transport.md | 129 +++++++ .../decisions/adr-002-main-process-session.md | 144 ++++++++ .../architecture/acappella/system-overview.md | 213 +++++++++++ .../acappella/voice-session-protocol.md | 341 ++++++++++++++++++ 5 files changed, 832 insertions(+) create mode 100644 docs/architecture/acappella/decisions/adr-001-webrtc-transport.md create mode 100644 docs/architecture/acappella/decisions/adr-002-main-process-session.md create mode 100644 docs/architecture/acappella/system-overview.md create mode 100644 docs/architecture/acappella/voice-session-protocol.md diff --git a/docs/.mintignore b/docs/.mintignore index caafba3181..c3fbc8024e 100644 --- a/docs/.mintignore +++ b/docs/.mintignore @@ -3,3 +3,8 @@ # `{...}` examples that are valid Markdown but not valid MDX, so exclude the whole # directory from the Mintlify build to keep the deploy green. agent-guides/ + +# Internal architecture notes and ADRs (structured markdown with YAML front +# matter and `[[wiki-links]]` for DocGraph, not part of docs.json navigation). +# Same MDX-hostile `{...}` type examples as above. +architecture/ diff --git a/docs/architecture/acappella/decisions/adr-001-webrtc-transport.md b/docs/architecture/acappella/decisions/adr-001-webrtc-transport.md new file mode 100644 index 0000000000..533bbf2ce9 --- /dev/null +++ b/docs/architecture/acappella/decisions/adr-001-webrtc-transport.md @@ -0,0 +1,129 @@ +--- +type: decision +title: 'ADR-001: WebRTC for the phone audio leg' +created: 2026-08-14 +tags: + - voice + - architecture + - acappella + - adr +related: + - '[[system-overview]]' + - '[[voice-session-protocol]]' + - '[[adr-002-main-process-session]]' +--- + +# ADR-001: WebRTC for the phone audio leg + +**Status:** Accepted +**Date:** 2026-08-14 +**Deciders:** A Cappella Phase 01 + +## Context + +A Cappella's iPhone client carries live microphone audio to the desktop and live speech back. The +desktop already runs an authenticated WebSocket at `/$TOKEN/ws` +(`src/main/web-server/routes/wsRoute.ts`) with an established client registry and a broadcast +fan-out (`src/main/web-server/services/broadcastService.ts`). The obvious cheap move is to encode +Opus frames and push them over that socket in both directions. + +The obvious cheap move is wrong, and it is worth writing down why so nobody re-proposes it in a +later phase. + +## Decision + +**The phone's media leg is WebRTC. The existing authenticated WebSocket carries signaling only.** + +Signaling (offer, answer, ICE candidates) rides `/$TOKEN/ws`, so the phone leg inherits the token +auth, the client registry, and the connection lifecycle that already exist. Media does not touch +that socket. + +Control events from [[voice-session-protocol]] also ride the WebSocket. Only audio moves to the +peer connection. + +## Rationale + +### Acoustic echo cancellation + +This is the decisive reason. The phone plays the assistant's speech through its speaker while its +microphone is open for barge-in. Without echo cancellation the microphone hears the assistant, +the STT transcribes it, and the system talks to itself. Barge-in, the single most important +interaction in the whole feature, becomes impossible. + +WebRTC's audio pipeline provides AEC, noise suppression, and automatic gain control as part of +`getUserMedia` plus the peer connection, tuned by people who have spent two decades on it. A raw +WebSocket carrying Opus frames provides none of them. Building AEC by hand is not a weekend +project; it is the reason conferencing products have audio teams. + +### Jitter buffer + +Networks deliver audio unevenly. WebRTC maintains an adaptive jitter buffer that trades a few +milliseconds of latency for smooth playback and adapts as conditions change. Over a WebSocket we +would write our own, and a hand-rolled fixed buffer is either too small (audible gaps) or too +large (barge-in feels laggy, which reads as the assistant ignoring you). + +### Packet loss concealment + +Opus over WebRTC conceals lost packets by synthesizing plausible audio for the gap. A lost frame +degrades quality slightly. TCP has no concept of concealment: a lost segment is retransmitted, +and the whole stream **stalls** behind it (head-of-line blocking). On a phone walking between WiFi +and cellular, that turns a small loss into a visible freeze in a conversation. + +### UDP versus TCP + +WebSockets are TCP. For real-time media, TCP's reliability guarantee is the wrong guarantee: a +retransmitted audio packet that arrives 400 ms late is worthless, and waiting for it delays every +packet behind it. WebRTC uses UDP, where late audio is simply dropped and the conversation keeps +moving. Plus ICE/STUN gives NAT traversal and network-change survival (WiFi to cellular) that a +plain socket would need re-connect logic to approximate. + +## Alternatives considered + +### WebSocket plus Opus frames + +Rejected. Cheapest to build, and it fails on all four axes above. The critical failure is AEC: +without it, open-mic barge-in cannot work at all, and barge-in is not optional in this design +(see [[voice-session-protocol]], "Invariants"). + +### Native iOS audio session with a custom UDP protocol + +Rejected. This is re-implementing WebRTC with fewer eyes on it, and it forfeits the browser-based +fallback client entirely. + +### Push to talk only, half duplex, over the existing socket + +Rejected as the architecture, though it remains a usable **mode**. Half duplex sidesteps echo by +never having the microphone open during playback, which makes a WebSocket viable. But it removes +barge-in, and barge-in is the difference between talking to a system and waiting for one. Half +duplex can ship as a low-bandwidth fallback on top of the WebRTC design; the reverse (adding full +duplex to a WebSocket design later) means rebuilding the transport. + +## Consequences + +**Positive** + +- Barge-in works with the speaker on, which is the whole interaction model. +- Network changes and packet loss degrade gracefully instead of freezing. +- The browser gets the same client for free, so a laptop can be a voice client too. +- Signaling reuses the existing authenticated socket, so there is no second auth surface. + +**Negative** + +- More moving parts: ICE, STUN, and possibly TURN for hostile networks. The desktop needs a + WebRTC peer implementation in the main process, which is a real dependency rather than a few + lines of `ws`. +- Local network discovery and certificate handling need design work in the phone phase. +- Debugging is harder: media that does not flow has more possible causes than a socket that is + closed. + +**Neutral** + +- The [[voice-session-protocol]] event stream is unaffected. It was designed transport-agnostic + precisely so this decision could be made independently, and so a client with no media leg at all + (the desktop dev harness) is still a first-class client. + +## Related + +- [[system-overview]] - where the phone sits in the client model. +- [[voice-session-protocol]] - the control-plane events that ride the WebSocket. +- [[adr-002-main-process-session]] - why there is a session in main for the phone to be a peer of. diff --git a/docs/architecture/acappella/decisions/adr-002-main-process-session.md b/docs/architecture/acappella/decisions/adr-002-main-process-session.md new file mode 100644 index 0000000000..4884bbfd05 --- /dev/null +++ b/docs/architecture/acappella/decisions/adr-002-main-process-session.md @@ -0,0 +1,144 @@ +--- +type: decision +title: 'ADR-002: The voice session is headless in the main process' +created: 2026-08-14 +tags: + - voice + - architecture + - acappella + - adr +related: + - '[[system-overview]]' + - '[[voice-session-protocol]]' + - '[[adr-001-webrtc-transport]]' +--- + +# ADR-002: The voice session is headless in the main process + +**Status:** Accepted +**Date:** 2026-08-14 +**Deciders:** A Cappella Phase 01 + +## Context + +A voice session owns a state machine, a monotonic event sequence, three providers, and a dispatch +path into agents and tabs. It has to live somewhere. + +The renderer is the tempting home. Maestro already does Web Speech dictation there +(`src/renderer/hooks/utils/useVoiceInput.ts`), the microphone and speaker are trivially reachable +from a renderer, `MediaStream` and `AudioWorklet` are browser APIs, and all the state stores are +already there. Building it in the renderer would be faster in Phase 01. + +## Decision + +**`VoiceSessionService` lives in `src/main/acappella/`, is headless, and is transport-agnostic.** +It may not reference a `BrowserWindow`, a React store, or the DOM. Every UI, on every device, is a +client of [[voice-session-protocol]]. + +## Rationale + +### The iPhone is a peer, not a port + +This is the reason that outranks the others. If the session lives in the renderer, the phone +either talks to the renderer (making one window a server for a physical device, which breaks the +moment that window closes or Maestro goes multi-window) or gets a second, parallel session +implementation in main. A second implementation means two state machines that drift, two routing +paths, two barge-in semantics, and two sets of bugs. + +With the session in main, the phone and the desktop HUD are the same kind of thing: subscribers to +one event stream, senders of the same three commands. Adding a client is adding a transport +adapter, not a feature. + +### Renderer lifetime is not session lifetime + +A renderer can be reloaded, hidden, closed, or moved between windows. Maestro is already +multi-window, and windows own subsets of agents. A voice session that dies because the user closed +the window it happened to start in is broken by construction, and one that has to be handed +between windows is worse. Conversations are longer-lived than views. + +### Dispatch authority already lives in main + +Routing a decision into agents and tabs needs the roster, and main already holds it: the sessions +store (`src/main/stores/getters.ts`, read the same way `registerSessionCallbacks` reads it) plus +the `remote:*` bridge the web server uses to ask the renderer to act +(`src/main/web-server/callbacks/tabCallbacks.ts`). A renderer-owned session would either duplicate +that or reach back into main for every step anyway. + +Note the asymmetry this creates and accepts: main holds the roster but has **no** tab authority. +Tab state lives in the renderer, so `executeRouteDecision()` sends `remote:newTab` / +`remote:selectTab` / `remote:renameTab` and waits for confirmation, exactly as the web path does. +Main is the decider; the renderer stays the executor. + +### Providers are Node-shaped + +Local Whisper and Kokoro are native modules or child processes. Cloud realtime sessions want a +long-lived socket with a secret key. A renderer has neither native module access nor a safe place +for keys, so provider work would end up proxied through IPC in every case. Putting the session +where the providers already have to run removes a whole layer of marshalling from the hot path, +which is latency the user hears. + +### Precedent + +Every comparable Maestro subsystem is already main-owned and renderer-projected: the Cue engine, +the web server, Pianola's supervised loop, the plugin broker. Cadenza's HUD window +(`src/main/app-lifecycle/cadenza-hud-window.ts`) is the closest visual analogue, and even it is a +main-created window that buffers payloads until the renderer signals ready. Voice would be the +odd one out. + +## Alternatives considered + +### Session in the renderer, main as a thin relay + +Rejected. Fastest to Phase 01 and worst for Phases 02 onward: it forces a second implementation +for the phone, ties session lifetime to a window, and puts provider secrets and native modules on +the wrong side of the bridge. + +### Session in a dedicated hidden `BrowserWindow` + +Considered seriously. A hidden renderer would give the session `MediaStream`, `AudioWorklet`, and +`RTCPeerConnection` for free, which is genuinely attractive given [[adr-001-webrtc-transport]]. + +Rejected because it trades one problem for a worse one: a hidden window is still a renderer with a +lifecycle, a crash surface, and IPC latency between it and the dispatch authority, and it makes +"who owns the session" ambiguous again. It also cannot be reached from the CLI or a headless +context. + +The capability gap is real but narrow, and it is solvable with a Node WebRTC implementation in +main. If that turns out to be untenable, the fallback is a hidden window acting as a **media +endpoint only**, still driven by the main-process session, not a relocation of the session itself. + +### Session in a separate process + +Rejected as premature. It would add IPC to the dispatch path and process supervision to the +lifecycle for isolation nobody has asked for. Revisit only if a provider proves unstable enough to +threaten app stability. + +## Consequences + +**Positive** + +- One session implementation for every client, forever. +- Sessions survive window reload, window close, and multi-window moves. +- Providers sit next to their native modules and secrets. +- The service is trivially testable: no DOM, no React, no Electron window. Feed it `startSession` + and `submitUtterance`, assert on the event stream. + +**Negative** + +- Audio device access in main needs deliberate work. The renderer's easy `getUserMedia` path is + not available, so Phase 01 ships mock providers and the real capture path is a later phase's + problem. +- WebRTC in main requires a Node implementation rather than the browser's built-in one. +- Every UI interaction costs an IPC hop. Acceptable: the events are small and infrequent compared + to the audio itself, which never crosses this boundary. + +**Neutral** + +- The renderer keeps `useVoiceInput.ts` for composer dictation until the local STT tier lands. + Two voice paths coexist briefly, with a clear owner for each. + +## Related + +- [[system-overview]] - the service, the tiers, and the client model. +- [[voice-session-protocol]] - the contract every client speaks. +- [[adr-001-webrtc-transport]] - the transport this decision makes possible. diff --git a/docs/architecture/acappella/system-overview.md b/docs/architecture/acappella/system-overview.md new file mode 100644 index 0000000000..684ba92581 --- /dev/null +++ b/docs/architecture/acappella/system-overview.md @@ -0,0 +1,213 @@ +--- +type: architecture +title: A Cappella System Overview +created: 2026-08-14 +tags: + - voice + - architecture + - acappella +related: + - '[[voice-session-protocol]]' + - '[[adr-001-webrtc-transport]]' + - '[[adr-002-main-process-session]]' +--- + +# A Cappella System Overview + +A Cappella turns Maestro into something you talk to. You say "start a new tab on the backend +agent about the auth refactor" and a correctly named tab appears on the right agent, primed with +the prompt. The agent answers, and the answer is spoken back in conversational form rather than +read out as raw terminal output. + +This document describes the skeleton that Phase 01 lands. Everything real that arrives later +(Whisper, Kokoro, OpenAI Realtime, ElevenLabs, the iPhone leg) drops in behind the interfaces +named here. + +## The one architectural rule + +**The voice session lives in the MAIN process and is transport-agnostic.** + +The desktop renderer is a client of the session. The iPhone will be a second client on the same +protocol, not a port of the desktop UI. Nothing in the session service may reference a +`BrowserWindow`, a React store, or the DOM. The rationale is recorded in +[[adr-002-main-process-session]]. + +```mermaid +flowchart LR + subgraph clients[Clients] + R[Desktop renderer
Voice HUD] + P[iPhone
WebRTC peer] + C[maestro-cli
future] + end + subgraph main[Main process] + S[VoiceSessionService
state machine + seq] + X[route-executor] + PR[provider registry] + end + subgraph prov[Providers] + STT[SttProvider] + TTS[TtsProvider] + B[BrainProvider] + end + R <-->|IPC acappella:*| S + P <-->|WS signaling + WebRTC media| S + C <-->|IPC/WS| S + S --> PR + PR --> STT + PR --> TTS + PR --> B + S --> X + X -->|remote:* to the renderer| R +``` + +## Surveyed precedent + +Phase 01 is built on top of code that already exists. The survey below records what was read and +what each piece contributes. + +| Existing code | What it contributes | +| --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `src/renderer/hooks/utils/useVoiceInput.ts` | Web Speech dictation for the AI composer on touch devices. It appends a transcript to a draft; it has no session, no routing, and no speech out. A Cappella eventually subsumes it, but it stays until the local STT tier lands. | +| `src/main/global-hotkey-manager.ts` | A singleton owning exactly one system-wide accelerator (`setGlobalShowHotkey`). Push-to-talk needs a second binding, so Phase 06 generalizes this from a singleton into a registry. | +| `src/main/app-lifecycle/cadenza-hud-window.ts` | The precedent for an auxiliary always-on-top window: transparent, frameless, `focusable: false`, click-through by default, hover hit-testing polled in main because `setIgnoreMouseEvents(..., { forward: true })` is unsupported on Linux. A floating voice HUD that survives app-switching reuses this shape. | +| `src/main/plugins/consent-window.ts` | The precedent for a dedicated auxiliary window with its OWN minimal preload and page, so a surface cannot reach the full IPC bridge. | +| `src/main/web-server/routes/wsRoute.ts` | The authenticated WebSocket at `/$TOKEN/ws`. Clients connect with an optional `?sessionId=`, receive a `connected` frame, then an initial state sync. The phone's WebRTC signaling rides this socket, so it inherits the existing token auth instead of opening a second listener. | +| `src/main/web-server/services/broadcastService.ts` | `broadcastToAll` / `broadcastToSession` over the connected client map. The fan-out shape the voice event stream copies for non-IPC clients. | +| `src/shared/plugins/first-party.ts` | The Encore feature registry, keyed by Encore flag, carrying an honest broker-permission disclosure and any supervised background services. | +| `src/renderer/components/Settings/Extensions/extensionModel.ts` | Projects first-party definitions into marketplace tiles; `beta` is a presentation concern that lives here, not in the shared registry. | + +## Where tab state actually lives + +This is the constraint that shapes the dispatch executor, and it is easy to get wrong. + +**Main has no tab authority.** Tab state lives in the renderer. Even a web or CLI request to open +a tab is forwarded to the renderer for execution: `src/main/web-server/callbacks/tabCallbacks.ts` +sends `remote:newTab`, `remote:selectTab`, `remote:renameTab`, `remote:closeTab` and, for the +create case, waits on a one-shot `remote:newTab:response:` channel with a 5 second timeout. +`src/main/ipc/handlers/tabs.ts` exists only so the renderer can report a close back to main. + +So `executeRouteDecision()` does not create tabs. It resolves a decision into the same +`remote:*` messages the web server already uses, and reads its roster from the persisted +sessions store (`sessionsStore.get('sessions', [])`, the same source +`registerSessionCallbacks` uses). Hand-rolling a parallel tab creation path in main would produce +tabs the renderer does not know about. + +The corollary for the phone: the iPhone never talks to tabs either. It sends a voice event, main +routes it, and the renderer performs it. One execution path, three possible originators. + +## Provider tiers + +A Cappella supports two fundamentally different pipeline shapes behind one set of interfaces. + +### Cascade tier (STT then Brain then TTS) + +Three independent providers, swappable individually: + +- `SttProvider`: streaming `feed(pcm)` with `partial` and `final` callbacks. +- `BrainProvider`: `route(input, context)` returns a `RouteDecision`; `converse(agentText, context)` + reshapes an agent's terminal-shaped answer into spoken-form text. +- `TtsProvider`: `speak(text)` returns an async iterable of audio chunks, plus `cancel()`. + +The cascade tier is the default because each stage is independently substitutable: local Whisper +with cloud TTS, or a cloud brain with local everything else. Latency is the sum of the stages, +which is why barge-in matters so much (see below). + +### Realtime tier (speech to speech) + +A single provider owns microphone audio in and speaker audio out, with routing expressed as tool +calls. Latency is far lower and prosody is far better, but the stages are no longer separable and +the audio must leave the machine. + +The realtime tier implements the same three interfaces as a **fused adapter**: `SttProvider` +emits transcripts the realtime session already produces, `BrainProvider.route()` maps to the +session's tool-call channel, and `TtsProvider` becomes a passthrough. The session service cannot +tell the tiers apart, which is the point. + +### Provider resolution rules + +`provider-registry.ts` resolves the active trio from settings. Two rules are non-negotiable: + +1. When nothing is configured, resolve the **mock** trio. The pipeline must always be runnable. +2. **Never silently substitute a cloud provider for a missing local one.** If the user asked for + local Whisper and the model is not downloaded, that is a `session-error` with a clear reason, + not a quiet upload of their microphone to a vendor. + +## Client model + +Both clients are peers on [[voice-session-protocol]]. Neither owns session state. + +| | Desktop renderer | iPhone | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| Transport | Electron IPC (`acappella:*` plus a push `acappella:event` channel) | Authenticated WebSocket for signaling, WebRTC for media | +| Audio capture | Local device, or none in dev-harness mode | Device microphone over WebRTC | +| Responsibilities | Render HUD state, transcript, dispatch narration; send `submitUtterance` / `interrupt` / `stopWord`; execute `remote:*` tab operations | Capture and play audio, render the project wheel from `agent-roster`, send the same three commands | +| State ownership | None. Mirrors the event stream into `voiceSessionStore`. | None. | + +Because clients only mirror, a client can join mid-session and catch up: it reads `get-state` and +`get-roster`, then follows `seq` for gaps. A gap means the client missed events and should +re-read state rather than guess. + +Transport choice for the phone leg is argued in [[adr-001-webrtc-transport]]. + +## Session lifecycle and the state machine + +States: `idle | arming | listening | transcribing | routing | dispatching | speaking | interrupted | error`. + +The legal transitions live in a `VOICE_STATE_TRANSITIONS` table in +`src/shared/acappella/session-state.ts`. An illegal transition throws rather than smearing state, +because a voice UI that is silently in two states at once is unfixable in the field. + +```mermaid +stateDiagram-v2 + [*] --> idle + idle --> arming: startSession + arming --> listening: listen-start + listening --> transcribing: final-transcript + transcribing --> routing: brain.route + routing --> dispatching: route-decision + dispatching --> speaking: agent-reply + speaking --> interrupted: barge-in + interrupted --> listening: floor retained + speaking --> listening: speak-end + listening --> idle: stop-word / stopSession + arming --> error: provider unavailable + routing --> error: no agent matched + error --> idle: stopSession +``` + +### Barge-in and stop are different + +This distinction is load-bearing and every phase inherits it: + +- **Barge-in** cancels TTS and **keeps the floor**. The session goes `speaking -> interrupted -> +listening`. You talked over the assistant; it shuts up and keeps listening. +- **Stop word** ends the session and returns to `idle`. The floor is released. + +Conflating them produces the single most annoying failure mode in voice interfaces: interrupting +the assistant hangs up on it. + +## Error policy + +Per the repo's Sentry policy, unexpected exceptions bubble. Only known failure modes are +classified into `session-error` events: + +- `provider-unavailable`: a configured provider cannot start (model missing, no API key, device + busy). +- `no-agent-matched`: the brain could not resolve a target and the conductor fallback is disabled. +- `dispatch-failed`: the renderer did not answer a `remote:newTab` within its timeout. + +Anything else is a bug and should reach Sentry with context. + +## What Phase 01 deliberately does not do + +- No network calls, no API keys, no model downloads. The mock tier proves the pipeline end to end. +- Enabling the Encore flag opens no device and starts no service. It only makes the Voice Setup + surface reachable. +- No WebRTC yet. The transport decision is recorded now so the protocol is designed for it, but + the phone leg is a later phase. + +## Related + +- [[voice-session-protocol]] - every event, payload, and direction. +- [[adr-001-webrtc-transport]] - why WebRTC beats WebSocket plus Opus for the phone leg. +- [[adr-002-main-process-session]] - why the session is headless in main. diff --git a/docs/architecture/acappella/voice-session-protocol.md b/docs/architecture/acappella/voice-session-protocol.md new file mode 100644 index 0000000000..85a53d450f --- /dev/null +++ b/docs/architecture/acappella/voice-session-protocol.md @@ -0,0 +1,341 @@ +--- +type: architecture +title: A Cappella Voice Session Protocol +created: 2026-08-14 +tags: + - voice + - architecture + - acappella + - protocol +related: + - '[[system-overview]]' + - '[[adr-001-webrtc-transport]]' + - '[[adr-002-main-process-session]]' +--- + +# Voice Session Protocol + +The protocol is the contract between the headless session service in the main process and every +client (desktop renderer today, iPhone later, CLI if it ever wants in). It is defined in +`src/shared/acappella/protocol.ts` as a discriminated union on `type`. + +The protocol is deliberately transport-agnostic. The same object graph travels over Electron IPC +(`acappella:event`) and over the authenticated WebSocket at `/$TOKEN/ws`. Nothing in it may refer +to a `BrowserWindow`, a DOM node, or a React store. See [[adr-002-main-process-session]]. + +## Envelope + +Every event carries the same three fields: + +```ts +interface VoiceEventBase { + /** The voice session this event belongs to. Not an agent id. */ + sessionId: string; + /** Monotonic per voice session, starting at 1. A gap means events were lost. */ + seq: number; + /** Emission time, epoch ms. */ + ts: number; +} +``` + +`sessionId` is the **voice** session id, minted by `startSession()`. It is not an agent id and it +is not a provider session id. When a voice session is bound to an agent, the agent id travels in +the scope, never in `sessionId`. + +`seq` exists so a client can detect gaps. It is monotonic and per voice session, incremented by +the service for every emitted event regardless of subscriber count. A client that sees `seq` jump +must not interpolate: it re-reads `acappella:get-state` and `acappella:get-roster` and resumes. + +## Direction + +`client -> service` events are commands. `service -> client` events are announcements, fanned out +to every subscriber so two clients watching the same session see identical state. + +Three events travel **both** ways. When a client sends one, the service validates it against the +state machine and, if it is legal, echoes it outward with a fresh `seq` so every other client sees +it. The echoed copy is the authoritative one; a client must render its own optimistic state only +until the echo lands. + +## Event catalogue + +| Event | Direction | Emitted when | +| -------------------- | ----------------- | ---------------------------------------------------------------------------------- | +| `wake` | both | A wake word or push-to-talk key fires on a client, then echoed as the session arms | +| `listen-start` | service -> client | The floor opens and audio is being consumed | +| `listen-stop` | service -> client | The floor closes (endpointed, stopped, or interrupted) | +| `partial-transcript` | service -> client | STT produces an interim hypothesis | +| `final-transcript` | both | STT settles an utterance; inbound when the client owns STT | +| `route-decision` | service -> client | The brain resolves a target, tab action, and prompt | +| `dispatch` | service -> client | The decision was executed against a real agent and tab | +| `agent-reply` | service -> client | The agent produced text worth speaking | +| `speak-start` | service -> client | TTS begins a reply | +| `speak-sentence` | service -> client | One sentence of the reply is spoken | +| `speak-end` | service -> client | TTS finishes or is cancelled | +| `barge-in` | both | The user speaks or clicks over active speech | +| `stop-word` | both | The stop word or Stop button ends the session | +| `session-error` | service -> client | A classified, known failure mode | +| `tab-state` | service -> client | The bound agent's tab set or active tab changes | +| `agent-roster` | service -> client | Roster snapshot, on subscribe and on change | + +## Payloads + +### `wake` (both) + +```ts +{ + type: 'wake'; + source: 'wake-word' | 'hotkey' | 'client-button'; + scope: VoiceScope; +} +``` + +`VoiceScope` is `{ kind: 'conductor' }` or `{ kind: 'agent'; sessionId: string }`, where that +`sessionId` is an **agent** id. Inbound, `wake` requests a session in that scope. Outbound, it +announces `idle -> arming`. + +### `listen-start` (service -> client) + +```ts +{ + type: 'listen-start'; + scope: VoiceScope; + sttProviderId: string; +} +``` + +State becomes `listening`. `sttProviderId` lets the HUD show which tier is active (`mock`, +`whisper-local`, `openai-realtime`, ...), which matters because provider substitution is never +silent. + +### `listen-stop` (service -> client) + +```ts +{ + type: 'listen-stop'; + reason: 'endpoint' | 'stopped' | 'interrupted' | 'error'; +} +``` + +### `partial-transcript` (service -> client) + +```ts +{ + type: 'partial-transcript'; + text: string; + stability: number; +} +``` + +`text` is the full hypothesis so far, not a delta, so a client that missed one partial still +renders correctly. `stability` is 0 to 1; the mock STT emits two partials with rising stability +before the final. + +### `final-transcript` (both) + +```ts +{ type: 'final-transcript'; text: string; confidence: number; durationMs?: number } +``` + +Outbound in the cascade tier. **Inbound** when a client owns transcription, which is exactly the +iPhone case if on-device dictation ever beats the desktop pipeline. Either way it lands on the +same seam as `submitUtterance(text)`, so the dev harness and a real microphone are +indistinguishable to everything downstream. + +### `route-decision` (service -> client) + +```ts +{ + type: 'route-decision'; + decision: RouteDecision; + brainProviderId: string; + latencyMs: number; +} +``` + +`RouteDecision` is defined in `src/shared/acappella/route-decision.ts`: + +```ts +{ + target: 'conductor' | { sessionId: string }; + tabAction: 'current' | 'new' | 'recall'; + tabId?: string; + tabName?: string; + prompt: string; + confidence: number; +} +``` + +The same file exports a JSON Schema constant for this shape. Phase 07 compiles that schema into a +GBNF grammar so a local model is structurally incapable of emitting an invalid decision. Keeping +the schema next to the type is what makes that possible without a second, drifting definition. + +### `dispatch` (service -> client) + +```ts +{ + type: 'dispatch'; + agentSessionId: string; + agentName: string; + tabId: string; + tabName?: string; + action: 'focused' | 'created' | 'recalled'; + promptSent: boolean; +} +``` + +This is what lets any client narrate "opened a new tab named Auth Refactor on agent Backend". It +is emitted **after** the renderer confirms the operation, not when it is requested, because tab +creation is a round trip that can time out (see [[system-overview]], "Where tab state actually +lives"). + +### `agent-reply` (service -> client) + +```ts +{ + type: 'agent-reply'; + agentSessionId: string; + tabId: string; + text: string; + spokenText: string; +} +``` + +`text` is what the agent actually wrote. `spokenText` is `BrainProvider.converse()` output: the +same content reshaped for the ear, because reading a diff aloud is useless. Clients that show a +transcript should show `text` and speak `spokenText`. + +### `speak-start` / `speak-sentence` / `speak-end` (service -> client) + +```ts +{ + type: 'speak-start'; + utteranceId: string; + sentenceCount: number; + ttsProviderId: string; +} +{ + type: 'speak-sentence'; + utteranceId: string; + index: number; + text: string; +} +{ + type: 'speak-end'; + utteranceId: string; + reason: 'complete' | 'cancelled' | 'error'; +} +``` + +`utteranceId` scopes a speech run so a late `speak-sentence` from a cancelled run can be dropped +rather than rendered after the next reply started. Sentence granularity is what makes barge-in +feel instant: the client already knows the sentence boundary it was cut at. + +### `barge-in` (both) + +```ts +{ type: 'barge-in'; source: 'voice' | 'client-button'; cancelledUtteranceId?: string } +``` + +Cancels TTS and **keeps the floor**: `speaking -> interrupted -> listening`. It does not end the +session. This is the difference that makes the interface usable. + +### `stop-word` (both) + +```ts +{ type: 'stop-word'; phrase?: string; source: 'voice' | 'client-button' } +``` + +Ends the session: any state `-> idle`. TTS is cancelled, the floor is released, providers are torn +down. + +### `session-error` (service -> client) + +```ts +{ + type: 'session-error'; + code: 'provider-unavailable' | 'no-agent-matched' | 'dispatch-failed'; + message: string; + recoverable: boolean; + providerId?: string; +} +``` + +The union is closed on purpose. Only classified, known failure modes become events; anything else +bubbles to Sentry per the repo error policy. A new code means a new, deliberately handled failure +mode, not a catch-all. + +### `tab-state` (service -> client) + +```ts +{ type: 'tab-state'; agentSessionId: string; tabs: RosterTab[]; activeTabId: string | null } +``` + +### `agent-roster` (service -> client) + +```ts +{ type: 'agent-roster'; agents: RosterAgent[] } + +interface RosterAgent { + sessionId: string; + name: string; + agentType: string; + cwd: string; + tabs: RosterTab[]; +} + +interface RosterTab { + id: string; + name: string | null; + lastActiveAt: number | null; +} +``` + +The roster is the brain's routing context and, later, the phone's project wheel. It is built by +`buildAgentRoster()` in `src/main/acappella/dispatch/route-executor.ts` from the main-process +stores via `src/main/stores/getters.ts`, the same source the web server's session callbacks read. +It is deliberately compact: no logs, no usage stats, nothing that would make it expensive to push +on every change or to send to a model as context. + +## Flow + +A complete turn, with the mock tier: + +```mermaid +sequenceDiagram + participant C as Client (HUD) + participant S as VoiceSessionService + participant B as BrainProvider + participant R as Renderer (tabs) + participant T as TtsProvider + C->>S: wake / startSession + S-->>C: wake, listen-start (seq 1,2) + C->>S: submitUtterance("new tab on backend about auth") + S-->>C: partial-transcript x2, final-transcript (seq 3-5) + S->>B: route(text, roster) + S-->>C: route-decision (seq 6) + S->>R: remote:newTab + remote:renameTab + R-->>S: response channel + S-->>C: dispatch (seq 7) + S-->>C: agent-reply (seq 8) + S->>T: speak(spokenText) + S-->>C: speak-start, speak-sentence..., speak-end (seq 9+) + C->>S: barge-in + S-->>C: speak-end(cancelled), listen-start +``` + +## Invariants + +1. **`seq` is never reused and never goes backwards** within a voice session. +2. **Clients hold no authoritative state.** Every client is a projection of the event stream. +3. **Barge-in keeps the floor; stop releases it.** No event may blur the two. +4. **An illegal state transition throws.** The transition table in `session-state.ts` is the only + place legality is defined. +5. **No provider substitution is silent.** Every provider-bearing event names its provider id. +6. **The union is closed.** Adding an event means adding it to `protocol.ts`, not stuffing an + extra field into an existing payload. + +## Related + +- [[system-overview]] - the service, tiers, and client model. +- [[adr-001-webrtc-transport]] - why the phone's media leg is WebRTC. +- [[adr-002-main-process-session]] - why the session is headless in main. From 0116127b8d1015b59d58bfbd5c220f11d315b8b0 Mon Sep 17 00:00:00 2001 From: Pedram Amini Date: Fri, 14 Aug 2026 22:38:49 -0500 Subject: [PATCH 02/66] MAESTRO: add A Cappella shared protocol types (protocol, state machine, providers, route decision) --- .../acappella/voice-session-protocol.md | 3 +- .../shared/acappella-protocol.test.ts | 174 +++++++++++ src/shared/acappella/protocol.ts | 291 ++++++++++++++++++ src/shared/acappella/providers.ts | 136 ++++++++ src/shared/acappella/route-decision.ts | 78 +++++ src/shared/acappella/session-state.ts | 85 +++++ 6 files changed, 766 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/shared/acappella-protocol.test.ts create mode 100644 src/shared/acappella/protocol.ts create mode 100644 src/shared/acappella/providers.ts create mode 100644 src/shared/acappella/route-decision.ts create mode 100644 src/shared/acappella/session-state.ts diff --git a/docs/architecture/acappella/voice-session-protocol.md b/docs/architecture/acappella/voice-session-protocol.md index 85a53d450f..be45027bcd 100644 --- a/docs/architecture/acappella/voice-session-protocol.md +++ b/docs/architecture/acappella/voice-session-protocol.md @@ -51,7 +51,8 @@ must not interpolate: it re-reads `acappella:get-state` and `acappella:get-roste `client -> service` events are commands. `service -> client` events are announcements, fanned out to every subscriber so two clients watching the same session see identical state. -Three events travel **both** ways. When a client sends one, the service validates it against the +Four events travel **both** ways (`wake`, `final-transcript`, `barge-in`, `stop-word`). When a +client sends one, the service validates it against the state machine and, if it is legal, echoes it outward with a fresh `seq` so every other client sees it. The echoed copy is the authoritative one; a client must render its own optimistic state only until the echo lands. diff --git a/src/__tests__/shared/acappella-protocol.test.ts b/src/__tests__/shared/acappella-protocol.test.ts new file mode 100644 index 0000000000..115a910a41 --- /dev/null +++ b/src/__tests__/shared/acappella-protocol.test.ts @@ -0,0 +1,174 @@ +/** + * Tests for the A Cappella shared protocol module (src/shared/acappella/). + * + * These cover the runtime pieces only: the state transition table, the event + * direction map, and the route decision helpers plus JSON Schema. The session + * service's use of them is tested separately in the main-process suites. + */ + +import { describe, it, expect } from 'vitest'; +import { + VOICE_EVENT_DIRECTIONS, + isClientVoiceEvent, + isContiguousVoiceSeq, + type VoiceEvent, + type VoiceEventType, +} from '../../shared/acappella/protocol'; +import { + VOICE_SESSION_STATES, + VOICE_STATE_TRANSITIONS, + assertVoiceStateTransition, + canTransitionVoiceState, + isVoiceSessionActive, + InvalidVoiceStateTransitionError, + type VoiceSessionState, +} from '../../shared/acappella/session-state'; +import { + ROUTE_DECISION_JSON_SCHEMA, + ROUTE_TAB_ACTIONS, + isConductorTarget, + routeTargetSessionId, +} from '../../shared/acappella/route-decision'; + +const ALL_EVENT_TYPES: VoiceEventType[] = [ + 'wake', + 'listen-start', + 'listen-stop', + 'partial-transcript', + 'final-transcript', + 'route-decision', + 'dispatch', + 'agent-reply', + 'speak-start', + 'speak-sentence', + 'speak-end', + 'barge-in', + 'stop-word', + 'session-error', + 'tab-state', + 'agent-roster', +]; + +function makeEvent(type: VoiceEventType): VoiceEvent { + return { type, sessionId: 'voice-1', seq: 1, ts: 0 } as VoiceEvent; +} + +describe('VOICE_STATE_TRANSITIONS', () => { + it('should cover every state as a source', () => { + expect(Object.keys(VOICE_STATE_TRANSITIONS).sort()).toEqual([...VOICE_SESSION_STATES].sort()); + }); + + it('should only name known states as targets', () => { + for (const targets of Object.values(VOICE_STATE_TRANSITIONS)) { + for (const target of targets) { + expect(VOICE_SESSION_STATES).toContain(target); + } + } + }); + + it('should let every non-idle state stop to idle', () => { + for (const state of VOICE_SESSION_STATES) { + if (state === 'idle') continue; + expect(canTransitionVoiceState(state, 'idle')).toBe(true); + } + }); + + it('should keep the floor on barge-in: speaking -> interrupted -> listening', () => { + expect(canTransitionVoiceState('speaking', 'interrupted')).toBe(true); + expect(canTransitionVoiceState('interrupted', 'listening')).toBe(true); + }); + + it('should reject skipping the pipeline', () => { + expect(canTransitionVoiceState('idle', 'speaking')).toBe(false); + expect(canTransitionVoiceState('listening', 'dispatching')).toBe(false); + expect(canTransitionVoiceState('error', 'listening')).toBe(false); + }); +}); + +describe('assertVoiceStateTransition', () => { + it('should not throw on a legal edge', () => { + expect(() => assertVoiceStateTransition('idle', 'arming')).not.toThrow(); + }); + + it('should throw a typed error naming both states', () => { + let caught: unknown; + try { + assertVoiceStateTransition('idle', 'speaking'); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(InvalidVoiceStateTransitionError); + const typed = caught as InvalidVoiceStateTransitionError; + expect(typed.from).toBe('idle'); + expect(typed.to).toBe('speaking'); + expect(typed.message).toContain('idle -> speaking'); + }); +}); + +describe('isVoiceSessionActive', () => { + it('should treat idle and error as not holding resources', () => { + const inactive: VoiceSessionState[] = ['idle', 'error']; + for (const state of VOICE_SESSION_STATES) { + expect(isVoiceSessionActive(state)).toBe(!inactive.includes(state)); + } + }); +}); + +describe('VOICE_EVENT_DIRECTIONS', () => { + it('should have an entry for every event type', () => { + expect(Object.keys(VOICE_EVENT_DIRECTIONS).sort()).toEqual([...ALL_EVENT_TYPES].sort()); + }); + + it('should mark exactly the four client-originated events as both-way', () => { + const both = ALL_EVENT_TYPES.filter((type) => VOICE_EVENT_DIRECTIONS[type] === 'both'); + expect(both.sort()).toEqual(['barge-in', 'final-transcript', 'stop-word', 'wake']); + }); + + it('should classify events through isClientVoiceEvent', () => { + expect(isClientVoiceEvent(makeEvent('barge-in'))).toBe(true); + expect(isClientVoiceEvent(makeEvent('speak-sentence'))).toBe(false); + }); +}); + +describe('isContiguousVoiceSeq', () => { + it('should accept the next sequence number and reject gaps or replays', () => { + expect(isContiguousVoiceSeq(4, 5)).toBe(true); + expect(isContiguousVoiceSeq(4, 6)).toBe(false); + expect(isContiguousVoiceSeq(4, 4)).toBe(false); + }); +}); + +describe('route decision helpers', () => { + it('should identify the conductor target', () => { + expect(isConductorTarget('conductor')).toBe(true); + expect(isConductorTarget({ sessionId: 'agent-1' })).toBe(false); + }); + + it('should extract the agent id, or null for the conductor', () => { + expect(routeTargetSessionId({ sessionId: 'agent-1' })).toBe('agent-1'); + expect(routeTargetSessionId('conductor')).toBeNull(); + }); +}); + +describe('ROUTE_DECISION_JSON_SCHEMA', () => { + it('should describe the same tab actions as the type', () => { + expect(ROUTE_DECISION_JSON_SCHEMA.properties.tabAction.enum).toEqual(ROUTE_TAB_ACTIONS); + }); + + it('should require the fields a dispatch cannot run without', () => { + expect(ROUTE_DECISION_JSON_SCHEMA.required).toEqual([ + 'target', + 'tabAction', + 'prompt', + 'confidence', + ]); + }); + + it('should stay GBNF-friendly: closed object, no $ref, bounded confidence', () => { + const serialized = JSON.stringify(ROUTE_DECISION_JSON_SCHEMA); + expect(serialized).not.toContain('$ref'); + expect(ROUTE_DECISION_JSON_SCHEMA.additionalProperties).toBe(false); + expect(ROUTE_DECISION_JSON_SCHEMA.properties.confidence.minimum).toBe(0); + expect(ROUTE_DECISION_JSON_SCHEMA.properties.confidence.maximum).toBe(1); + }); +}); diff --git a/src/shared/acappella/protocol.ts b/src/shared/acappella/protocol.ts new file mode 100644 index 0000000000..a0f90eead7 --- /dev/null +++ b/src/shared/acappella/protocol.ts @@ -0,0 +1,291 @@ +/** + * A Cappella Voice Session Protocol - the contract between the headless session + * service in the main process and every client (desktop renderer today, iPhone + * later, CLI if it ever wants in). + * + * The protocol is transport-agnostic on purpose: the same object graph travels + * over Electron IPC (`acappella:event`) and over the authenticated WebSocket at + * `/$TOKEN/ws`. Nothing here may refer to a BrowserWindow, a DOM node, or a + * React store. + * + * Full narrative, including the flow diagram and invariants, lives in + * docs/architecture/acappella/voice-session-protocol.md. + */ + +import type { RouteDecision } from './route-decision'; + +/** + * What a voice session is bound to. The `sessionId` inside an agent scope is an + * AGENT id; the envelope's `sessionId` is the voice session id. They are never + * the same value and are never interchangeable. + */ +export type VoiceScope = { kind: 'conductor' } | { kind: 'agent'; sessionId: string }; + +/** Every event carries the same three fields. */ +export interface VoiceEventBase { + /** The voice session this event belongs to. Not an agent id. */ + sessionId: string; + /** Monotonic per voice session, starting at 1. A gap means events were lost. */ + seq: number; + /** Emission time, epoch ms. */ + ts: number; +} + +// --------------------------------------------------------------------------- +// Roster +// --------------------------------------------------------------------------- + +/** One AI tab, compact enough to push on every change and to feed a model. */ +export interface RosterTab { + id: string; + name: string | null; + lastActiveAt: number | null; +} + +/** One agent as the Brain sees it, and later as the phone's project wheel shows it. */ +export interface RosterAgent { + sessionId: string; + name: string; + agentType: string; + cwd: string; + tabs: RosterTab[]; +} + +// --------------------------------------------------------------------------- +// Events +// --------------------------------------------------------------------------- + +/** How a session was woken. */ +export type WakeSource = 'wake-word' | 'hotkey' | 'client-button'; + +/** + * Inbound, requests a session in `scope`. Outbound, announces `idle -> arming`. + */ +export interface WakeEvent extends VoiceEventBase { + type: 'wake'; + source: WakeSource; + scope: VoiceScope; +} + +/** The floor is open and audio is being consumed. */ +export interface ListenStartEvent extends VoiceEventBase { + type: 'listen-start'; + scope: VoiceScope; + /** Named so provider substitution can never be silent. */ + sttProviderId: string; +} + +export type ListenStopReason = 'endpoint' | 'stopped' | 'interrupted' | 'error'; + +/** The floor closed: endpointed, stopped, or interrupted. */ +export interface ListenStopEvent extends VoiceEventBase { + type: 'listen-stop'; + reason: ListenStopReason; +} + +/** An interim STT hypothesis. */ +export interface PartialTranscriptEvent extends VoiceEventBase { + type: 'partial-transcript'; + /** The full hypothesis so far, not a delta, so a missed partial is harmless. */ + text: string; + /** 0 to 1. Rises across successive partials for the same utterance. */ + stability: number; +} + +/** + * A settled utterance. Inbound when the client owns transcription (on-device + * iPhone dictation); either way it lands on the same seam as + * `submitUtterance(text)`, so a real microphone and the dev harness are + * indistinguishable downstream. + */ +export interface FinalTranscriptEvent extends VoiceEventBase { + type: 'final-transcript'; + text: string; + confidence: number; + durationMs?: number; +} + +/** The Brain resolved a target, a tab action, and a prompt. */ +export interface RouteDecisionEvent extends VoiceEventBase { + type: 'route-decision'; + decision: RouteDecision; + brainProviderId: string; + latencyMs: number; +} + +export type DispatchAction = 'focused' | 'created' | 'recalled'; + +/** + * The decision was executed against a real agent and tab. Emitted AFTER the + * renderer confirms, never when the operation is requested: main has no tab + * authority and the round trip can time out. + */ +export interface DispatchEvent extends VoiceEventBase { + type: 'dispatch'; + agentSessionId: string; + agentName: string; + tabId: string; + tabName?: string; + action: DispatchAction; + promptSent: boolean; +} + +/** The agent produced text worth speaking. */ +export interface AgentReplyEvent extends VoiceEventBase { + type: 'agent-reply'; + agentSessionId: string; + tabId: string; + /** What the agent actually wrote. Show this. */ + text: string; + /** `BrainProvider.converse()` output, reshaped for the ear. Speak this. */ + spokenText: string; +} + +export interface SpeakStartEvent extends VoiceEventBase { + type: 'speak-start'; + /** Scopes a speech run so late sentences from a cancelled run can be dropped. */ + utteranceId: string; + sentenceCount: number; + ttsProviderId: string; +} + +export interface SpeakSentenceEvent extends VoiceEventBase { + type: 'speak-sentence'; + utteranceId: string; + index: number; + text: string; +} + +export type SpeakEndReason = 'complete' | 'cancelled' | 'error'; + +export interface SpeakEndEvent extends VoiceEventBase { + type: 'speak-end'; + utteranceId: string; + reason: SpeakEndReason; +} + +export type InterruptSource = 'voice' | 'client-button'; + +/** + * The user spoke or clicked over active speech. Cancels TTS and KEEPS the + * floor (`speaking -> interrupted -> listening`). It never ends the session. + */ +export interface BargeInEvent extends VoiceEventBase { + type: 'barge-in'; + source: InterruptSource; + cancelledUtteranceId?: string; +} + +/** Ends the session from any state. TTS is cancelled and the floor is released. */ +export interface StopWordEvent extends VoiceEventBase { + type: 'stop-word'; + phrase?: string; + source: InterruptSource; +} + +/** + * Closed on purpose. Only classified, known failure modes become events; + * anything else bubbles to Sentry per the repo error policy. + */ +export const VOICE_SESSION_ERROR_CODES = [ + 'provider-unavailable', + 'no-agent-matched', + 'dispatch-failed', +] as const; + +export type VoiceSessionErrorCode = (typeof VOICE_SESSION_ERROR_CODES)[number]; + +export interface SessionErrorEvent extends VoiceEventBase { + type: 'session-error'; + code: VoiceSessionErrorCode; + message: string; + recoverable: boolean; + providerId?: string; +} + +/** The bound agent's tab set or active tab changed. */ +export interface TabStateEvent extends VoiceEventBase { + type: 'tab-state'; + agentSessionId: string; + tabs: RosterTab[]; + activeTabId: string | null; +} + +/** Roster snapshot, sent on subscribe and on change. */ +export interface AgentRosterEvent extends VoiceEventBase { + type: 'agent-roster'; + agents: RosterAgent[]; +} + +/** The whole protocol, discriminated on `type`. */ +export type VoiceEvent = + | WakeEvent + | ListenStartEvent + | ListenStopEvent + | PartialTranscriptEvent + | FinalTranscriptEvent + | RouteDecisionEvent + | DispatchEvent + | AgentReplyEvent + | SpeakStartEvent + | SpeakSentenceEvent + | SpeakEndEvent + | BargeInEvent + | StopWordEvent + | SessionErrorEvent + | TabStateEvent + | AgentRosterEvent; + +export type VoiceEventType = VoiceEvent['type']; + +/** The payload of an event before the service stamps `sessionId`, `seq`, and `ts`. */ +export type VoiceEventPayload = Omit< + Extract, + keyof VoiceEventBase +>; + +// --------------------------------------------------------------------------- +// Direction +// --------------------------------------------------------------------------- + +/** + * `client-to-service` events are commands; the service validates one against + * the state machine and, if legal, echoes it outward with a fresh `seq` so every + * other client sees it. The echo is authoritative: a client renders its own + * optimistic state only until the echo lands. + */ +export type VoiceEventDirection = 'service-to-client' | 'both'; + +export const VOICE_EVENT_DIRECTIONS: Record = { + wake: 'both', + 'listen-start': 'service-to-client', + 'listen-stop': 'service-to-client', + 'partial-transcript': 'service-to-client', + 'final-transcript': 'both', + 'route-decision': 'service-to-client', + dispatch: 'service-to-client', + 'agent-reply': 'service-to-client', + 'speak-start': 'service-to-client', + 'speak-sentence': 'service-to-client', + 'speak-end': 'service-to-client', + 'barge-in': 'both', + 'stop-word': 'both', + 'session-error': 'service-to-client', + 'tab-state': 'service-to-client', + 'agent-roster': 'service-to-client', +}; + +/** The four events a client is allowed to originate. */ +export type ClientVoiceEvent = Extract< + VoiceEvent, + { type: 'wake' | 'final-transcript' | 'barge-in' | 'stop-word' } +>; + +export function isClientVoiceEvent(event: VoiceEvent): event is ClientVoiceEvent { + return VOICE_EVENT_DIRECTIONS[event.type] === 'both'; +} + +/** True when `next` continues `previous` without a gap. */ +export function isContiguousVoiceSeq(previous: number, next: number): boolean { + return next === previous + 1; +} diff --git a/src/shared/acappella/providers.ts b/src/shared/acappella/providers.ts new file mode 100644 index 0000000000..ecdd095ed5 --- /dev/null +++ b/src/shared/acappella/providers.ts @@ -0,0 +1,136 @@ +/** + * A Cappella provider interfaces - the three seams every speech tier plugs into. + * + * The session service never imports a concrete provider; it takes a trio at + * construction. That is what lets the cascade tier (independent STT / Brain / + * TTS, individually swappable) and the realtime speech-to-speech tier (one + * provider wearing all three hats as a fused adapter) look identical from the + * service's side. + * + * See docs/architecture/acappella/system-overview.md, "Provider tiers". + */ + +import type { RosterAgent, VoiceScope } from './protocol'; +import type { RouteDecision } from './route-decision'; + +/** + * Where a provider runs. The registry uses this to enforce the rule that a + * cloud provider is NEVER silently substituted for a missing local one. + */ +export type VoiceProviderTier = 'mock' | 'local' | 'cloud'; + +export interface VoiceProviderInfo { + /** Stable id carried in the protocol (`mock-stt`, `whisper-local`, ...). */ + readonly id: string; + /** Human label for the HUD and Voice Setup. */ + readonly label: string; + readonly tier: VoiceProviderTier; +} + +// --------------------------------------------------------------------------- +// Speech to text +// --------------------------------------------------------------------------- + +export interface SttCallbacks { + /** `text` is the full hypothesis so far, not a delta. `stability` is 0 to 1. */ + onPartial(text: string, stability: number): void; + onFinal(text: string, confidence: number, durationMs?: number): void; + /** Known, classified failures only. Anything else should throw and reach Sentry. */ + onError(error: Error): void; +} + +export interface SttProvider extends VoiceProviderInfo { + /** Sample rate `feed()` expects, in Hz. */ + readonly sampleRate: number; + /** Acquire the device or session. Throws when the provider cannot start. */ + start(callbacks: SttCallbacks): Promise; + /** Push one buffer of 16-bit mono PCM at `sampleRate`. */ + feed(pcm: Int16Array): void; + /** Force endpointing of the current utterance. */ + flush(): Promise; + stop(): Promise; + /** + * Text-in seam for providers with no audio path: the dev harness, and a + * client that did its own transcription (on-device iPhone dictation). It + * lands on exactly the same callbacks as audio, so nothing downstream can + * tell the difference. Audio-only providers omit it. + */ + injectUtterance?(text: string): void; +} + +// --------------------------------------------------------------------------- +// Brain +// --------------------------------------------------------------------------- + +export interface VoiceRouteContext { + /** The routing context: every agent and its tabs. */ + roster: RosterAgent[]; + /** What the session is bound to. An agent scope biases routing toward it. */ + scope: VoiceScope; + /** Agent the user is looking at, when a client reported one. */ + activeAgentSessionId?: string | null; + /** Most recent utterances this session, oldest first, for "back to" style references. */ + recentUtterances?: string[]; +} + +export interface VoiceConverseContext { + /** The agent whose text is being reshaped. */ + agentSessionId: string; + tabId: string; + /** Rough budget for the spoken form. Reading a diff aloud is useless. */ + maxSentences?: number; +} + +export interface BrainProvider extends VoiceProviderInfo { + /** Resolve an utterance into a target, a tab action, and a prompt. */ + route(input: string, context: VoiceRouteContext): Promise; + /** Reshape an agent's terminal-shaped answer into spoken-form text. */ + converse(agentText: string, context: VoiceConverseContext): Promise; +} + +// --------------------------------------------------------------------------- +// Text to speech +// --------------------------------------------------------------------------- + +/** `none` is the mock tier: sentence text with no audio behind it. */ +export type TtsAudioFormat = 'none' | 'pcm16' | 'mp3' | 'opus'; + +/** + * One sentence of a speech run. Sentence granularity is what makes barge-in + * feel instant: the client already knows the boundary it was cut at. + */ +export interface TtsChunk { + utteranceId: string; + index: number; + /** The sentence being spoken, for the transcript UI. */ + text: string; + format: TtsAudioFormat; + /** Null in the mock tier, which speaks nothing. */ + audio: Uint8Array | null; +} + +export interface TtsSpeakOptions { + /** Correlates every chunk of one run so a cancelled run's stragglers can be dropped. */ + utteranceId: string; + voiceId?: string; + /** 1 is the provider's natural rate. */ + rate?: number; +} + +export interface TtsProvider extends VoiceProviderInfo { + /** Stream a reply. Iteration ends early when `cancel()` is called. */ + speak(text: string, options: TtsSpeakOptions): AsyncIterable; + /** Barge-in. Must cut the current run off mid-sentence, not at the next boundary. */ + cancel(): void; +} + +// --------------------------------------------------------------------------- +// Trio +// --------------------------------------------------------------------------- + +/** The bag injected into `VoiceSessionService`. Resolved by `provider-registry.ts`. */ +export interface VoiceProviderTrio { + stt: SttProvider; + tts: TtsProvider; + brain: BrainProvider; +} diff --git a/src/shared/acappella/route-decision.ts b/src/shared/acappella/route-decision.ts new file mode 100644 index 0000000000..38ae20fca5 --- /dev/null +++ b/src/shared/acappella/route-decision.ts @@ -0,0 +1,78 @@ +/** + * A Cappella route decision - what the Brain decided to do with an utterance. + * + * This is the one shape that crosses every layer: a Brain provider produces it, + * the session service validates and announces it, the dispatch executor performs + * it, and every client narrates it. The JSON Schema below is exported alongside + * the type on purpose: Phase 07 compiles it into a GBNF grammar so a local model + * is structurally incapable of emitting an invalid decision, and a second, + * drifting copy of the shape would defeat that. + */ + +/** Where the prompt goes: the conductor, or one specific agent by id. */ +export type RouteTarget = 'conductor' | { sessionId: string }; + +/** + * What to do with tabs on the target: + * current - use whatever tab is active + * new - open a fresh AI tab, named `tabName` when the Brain suggested one + * recall - return to an existing tab by `tabId` ("back to the auth one") + */ +export const ROUTE_TAB_ACTIONS = ['current', 'new', 'recall'] as const; + +export type RouteTabAction = (typeof ROUTE_TAB_ACTIONS)[number]; + +export interface RouteDecision { + target: RouteTarget; + tabAction: RouteTabAction; + /** Required by `recall`; ignored otherwise. */ + tabId?: string; + /** Suggested name for a `new` tab. */ + tabName?: string; + /** What to actually send the agent, cleaned of routing chatter. */ + prompt: string; + /** 0 to 1. A low-confidence decision may be confirmed out loud before dispatch. */ + confidence: number; +} + +/** True when the decision targets the conductor rather than a specific agent. */ +export function isConductorTarget(target: RouteTarget): target is 'conductor' { + return target === 'conductor'; +} + +/** The agent id a decision targets, or null for the conductor. */ +export function routeTargetSessionId(target: RouteTarget): string | null { + return isConductorTarget(target) ? null : target.sessionId; +} + +/** + * JSON Schema (draft-07 subset) for `RouteDecision`, kept deliberately plain so + * it survives compilation to a GBNF grammar: no `$ref`, no conditionals, and a + * closed `oneOf` for the target. Keep this in sync with the type above; the + * shape is small enough that duplication is cheaper than a code generator. + */ +export const ROUTE_DECISION_JSON_SCHEMA = { + $schema: 'http://json-schema.org/draft-07/schema#', + title: 'RouteDecision', + type: 'object', + properties: { + target: { + oneOf: [ + { type: 'string', const: 'conductor' }, + { + type: 'object', + properties: { sessionId: { type: 'string' } }, + required: ['sessionId'], + additionalProperties: false, + }, + ], + }, + tabAction: { type: 'string', enum: ROUTE_TAB_ACTIONS }, + tabId: { type: 'string' }, + tabName: { type: 'string' }, + prompt: { type: 'string' }, + confidence: { type: 'number', minimum: 0, maximum: 1 }, + }, + required: ['target', 'tabAction', 'prompt', 'confidence'], + additionalProperties: false, +} as const; diff --git a/src/shared/acappella/session-state.ts b/src/shared/acappella/session-state.ts new file mode 100644 index 0000000000..ae9c3c8a06 --- /dev/null +++ b/src/shared/acappella/session-state.ts @@ -0,0 +1,85 @@ +/** + * A Cappella voice session state machine. + * + * The transition table below is the ONLY place transition legality is defined. + * An illegal transition throws rather than smearing state, because a voice UI + * that is silently in two states at once is unfixable in the field: the user + * hears nothing and there is no screen to read. + * + * See docs/architecture/acappella/system-overview.md for the diagram. + */ + +export type VoiceSessionState = + | 'idle' + | 'arming' + | 'listening' + | 'transcribing' + | 'routing' + | 'dispatching' + | 'speaking' + | 'interrupted' + | 'error'; + +export const VOICE_SESSION_STATES: readonly VoiceSessionState[] = [ + 'idle', + 'arming', + 'listening', + 'transcribing', + 'routing', + 'dispatching', + 'speaking', + 'interrupted', + 'error', +] as const; + +/** + * Legal transitions, keyed by source state. + * + * Two rules explain most of the table: + * - Every non-idle state may go to `idle`, because the stop word ends a + * session from wherever it is. Barge-in does NOT appear here as a path to + * idle: it goes `speaking -> interrupted -> listening` and keeps the floor. + * - Every state that runs a provider may go to `error`, because a classified + * provider failure is a state, not an exception. + */ +export const VOICE_STATE_TRANSITIONS: Record = { + idle: ['arming'], + arming: ['listening', 'idle', 'error'], + listening: ['transcribing', 'idle', 'error'], + /** Back to `listening` when the utterance was empty and there is nothing to route. */ + transcribing: ['routing', 'listening', 'idle', 'error'], + routing: ['dispatching', 'idle', 'error'], + /** Back to `listening` when the dispatch produced no reply worth speaking. */ + dispatching: ['speaking', 'listening', 'idle', 'error'], + speaking: ['interrupted', 'listening', 'idle', 'error'], + /** Barge-in retains the floor, so the only forward path is back to listening. */ + interrupted: ['listening', 'idle', 'error'], + error: ['idle'], +}; + +/** States in which a session holds resources (providers, the audio floor, or both). */ +export function isVoiceSessionActive(state: VoiceSessionState): boolean { + return state !== 'idle' && state !== 'error'; +} + +export function canTransitionVoiceState(from: VoiceSessionState, to: VoiceSessionState): boolean { + return VOICE_STATE_TRANSITIONS[from].includes(to); +} + +/** Thrown by `assertVoiceStateTransition`. Carries both states so Sentry gets the edge. */ +export class InvalidVoiceStateTransitionError extends Error { + constructor( + public readonly from: VoiceSessionState, + public readonly to: VoiceSessionState + ) { + super(`Illegal voice session transition: ${from} -> ${to}`); + this.name = 'InvalidVoiceStateTransitionError'; + } +} + +/** Throws `InvalidVoiceStateTransitionError` unless the edge is in the table. */ +export function assertVoiceStateTransition(from: VoiceSessionState, to: VoiceSessionState): void { + if (!canTransitionVoiceState(from, to)) { + throw new InvalidVoiceStateTransitionError(from, to); + } +} From ef506ebec29c0f123518465efe9bdd4f3be4c69b Mon Sep 17 00:00:00 2001 From: Pedram Amini Date: Fri, 14 Aug 2026 22:48:15 -0500 Subject: [PATCH 03/66] MAESTRO: register A Cappella as a default-off Encore Feature Adds the aCappella Encore flag and its first-party plugin definition (com.maestro.acappella) so the voice interface is listed in the Extensions marketplace, off by default, and searchable in Settings. Enabling the flag only makes the Voice Setup surface reachable: no device is opened, no model downloaded, and no socket dialled until a session is explicitly started, so the definition declares no background service. agents:dispatch is deliberately absent from the permission disclosure, the same precedent as Pianola and Concerto: a spoken request resolves to an agent and tab at runtime, which a static allowlist scope cannot name. --- .../Extensions/extensionModel.test.ts | 21 ++++++++ .../pianola-first-party-plugin.test.ts | 1 + .../Settings/Extensions/ExtensionsGrid.tsx | 2 + .../Settings/Extensions/ExtensionsView.tsx | 1 + .../Settings/Extensions/extensionModel.ts | 3 +- .../Settings/searchableSettingsEncore.ts | 26 ++++++++++ src/renderer/stores/settingsStore.ts | 6 +++ src/renderer/types/index.ts | 6 +++ src/shared/plugins/first-party.ts | 49 ++++++++++++++++++- src/shared/settingsMetadataFeatures.ts | 1 + 10 files changed, 114 insertions(+), 2 deletions(-) diff --git a/src/__tests__/renderer/components/Settings/Extensions/extensionModel.test.ts b/src/__tests__/renderer/components/Settings/Extensions/extensionModel.test.ts index 1064eb6b24..0467668865 100644 --- a/src/__tests__/renderer/components/Settings/Extensions/extensionModel.test.ts +++ b/src/__tests__/renderer/components/Settings/Extensions/extensionModel.test.ts @@ -96,6 +96,7 @@ describe('extensionModel first-party projection (all Encore features)', () => { 'opencodeServer', 'concerto', 'groupsPlus', + 'aCappella', ]); for (const def of BUILTIN_FEATURES) { @@ -134,6 +135,26 @@ describe('extensionModel first-party projection (all Encore features)', () => { expect(builtinExtension(groupsPlus!, flags({ groupsPlus: true })).state).toBe('enabled'); }); + it('projects A Cappella as a beta, disabled-by-default plugin-backed UI extension', () => { + const aCappella = BUILTIN_FEATURES.find((def) => def.flag === 'aCappella'); + expect(aCappella).toBeDefined(); + expect(aCappella!.beta).toBe(true); + + expect(builtinExtension(aCappella!, flags())).toMatchObject({ + key: 'builtin:aCappella', + state: 'not-installed', + name: 'A Cappella', + category: 'ui', + pluginId: 'com.maestro.acappella', + settingsNamespace: 'aCappella', + }); + expect(builtinExtension(aCappella!, flags({ aCappella: true })).state).toBe('enabled'); + + // Enabling voice must not imply a running background service: nothing + // listens until the user explicitly starts a session. + expect(builtinExtension(aCappella!, flags()).backgroundServiceId).toBeUndefined(); + }); + it('surfaces the plan-table identities on the details pane fields', () => { const byFlag = (flag: keyof EncoreFeatureFlags) => builtinExtension(BUILTIN_FEATURES.find((d) => d.flag === flag)!, flags()); diff --git a/src/__tests__/shared/pianola/pianola-first-party-plugin.test.ts b/src/__tests__/shared/pianola/pianola-first-party-plugin.test.ts index 32a3047418..545338c2fd 100644 --- a/src/__tests__/shared/pianola/pianola-first-party-plugin.test.ts +++ b/src/__tests__/shared/pianola/pianola-first-party-plugin.test.ts @@ -147,6 +147,7 @@ describe('first-party plugin registry', () => { ['opencodeServer', 'com.maestro.opencode-server'], ['concerto', 'com.maestro.concerto'], ['groupsPlus', 'com.maestro.groups-plus'], + ['aCappella', 'com.maestro.acappella'], ]); }); diff --git a/src/renderer/components/Settings/Extensions/ExtensionsGrid.tsx b/src/renderer/components/Settings/Extensions/ExtensionsGrid.tsx index 3bd23d5cad..2170d8c089 100644 --- a/src/renderer/components/Settings/Extensions/ExtensionsGrid.tsx +++ b/src/renderer/components/Settings/Extensions/ExtensionsGrid.tsx @@ -20,6 +20,7 @@ import { Zap, Clapperboard, Bot, + Mic, ShieldCheck, ShieldAlert, ShieldX, @@ -55,6 +56,7 @@ const BUILTIN_ICONS: Record = { maestroCue: Zap, directorNotes: Clapperboard, pianola: Bot, + aCappella: Mic, }; const TRUST_META: Record< diff --git a/src/renderer/components/Settings/Extensions/ExtensionsView.tsx b/src/renderer/components/Settings/Extensions/ExtensionsView.tsx index 2b806d32f1..0ed8c6f2b2 100644 --- a/src/renderer/components/Settings/Extensions/ExtensionsView.tsx +++ b/src/renderer/components/Settings/Extensions/ExtensionsView.tsx @@ -116,6 +116,7 @@ export function ExtensionsView({ theme, settingsBodies }: ExtensionsViewProps) {