diff --git a/.gitignore b/.gitignore index d086eb67a3..f42d998afc 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,16 @@ yarn-error.log* # Bundled-plugin signing artifact (release-time build output; dev signs locally) examples/plugins/*/signature.json + +# A Cappella model weights and partial downloads. +# +# Models live in userData and are downloaded by the user, never bundled into the +# installer: one of them is 1.1 GB, and shipping weights would multiply the +# download for everyone including people who never turn voice on. These patterns +# exist so a stray copy in the working tree (a manual download, a test fixture, a +# .part file from an interrupted transfer) cannot be committed by accident. +*.gguf +*.onnx +*.part +ggml-*.bin +models/acappella/ diff --git a/CLAUDE.md b/CLAUDE.md index 4ab87d73a1..e549ba7f84 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -198,6 +198,19 @@ Use "agent" in user-facing language. Reserve "session" for provider-level conver - **Cue** - Event-driven automation system (Maestro Cue), gated as an Encore Feature. Watches for file changes, time intervals, agent completions, GitHub PRs/issues, and pending markdown tasks to trigger automated prompts. Configured via `.maestro/cue.yaml` per project. - **Cue Modal** - Dashboard for managing Cue subscriptions and viewing activity (`CueModal.tsx`) +### Voice + +- **A Cappella** - The voice interface, gated as an Encore Feature and off by default. Speech in, routing to an agent and tab, spoken reply back. The session itself lives in the MAIN process (`src/main/acappella/voice-session-service.ts`); every renderer holds a projection of its event stream, never authority. Architecture: [docs/architecture/acappella/](docs/architecture/acappella/). User docs: [docs/voice-mode.md](docs/voice-mode.md). +- **Conductor** - The routing model that decides which agent and tab an utterance meant. It is a router, not a second assistant: it never answers a question itself. +- **Voice HUD** - The one on-screen surface for a session (`VoiceHud.tsx`). Minimize keeps the session and hands the indicator to the Left Bar; close ENDS it. +- **Voice floor** - Which microphone currently holds the session. One floor app-wide, held locally or by a paired device. + +Three invariants worth knowing before touching any of it, because each was a shipped bug: + +1. **Nothing starts itself.** Enabling the feature opens no device, downloads nothing, registers no hotkey, and renders no surface. A session begins only from an explicit trigger (composer microphone, global hotkey, command palette, Left Bar menu, wake word, paired phone). +2. **One session belongs to ONE window.** Voice events are broadcast to every renderer like every other main -> renderer push, so a surface that does not gate on `useOwnsVoiceSession()` appears in all of them at once. +3. **A missing capability is refused by name, never substituted.** The gate (`models/capability-gate.ts`) reports the specific slot and its recovery. It never quietly swaps in a cloud provider for a local one that will not load - that would spend the user's money and ship their microphone somewhere they did not choose. + ### Agent States (color-coded) - **Green** - Ready/idle @@ -322,57 +335,60 @@ src/ ## Key Files for Common Tasks -| Task | Primary Files | -| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Add IPC handler | `src/main/index.ts`, `src/main/preload.ts` | -| Add UI component | `src/renderer/components/` | -| Add browser/mobile touch affordance | `src/renderer/components/`, `src/renderer/utils/touch.ts`, `src/renderer/hooks/utils/` (gate on `isCoarsePointer()`; PWA assets in `src/web/public/`) | -| Add keyboard shortcut | `src/renderer/constants/shortcuts.ts`, `App.tsx` | -| Add theme | `src/renderer/constants/themes.ts` | -| Add modal | Component + `src/renderer/constants/modalPriorities.ts` | -| Add tab overlay menu | See Tab Hover Overlay Menu pattern in [[CLAUDE-PATTERNS.md]] | -| Add setting | `src/shared/settingsMetadata.ts` (metadata), `src/renderer/stores/settingsStore.ts`, `src/main/stores/defaults.ts`, AND `src/renderer/components/Settings/searchableSettings.ts` + `data-setting-id` wrapper on rendered control (see [[CLAUDE-PATTERNS.md]] §3) | -| Add template variable | `src/shared/templateVariables.ts`, `src/renderer/utils/templateVariables.ts` | -| Modify system prompts | `src/prompts/*.md` (wizard, Auto Run, etc.) or edit via **Maestro Prompts** tab in Settings | -| Customize prompts | Use **Maestro Prompts** tab in Settings, or edit `userData/core-prompts-customizations.json` | -| Add new prompt | `src/prompts/*.md`, `src/shared/promptDefinitions.ts` (add to `CORE_PROMPTS` array and `PROMPT_IDS`) | -| Add Spec-Kit command | `src/prompts/speckit/`, `src/main/speckit-manager.ts` | -| Add OpenSpec command | `src/prompts/openspec/`, `src/main/openspec-manager.ts` | -| Add CLI command | `src/cli/commands/`, `src/cli/index.ts` | -| Add new agent | `src/shared/agentIds.ts`, `src/main/agents/definitions.ts`, `src/main/agents/capabilities.ts`, `src/shared/agentMetadata.ts` - see [AGENT_SUPPORT.md](AGENT_SUPPORT.md) | -| Add agent output parser | `src/main/parsers/`, `src/main/parsers/index.ts` | -| Add agent session storage | `src/main/storage/` (extend `BaseSessionStorage`), `src/main/storage/index.ts` | -| Add agent error patterns | `src/main/parsers/error-patterns.ts` | -| Add agent context window | `src/shared/agentConstants.ts` (`DEFAULT_CONTEXT_WINDOWS`, `FALLBACK_CONTEXT_WINDOW`) | -| Add playbook feature | `src/cli/services/playbooks.ts` | -| Add marketplace playbook | `src/main/ipc/handlers/marketplace.ts` (import from GitHub) | -| Playbook import/export | `src/main/ipc/handlers/playbooks.ts` (ZIP handling with assets) | -| Modify wizard flow | `src/renderer/components/Wizard/` (see [[CLAUDE-WIZARD.md]]) | -| Add tour step | `src/renderer/components/Wizard/tour/tourSteps.ts` | -| Modify file linking | `src/renderer/utils/remarkFileLinks.ts` (remark plugin for `[[wiki]]` and path links) | -| Add documentation page | `docs/*.md`, `docs/docs.json` (navigation) | -| Add documentation screenshot | `docs/screenshots/` (PNG, kebab-case naming) | -| MCP server integration | See [MCP Server docs](https://docs.runmaestro.ai/mcp-server) | -| Add stats/analytics feature | `src/main/stats-db.ts`, `src/main/ipc/handlers/stats.ts` | -| Add Usage Dashboard chart | `src/renderer/components/UsageDashboard/` | -| Add Document Graph feature | `src/renderer/components/DocumentGraph/`, `src/main/ipc/handlers/documentGraph.ts` | -| Add colorblind palette | `src/renderer/constants/colorblindPalettes.ts` | -| Add performance metrics | `src/shared/performance-metrics.ts` | -| Capture/analyze perf trace | `src/main/profiling/` (Chromium contentTracing capture), `scripts/analyze-perf-trace.mjs` (offline analysis), `CLAUDE-PERFORMANCE.md` -> Field Performance Traces | -| Add power management | `src/main/power-manager.ts`, `src/main/ipc/handlers/system.ts` | -| Spawn agent with SSH support | `src/main/utils/ssh-spawn-wrapper.ts` (required for SSH remote execution) | -| Modify file preview tabs | `TabBar.tsx`, `FilePreview.tsx`, `MainPanel.tsx` (see ARCHITECTURE.md → File Preview Tab System) | -| Add Director's Notes feature | `src/renderer/components/DirectorNotes/`, `src/main/ipc/handlers/director-notes.ts` | -| Add Encore Feature | `src/renderer/types/index.ts` (flag), `useSettings.ts` (state), `SettingsModal.tsx` (toggle UI), gate in `App.tsx` + keyboard handler | -| Modify history components | `src/renderer/components/History/` | -| Modify history activity graph | `src/renderer/components/History/ActivityGraph.tsx`, `src/main/utils/history-bucket-cache.ts` (disk-cached aggregates), `src/main/utils/history-bucket-builder.ts` | -| Modify Auto Run Thought Stream | `src/renderer/stores/thoughtStreamStore.ts` (in-memory capture + `groupThoughtsIntoBlocks`), `src/renderer/components/ThoughtStreamPanel.tsx` (panel), `src/renderer/hooks/agent/internal/useThoughtStreamCaptureListener.ts` (taps `process:thinking-chunk`) | -| Add Cue event type | `src/main/cue/cue-types.ts`, `src/main/cue/cue-engine.ts` | -| Add Cue template variable | `src/shared/templateVariables.ts`, `src/main/cue/cue-executor.ts` | -| Modify Cue modal | `src/renderer/components/CueModal.tsx` | -| Configure Cue engine | `src/main/cue/cue-engine.ts`, `src/main/ipc/handlers/cue.ts` | -| Add terminal feature | `src/renderer/components/XTerminal.tsx`, `src/renderer/components/TerminalView.tsx` | -| Modify terminal tabs | `src/renderer/utils/terminalTabHelpers.ts`, `src/renderer/stores/tabStore.ts` | +| Task | Primary Files | +| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Add IPC handler | `src/main/index.ts`, `src/main/preload.ts` | +| Add UI component | `src/renderer/components/` | +| Add browser/mobile touch affordance | `src/renderer/components/`, `src/renderer/utils/touch.ts`, `src/renderer/hooks/utils/` (gate on `isCoarsePointer()`; PWA assets in `src/web/public/`) | +| Add keyboard shortcut | `src/renderer/constants/shortcuts.ts`, `App.tsx` | +| Add theme | `src/renderer/constants/themes.ts` | +| Add modal | Component + `src/renderer/constants/modalPriorities.ts` | +| Add tab overlay menu | See Tab Hover Overlay Menu pattern in [[CLAUDE-PATTERNS.md]] | +| Add setting | `src/shared/settingsMetadata.ts` (metadata), `src/renderer/stores/settingsStore.ts`, `src/main/stores/defaults.ts`, AND `src/renderer/components/Settings/searchableSettings.ts` + `data-setting-id` wrapper on rendered control (see [[CLAUDE-PATTERNS.md]] §3) | +| Add template variable | `src/shared/templateVariables.ts`, `src/renderer/utils/templateVariables.ts` | +| Modify system prompts | `src/prompts/*.md` (wizard, Auto Run, etc.) or edit via **Maestro Prompts** tab in Settings | +| Customize prompts | Use **Maestro Prompts** tab in Settings, or edit `userData/core-prompts-customizations.json` | +| Add new prompt | `src/prompts/*.md`, `src/shared/promptDefinitions.ts` (add to `CORE_PROMPTS` array and `PROMPT_IDS`) | +| Add Spec-Kit command | `src/prompts/speckit/`, `src/main/speckit-manager.ts` | +| Add OpenSpec command | `src/prompts/openspec/`, `src/main/openspec-manager.ts` | +| Add CLI command | `src/cli/commands/`, `src/cli/index.ts` | +| Add new agent | `src/shared/agentIds.ts`, `src/main/agents/definitions.ts`, `src/main/agents/capabilities.ts`, `src/shared/agentMetadata.ts` - see [AGENT_SUPPORT.md](AGENT_SUPPORT.md) | +| Add agent output parser | `src/main/parsers/`, `src/main/parsers/index.ts` | +| Add agent session storage | `src/main/storage/` (extend `BaseSessionStorage`), `src/main/storage/index.ts` | +| Add agent error patterns | `src/main/parsers/error-patterns.ts` | +| Add agent context window | `src/shared/agentConstants.ts` (`DEFAULT_CONTEXT_WINDOWS`, `FALLBACK_CONTEXT_WINDOW`) | +| Add playbook feature | `src/cli/services/playbooks.ts` | +| Add marketplace playbook | `src/main/ipc/handlers/marketplace.ts` (import from GitHub) | +| Playbook import/export | `src/main/ipc/handlers/playbooks.ts` (ZIP handling with assets) | +| Modify wizard flow | `src/renderer/components/Wizard/` (see [[CLAUDE-WIZARD.md]]) | +| Add tour step | `src/renderer/components/Wizard/tour/tourSteps.ts` | +| Modify file linking | `src/renderer/utils/remarkFileLinks.ts` (remark plugin for `[[wiki]]` and path links) | +| Add documentation page | `docs/*.md`, `docs/docs.json` (navigation) | +| Add documentation screenshot | `docs/screenshots/` (PNG, kebab-case naming) | +| MCP server integration | See [MCP Server docs](https://docs.runmaestro.ai/mcp-server) | +| Add stats/analytics feature | `src/main/stats-db.ts`, `src/main/ipc/handlers/stats.ts` | +| Add Usage Dashboard chart | `src/renderer/components/UsageDashboard/` | +| Add Document Graph feature | `src/renderer/components/DocumentGraph/`, `src/main/ipc/handlers/documentGraph.ts` | +| Add colorblind palette | `src/renderer/constants/colorblindPalettes.ts` | +| Add performance metrics | `src/shared/performance-metrics.ts` | +| Capture/analyze perf trace | `src/main/profiling/` (Chromium contentTracing capture), `scripts/analyze-perf-trace.mjs` (offline analysis), `CLAUDE-PERFORMANCE.md` -> Field Performance Traces | +| Add power management | `src/main/power-manager.ts`, `src/main/ipc/handlers/system.ts` | +| Spawn agent with SSH support | `src/main/utils/ssh-spawn-wrapper.ts` (required for SSH remote execution) | +| Modify file preview tabs | `TabBar.tsx`, `FilePreview.tsx`, `MainPanel.tsx` (see ARCHITECTURE.md → File Preview Tab System) | +| Add Director's Notes feature | `src/renderer/components/DirectorNotes/`, `src/main/ipc/handlers/director-notes.ts` | +| Add Encore Feature | `src/renderer/types/index.ts` (flag), `useSettings.ts` (state), `SettingsModal.tsx` (toggle UI), gate in `App.tsx` + keyboard handler | +| Modify history components | `src/renderer/components/History/` | +| Modify history activity graph | `src/renderer/components/History/ActivityGraph.tsx`, `src/main/utils/history-bucket-cache.ts` (disk-cached aggregates), `src/main/utils/history-bucket-builder.ts` | +| Modify Auto Run Thought Stream | `src/renderer/stores/thoughtStreamStore.ts` (in-memory capture + `groupThoughtsIntoBlocks`), `src/renderer/components/ThoughtStreamPanel.tsx` (panel), `src/renderer/hooks/agent/internal/useThoughtStreamCaptureListener.ts` (taps `process:thinking-chunk`) | +| Add A Cappella voice feature | `src/main/acappella/` (session, providers, router, audio, wake, transport), `src/main/ipc/handlers/acappella.ts`, `src/renderer/components/ACappella/` - read [docs/architecture/acappella/system-overview.md](docs/architecture/acappella/system-overview.md) first | +| Add a voice provider (STT/TTS/brain) | `src/main/acappella/providers/` + `src/shared/acappella/provider-catalog.ts`. Register in `provider-registry.ts`; NEVER substitute one provider for another (see `capability-gate.ts`) | +| Surface a voice session in the UI | `useOwnsVoiceSession()` in `src/renderer/components/ACappella/` - voice events are broadcast to EVERY window, so any new voice surface must gate on it or it renders in all of them | +| Add Cue event type | `src/main/cue/cue-types.ts`, `src/main/cue/cue-engine.ts` | +| Add Cue template variable | `src/shared/templateVariables.ts`, `src/main/cue/cue-executor.ts` | +| Modify Cue modal | `src/renderer/components/CueModal.tsx` | +| Configure Cue engine | `src/main/cue/cue-engine.ts`, `src/main/ipc/handlers/cue.ts` | +| Add terminal feature | `src/renderer/components/XTerminal.tsx`, `src/renderer/components/TerminalView.tsx` | +| Modify terminal tabs | `src/renderer/utils/terminalTabHelpers.ts`, `src/renderer/stores/tabStore.ts` | --- diff --git a/build/entitlements.mac.plist b/build/entitlements.mac.plist index a84699465e..54b94314a9 100644 --- a/build/entitlements.mac.plist +++ b/build/entitlements.mac.plist @@ -8,8 +8,11 @@ com.apple.security.cs.disable-library-validation + com.apple.security.device.audio-input - + com.apple.security.automation.apple-events diff --git a/docs/.mintignore b/docs/.mintignore index caafba3181..c3fbc8024e 100644 --- a/docs/.mintignore +++ b/docs/.mintignore @@ -3,3 +3,8 @@ # `{...}` examples that are valid Markdown but not valid MDX, so exclude the whole # directory from the Mintlify build to keep the deploy green. agent-guides/ + +# Internal architecture notes and ADRs (structured markdown with YAML front +# matter and `[[wiki-links]]` for DocGraph, not part of docs.json navigation). +# Same MDX-hostile `{...}` type examples as above. +architecture/ diff --git a/docs/agent-guides/SHARED-UTILS.md b/docs/agent-guides/SHARED-UTILS.md index a0871d36a6..b1cdc0f780 100644 --- a/docs/agent-guides/SHARED-UTILS.md +++ b/docs/agent-guides/SHARED-UTILS.md @@ -508,6 +508,35 @@ A checked task is stepped over entirely, marker and all. That keeps a half-finis --- +## A Cappella Encore Flag (`src/shared/acappella/feature-flag.ts` - Both) + +The ONE reader of the `encoreFeatures.aCappella` flag. Do NOT hand-roll +`flags.aCappella === true` at a new call site: the surfaces that gate on it are +not one system (IPC handlers, the hotkey installation, the WebSocket signaling +adapter, the transport, the debug-package collector), they each control a real +resource - a microphone, a global shortcut, a Bonjour advert - and a surface that +disagrees with the rest leaves one of those running behind a switch the user +believes is off. Five byte-identical copies had already accumulated. + +| Function / Constant | Signature | Purpose | +| -------------------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `isACappellaEnabled(store)` | `(EncoreFlagStore) => bool` | True only for the literal `true`. A hand-edited `"true"` or `1` reads as OFF, which is the safe direction for a mic feature. | +| `requireACappellaEnabled(store)` | `(EncoreFlagStore) => void` | Throw `ACappellaDisabled` unless the flag is on. What a gated IPC handler calls. | +| `ACAPPELLA_DISABLED_ERROR` | `'ACappellaDisabled'` | The stable error string the renderer maps. Not prose - a sentence here would be a wire contract. | +| `EncoreFlagStore` | `{ get(key, default?) }` | The narrow store slice this needs, so an electron-store or a plain object both satisfy it. | + +**Turning the flag off is a teardown, not just a gate.** +`shutdownACappellaForDisable()` in `src/main/ipc/handlers/acappella.ts` is what +`main/index.ts` runs from its `encoreFeatures` watcher: it stops the session, +drops the audio bridge, disposes the inference pipeline (which is also what lets +reclaim-disk delete model files on Windows), and calls +`ACappellaTransport.standDown()` for the advert and the connected phones. It +deliberately does NOT dispose the transport or the hotkey installation - both are +built once per process, so tearing them down would mean switching the feature +back on did nothing until the next restart. + +--- + ## Synopsis Parsing (`src/shared/synopsis.ts` - Both) | Function / Constant | Signature | Purpose | diff --git a/docs/architecture/acappella/conversation-acceptance.md b/docs/architecture/acappella/conversation-acceptance.md new file mode 100644 index 0000000000..c0a04d90dc --- /dev/null +++ b/docs/architecture/acappella/conversation-acceptance.md @@ -0,0 +1,207 @@ +--- +type: reference +title: A Cappella Conversation Acceptance Checklist +created: 2026-08-15 +tags: + - acappella + - architecture + - testing + - speech +related: + - '[[latency-baseline]]' + - '[[system-overview]]' + - '[[voice-session-protocol]]' +--- + +# A Cappella Conversation Acceptance Checklist + +The speech layer is the one part of A Cappella that automated tests cannot sign off. A scheduler +can be proven to deliver sentences in order without a human confirming that the result sounds like +a person talking, and a barge-in controller can tear down in the right order while still feeling +laggy to the person doing the interrupting. This document is the by-hand pass: what to say, what to +listen for, and which module to open when a check fails. + +Run it on the oldest machine available. Every check below is a latency judgement in disguise, and a +fast machine hides the failures this list exists to find. + +## Precondition: the layer has to be reachable + +The speech layer is wired as of 2026-08-15. `VoiceSessionService` composes the translator, the +scheduler, the barge-in controller, the detail buffer, and the background announcer directly, and +`src/main/ipc/handlers/acappella.ts` builds the agent-output tap over the process manager and hands +it in as `agentReplyStream`. Confirm both seams before treating any failure below as a bug: + +```bash +grep -rn "ConversationalTranslator\|SpeechScheduler\|BargeInController" \ + src/main/acappella/voice-session-service.ts +grep -rn "createAgentOutputTap\|agentReplyStream" src/main/ipc/handlers/acappella.ts +``` + +Two things can still leave the layer inert at runtime, and both are silent: + +- **No process manager.** `agentReplyStream` is optional, and without it the session waits for a + whole reply through `submitAgentReply()` - the `buffered` counterfactual the latency harness + measures the shipped path against in [[latency-baseline]]. This is the mock tier and the dev + harness. In the packaged app the manager is passed from `src/main/ipc/bootstrap/index.ts`. +- **A focus-only dispatch.** The tap is only armed when the dispatch actually sent a prompt, so + "switch to the backend agent" is correctly followed by silence rather than by the tab's previous + output being read aloud. + +One thing is genuinely not wired yet: `focusTarget`, which is what a "show me" needs to put a tab on +screen. Nothing supplies it, so check 5's `show` case focuses nothing today. That is a renderer +round trip and belongs with the Phase 09 tab affordances. + +## What else you need + +Four things, none of which live in the repo: + +1. The three native runtimes installed. They are declared in + `src/shared/acappella/native-runtimes.ts` but are not yet `package.json` dependencies. See + [[packaging-notes]]. +2. The model set downloaded, from **Settings > Plugins > A Cappella > Models**. +3. API keys for OpenAI and ElevenLabs if you are testing anything other than the fully local + configuration. +4. A microphone, and a quiet room. Barge-in checks are meaningless over a fan. + +## The checks + +Each check is one utterance, one thing to listen for, and one place to look when it fails. + +### 1. The first word arrives before the agent has finished writing + +**Say:** "ask backend to summarise everything that changed in the router this week" + +**Listen for:** speech starting while the agent is visibly still writing in its tab. The point is +not that speech is fast, it is that speech and writing overlap. If the reply finishes rendering in +silence and only then does anything get said, the tap is not cutting at a completed thought. + +**When it fails:** `speech/agent-output-tap.ts`. `DEFAULT_MIN_CHUNK_CHARS` is 200, so a reply whose +first paragraph is shorter than that waits for a paragraph break. Raise the agent's verbosity before +concluding the tap is broken. + +### 2. Sentences play without gaps + +**Say:** anything that produces a four or five sentence answer. + +**Listen for:** the seam between sentence one and sentence two. A pause there is a provider round +trip that the lookahead should have hidden. + +**When it fails:** `speech/speech-scheduler.ts`. `DEFAULT_LOOKAHEAD` is 1, meaning two sentences are +in flight at once. A slow TTS provider may need 2. Note that the scheduler delivers strictly in +order regardless of lookahead, so raising it cannot reorder speech. + +### 3. Interrupting stops audio immediately and captures your first word + +**Say:** anything long, then talk over it mid-sentence. Start your interruption with a distinct word +you can check for, such as "stop, actually, what about the tests". + +**Listen for:** audio dropping within a beat, not at the end of the current sentence. Then check the +transcript: your first word has to be there. "Actually, what about the tests" with the leading +"stop" missing means the pre-roll is not reaching the reopened floor. + +**When it fails:** `speech/barge-in.ts`. Ducking is a 20 ms ramp to `DEFAULT_DUCK_GAIN` 0.15, so what +you hear should be a fast fade rather than a hard cut. The order matters and is deliberate: duck, +flush playback, cancel synthesis, cancel the translator stream, reopen the floor. A missing first +word is the last step; audio that keeps playing is one of the first two. + +Also confirm the negative case: the assistant must not interrupt itself. Let a long reply play in +full without speaking. Any self-interrupt means AEC leakage is beating the 250 ms +`DEFAULT_GUARD_MS` window. + +### 4. "Tell me more" drills into real detail instantly + +**Say:** after any substantial reply, "tell me more". + +**Listen for:** detail arriving with no perceptible think time and no new work in the agent's tab. +The whole point of the detail buffer is that the follow-up costs nothing. If the agent tab shows a +new turn, the utterance was routed instead of matched as a follow-up. + +**Then check the siblings:** "read that again" repeats what was actually spoken, not what was +queued. "What was the file" speaks a basename, never a path read character by character. "Show me" +focuses the tab and says nothing at all. + +**When it fails:** `speech/drill-down.ts`. Intent matching is ordered, with `show` ahead of `file`. + +### 5. Nothing markdown-shaped is ever read aloud + +**Say:** something that forces a code-heavy answer, such as "ask backend to show me the diff for the +router change". + +**Listen for:** silence over the diff. The intro line should be spoken, the fence never. This is the +check most likely to surface something ugly, because it is where the tap's filtering and the +translator's markdown stripping have to agree. + +**Known rough edge:** a diff-heavy reply produces a real multi-second silence mid-turn. The intro is +spoken, the fence is correctly skipped, and the hang notice cannot cover the gap because the diff +keeps arriving as `data` and keeps resetting the 20 s timer. Nothing is malfunctioning and it is +still the worst listening experience the harness produces. See [[latency-baseline]]. + +### 6. A background completion waits for a pause + +**Say:** dispatch to a second agent, then start a conversation with the first while the second +works. + +**Listen for:** the second agent's completion never landing on top of your conversation, and when it +does land, naming its source ("the backend agent finished the migration"). + +**When it fails:** `speech/background-announcer.ts`. The setting is `speakBackgroundCompletions` +under the `acappella` settings key, with `on | off | auto`. `auto` is the default and resolves to on +for the Conductor scope, off inside a focused agent session, so a silent announcement inside a +focused session is correct behaviour rather than a bug. + +## The visual checks + +The six checks above are about what you hear. These are about what you see, and they are equally +outside what an automated test can sign off: jsdom has no layout engine, so every assertion about +clipping, readability, and contrast in the test suite is an assertion about VALUES rather than about +pixels. The exception is colour contrast, which `VoiceAccessibility.test.tsx` verifies against every +shipped theme with `contrastRatio()`, so this pass is looking for layout and legibility rather than +re-checking the numbers. + +Run these in at least **three themes, one of which must be a light theme**. Light themes are where a +widget built against a dark default falls apart, and A Cappella's HUD is drawn almost entirely from +theme colours. + +### 7. The HUD is readable and nothing clips + +**Do:** open a session, drag the HUD to each corner, and let a turn run through listening, thinking, +and speaking in each theme. + +**Look for:** the five indicator states distinguishable at a glance and by SHAPE, not only by hue +(outlined ring, filled disc, dashed spinner ring, error ring). The bound scope in the agent's own +colour, legible against the panel. Nothing spilling out of the widget. + +### 8. Minimize keeps the audio, close stops it + +**Do:** while a reply is being spoken, press the `-` button. Then restore, and press the ESC pill. + +**Look for:** minimize collapsing the HUD to a small indicator with the reply STILL AUDIBLE and a +visible way back. Close stopping the speech and ending the session. + +A control that hides itself must not silently leave a hot microphone, and a close button that only +hides leaves audio coming from nowhere. This is the one pair in the feature where getting it +backwards is a safety problem rather than a papercut. + +### 9. The transcript survives a restart and does not interrupt + +**Do:** turn the transcript on from the HUD, quit and reopen Maestro, then turn it off mid-reply. + +**Look for:** the transcript still open after the restart, and turning it off leaving the +conversation running - the speech does not stop, the floor is not released, and the next sentence +still arrives. + +### 10. Reduced motion actually stops the motion + +**Do:** turn on the OS "reduce motion" setting while a session is live (macOS: System Settings -> +Accessibility -> Display -> Reduce motion). + +**Look for:** the animations stopping WITHOUT a restart, and each state still distinguishable +without them. This widget is designed to be left on screen all day, which is exactly why a +permanently animating one is a real problem rather than a preference. + +## Recording the result + +Numbers from the same session go in the **Measured results** table in [[latency-baseline]]: press +**Read last turn** on the Models page and paste the copied JSON. Three turns per configuration, +record the median. The first turn of a local configuration includes the model load and is not +representative of a conversation. diff --git a/docs/architecture/acappella/decisions/adr-001-webrtc-transport.md b/docs/architecture/acappella/decisions/adr-001-webrtc-transport.md new file mode 100644 index 0000000000..fac451e1f3 --- /dev/null +++ b/docs/architecture/acappella/decisions/adr-001-webrtc-transport.md @@ -0,0 +1,132 @@ +--- +type: decision +title: 'ADR-001: WebRTC for the phone audio leg' +created: 2026-08-14 +tags: + - voice + - architecture + - acappella + - adr +related: + - '[[system-overview]]' + - '[[voice-session-protocol]]' + - '[[adr-002-main-process-session]]' + - '[[transport-and-pairing]]' +--- + +# ADR-001: WebRTC for the phone audio leg + +**Status:** Accepted +**Date:** 2026-08-14 +**Deciders:** A Cappella Phase 01 + +## Context + +A Cappella's iPhone client carries live microphone audio to the desktop and live speech back. The +desktop already runs an authenticated WebSocket at `/$TOKEN/ws` +(`src/main/web-server/routes/wsRoute.ts`) with an established client registry and a broadcast +fan-out (`src/main/web-server/services/broadcastService.ts`). The obvious cheap move is to encode +Opus frames and push them over that socket in both directions. + +The obvious cheap move is wrong, and it is worth writing down why so nobody re-proposes it in a +later phase. + +## Decision + +**The phone's media leg is WebRTC. The existing authenticated WebSocket carries signaling only.** + +Signaling (offer, answer, ICE candidates) rides `/$TOKEN/ws`, so the phone leg inherits the token +auth, the client registry, and the connection lifecycle that already exist. Media does not touch +that socket. + +Control events from [[voice-session-protocol]] also ride the WebSocket. Only audio moves to the +peer connection. + +## Rationale + +### Acoustic echo cancellation + +This is the decisive reason. The phone plays the assistant's speech through its speaker while its +microphone is open for barge-in. Without echo cancellation the microphone hears the assistant, +the STT transcribes it, and the system talks to itself. Barge-in, the single most important +interaction in the whole feature, becomes impossible. + +WebRTC's audio pipeline provides AEC, noise suppression, and automatic gain control as part of +`getUserMedia` plus the peer connection, tuned by people who have spent two decades on it. A raw +WebSocket carrying Opus frames provides none of them. Building AEC by hand is not a weekend +project; it is the reason conferencing products have audio teams. + +### Jitter buffer + +Networks deliver audio unevenly. WebRTC maintains an adaptive jitter buffer that trades a few +milliseconds of latency for smooth playback and adapts as conditions change. Over a WebSocket we +would write our own, and a hand-rolled fixed buffer is either too small (audible gaps) or too +large (barge-in feels laggy, which reads as the assistant ignoring you). + +### Packet loss concealment + +Opus over WebRTC conceals lost packets by synthesizing plausible audio for the gap. A lost frame +degrades quality slightly. TCP has no concept of concealment: a lost segment is retransmitted, +and the whole stream **stalls** behind it (head-of-line blocking). On a phone walking between WiFi +and cellular, that turns a small loss into a visible freeze in a conversation. + +### UDP versus TCP + +WebSockets are TCP. For real-time media, TCP's reliability guarantee is the wrong guarantee: a +retransmitted audio packet that arrives 400 ms late is worthless, and waiting for it delays every +packet behind it. WebRTC uses UDP, where late audio is simply dropped and the conversation keeps +moving. Plus ICE/STUN gives NAT traversal and network-change survival (WiFi to cellular) that a +plain socket would need re-connect logic to approximate. + +## Alternatives considered + +### WebSocket plus Opus frames + +Rejected. Cheapest to build, and it fails on all four axes above. The critical failure is AEC: +without it, open-mic barge-in cannot work at all, and barge-in is not optional in this design +(see [[voice-session-protocol]], "Invariants"). + +### Native iOS audio session with a custom UDP protocol + +Rejected. This is re-implementing WebRTC with fewer eyes on it, and it forfeits the browser-based +fallback client entirely. + +### Push to talk only, half duplex, over the existing socket + +Rejected as the architecture, though it remains a usable **mode**. Half duplex sidesteps echo by +never having the microphone open during playback, which makes a WebSocket viable. But it removes +barge-in, and barge-in is the difference between talking to a system and waiting for one. Half +duplex can ship as a low-bandwidth fallback on top of the WebRTC design; the reverse (adding full +duplex to a WebSocket design later) means rebuilding the transport. + +## Consequences + +**Positive** + +- Barge-in works with the speaker on, which is the whole interaction model. +- Network changes and packet loss degrade gracefully instead of freezing. +- The browser gets the same client for free, so a laptop can be a voice client too. +- Signaling reuses the existing authenticated socket, so there is no second auth surface. + +**Negative** + +- More moving parts: ICE, STUN, and possibly TURN for hostile networks. The desktop needs a + WebRTC peer implementation in the main process, which is a real dependency rather than a few + lines of `ws`. +- Local network discovery and certificate handling need design work in the phone phase. +- Debugging is harder: media that does not flow has more possible causes than a socket that is + closed. + +**Neutral** + +- The [[voice-session-protocol]] event stream is unaffected. It was designed transport-agnostic + precisely so this decision could be made independently, and so a client with no media leg at all + (the desktop dev harness) is still a first-class client. + +## Related + +- [[system-overview]] - where the phone sits in the client model. +- [[voice-session-protocol]] - the control-plane events that ride the WebSocket. +- [[adr-002-main-process-session]] - why there is a session in main for the phone to be a peer of. +- [[transport-and-pairing]] - how the decision was actually built: pairing, signaling, ICE, TURN, + and the connection matrix. diff --git a/docs/architecture/acappella/decisions/adr-002-main-process-session.md b/docs/architecture/acappella/decisions/adr-002-main-process-session.md new file mode 100644 index 0000000000..4884bbfd05 --- /dev/null +++ b/docs/architecture/acappella/decisions/adr-002-main-process-session.md @@ -0,0 +1,144 @@ +--- +type: decision +title: 'ADR-002: The voice session is headless in the main process' +created: 2026-08-14 +tags: + - voice + - architecture + - acappella + - adr +related: + - '[[system-overview]]' + - '[[voice-session-protocol]]' + - '[[adr-001-webrtc-transport]]' +--- + +# ADR-002: The voice session is headless in the main process + +**Status:** Accepted +**Date:** 2026-08-14 +**Deciders:** A Cappella Phase 01 + +## Context + +A voice session owns a state machine, a monotonic event sequence, three providers, and a dispatch +path into agents and tabs. It has to live somewhere. + +The renderer is the tempting home. Maestro already does Web Speech dictation there +(`src/renderer/hooks/utils/useVoiceInput.ts`), the microphone and speaker are trivially reachable +from a renderer, `MediaStream` and `AudioWorklet` are browser APIs, and all the state stores are +already there. Building it in the renderer would be faster in Phase 01. + +## Decision + +**`VoiceSessionService` lives in `src/main/acappella/`, is headless, and is transport-agnostic.** +It may not reference a `BrowserWindow`, a React store, or the DOM. Every UI, on every device, is a +client of [[voice-session-protocol]]. + +## Rationale + +### The iPhone is a peer, not a port + +This is the reason that outranks the others. If the session lives in the renderer, the phone +either talks to the renderer (making one window a server for a physical device, which breaks the +moment that window closes or Maestro goes multi-window) or gets a second, parallel session +implementation in main. A second implementation means two state machines that drift, two routing +paths, two barge-in semantics, and two sets of bugs. + +With the session in main, the phone and the desktop HUD are the same kind of thing: subscribers to +one event stream, senders of the same three commands. Adding a client is adding a transport +adapter, not a feature. + +### Renderer lifetime is not session lifetime + +A renderer can be reloaded, hidden, closed, or moved between windows. Maestro is already +multi-window, and windows own subsets of agents. A voice session that dies because the user closed +the window it happened to start in is broken by construction, and one that has to be handed +between windows is worse. Conversations are longer-lived than views. + +### Dispatch authority already lives in main + +Routing a decision into agents and tabs needs the roster, and main already holds it: the sessions +store (`src/main/stores/getters.ts`, read the same way `registerSessionCallbacks` reads it) plus +the `remote:*` bridge the web server uses to ask the renderer to act +(`src/main/web-server/callbacks/tabCallbacks.ts`). A renderer-owned session would either duplicate +that or reach back into main for every step anyway. + +Note the asymmetry this creates and accepts: main holds the roster but has **no** tab authority. +Tab state lives in the renderer, so `executeRouteDecision()` sends `remote:newTab` / +`remote:selectTab` / `remote:renameTab` and waits for confirmation, exactly as the web path does. +Main is the decider; the renderer stays the executor. + +### Providers are Node-shaped + +Local Whisper and Kokoro are native modules or child processes. Cloud realtime sessions want a +long-lived socket with a secret key. A renderer has neither native module access nor a safe place +for keys, so provider work would end up proxied through IPC in every case. Putting the session +where the providers already have to run removes a whole layer of marshalling from the hot path, +which is latency the user hears. + +### Precedent + +Every comparable Maestro subsystem is already main-owned and renderer-projected: the Cue engine, +the web server, Pianola's supervised loop, the plugin broker. Cadenza's HUD window +(`src/main/app-lifecycle/cadenza-hud-window.ts`) is the closest visual analogue, and even it is a +main-created window that buffers payloads until the renderer signals ready. Voice would be the +odd one out. + +## Alternatives considered + +### Session in the renderer, main as a thin relay + +Rejected. Fastest to Phase 01 and worst for Phases 02 onward: it forces a second implementation +for the phone, ties session lifetime to a window, and puts provider secrets and native modules on +the wrong side of the bridge. + +### Session in a dedicated hidden `BrowserWindow` + +Considered seriously. A hidden renderer would give the session `MediaStream`, `AudioWorklet`, and +`RTCPeerConnection` for free, which is genuinely attractive given [[adr-001-webrtc-transport]]. + +Rejected because it trades one problem for a worse one: a hidden window is still a renderer with a +lifecycle, a crash surface, and IPC latency between it and the dispatch authority, and it makes +"who owns the session" ambiguous again. It also cannot be reached from the CLI or a headless +context. + +The capability gap is real but narrow, and it is solvable with a Node WebRTC implementation in +main. If that turns out to be untenable, the fallback is a hidden window acting as a **media +endpoint only**, still driven by the main-process session, not a relocation of the session itself. + +### Session in a separate process + +Rejected as premature. It would add IPC to the dispatch path and process supervision to the +lifecycle for isolation nobody has asked for. Revisit only if a provider proves unstable enough to +threaten app stability. + +## Consequences + +**Positive** + +- One session implementation for every client, forever. +- Sessions survive window reload, window close, and multi-window moves. +- Providers sit next to their native modules and secrets. +- The service is trivially testable: no DOM, no React, no Electron window. Feed it `startSession` + and `submitUtterance`, assert on the event stream. + +**Negative** + +- Audio device access in main needs deliberate work. The renderer's easy `getUserMedia` path is + not available, so Phase 01 ships mock providers and the real capture path is a later phase's + problem. +- WebRTC in main requires a Node implementation rather than the browser's built-in one. +- Every UI interaction costs an IPC hop. Acceptable: the events are small and infrequent compared + to the audio itself, which never crosses this boundary. + +**Neutral** + +- The renderer keeps `useVoiceInput.ts` for composer dictation until the local STT tier lands. + Two voice paths coexist briefly, with a clear owner for each. + +## Related + +- [[system-overview]] - the service, the tiers, and the client model. +- [[voice-session-protocol]] - the contract every client speaks. +- [[adr-001-webrtc-transport]] - the transport this decision makes possible. diff --git a/docs/architecture/acappella/latency-baseline.md b/docs/architecture/acappella/latency-baseline.md new file mode 100644 index 0000000000..21cab50c5c --- /dev/null +++ b/docs/architecture/acappella/latency-baseline.md @@ -0,0 +1,224 @@ +--- +type: reference +title: A Cappella Latency Baseline +created: 2026-08-15 +tags: + - acappella + - architecture + - latency + - providers +related: + - '[[system-overview]]' + - '[[model-manager]]' + - '[[voice-session-protocol]]' + - '[[conversation-acceptance]]' +--- + +# A Cappella Latency Baseline + +Voice is the one Maestro surface where latency is not a quality-of-life detail. A user talking to +an agent has no screen to read while they wait, so a slow turn is indistinguishable from a broken +one. This document records what each provider configuration costs per hop, how those numbers are +measured, and which hop to suspect first when someone reports that voice feels slow. + +## How a turn is measured + +Every turn is timed by `src/main/acappella/telemetry/turn-metrics.ts`. The zero point is the +moment the voice activity detector decides the user stopped talking, published by the audio +pipeline as `onSpeechEnd`. That point matters: a timer anchored on the transcript would exclude the +decode, which is exactly the hop most often to blame. + +Six milestones are stamped per turn: + +| Span | What it measures | +| ------------------------------- | ---------------------------------------------------------------------- | +| **Speech end to first partial** | How long before anything at all appears. The gap the user feels most. | +| **Final transcript** | Endpointing to a settled utterance. | +| **Route decision** | Brain latency: which agent, which tab, which prompt. | +| **Agent first token** | Dispatch to the agent producing text. Not ours, but it is in the turn. | +| **First spoken sentence** | Reply text to the first audible sentence. | +| **Total turn** | Speech end to first audible sentence. | + +Spans are recorded cumulatively (one subtraction per mark, which is all a hot path should do) and +converted to per-hop deltas for display. They are formatted with `formatDuration()` from +`src/shared/performance-metrics.ts`; there is no second duration helper. + +The last twenty turns are retained in memory. The most recent breakdown is readable from +**Settings > Plugins > A Cappella > Models > Turn latency**, with a Copy button, so a bug report +carries numbers rather than an adjective. + +## Configurations + +Four shapes, in the order a user is likely to try them. + +### 1. Fully local (Whisper, Qwen3, Kokoro) + +Audio never leaves the machine. Every hop is CPU or GPU work on the user's hardware, so the numbers +vary by an order of magnitude across machines and this configuration is the one worth measuring on +the oldest machine you can find rather than the newest. + +### 2. OpenAI STT, local Brain, ElevenLabs TTS + +The mixed case. Two network hops and one local inference. Usually the fastest cascade on a laptop, +because a hosted transcription of a short utterance beats a local decode on a CPU without an +accelerator. + +### 3. Fully hosted cascade (OpenAI STT, OpenAI Brain, ElevenLabs TTS) + +Three network hops in series. Predictable, and bounded by the slowest provider on the day. + +### 4. Realtime (OpenAI speech to speech) + +One socket, the provider's own endpointing, and no serial hops at all. The tradeoff is stated in +the settings copy where the choice is made: the assistant speaks in that provider's voice, and the +microphone's samples go to their servers. + +## Measured results + +**Not yet measured. This table is deliberately empty rather than filled with plausible numbers.** + +Recording it requires four things this phase did not have on the build machine: the three native +runtimes installed (they are declared in `src/shared/acappella/native-runtimes.ts` but not yet in +`package.json` dependencies, see [[packaging-notes]]), the model set downloaded, API keys for +OpenAI and ElevenLabs, and a microphone. Inventing a baseline would be worse than having none: the +whole purpose of this document is to be the thing a regression is measured against. + +| Configuration | First partial | Final | Route | Agent first token | First spoken sentence | Total | +| ------------------------ | ------------- | ----- | ----- | ----------------- | --------------------- | ----- | +| Fully local | - | - | - | - | - | - | +| OpenAI STT + local Brain | - | - | - | - | - | - | +| Fully hosted cascade | - | - | - | - | - | - | +| Realtime | - | - | - | - | - | - | + +To fill it in: enable the Encore Feature, configure the slots in **Voice Providers**, speak one +short instruction ("ask backend what changed"), then press **Read last turn** on the Models page +and paste the copied JSON into the row. Repeat three times per configuration and record the median, +because the first turn of a local configuration includes the model load and is not representative +of a conversation. + +The by-hand pass that produces these numbers, including what to say and what to listen for on each +check, is [[conversation-acceptance]]. Read its precondition section first: a session built without +an `agentReplyStream` still waits for a whole reply, which measures the `buffered` arm below rather +than the streamed one. + +## What each hop tells you + +- **First partial is slow, local STT.** The decode is CPU-bound. Check whether the machine has an + accelerator whisper.cpp can use, and check the partial interval: re-transcribing the whole + utterance every 900 ms is the design, and on a slow machine each pass takes longer than the + interval, so passes are skipped rather than queued. +- **First partial is slow, hosted STT.** Network, or the utterance was long. The upload happens on + endpointing, so a long utterance costs upload time no partial can hide. +- **Route decision is slow, local Brain.** Almost always a model load: the Qwen3 context unloads + after five idle minutes, and the next turn pays for it once. Two consecutive turns will show the + difference immediately. +- **Route decision is slow, hosted Brain.** Look for a retry. The transport backs off on 429 and + 5xx up to three attempts, which can add seconds; the `provider-quota-exceeded` session error + fires only after the last attempt. +- **First spoken sentence is slow.** The TTS provider is synthesising the whole first sentence + before any audio exists. Both real providers work sentence by sentence for exactly this reason, + so a long first sentence is the usual cause. +- **Total is fine but it FEELS slow.** Check barge-in rather than latency. Cutting the assistant + off has to be instant, and a cancel that waits out a sentence reads as lag even when every number + above is good. + +## Time to first spoken word + +The number a user actually feels is not the total turn: it is how long they stand in silence before +anything is said. Everything in `src/main/acappella/speech/` exists to shorten that one span, and it +is measured as `speak-sentence` index 0 minus the detector's speech end, which is the +**First spoken sentence** column above. + +Three mechanisms move it, in descending order of effect: + +1. **The agent output tap cuts at a completed thought, not at the end of the reply.** A four hundred + line summary is spoken from its first paragraph while the agent is still writing the rest. This + is worth more than every other optimisation combined, because it removes the agent's own write + time from the span rather than shaving a hop. +2. **The translator rewrites that piece alone.** One short rewrite instead of one long one, and a + reply that is already conversational ("yes, the tests pass") skips the model entirely. + `ConversationalTranslator.stats` reports the translations-to-passthroughs split, which is the + number to check when short answers feel slower than they should. +3. **The scheduler synthesizes one sentence ahead of the one being delivered.** This does not shorten + the first word; it removes the provider round trip that would otherwise fall between every pair + of sentences. A reply that starts fast and then stutters is this, not the tap. + +### Measurements + +Two instruments, because they answer different questions and neither replaces the other. + +**The harness** (`npm run acappella:latency`) measures the span this layer owns, with the providers +replaced by stubs whose costs are declared in the script. It exists because a microphone session +measures four things at once - the decode, the model on the day, the network on the day, and the +streaming layer - and only the last of those is ours to regress. Everything between the stubs is the +shipping code: `AgentOutputTap`, `ConversationalTranslator`, `SpeechScheduler`, and the splitter. +Each fixture runs twice: `streamed` is the shipped path, `buffered` is the counterfactual the layer +replaced (wait for the whole reply, rewrite the whole thing, then speak). The agent writes at the +same rate in both arms, so the difference between them is the tap and nothing else. + +Recorded 2026-08-15, one run per cell, zero point the agent's first token. **First sound** and +**first word of the answer** are different numbers on a long reply: the buffered arm makes a noise at +twenty seconds because the tap refuses to go silent and says the agent is still working, which is the +safety net firing rather than an answer arriving. + +| Profile | Fixture | Arm | First sound | First word of the answer | Longest mid-turn silence | +| ------- | ------------ | -------- | ----------- | ------------------------ | ------------------------ | +| local | long summary | streamed | 220 ms | 220 ms | 0 ms | +| local | long summary | buffered | 20221 ms | 33734 ms | 11227 ms | +| local | diff-heavy | streamed | 226 ms | 226 ms | 11903 ms | +| local | diff-heavy | buffered | 14248 ms | 14248 ms | 0 ms | +| local | confirmation | streamed | 206 ms | 206 ms | 0 ms | +| local | confirmation | buffered | 206 ms | 206 ms | 0 ms | +| hosted | long summary | streamed | 280 ms | 280 ms | 0 ms | +| hosted | long summary | buffered | 20283 ms | 33795 ms | 11226 ms | +| hosted | diff-heavy | streamed | 284 ms | 284 ms | 12061 ms | +| hosted | diff-heavy | buffered | 14307 ms | 14307 ms | 0 ms | +| hosted | confirmation | streamed | 271 ms | 271 ms | 0 ms | +| hosted | confirmation | buffered | 275 ms | 275 ms | 0 ms | + +Three things to read out of it: + +- **The tap is worth 33 seconds on a long reply and 14 on a diff-heavy one.** That is the whole + argument for the layer, and it is not a hop that was shaved - it is the agent's own write time + removed from the span. Both profiles land within 60 ms of each other on the streamed arm, which is + the point: once the first spoken word costs one short rewrite, the choice of provider stops + mattering to the number the user feels. +- **A one-line confirmation is identical in both arms.** No model hop, because the passthrough test + catches it. If that row ever shows the streamed arm slower, the passthrough stopped firing. +- **The diff-heavy streamed row has a 12 second silence in the middle of the turn.** The intro line + is spoken at 226 ms, the fence is correctly never spoken, and nothing is said again until the + closing prose. The hang notice does not cover it, because the diff keeps arriving as `data` and + keeps resetting the timer. Nothing here is behaving incorrectly, and it is still the worst listening + experience the harness produces - the thing to watch if a user reports the assistant "stopping + halfway". + +**The real pipeline**, which the harness deliberately does not stand in for. Record the median of +three turns per configuration with one short instruction ("ask backend what changed"), taken from +**Settings > Plugins > A Cappella > Models > Turn latency**. Zero point is the detector's speech end, +so these include the decode and the route that the harness excludes. + +| Configuration | Time to first spoken word | Inter-sentence gap | Notes | +| -------------------- | ------------------------- | ------------------ | ----- | +| Fully local cascade | - | - | - | +| Fully hosted cascade | - | - | - | +| Realtime | - | - | - | + +The realtime pipeline bypasses this layer: its provider produces speech directly, so the tap and the +translator do not run and the span is the provider's own. That is the comparison the table exists to +make - when the cascade's time to first spoken word is within a couple of hundred milliseconds of +realtime, the streaming layer is doing its job and there is no reason to send audio to a third party +for speed alone. It is also why the harness has no realtime arm: there would be nothing of ours in it. + +### Barge-in + +Measured separately, from the detector's `speech-start` to playback going quiet. The teardown is +ordered so the ducking is first (see `speech/barge-in.ts`), which puts the number the user perceives +at roughly the duck ramp, about 20 ms, regardless of how long the cancellation behind it takes. The +guard window (250 ms after speech starts) is deliberately excluded: it is the one span where a +barge-in is refused on purpose. + +## Related + +- [[system-overview]] - the two pipeline shapes and the provider resolution rules. +- [[model-manager]] - what the local tier downloads and how readiness is decided. +- [[packaging-notes]] - the native runtimes the local tier needs. diff --git a/docs/architecture/acappella/model-manager.md b/docs/architecture/acappella/model-manager.md new file mode 100644 index 0000000000..81fca61055 --- /dev/null +++ b/docs/architecture/acappella/model-manager.md @@ -0,0 +1,135 @@ +--- +type: reference +title: A Cappella Model Manager +created: 2026-08-15 +tags: + - acappella + - architecture + - models +related: + - '[[system-overview]]' + - '[[voice-session-protocol]]' +--- + +# A Cappella Model Manager + +A Cappella is the first Maestro feature that ships a binary payload the user has to fetch. This +document describes how that stays honest: what is downloaded, when, from where, and how the app +proves the bytes are what it promised. + +## The four rules + +1. **Enabling the Encore Feature touches the network never.** Registering the model IPC handlers + constructs nothing and opens no socket. `models:list` is a disk read against a frozen local + catalog. The first byte of traffic in the whole subsystem comes from a `models:download` the + user pressed a button to send. +2. **Pinned revisions, never `main`.** Every source URL is `/resolve/<40-hex commit>/`. A moving + ref would mean the bytes behind a hash can change under us, which turns SHA-256 verification + into a superstition. +3. **A file appears at its final path only after its hash matched.** Until then the bytes live in + `.part`. A killed app therefore leaves a resumable partial, never a truncated file that + passes an existence check and detonates weeks later inside a model runtime. +4. **No silent substitution, ever.** When a required model is missing or corrupt, voice mode + refuses to start and says which piece is missing and what to do about it. It does not reach for + a cloud provider the user did not choose: that is both an unasked-for charge and a privacy + break. + +## Files + +| File | Responsibility | +| ----------------------------------------------- | ---------------------------------------------------------------------------------- | +| `src/shared/acappella/model-catalog.ts` | The frozen bill of materials: id, revision, per-file URL + SHA-256 + size, license | +| `src/shared/acappella/readiness.ts` | Readiness shapes, shared with the renderer | +| `src/main/acappella/models/model-store.ts` | Install layout, manifests, `isInstalled`, `verify`, `remove`, `totalFootprint` | +| `src/main/acappella/models/model-downloader.ts` | Resumable range downloads, streaming SHA-256, pause/resume/cancel, progress | +| `src/main/acappella/models/capability-gate.ts` | `resolveVoiceReadiness()`: which slots are satisfied, and why not | +| `src/main/ipc/handlers/acappella-models.ts` | The `models:*` channels and the `models:progress` broadcast | +| `src/renderer/components/Settings/ACappella/` | Voice Setup and the Models page | + +## The catalog + +Four models, all fetched from Hugging Face at a pinned commit: + +| Model | Role | Source | License | Size | +| -------------------------- | --------- | ---------------------------------------------- | ---------- | -------- | +| `whisper-base-en` | STT | `ggerganov/whisper.cpp@5359861` | MIT | 141.1 MB | +| `openwakeword-base` | Wake word | `littlebearlabs/openwakeword-features@5e032d9` | Apache-2.0 | 2.3 MB | +| `kokoro-82m` | TTS | `onnx-community/Kokoro-82M-v1.0-ONNX@1939ad2` | Apache-2.0 | 311.0 MB | +| `qwen3-1.7b-instruct-q4km` | Brain | `unsloth/Qwen3-1.7B-GGUF@d7f544e` | Apache-2.0 | 1.0 GB | + +Every hash is the Hugging Face LFS object id, which IS the SHA-256 of the file contents, read from +`/api/models//paths-info/` at the pinned commit. Do not hand-edit one. + +**A model is one or more FILES.** Two of the four genuinely need more than one: the wake word needs +its mel front end and its embedding head, and Kokoro needs a voice pack alongside the graph. +`sourceUrl` / `sha256` / `bytes` therefore live per file; the entry carries the computed total, and +`MODEL_SETS` totals are computed from those. No size string is written by hand anywhere, so a +revision bump cannot leave the UI quoting a stale number. + +`MODEL_SETS` names two bundles: `hands-free-local` (STT + wake word + TTS) and `fully-local` (that +plus the Brain). + +## Install layout + +``` +userData/models/acappella// + manifest.json id, revision, sha256, bytes, sourceUrl, license, installedAt, verifiedAt + the model itself +``` + +`isInstalled(id)` requires a manifest whose digest matches the current catalog AND every file's +byte length on disk to match exactly. It is never a bare `existsSync`, because a truncated file +exists. `verify(id)` is the slower check: it re-hashes and reports a mismatch as CORRUPT with both +hashes, and repairs nothing. Silently re-downloading would spend a gigabyte without asking and hide +the fact that something on this machine is modifying model files. + +Manifests are written through `atomicWriteJson` plus a per-model write queue +(`src/main/utils/atomic-json-store.ts`). Concurrent non-atomic writes have corrupted JSON state in +this codebase before. + +## Download lifecycle + +- Resume offset is the `.part` file's length, sent as `Range: bytes=N-`. A server that answers 200 + instead of 206 ignored the range, so the partial is discarded rather than appended to. +- SHA-256 is computed as bytes stream past; on resume the existing partial is re-hashed first so + the digest covers the whole file. +- A mismatch deletes the `.part` and reports both hashes. Keeping it would resume a file already + known to be wrong, forever. +- Pause keeps the partial; cancel deletes the model directory. Both leave on-disk state coherent, + and neither writes a manifest, because there is no manifest until success. +- Transient network errors retry with exponential backoff. A 404 or a hash mismatch does not. +- At most two models transfer at once: a 1.4 GB set downloaded four-wide saturates a domestic + uplink and makes every file slower, which reads to the user as a hang. +- Progress is throttled at the source (~4 Hz) and broadcast on `models:progress`. The renderer adds + a second `useThrottledCallback` stage for its own repaints. + +## The capability gate + +`resolveVoiceReadiness(settings)` returns a structured verdict rather than a boolean: per slot +(STT, TTS, Brain, wake word) either satisfied, or a reason plus a suggested action. The reasons are +closed: `model-not-installed`, `model-corrupt`, `api-key-missing`, `provider-unreachable`. + +`VoiceSessionService.startSession()` consults it BEFORE opening the microphone, and on failure +emits `session-error(provider-unavailable)` naming the missing piece and its recovery. The gate has +no code path that can return a different provider than the one configured; choosing providers is +the registry's job, and the registry's only fallback is the mock. + +**The wake word does not gate a session.** Hands-free means something is always listening, and that +is a real capability with a real requirement. Click-to-talk is not, so refusing a session the user +explicitly asked for because an optional always-on model is missing would be the gate getting in +the way. `canStartSession` and `canRunHandsFree` are reported separately. + +## Disk lifecycle + +`totalFootprint()` walks the models root rather than the catalog, so a directory left behind by a +model since dropped from the catalog is still counted and still reclaimable. Disk the user cannot +see is disk they cannot get back. + +`models:remove`, `models:remove-all`, and `models:footprint` stay callable when the Encore Feature +is OFF, following the `stop-session` precedent. The Models page offers to reclaim the space exactly +then, with a confirmation step, and deletes only `userData/models/acappella`. + +## Related + +- [[system-overview]] - the provider tiers and the session pipeline these models plug into. +- [[voice-session-protocol]] - every event, payload, and direction. diff --git a/docs/architecture/acappella/packaging-notes.md b/docs/architecture/acappella/packaging-notes.md new file mode 100644 index 0000000000..3765f19038 --- /dev/null +++ b/docs/architecture/acappella/packaging-notes.md @@ -0,0 +1,112 @@ +--- +type: reference +title: A Cappella Packaging, Signing, and Permissions +created: 2026-08-15 +tags: + - acappella + - architecture + - packaging + - notarization + - permissions +related: + - '[[system-overview]]' + - '[[model-manager]]' +--- + +# A Cappella Packaging, Signing, and Permissions + +Local inference means native binaries, and native binaries in an Electron app mean code signing, hardened runtime entitlements, notarization, and three separate platform stories. This page is the record of what was decided, what was verified, and what is still open, so the next person cutting a release does not rediscover it from a crash report. + +The load-bearing fact: every failure in this area is invisible in development. A native module left inside `app.asar`, an unsigned nested dylib, a missing per-platform prebuild - all of them work from source and fail only in the installed, signed app, on someone else's machine, after release. + +## The runtime registry is the single source of truth + +`src/shared/acappella/native-runtimes.ts` holds one descriptor per native runtime: the npm package, an exact version pin, the per-platform prebuild story, the `asarUnpack` globs, and the binaries a packaged app must contain. Four consumers read it and none of them keep their own copy: + +| Consumer | What it uses the registry for | +| ------------------------------------------------ | --------------------------------------------------------------- | +| `src/main/acappella/runtime/native-loader.ts` | The only module allowed to import these packages | +| `src/main/acappella/runtime/runtime-selftest.ts` | "Run voice self-test" on the Models page | +| `scripts/verify-native-packaging.mjs` | Post-packaging assertion, reads the compiled copy from `dist/` | +| `src/main/acappella/models/capability-gate.ts` | Reports a runtime that will not load as its own blocking reason | + +`src/__tests__/shared/acappella-native-runtimes.test.ts` asserts the registry against `package.json`: version pins are exact, every `asarUnpack` glob is present in the electron-builder config, and `declared` matches the actual dependency list. + +## The three runtimes + +| Runtime | Package | Version | Slots | Prebuilds | Electron rebuild | +| ------------ | ------------------ | ------- | --------------- | ----------------------------------------------------- | ---------------- | +| llama.cpp | `node-llama-cpp` | 3.20.0 | Conductor Brain | Prebuilt for all four targets via `@node-llama-cpp/*` | No | +| whisper.cpp | `smart-whisper` | 0.8.1 | Speech-to-Text | **None. Compiles from source at install** | No | +| ONNX Runtime | `onnxruntime-node` | 1.27.0 | TTS + wake word | Prebuilt, `bin/napi-v6///` | No | + +None of the three needs `electron-rebuild`. All three are Node-API addons, and Node-API is ABI-stable across Node and Electron by design, which is why they are absent from the `postinstall` rebuild list that carries `node-pty` and `better-sqlite3`. Adding a non-Node-API addon later means setting `requiresElectronRebuild: true` AND adding it to that list; the registry test fails if the two disagree. + +### Open question: whisper has no prebuilds + +`smart-whisper` runs `node-gyp rebuild` in its install script on every platform. That makes a C++ toolchain and CMake a build requirement for every contributor and both CI legs, not just for release machines. It is recorded here rather than worked around because the decision belongs with the phase that first executes the runtime: + +- Accept the source build and add the toolchain to CI, or +- Produce prebuilds ourselves and consume them, or +- Choose a different whisper.cpp binding, or run STT through ONNX instead and change the model catalog entry (the catalog currently pins `ggml-base.en.bin`, which is a whisper.cpp format). + +### Deliberately not yet dependencies + +All three descriptors carry `declared: false`, and none of the packages is in `package.json` dependencies yet. They land in the phase that first executes them (Phase 05, the real providers), and `declared` flips in that same commit. + +The reason is cost with no benefit: these packages are large, one of them compiles from source, and until a provider calls them, adding them would slow every `npm ci` and both CI legs to install code nothing runs. The loader reports `not-a-dependency`, which is a distinct and truthful answer from "your install is broken", the self-test reports `skipped` rather than `fail`, and the packaging script skips them unless run with `--require-all`. The packaging configuration (asarUnpack globs, entitlements, Info.plist, the assertion script) is already in place, so the phase that adds the dependencies changes one boolean per runtime and one dependency line, not the build. + +## macOS: entitlements, Info.plist, notarization + +`build/entitlements.mac.plist` gained `com.apple.security.device.audio-input` (it was present but set to `false`, which denies capture under the hardened runtime with no prompt shown - a session that starts and stays silent forever). + +`com.apple.security.cs.allow-jit`, `allow-unsigned-executable-memory`, and `disable-library-validation` were already enabled for reasons that predate A Cappella. They were NOT added for the native runtimes and should not be justified by them; each one weakens the app, and any future addition needs a runtime that provably requires it. + +`NSMicrophoneUsageDescription` is set through `build.mac.extendInfo` in `package.json`. It names A Cappella specifically and states that audio is processed on the machine when local providers are selected. A registry test asserts both properties, because "Maestro would like to access the microphone" answers neither question a user has at the moment of the prompt. + +Every nested binary must be signed with the same identity: notarization rejects a bundle containing an unsigned nested binary, and `node-llama-cpp` ships several ggml dylibs beside its addon. `scripts/verify-native-packaging.mjs` runs `codesign --verify --strict` on each expected binary rather than a single `--deep` pass, because `--deep` stops at the first failure and the useful output is the full list. + +### Verification, and what has not been run + +Automated, and wired into `npm run package:mac` / `package:win` / `package:linux`: + +``` +npm run verify:native-packaging # after any electron-builder target +node scripts/verify-native-packaging.mjs --require-all # release builds, once the runtimes ship +``` + +**A real notarized build has not been run for this phase.** The signing identity and Apple credentials are not available in this environment, so `spctl --assess` and `codesign --verify --deep --strict` against a stapled artifact remain to be done on a machine that has them, along with installing the result on a machine that has never run Maestro from source. Since no native runtime is a dependency yet, that build would exercise the entitlement and Info.plist changes but not the nested-binary signing path, which is the part worth proving. The honest sequencing is to run it in Phase 05, when there is a dylib in the bundle to sign. + +## Windows + +- The prebuilt binaries load from the installed location once they are unpacked from the asar, which is what the `asarUnpack` entries and the packaging assertion enforce. +- Paths with spaces and non-ASCII characters: the loader never builds a path. It hands a bare package specifier to the module system, so resolution is Node's, which handles both. Model files are a separate matter and are already handled by the model store. +- No Visual C++ redistributable is expected: Electron ships the CRT the renderer needs, and all three runtimes are Node-API addons built against it. If one is missing anyway, the loader detects the OS's "The specified module could not be found" (Windows error 126) and reports it as a distinct load failure that names the redistributable, which reaches the user through the capability gate instead of reading like a corrupt install. + +## Linux + +- AppImage and deb both extract to a real filesystem path before launch, so the unpacked binaries are dlopen-able for the same reason they are on the other platforms. +- PulseAudio and PipeWire are both reached through Chromium's audio stack in the hidden audio host window, not directly, so there is nothing platform-specific in A Cappella's own code. +- Linux has no microphone permission API and no privacy-pane deep link that works across desktops. `micSettingsUrl()` returns null there and `getMicPermission()` reports `unknown` until a capture actually fails. + +Neither the AppImage nor the deb has been verified with a real capture in this phase; both are listed above as what to run when the runtimes land. + +## Platform branching + +Main-process code uses `isWindows()`, `isMacOS()`, `isLinux()` from `src/shared/platformDetection.ts`. Renderer code must never read `process.platform`: the renderer's `process` shim reports the sentinel `'browser'`, so `platformDetection` rejects it and renderer code uses `platformUtils` instead. + +## The microphone permission is not a model problem + +`src/main/acappella/permissions/mic-permission.ts` answers one question, and the capability gate turns it into its own slot with its own reason codes (`mic-permission-denied`, `mic-permission-restricted`). "Voice unavailable" in front of someone who has already downloaded 1.4 GB of models, when the real problem is a TCC checkbox, is a support ticket the app could have answered itself. + +Four states are kept apart because the recovery differs for each: `not-determined` (nobody has asked, blocks nothing), `granted`, `denied` (one checkbox), `restricted` (policy, and the user cannot fix it, so no privacy-pane button is offered). + +**When the prompt happens.** At the first real session start, in the `acappella:start-session` handler. Not at app launch, and not when the Encore Feature is switched on. `getMicPermission()` is a pure query and never prompts, which is what makes it safe for the capability gate to call on every Settings render. + +**Why a remembered denial is not sticky.** Windows and Linux learn about a denial only from a failed capture. That observation fills the gap where the OS has no answer, but the OS wins wherever it has one, and a fresh session start clears it. The alternative deadlocks: a denial that outranks a `granted` query, or that survives the user fixing the setting, blocks every future session through the gate, and the only thing that could clear it is the successful capture the gate is now preventing. + +## The self-test + +`Settings > Plugins > A Cappella > Models > Run voice self-test` loads each runtime through the same loader the providers use, runs a trivial operation against its API, and reports per-runtime pass/fail with timings plus the microphone permission. It loads no model and opens no device, so it is free to run on a machine where nothing has been downloaded. The result is also collected into the debug package as `voice-runtime.json`, so a support report carries it without anyone having to ask. + +A probe checks the export the provider will actually call (`getLlama`, `Whisper`, `InferenceSession`), so a version bump that moves the API fails here rather than mid-session. Every probe races a timeout, because "the button did nothing" is the bug being diagnosed and a diagnostic that reproduces it is not a diagnostic. diff --git a/docs/architecture/acappella/routing-evaluation.md b/docs/architecture/acappella/routing-evaluation.md new file mode 100644 index 0000000000..d994e6cc0d --- /dev/null +++ b/docs/architecture/acappella/routing-evaluation.md @@ -0,0 +1,214 @@ +--- +type: reference +title: A Cappella Routing Evaluation +created: 2026-08-15 +tags: + - acappella + - architecture + - routing + - conductor +related: + - '[[system-overview]]' + - '[[voice-session-protocol]]' +--- + +# A Cappella Routing Evaluation + +Routing is the only part of A Cappella that can be confidently wrong. Every other failure announces +itself: a dead microphone produces no transcript, a missing model refuses to start, a broken voice +says nothing. A misroute produces a completed turn, a spoken confirmation, and a prompt sitting in +the wrong repository. So routing quality has to be a measured number rather than a feeling, and this +document is where that number lives. + +## What is measured, and where + +Three separate things get called "routing quality", and they are measured in three different places. +Conflating them is how a router with a 60% hit rate ends up described as working. + +| Layer | Question it answers | Where it is measured | +| ----------------- | ---------------------------------------------------------- | -------------------------------------- | +| Decision rules | Given a decision, does the router do the right thing? | `src/__tests__/main/acappella/router/` | +| Model in the loop | Given an utterance, does the Brain pick the right target? | The script below, run by hand | +| Field | Over real use, how often does the user have to correct it? | The routing log (`routingQuality()`) | + +The first is deterministic and runs in CI. The second needs a real model and a real roster. The +third accumulates on its own once people are talking to it. + +## The routing log is the field instrument + +`src/main/acappella/router/routing-log.ts` records every turn: the utterance, the serialized size of +the context the Brain saw, the decision, the confidence, the latency, and what became of it. The +outcome is what makes the number honest: + +- `dispatched` - it landed, and the user let it stand. +- `corrected` - it landed and the user moved it. **A miss**, even though nothing errored. +- `clarified` - the router asked instead of guessing. **Neither**, and excluded from the hit rate: + asking is the correct behaviour below the confidence threshold, and counting it either way makes + the threshold impossible to tune. +- `failed` - the dispatch could not be performed at all. + +`hitRate = dispatched / (dispatched + corrected)`. Read it with +`window.maestro.voice.routingLog()`, or from the `acappella:routing-log` IPC channel. + +## The evaluation script + +Fifteen utterances against a fixed roster of four agents and twelve tabs. The set is deliberately +weighted toward the cases that are ambiguous rather than the ones that are obvious: a script of +fifteen "tell the backend agent to run the tests" would score 100% and prove nothing. + +It runs headless. `scripts/acappella-routing-eval.ts` encodes the roster and the script below and +drives the real `createConductorRouter` against a real Brain: + +```bash +npm run acappella:eval # the Conductor-agent Brain +npm run acappella:eval -- --brain anthropic # ANTHROPIC_API_KEY +npm run acappella:eval -- --brain openai # OPENAI_API_KEY +npm run acappella:eval -- --brain local --model-path /path/to/qwen3-1.7b.gguf +``` + +This was originally written down as a microphone session with four live agents, which is the wrong +instrument for the thing being measured. Routing takes a TRANSCRIPT and a ROSTER, both of which are +data; speaking the script aloud adds speech recognition and four real agents as uncontrolled +variables and makes the result unrepeatable. Everything below the Brain in the harness is shipping +code - the prompt from `src/prompts/acappella-router.md` (read through `initializePrompts()`, so a +local edit is what gets measured), `parseRouteDecision`, the grammar validator, the recall ranker, +the confidence and recall policies, and the routing log itself - so the only thing being varied is +the model. The harness plays the user who corrects a misroute: a decision that misses its +expectation is marked `corrected` in the log, exactly as the HUD's correction control does, so +`routingQuality()` produces the reported number rather than a second tally beside it. + +An unusable Brain fails once, before the script starts, rather than fifteen times inside the results +table. That is how the local tier reports itself in a checkout without the native runtime. + +### The fixture roster + +| Agent | Type | Project path | Tabs | +| ---------- | ----------- | -------------------- | ------------------------------------------------------------------ | +| `Backend` | claude-code | `/repo/payments-api` | Auth Refactor (active), DB Migrations, Rate Limit Spike (snoozed) | +| `API` | codex | `/repo/gateway` | Gateway Routing (active), Webhook Retries, Old Auth Spike (closed) | +| `Frontend` | claude-code | `/repo/web` | Sidebar Collapse (active), Checkout Flow, Dark Mode | +| `Infra` | opencode | `/repo/terraform` | Cluster Upgrade (active), Cost Report, Log Retention | + +### The utterances + +| # | Utterance | Expected target | Expected action | Tests | +| --- | ------------------------------------------------ | --------------- | ------------------ | ------------------------ | +| 1 | "run the tests" | active agent | `current` | same-topic continuation | +| 2 | "what broke" | active agent | `current` | pronoun-free follow-up | +| 3 | "add a rate limiter to the public endpoints" | Backend | `new` | topic switch | +| 4 | "ask the frontend agent about the checkout flow" | Frontend | `recall` | explicit agent naming | +| 5 | "tell infra to bump the cluster version" | Infra | `current` | explicit agent naming | +| 6 | "back to the auth thing" | Backend | `recall` | vague recall | +| 7 | "what did we decide about webhook retries" | API | `recall` | recall by topic | +| 8 | "the gateway one" | API | `current` | recall by project path | +| 9 | "pick up that rate limit spike again" | Backend | `recall` (snoozed) | snoozed-tab wake | +| 10 | "go back to the old auth spike" | API | `recall` (closed) | closed-tab reopen offer | +| 11 | "how many agents do I have running" | conductor | - | Maestro-level question | +| 12 | "which one is busy right now" | conductor | - | fleet-level question | +| 13 | "make the dark mode toggle stick" | Frontend | `recall` | topic match over recency | +| 14 | "do the auth one" | ambiguous | clarification | low confidence | +| 15 | "no, the other one" | - | correction | correction path | + +Utterance 14 is the interesting one: with an "Auth Refactor" tab on Backend and an "Old Auth Spike" +tab on API, the correct behaviour is a spoken "Backend or API?" rather than a coin flip. Which agent +the router leaned toward while asking is deliberately NOT scored - when it is right to be unsure, +penalising the lean would penalise the behaviour the threshold exists to produce. The harness then +routes the answer ("the backend one") with `clarification` set, which is the round trip that stops a +two-word reply from becoming a tab called "the backend one". Utterance 15 is not routed at all: it is +recognised from the utterance alone and turned into a correction plan. + +Fourteen of the fifteen are routed and scored. Utterance 15 is a recognition check, and the +disambiguation answer is reported beside the script rather than inside it. + +Record each run as a new row below, with the date and the Brain, so a prompt change or a model swap +can be compared against what came before rather than argued about. + +## Results + +### Deterministic layer + +Green as of 2026-08-15, inside a whole-repo run of 37,762 tests. 102 assertions across +`grammar.test.ts`, `conductor-router.test.ts`, `tab-recall.test.ts`, `routing-context.test.ts`, +`routing-log.test.ts` and `conductor-agent.test.ts`, plus the executor and session-service suites. +This layer proves the router's rules, not the model's judgement: every case in it is driven by a +scripted Brain, so a 100% pass says the decision handling is correct and says nothing about whether a +real model would have produced those decisions. + +### Model in the loop + +| Date | Brain | Hits | Corrections | Clarifications | Hit rate | Mean latency | +| ---------- | ----------------- | ----- | ----------- | -------------- | -------- | ------------ | +| 2026-08-15 | `conductor-agent` | 10/14 | 3 | 2 | 77% | 5485 ms | +| 2026-08-15 | `conductor-agent` | 11/14 | 2 | 2 | 85% | 6703 ms | +| 2026-08-15 | `conductor-agent` | 10/14 | 3 | 2 | 77% | 5854 ms | + +Three runs of the Conductor-agent Brain (Claude Code, `--output-format json`, read-only). "Hits" is +`dispatched` from the routing log; the script matched 11, 12 and 11 of its fourteen expectations, and +the two counts differ because a correct clarification is a hit for the script and neither for the hit +rate. + +Two misses are reproducible across all three runs, and they are the reason for running this at all: + +- **"add a rate limiter to the public endpoints"** (expected `new`) recalls the snoozed + `Rate Limit Spike` tab every time, at 0.60. The agent is right; the tab is not. A NEW request that + shares words with an abandoned conversation is currently pulled into it, because nothing in the + prompt says that a tab's topic being ABOUT a subject is weaker evidence than the utterance being a + fresh instruction. This is the most useful finding here and it is a prompt fix, not a code fix. +- **"the gateway one"** (expected a confident `current` on API) lands on API but asks, at 0.40. The + target is right every time and only the confidence is under the threshold. Defensible behaviour for + a three-word fragment, and arguably the script's expectation is the thing that is wrong; recorded + rather than tuned away, because a threshold moved to make a table look better is a threshold that + is no longer measuring anything. + +One miss is a threshold flake: "ask the frontend agent about the checkout flow" scored 0.50 on the +third run and 0.90 on the other two, so it asked once for an utterance that names its agent out loud. + +**Latency is the headline.** 5.5 to 6.7 seconds mean, with individual turns to 20 s. That is an order +of magnitude outside the routing budget in [[latency-baseline]], and it is the expected cost of this +tier: a full agent run is being paid for a classification. The Conductor-agent Brain is for people who +want routing that reasons about their projects and will accept the wait for it. Voice-paced routing is +the local and hosted tiers, and their numbers belong in the table above before any claim about +A Cappella's felt latency is made. + +**Not yet measured: the local and hosted tiers, and shape parity between them.** Both fail closed +here and say so: + +``` +Routing evaluation failed: Qwen3 1.7B (local) is not usable here: + llama.cpp (Conductor Brain) is not part of this build yet. +Routing evaluation failed: Anthropic (hosted) is not usable here: + No Anthropic API key is configured. +``` + +`node-llama-cpp` is loaded dynamically and is not installed in this checkout, and no hosted key is +configured. Filling those rows in needs the native runtime plus the Qwen3 1.7B GGUF for the first and +one API key for the second; the harness needs nothing else, and each run is about ninety seconds. +What to look for when they are run: + +1. **Shape parity.** The local grammar-constrained Brain and a hosted Brain must produce the same + decision SHAPE for the same input - the same `target`, `tabAction`, and `tabId` - even where their + confidences differ. Divergence here means the prompt reads differently to the two models, and that + is a prompt bug rather than a model difference. Run both with `--json` and diff the `results` + arrays. +2. **Latency per Brain.** Routing sits between a finished sentence and anything visible happening, so + it is felt directly. The routing log records `latencyMs` per turn; `routingQuality().meanLatencyMs` + aggregates it. Compare against the per-hop budget in [[latency-baseline]]. + +A caveat that belongs on every row: the Conductor-agent Brain is not deterministic, so a single run is +a sample rather than a score. Three runs moved between 77% and 85% on fourteen utterances with no code +change between them. Read a one-row difference as noise; read a reproducible per-utterance miss, like +the rate-limiter one above, as a finding. + +## Tuning + +`DEFAULT_CONFIDENCE_THRESHOLD` in `src/main/acappella/router/conductor-router.ts` is 0.55. It is the +one number worth moving once there is field data, and the log is arranged to make that decidable: + +- Many `corrected` entries with a confidence above the threshold means the threshold is too low - the + router is acting on beliefs it should be checking. +- Many `clarified` entries the user answers with the router's own first guess means it is too high - + it is asking questions whose answer it already had. + +The routing prompt itself is `src/prompts/acappella-router.md`, editable in Settings > Maestro +Prompts. Someone whose agents are all called "api" can teach the Conductor how to tell them apart +without touching the code, and the log is how they find out whether it worked. diff --git a/docs/architecture/acappella/system-overview.md b/docs/architecture/acappella/system-overview.md new file mode 100644 index 0000000000..8da5aff4ba --- /dev/null +++ b/docs/architecture/acappella/system-overview.md @@ -0,0 +1,287 @@ +--- +type: architecture +title: A Cappella System Overview +created: 2026-08-14 +tags: + - voice + - architecture + - acappella +related: + - '[[voice-session-protocol]]' + - '[[adr-001-webrtc-transport]]' + - '[[adr-002-main-process-session]]' + - '[[transport-and-pairing]]' +--- + +# A Cappella System Overview + +A Cappella turns Maestro into something you talk to. You say "start a new tab on the backend +agent about the auth refactor" and a correctly named tab appears on the right agent, primed with +the prompt. The agent answers, and the answer is spoken back in conversational form rather than +read out as raw terminal output. + +This document describes the skeleton that Phase 01 lands. Everything real that arrives later +(Whisper, Kokoro, OpenAI Realtime, ElevenLabs, the iPhone leg) drops in behind the interfaces +named here. + +## The one architectural rule + +**The voice session lives in the MAIN process and is transport-agnostic.** + +The desktop renderer is a client of the session. The iPhone will be a second client on the same +protocol, not a port of the desktop UI. Nothing in the session service may reference a +`BrowserWindow`, a React store, or the DOM. The rationale is recorded in +[[adr-002-main-process-session]]. + +```mermaid +flowchart LR + subgraph clients[Clients] + R[Desktop renderer
Voice HUD] + P[iPhone
WebRTC peer] + C[maestro-cli
future] + end + subgraph main[Main process] + S[VoiceSessionService
state machine + seq] + X[route-executor] + PR[provider registry] + end + subgraph prov[Providers] + STT[SttProvider] + TTS[TtsProvider] + B[BrainProvider] + end + R <-->|IPC acappella:*| S + P <-->|WS signaling + WebRTC media| S + C <-->|IPC/WS| S + S --> PR + PR --> STT + PR --> TTS + PR --> B + S --> X + X -->|remote:* to the renderer| R +``` + +## Surveyed precedent + +Phase 01 is built on top of code that already exists. The survey below records what was read and +what each piece contributes. + +| Existing code | What it contributes | +| --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `src/renderer/hooks/utils/useVoiceInput.ts` | Web Speech dictation for the AI composer on touch devices. It appends a transcript to a draft; it has no session, no routing, and no speech out. A Cappella eventually subsumes it, but it stays until the local STT tier lands. | +| `src/main/global-hotkey-manager.ts` | A singleton owning exactly one system-wide accelerator (`setGlobalShowHotkey`). Push-to-talk needs a second binding, so Phase 06 generalizes this from a singleton into a registry. | +| `src/main/app-lifecycle/cadenza-hud-window.ts` | The precedent for an auxiliary always-on-top window: transparent, frameless, `focusable: false`, click-through by default, hover hit-testing polled in main because `setIgnoreMouseEvents(..., { forward: true })` is unsupported on Linux. A floating voice HUD that survives app-switching reuses this shape. | +| `src/main/plugins/consent-window.ts` | The precedent for a dedicated auxiliary window with its OWN minimal preload and page, so a surface cannot reach the full IPC bridge. | +| `src/main/web-server/routes/wsRoute.ts` | The authenticated WebSocket at `/$TOKEN/ws`. Clients connect with an optional `?sessionId=`, receive a `connected` frame, then an initial state sync. The phone's WebRTC signaling rides this socket, so it inherits the existing token auth instead of opening a second listener. | +| `src/main/web-server/services/broadcastService.ts` | `broadcastToAll` / `broadcastToSession` over the connected client map. The fan-out shape the voice event stream copies for non-IPC clients. | +| `src/shared/plugins/first-party.ts` | The Encore feature registry, keyed by Encore flag, carrying an honest broker-permission disclosure and any supervised background services. | +| `src/renderer/components/Settings/Extensions/extensionModel.ts` | Projects first-party definitions into marketplace tiles; `beta` is a presentation concern that lives here, not in the shared registry. | + +## Where tab state actually lives + +This is the constraint that shapes the dispatch executor, and it is easy to get wrong. + +**Main has no tab authority.** Tab state lives in the renderer. Even a web or CLI request to open +a tab is forwarded to the renderer for execution: `src/main/web-server/callbacks/tabCallbacks.ts` +sends `remote:newTab`, `remote:selectTab`, `remote:renameTab`, `remote:closeTab` and, for the +create case, waits on a one-shot `remote:newTab:response:` channel with a 5 second timeout. +`src/main/ipc/handlers/tabs.ts` exists only so the renderer can report a close back to main. + +So `executeRouteDecision()` does not create tabs. It resolves a decision into the same +`remote:*` messages the web server already uses, and reads its roster from the persisted +sessions store (`sessionsStore.get('sessions', [])`, the same source +`registerSessionCallbacks` uses). Hand-rolling a parallel tab creation path in main would produce +tabs the renderer does not know about. + +The corollary for the phone: the iPhone never talks to tabs either. It sends a voice event, main +routes it, and the renderer performs it. One execution path, three possible originators. + +### What the executor actually does + +`src/main/acappella/dispatch/route-executor.ts` maps each `tabAction` onto one existing channel: + +| Decision | Channel(s) | Reported action | +| -------------------- | ------------------------------------------------------------------ | --------------- | +| `new` with a prompt | `remote:newAITabWithPrompt`, then `remote:renameTab` for `tabName` | `created` | +| `new` with no prompt | `remote:newTab`, then `remote:renameTab` | `created` | +| `current` | `remote:selectSession`, then `remote:executeCommand` | `focused` | +| `recall` | `remote:selectSession`, then `remote:executeCommand` | `recalled` | + +Creation and prompt delivery are one atomic renderer operation because a separate create-then-send +leaves an orphan tab behind whenever the send is dropped. The `dispatch` event is emitted only +after the renderer answers, so "opened a new tab named Auth Refactor on agent Backend" is a report, +not a hope. + +Four rules the executor holds to, all of them about refusing to guess: + +- **The roster is re-read at dispatch time**, not carried over from routing. The user can close a + tab while a decision is in flight, and the fresh read is the authority. +- **A recalled tab that is gone is a `dispatch-failed`**, never a quiet landing in some other tab. + Recall is a promise to return somewhere specific. +- **A `conductor` target resolves to the session's bound agent, then the agent the desktop is + showing, then the only agent there is.** With several agents and no signal, it fails: a spoken + instruction in the wrong repository is worse than an error. +- **A rejected delivery receipt is a failure, not a `promptSent: false` footnote.** The session + holds the floor open waiting for a reply, so a dropped prompt has to be reported as one. + +The renderer round trip sits behind a `VoiceRendererBridge` interface, for the same reason the +session service takes its providers injected: the routing rules are testable without an Electron +window, and the phone leg gets the same executor with a different bridge. + +## Provider tiers + +A Cappella supports two fundamentally different pipeline shapes behind one set of interfaces. + +### Cascade tier (STT then Brain then TTS) + +Three independent providers, swappable individually: + +- `SttProvider`: streaming `feed(pcm)` with `partial` and `final` callbacks. +- `BrainProvider`: `route(input, context)` returns a `RouteDecision`; `converse(agentText, context)` + reshapes an agent's terminal-shaped answer into spoken-form text. +- `TtsProvider`: `speak(text)` returns an async iterable of audio chunks, plus `cancel()`. + +The cascade tier is the default because each stage is independently substitutable: local Whisper +with cloud TTS, or a cloud brain with local everything else. Latency is the sum of the stages, +which is why barge-in matters so much (see below). + +### Realtime tier (speech to speech) + +A single provider owns microphone audio in and speaker audio out, with routing expressed as tool +calls. Latency is far lower and prosody is far better, but the stages are no longer separable and +the audio must leave the machine. + +The realtime tier implements the same three interfaces as a **fused adapter**: `SttProvider` +emits transcripts the realtime session already produces, `BrainProvider.route()` maps to the +session's tool-call channel, and `TtsProvider` becomes a passthrough. The session service cannot +tell the tiers apart, which is the point. + +### Provider resolution rules + +`src/main/acappella/providers/provider-registry.ts` resolves the active trio from settings, and is +the only module allowed to import a concrete provider. Two rules are non-negotiable: + +1. When nothing is configured, resolve the **mock** trio. The pipeline must always be runnable. + The one per-build exception is STT in a development build, which defaults to `echo-stt`: it + consumes real PCM and reports the speech segments it heard, so `npm run dev` exercises the whole + audio path without a settings edit. A default is not a substitution and is not reported as one. + Whether a microphone is opened at all follows from `SttProvider.acceptsAudio` rather than from a + list of provider ids, so a text-in provider never costs the user a permission prompt. +2. **Never substitute anything for a provider that cannot be built.** Not a cloud provider, which + would spend the user's money and send their microphone somewhere they did not choose; and not + the mock either, because a session that transcribes nothing while looking healthy hides the + reason. A slot whose provider is unknown or unrunnable resolves to an `Unresolved*` provider + that refuses BY NAME the first time anything asks it to work, and the resolution carries a + `VoiceProviderSubstitution` recording what was asked for and why it could not run. That record + is logged and handed to the caller to put in front of the user. + +There is no search over the catalog that could land on a different tier, which is what makes rule 2 +structural rather than a promise. The mock tier is selected, never substituted in: it is what an +unconfigured install runs on purpose, and it is what the tests and the dev harness drive. + +Phase 05 added the concrete backends behind these rules: `providers/local/` (Whisper, Kokoro, +Qwen3 through `runtime/native-loader.ts`), `providers/hosted/` (OpenAI STT and Brain, Anthropic +Brain, ElevenLabs TTS, all through one retrying and classifying transport), and +`providers/realtime/` for the speech-to-speech tier. Which engine fills which slot, and what each +one needs and sends, is declared once in `src/shared/acappella/provider-catalog.ts`, so the +capability gate, the registry, the credential layer, and the settings panel cannot drift apart. + +API keys live in the OS keychain (`providers/credentials.ts`) and never in `settings.json`. Every +turn is timed per hop in `telemetry/turn-metrics.ts`; see [[latency-baseline]]. + +## Client model + +Both clients are peers on [[voice-session-protocol]]. Neither owns session state. + +| | Desktop renderer | iPhone | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| Transport | Electron IPC (`acappella:*` plus a push `acappella:event` channel) | Authenticated WebSocket for signaling, WebRTC for media | +| Audio capture | Local device, or none in dev-harness mode | Device microphone over WebRTC | +| Responsibilities | Render HUD state, transcript, dispatch narration; send `submitUtterance` / `interrupt` / `stopWord`; execute `remote:*` tab operations | Capture and play audio, render the project wheel from `agent-roster`, send the same three commands | +| State ownership | None. Mirrors the event stream into `voiceSessionStore`. | None. | + +Because clients only mirror, a client can join mid-session and catch up: it reads `get-state` and +`get-roster`, then follows `seq` for gaps. A gap means the client missed events and should +re-read state rather than guess. + +Transport choice for the phone leg is argued in [[adr-001-webrtc-transport]]. + +The desktop transport is `src/main/ipc/handlers/acappella.ts` plus the `window.maestro.voice.*` +preload namespace. Its channel table and the four properties that binding has to hold (lazy +construction, broadcast fan-out, the `ACappellaDisabled` gate, and a null snapshot before the +first start) are in [[voice-session-protocol]]. + +## Session lifecycle and the state machine + +States: `idle | arming | listening | transcribing | routing | dispatching | speaking | interrupted | error`. + +The legal transitions live in a `VOICE_STATE_TRANSITIONS` table in +`src/shared/acappella/session-state.ts`. An illegal transition throws rather than smearing state, +because a voice UI that is silently in two states at once is unfixable in the field. + +```mermaid +stateDiagram-v2 + [*] --> idle + idle --> arming: startSession + arming --> listening: listen-start + listening --> transcribing: final-transcript + transcribing --> routing: brain.route + routing --> dispatching: route-decision + dispatching --> speaking: agent-reply + speaking --> interrupted: barge-in + interrupted --> listening: floor retained + speaking --> listening: speak-end + listening --> idle: stop-word / stopSession + arming --> error: provider unavailable + routing --> error: no agent matched + error --> idle: stopSession +``` + +### Barge-in and stop are different + +This distinction is load-bearing and every phase inherits it: + +- **Barge-in** cancels TTS and **keeps the floor**. The session goes `speaking -> interrupted -> +listening`. You talked over the assistant; it shuts up and keeps listening. +- **Stop word** ends the session and returns to `idle`. The floor is released. + +Conflating them produces the single most annoying failure mode in voice interfaces: interrupting +the assistant hangs up on it. + +## Error policy + +Per the repo's Sentry policy, unexpected exceptions bubble. Only known failure modes are +classified into `session-error` events: + +- `provider-unavailable`: a configured provider cannot start (model missing, no API key, device + busy). +- `no-agent-matched`: the brain could not resolve a target and the conductor fallback is disabled. +- `dispatch-failed`: the renderer did not answer a `remote:newTab` within its timeout. + +Anything else is a bug and should reach Sentry with context. + +Two places have no caller to bubble to, so they report explicitly instead of rethrowing: the turn +pipeline (which runs from an STT callback) and the speech run (whose only caller is the reply +seam, and from there an IPC handler). Both call `captureException` with the session context, emit +`listen-stop(error)`, and park the session in `error`. Swallowing is not the point: an escaping +rejection there would reach the process handler stripped of session context AND leave the session +holding a floor nothing will ever hand back, which reads to the user as a frozen HUD. + +## What Phase 01 deliberately does not do + +- No network calls, no API keys, no model downloads. The mock tier proves the pipeline end to end. +- Enabling the Encore flag opens no device and starts no service. It only makes the Voice Setup + surface reachable. +- No WebRTC yet. The transport decision is recorded now so the protocol is designed for it, but + the phone leg is a later phase. + +## Related + +- [[voice-session-protocol]] - every event, payload, and direction. +- [[model-manager]] - the local model catalog, downloads, verification, and the capability gate. +- [[packaging-notes]] - native runtimes, asar unpacking, entitlements, notarization, and the + microphone permission. +- [[latency-baseline]] - the per-hop latency budget for each provider configuration, and how it is + measured. +- [[adr-001-webrtc-transport]] - why WebRTC beats WebSocket plus Opus for the phone leg. +- [[adr-002-main-process-session]] - why the session is headless in main. diff --git a/docs/architecture/acappella/transport-and-pairing.md b/docs/architecture/acappella/transport-and-pairing.md new file mode 100644 index 0000000000..532c351f62 --- /dev/null +++ b/docs/architecture/acappella/transport-and-pairing.md @@ -0,0 +1,230 @@ +--- +type: architecture +title: A Cappella Transport and Pairing +created: 2026-08-15 +tags: + - voice + - architecture + - acappella + - webrtc + - pairing +related: + - '[[system-overview]]' + - '[[adr-001-webrtc-transport]]' + - '[[voice-session-protocol]]' +--- + +# A Cappella Transport and Pairing + +How a phone becomes a microphone and a speaker for a desktop that is somewhere else, and what +that costs on each kind of network. + +The design in one sentence: **the phone carries audio and nothing else.** One STT, one TTS, one +router, all still in the main process, so whatever provider you picked on the desktop is exactly +what you hear on the walk. A remote utterance is not a second pipeline; it is the same pipeline +with a different microphone attached. + +## The parts + +| Piece | Where it lives | What it owns | +| ------------------------ | ------------------------------------------------------------------- | -------------------------------------------------------------- | +| Pairing service | `src/main/acappella/pairing/pairing-service.ts` | Codes, desktop approval, hashed device tokens, revocation | +| Discovery | `src/main/acappella/pairing/discovery.ts` | The Bonjour `_maestro._tcp` advert, and its off switch | +| Signaling | `src/main/acappella/transport/signaling.ts` | Offer/answer/ICE over the existing authenticated WebSocket | +| ICE configuration | `src/main/acappella/transport/ice-config.ts` | STUN, TURN, candidate classification, the reach statement | +| Remote session semantics | `src/main/acappella/transport/remote-session.ts` | One floor, takeover, clean teardown on connection loss | +| Peer connection | `src/renderer/acappella-audio/peer-connection.ts` | `RTCPeerConnection`, Opus tuning, data channels, `getStats()` | +| Data-channel protocol | `src/shared/acappella/device-protocol.ts` | Version handshake, message shapes, reliable/unreliable routing | +| Device management UI | `src/renderer/components/Settings/ACappella/PairedDevicesPanel.tsx` | QR code, approval, device list, revoke, TURN, Test Connection | + +The peer terminates in the hidden audio window from Phase 02, because that window already owns +the `AudioContext`: a remote track has to meet the local microphone somewhere, and this is the +only place both exist. Electron ships Chromium's libwebrtc, so the desktop needs no new native +dependency for any of it. + +## The connection matrix + +Three paths, tried in this order, and honestly labelled everywhere they are shown: + +| Path | Candidate type | Infrastructure needed | Typical latency added | Works on | +| ----------------------- | -------------- | ------------------------------------ | ---------------------- | ---------------------------------------- | +| Host (LAN) | `lan` | None | Under 5 ms | Same WiFi, same wire | +| Host (overlay) | `lan` | Tailscale/ZeroTier | Overlay's own latency | Anywhere the overlay reaches | +| Server reflexive (STUN) | `stun` | A STUN server | None (media is direct) | Most home NATs | +| Relayed (TURN) | `relay` | **A TURN server you run or pay for** | One extra hop | Cellular, hotel WiFi, corporate networks | + +**The overlay row is the interesting one.** A Tailscale-style network hands both machines a +routable address for each other, so the connection is a plain host candidate and it connects +instantly, from anywhere, with no STUN, no TURN, and no port forwarding. If you already run one, +this is the whole answer and everything below it is a fallback. The pairing QR code carries every +local address the desktop has, overlay addresses included, so a phone that is on the overlay but +not on the WiFi still connects directly. + +### TURN is not optional for cellular + +A phone on a mobile network sits behind carrier-grade NAT. CGNAT shares one public address across +thousands of subscribers and does not support the endpoint-independent mapping that hole punching +needs. No amount of STUN gets through it. If you want voice to work while walking down the street +on LTE, **you need a TURN server**, somebody has to run it, and somebody has to pay for the +bandwidth every second of audio goes through it. + +Settings states this as a fact rather than hiding it behind a warning triangle, and the Test +Connection button proves it either way: a `relay` candidate can only be gathered by successfully +authenticating to a TURN server, so its presence is evidence rather than configuration. + +### The Cloudflare quick tunnel cannot carry this + +`src/main/tunnel-manager.ts` runs a Cloudflare quick tunnel so the browser interface is reachable +from outside your network. **It cannot carry the voice audio.** It is an HTTPS reverse proxy and +terminates TCP at Cloudflare; the media leg is a direct UDP association between two peers, chosen +by ICE. The two are separate paths that happen to be used by the same feature: + +- **Signaling** (offer, answer, ICE candidates) is WebSocket traffic on `/$TOKEN/ws`, so it goes + through the tunnel perfectly well. +- **Media** never touches the tunnel. It is LAN, STUN-punched, or relayed through TURN. + +So "the tunnel is up" tells you nothing about whether audio will flow, and a tunnel URL is not a +substitute for a TURN server. This is written into the settings copy (`TUNNEL_MEDIA_NOTE`) because +a user who does not know it will blame the wrong component every time. + +## Pairing + +``` +Device Desktop + | | + | scans QR / enters code | startPairing() -> 6-char code, 2 min TTL + |------ pair-claim ------------->| claim() consumes the code, creates a request + |<----- pair-pending ------------| + | | *** a human clicks Approve, looking at the + | | name and platform of the thing asking *** + |------ pair-poll -------------->| approve() mints a 32-byte token, + |<----- pair-approved -----------| persists ONLY its salted SHA-256 + | | + |------ auth (deviceId, token) ->| authenticate() -> signaling session + |<----- authenticated -----------| (ICE servers, negotiated protocol version) + |------ offer ------------------>| rate limited, 6 per minute, sliding window + |<----- answer ------------------| + |<===== ICE candidates =========>| + |<~~~~~ audio + data channels ~~>| +``` + +Four properties this flow is arranged around: + +1. **Knowing the code is not enough.** A code is short enough to read over a shoulder, so it buys + a row in a dialog and nothing else. Pairing completes only on an affirmative action on the + desktop. +2. **The window is short and one-shot.** Two minutes, and the code is spent by the FIRST claim + whether or not the human approves - a denied request must not leave a live code behind for + whoever was watching. +3. **The token is never stored in plain text.** `userData/acappella/devices.json` holds a salted + SHA-256. A stolen file discloses device names, which is the smallest disclosure that still lets + a returning device authenticate with no server involved. +4. **Revocation is immediate.** `revoke()` fires an event before the disk write completes; the + signaling service turns that into a torn-down peer connection and a closed voice session. A + revocation that only applied at the next connect would be useless in the one situation anybody + ever uses it. + +The QR payload carries the host candidates, the port, the server token, the pairing code, and a +fingerprint derived from the server token. The fingerprint is shown on both screens so a user can +compare four characters and notice a man in the middle. + +### Discovery + +The desktop advertises `_maestro._tcp` over Bonjour so a device on the same network finds it +without anybody typing an address. Three things about it: + +- It is a convenience, never the connection. The QR code carries the addresses directly, and + manual host entry always works. +- It is off-switchable, and the switch is real: broadcasting a machine name and a port to every + device on a network is a disclosure some people do not want to make. +- It carries no secret. The TXT record holds the app version, the protocol version, the machine + name, and the pairing fingerprint. Never the token and never a pairing code - an advert is + readable by everything on the network. + +The mDNS responder is loaded optionally (`bonjour-service` if present). When it is absent the +advert reports `unavailable` with the sentence that says so, rather than failing silently. + +## The data channels + +Two, because the two kinds of message have opposite failure preferences: + +| Channel | Config | Carries | +| ----------------- | ------------------------------ | ------------------------------------------------------------------------------------- | +| `acappella-state` | ordered, reliable | Agent roster, tab state, route decisions, dispatches, errors, floor state, revocation | +| `acappella-live` | unordered, `maxRetransmits: 0` | Audio level, partial transcripts, push-to-talk press/release, barge-in, link quality | + +Session events travel as `{ type: 'voice-event', event }` - a Phase 01 `VoiceEvent`, unchanged. +There is deliberately no parallel vocabulary: the phone reads the same object graph the desktop +renderer and the CLI read. Only the things a peer connection genuinely adds (the version +handshake, floor control, link quality) are new message types. + +Push-to-talk rides the lossy channel with the rest of the gesture traffic. A dropped RELEASE +cannot leave a hot microphone: the floor's idle timeout closes it, the next press is idempotent, +and the desktop re-sends the floor state it observes. + +Version negotiation happens at `auth`, before the credential is even checked, and a mismatch is +refused with a sentence naming which end has to update. An old client that half works is worse +than one that will not connect. + +## Remote session semantics + +- **One floor, last press wins.** A device that presses talk takes the floor from whoever had it, + and the displaced device is told immediately so its button snaps back. Every device here was + individually approved by the person doing the pressing; the alternative rule ends with a user + pressing talk on the phone in their hand and nothing happening because a laptop in another room + holds the floor. +- **A stale release cannot close a live floor.** Only the current holder's release does anything. +- **The path is identical.** A remote press drives the same `FloorController` the desktop hotkey + drives, and opens an ordinary session whose only difference is `VoiceOrigin`. The origin exists + so the desktop HUD can name the device that is listening, not so anything can branch on it. +- **A dropped connection ends the session cleanly.** Speech is cancelled first (the chunks are + already queued in the audio host), then the session closes. An ICE `disconnected` is NOT that + trigger: it is what a WiFi-to-LTE handover looks like, and hanging up there would hang up on + every user this transport exists for. +- **Wake word and stop word stay local to whichever device is capturing.** No audio leaves a + device before its wake phrase fires. This falls out of the design rather than being enforced: + the phone's microphone is not sent anywhere until the phone opens the floor. + +## Audio configuration + +Opus, mono, with in-band FEC and DTX on and a 24 kbps target: + +- **FEC** is what makes 5% packet loss sound like nothing instead of like a robot. +- **DTX** stops a phone in a pocket transmitting silence over a metered radio. +- **Mono at 24 kbps** is transparent for speech, and the pipeline downmixes to one channel anyway. + +Both the SDP `fmtp` parameters and `RTCRtpSender.setParameters` are set, because either one alone +is routinely ignored depending on which end negotiated what. + +**Echo cancellation for the remote path runs on the device, not on the desktop.** The echo happens +in the room the phone is in, and cancelling it requires the phone's own speaker output as the +reference signal - which only the phone has. The desktop asks for it +(`RemoteAudioConfig.requestRemoteEchoCancellation`) and applies its own AEC to its own microphone, +where it works. Any claim that the desktop cancels the phone's echo would be false. + +## Acceptance: what to check by hand + +The suites in `src/__tests__/main/acappella/` and +`src/__tests__/renderer/acappella-audio/peer-connection.test.ts` run in jsdom with a mocked +`RTCPeerConnection` and no network. They cannot prove audio flows. These steps can: + +- [ ] Pair a second machine over LAN with the QR code, approve it on the desktop, hold its + push-to-talk, and speak. The desktop routes and dispatches identically to a local utterance + and the reply is audible on the remote device in the configured voice. +- [ ] Record the candidate type the device list shows. On the same network it must say + `Direct (LAN or overlay)`. +- [ ] Repeat over a Tailscale-style overlay with the desktop off the device's WiFi. Still + `Direct (LAN or overlay)`, because an overlay address is a host candidate. +- [ ] With a TURN server configured, repeat over cellular. The device list must say + `Relayed (TURN)`, and Test Connection must report a relay candidate. +- [ ] Walk from WiFi to cellular mid-conversation. The peer renegotiates; the session survives. +- [ ] Revoke the device mid-conversation. The connection drops immediately, the session ends, and + no speaking state or open floor is left behind. +- [ ] Two devices: press talk on the second while the first holds the floor. The first shows the + takeover and its microphone closes; the second is heard. + +## Related + +- [[system-overview]] - where the phone sits in the client model. +- [[adr-001-webrtc-transport]] - why WebRTC rather than Opus frames over the WebSocket. +- [[voice-session-protocol]] - the event vocabulary the data channel carries unchanged. diff --git a/docs/architecture/acappella/voice-session-protocol.md b/docs/architecture/acappella/voice-session-protocol.md new file mode 100644 index 0000000000..e598f5cdf7 --- /dev/null +++ b/docs/architecture/acappella/voice-session-protocol.md @@ -0,0 +1,446 @@ +--- +type: architecture +title: A Cappella Voice Session Protocol +created: 2026-08-14 +tags: + - voice + - architecture + - acappella + - protocol +related: + - '[[system-overview]]' + - '[[adr-001-webrtc-transport]]' + - '[[adr-002-main-process-session]]' +--- + +# Voice Session Protocol + +The protocol is the contract between the headless session service in the main process and every +client (desktop renderer today, iPhone later, CLI if it ever wants in). It is defined in +`src/shared/acappella/protocol.ts` as a discriminated union on `type`. + +The protocol is deliberately transport-agnostic. The same object graph travels over Electron IPC +(`acappella:event`) and over the authenticated WebSocket at `/$TOKEN/ws`. Nothing in it may refer +to a `BrowserWindow`, a DOM node, or a React store. See [[adr-002-main-process-session]]. + +## Envelope + +Every event carries the same three fields: + +```ts +interface VoiceEventBase { + /** The voice session this event belongs to. Not an agent id. */ + sessionId: string; + /** Monotonic per voice session, starting at 1. A gap means events were lost. */ + seq: number; + /** Emission time, epoch ms. */ + ts: number; +} +``` + +`sessionId` is the **voice** session id, minted by `startSession()`. It is not an agent id and it +is not a provider session id. When a voice session is bound to an agent, the agent id travels in +the scope, never in `sessionId`. + +`seq` exists so a client can detect gaps. It is monotonic and per voice session, incremented by +the service for every emitted event regardless of subscriber count. A client that sees `seq` jump +must not interpolate: it re-reads `acappella:get-state` and `acappella:get-roster` and resumes. + +## Direction + +`client -> service` events are commands. `service -> client` events are announcements, fanned out +to every subscriber so two clients watching the same session see identical state. + +Four events travel **both** ways (`wake`, `final-transcript`, `barge-in`, `stop-word`). When a +client sends one, the service validates it against the +state machine and, if it is legal, echoes it outward with a fresh `seq` so every other client sees +it. The echoed copy is the authoritative one; a client must render its own optimistic state only +until the echo lands. + +## Event catalogue + +| Event | Direction | Emitted when | +| -------------------- | ----------------- | ---------------------------------------------------------------------------------- | +| `wake` | both | A wake word or push-to-talk key fires on a client, then echoed as the session arms | +| `listen-start` | service -> client | The floor opens and audio is being consumed | +| `listen-stop` | service -> client | The floor closes (endpointed, stopped, or interrupted) | +| `partial-transcript` | service -> client | STT produces an interim hypothesis | +| `final-transcript` | both | STT settles an utterance; inbound when the client owns STT | +| `route-decision` | service -> client | The brain resolves a target, tab action, and prompt | +| `dispatch` | service -> client | The decision was executed against a real agent and tab | +| `agent-reply` | service -> client | The agent produced text worth speaking | +| `speak-start` | service -> client | TTS begins a reply | +| `speak-sentence` | service -> client | One sentence of the reply is spoken | +| `speak-end` | service -> client | TTS finishes or is cancelled | +| `barge-in` | both | The user speaks or clicks over active speech | +| `stop-word` | both | The stop word or Stop button ends the session | +| `session-error` | service -> client | A classified, known failure mode | +| `audio-level` | service -> client | A downsampled input level, ~20 a second, so a client can draw a meter | +| `mic-state` | service -> client | The microphone's permission, device, or availability changes | +| `tab-state` | service -> client | The bound agent's tab set or active tab changes | +| `agent-roster` | service -> client | Roster snapshot, on subscribe and on change | + +## Payloads + +### `wake` (both) + +```ts +{ + type: 'wake'; + source: 'wake-word' | 'hotkey' | 'client-button'; + scope: VoiceScope; +} +``` + +`VoiceScope` is `{ kind: 'conductor' }` or `{ kind: 'agent'; sessionId: string }`, where that +`sessionId` is an **agent** id. Inbound, `wake` requests a session in that scope. Outbound, it +announces `idle -> arming`. + +### `listen-start` (service -> client) + +```ts +{ + type: 'listen-start'; + scope: VoiceScope; + sttProviderId: string; +} +``` + +State becomes `listening`. `sttProviderId` lets the HUD show which tier is active (`mock`, +`whisper-local`, `openai-realtime`, ...), which matters because provider substitution is never +silent. + +### `listen-stop` (service -> client) + +```ts +{ + type: 'listen-stop'; + reason: 'endpoint' | 'stopped' | 'interrupted' | 'error'; +} +``` + +### `partial-transcript` (service -> client) + +```ts +{ + type: 'partial-transcript'; + text: string; + stability: number; +} +``` + +`text` is the full hypothesis so far, not a delta, so a client that missed one partial still +renders correctly. `stability` is 0 to 1; the mock STT emits two partials with rising stability +before the final. + +### `final-transcript` (both) + +```ts +{ type: 'final-transcript'; text: string; confidence: number; durationMs?: number } +``` + +Outbound in the cascade tier. **Inbound** when a client owns transcription, which is exactly the +iPhone case if on-device dictation ever beats the desktop pipeline. Either way it lands on the +same seam as `submitUtterance(text)`, so the dev harness and a real microphone are +indistinguishable to everything downstream. + +### `route-decision` (service -> client) + +```ts +{ + type: 'route-decision'; + decision: RouteDecision; + brainProviderId: string; + latencyMs: number; +} +``` + +`RouteDecision` is defined in `src/shared/acappella/route-decision.ts`: + +```ts +{ + target: 'conductor' | { sessionId: string }; + tabAction: 'current' | 'new' | 'recall'; + tabId?: string; + tabName?: string; + prompt: string; + confidence: number; +} +``` + +The same file exports a JSON Schema constant for this shape. Phase 07 compiles that schema into a +GBNF grammar so a local model is structurally incapable of emitting an invalid decision. Keeping +the schema next to the type is what makes that possible without a second, drifting definition. + +### `dispatch` (service -> client) + +```ts +{ + type: 'dispatch'; + agentSessionId: string; + agentName: string; + tabId: string; + tabName?: string; + action: 'focused' | 'created' | 'recalled'; + promptSent: boolean; +} +``` + +This is what lets any client narrate "opened a new tab named Auth Refactor on agent Backend". It +is emitted **after** the renderer confirms the operation, not when it is requested, because tab +creation is a round trip that can time out (see [[system-overview]], "Where tab state actually +lives"). + +### `agent-reply` (service -> client) + +```ts +{ + type: 'agent-reply'; + agentSessionId: string; + tabId: string; + text: string; + spokenText: string; +} +``` + +`text` is what the agent actually wrote. `spokenText` is `BrainProvider.converse()` output: the +same content reshaped for the ear, because reading a diff aloud is useless. Clients that show a +transcript should show `text` and speak `spokenText`. + +### `speak-start` / `speak-sentence` / `speak-end` (service -> client) + +```ts +{ + type: 'speak-start'; + utteranceId: string; + sentenceCount: number; + ttsProviderId: string; +} +{ + type: 'speak-sentence'; + utteranceId: string; + index: number; + text: string; +} +{ + type: 'speak-end'; + utteranceId: string; + reason: 'complete' | 'cancelled' | 'error'; +} +``` + +`utteranceId` scopes a speech run so a late `speak-sentence` from a cancelled run can be dropped +rather than rendered after the next reply started. Sentence granularity is what makes barge-in +feel instant: the client already knows the sentence boundary it was cut at. + +### `barge-in` (both) + +```ts +{ type: 'barge-in'; source: 'voice' | 'client-button'; cancelledUtteranceId?: string } +``` + +Cancels TTS and **keeps the floor**: `speaking -> interrupted -> listening`. It does not end the +session. This is the difference that makes the interface usable. + +### `stop-word` (both) + +```ts +{ type: 'stop-word'; phrase?: string; source: 'voice' | 'client-button' } +``` + +Ends the session: any state `-> idle`. TTS is cancelled, the floor is released, providers are torn +down. + +### `session-error` (service -> client) + +```ts +{ + type: 'session-error'; + code: 'provider-unavailable' | 'no-agent-matched' | 'dispatch-failed' | 'audio-capture-failed'; + message: string; + recoverable: boolean; + providerId?: string; +} +``` + +The union is closed on purpose. Only classified, known failure modes become events; anything else +bubbles to Sentry per the repo error policy. A new code means a new, deliberately handled failure +mode, not a catch-all. + +### `audio-level` (service -> client) + +```ts +{ + type: 'audio-level'; + level: number; + speech: boolean; +} +``` + +The input level, downsampled to roughly 20 a second by `main/acappella/audio/level-meter.ts`, so a +client can draw a live meter without ever receiving PCM. `level` is a linear RMS from 0 to 1 and the +client picks its own curve; `speech` says whether the detector held the floor open across the +window, which is how a meter shows the difference between a loud room and a person talking. + +Silence is published once and then withheld until something moves. An open microphone in a quiet +room is the normal state of a session, and 20 identical zeros a second is traffic nobody can use. + +### `mic-state` (service -> client) + +```ts +{ + type: 'mic-state'; + permission: 'unknown' | 'granted' | 'denied'; + capturing: boolean; + deviceId: string | null; + deviceLabel: string | null; + issue: 'permission-denied' | 'no-device' | 'device-lost' | 'unavailable' | null; + deviceChanged: boolean; +} +``` + +Every transition is published, including the benign ones. The failure this exists to prevent is a +client showing a listening indicator over a microphone that will never produce a transcript: a +denied permission and a quiet room are indistinguishable from the rest of the stream. `deviceLabel` +is null until permission is granted, because Chromium redacts device labels before that. + +### `tab-state` (service -> client) + +```ts +{ type: 'tab-state'; agentSessionId: string; tabs: RosterTab[]; activeTabId: string | null } +``` + +### `agent-roster` (service -> client) + +```ts +{ type: 'agent-roster'; agents: RosterAgent[] } + +interface RosterAgent { + sessionId: string; + name: string; + agentType: string; + cwd: string; + tabs: RosterTab[]; +} + +interface RosterTab { + id: string; + name: string | null; + lastActiveAt: number | null; +} +``` + +The roster is the brain's routing context and, later, the phone's project wheel. It is built by +`buildAgentRoster()` in `src/main/acappella/dispatch/route-executor.ts` from the main-process +stores via `src/main/stores/getters.ts`, the same source the web server's session callbacks read. +It is deliberately compact: no logs, no usage stats, nothing that would make it expensive to push +on every change or to send to a model as context. + +## Flow + +A complete turn, with the mock tier: + +```mermaid +sequenceDiagram + participant C as Client (HUD) + participant S as VoiceSessionService + participant B as BrainProvider + participant R as Renderer (tabs) + participant T as TtsProvider + C->>S: wake / startSession + S-->>C: wake, listen-start (seq 1,2) + C->>S: submitUtterance("new tab on backend about auth") + S-->>C: partial-transcript x2, final-transcript (seq 3-5) + S->>B: route(text, roster) + S-->>C: route-decision (seq 6) + S->>R: remote:newTab + remote:renameTab + R-->>S: response channel + S-->>C: dispatch (seq 7) + S-->>C: agent-reply (seq 8) + S->>T: speak(spokenText) + S-->>C: speak-start, speak-sentence..., speak-end (seq 9+) + C->>S: barge-in + S-->>C: speak-end(cancelled), listen-start +``` + +## Electron IPC binding + +The desktop client speaks the protocol over `src/main/ipc/handlers/acappella.ts`, exposed to the +renderer as `window.maestro.voice.*` (`src/main/preload/acappella.ts`). The handlers are +registered from `setupIpcHandlers()` in `src/main/ipc/bootstrap/index.ts`, which is the path the +running app actually takes; `registerAllHandlers()` in `handlers/index.ts` is not called at +runtime, so a handler wired only there would be dead. + +| Channel | Preload method | Returns | +| ------------------------------ | -------------------------- | ------------------------------------------------------ | +| `acappella:start-session` | `voice.start(scope?)` | `{ snapshot, substitutions }` | +| `acappella:stop-session` | `voice.stop()` | `void` | +| `acappella:submit-utterance` | `voice.submitUtterance()` | `boolean` (false when the state cannot take one) | +| `acappella:interrupt` | `voice.interrupt(source)` | `boolean` (false when nothing is speaking) | +| `acappella:stop-word` | `voice.stopWord(payload?)` | `void` | +| `acappella:submit-agent-reply` | `voice.submitAgentReply()` | `boolean` (false when no reply was awaited) | +| `acappella:get-roster` | `voice.getRoster()` | `RosterAgent[]` | +| `acappella:get-state` | `voice.getState()` | `VoiceSessionSnapshot`, or null before the first start | +| `acappella:event` (push) | `voice.onEvent(handler)` | every `VoiceEvent`, in `seq` order | + +Five properties of this binding are deliberate: + +- **Registration builds nothing.** The service, its provider trio, and the dispatch executor are + all constructed on the first `start-session`. Enabling the Encore Feature opens no device and + downloads nothing. +- **Events are broadcast, not addressed.** `acappella:event` goes to every window and to the + web-desktop bridge through `safeSend`, matching the multi-window invariant in + `src/main/utils/safe-send.ts`. There is no per-window subscriber list: a client that does not + want the stream simply does not listen. +- **The Encore gate rejects with `ACappellaDisabled`**, so the renderer can tell "feature off" + from "no session". `stop-session` is the one ungated channel, because toggling the feature off + mid-session must still be able to release the floor. +- **`get-state` returns null before the first start.** Synthesising an idle snapshot would have to + name provider ids that nothing has resolved yet, and reporting a requested-but-unavailable + provider as the running one is the substitution lie in a different costume. +- **The floor is released on `will-quit`.** Registering the handlers also registers an + `app.on('will-quit')` that calls `disposeVoiceSessionService()`. The mock tier holds no OS + device, but a real microphone (Phase 05) does, and a live session must not outlive the last + window. + +A provider selection change in settings rebuilds the service on the next start, which is how Voice +Setup takes effect without an app restart. + +`submit-agent-reply` is the reply seam over the transport. Phase 05 feeds real agent output to +`VoiceSessionService.submitAgentReply()` in-process, but nothing outside main can push a session +past `dispatching` without this channel, and a session that never leaves `dispatching` never +speaks. It is what the dev harness uses to demonstrate speech, barge-in, and the difference +between barge-in and the stop word. + +### The renderer client + +`src/renderer/components/ACappella/` holds the desktop client, and it is deliberately thin: + +- `useVoiceSession(enabled)` owns the ONE subscription to `acappella:event`. A second subscriber + would apply every event twice, so the HUD is the only mount point and it renders the harness + itself. +- `voiceSessionStore` is a pure projection of the stream. It derives state from the event rather + than from the return value of an IPC call, flags a `seq` gap instead of smoothing it over, and + refuses to let a late `get-state` catch-up rewind a projection the stream has already carried + past. +- `VoiceHud` renders that projection: bound scope, state, a listening/speaking indicator that + differs in shape and not only in hue, the streaming transcript, and any provider substitution. + Closing it ENDS the session, because an open floor behind an invisible surface is a microphone + the user cannot see. + +## Invariants + +1. **`seq` is never reused and never goes backwards** within a voice session. +2. **Clients hold no authoritative state.** Every client is a projection of the event stream. +3. **Barge-in keeps the floor; stop releases it.** No event may blur the two. +4. **An illegal state transition throws.** The transition table in `session-state.ts` is the only + place legality is defined. +5. **No provider substitution is silent.** Every provider-bearing event names its provider id. +6. **The union is closed.** Adding an event means adding it to `protocol.ts`, not stuffing an + extra field into an existing payload. + +## Related + +- [[system-overview]] - the service, tiers, and client model. +- [[adr-001-webrtc-transport]] - why the phone's media leg is WebRTC. +- [[adr-002-main-process-session]] - why the session is headless in main. diff --git a/docs/architecture/acappella/wake-and-hotkeys.md b/docs/architecture/acappella/wake-and-hotkeys.md new file mode 100644 index 0000000000..b02137b234 --- /dev/null +++ b/docs/architecture/acappella/wake-and-hotkeys.md @@ -0,0 +1,169 @@ +--- +type: reference +title: A Cappella Wake Word, Stop Word, and Hotkeys +created: 2026-08-15 +tags: + - acappella + - architecture + - wake-word + - hotkeys +related: + - '[[system-overview]]' + - '[[voice-session-protocol]]' + - '[[model-manager]]' +--- + +# Wake Word, Stop Word, and the Global Hotkey Registry + +Three ways to open the floor, and one way to shut it. + +| Surface | Opens | Steals focus | Lives in | +| ---------------------------- | ---------------------------------- | ------------ | ----------------------------------------- | +| Wake word (global phrase) | Conductor-scoped session | No | `main/acappella/wake/wake-detector.ts` | +| Wake word (per-agent phrase) | Session bound to that agent | No | same | +| `voiceConductor` hotkey | Conductor-scoped session | No | `main/acappella/hotkeys/voice-hotkeys.ts` | +| `voiceCurrentAgent` hotkey | Session bound to the focused agent | Yes | same | +| Stop word | Nothing - it ENDS the session | No | `main/acappella/wake/stop-word.ts` | + +Every one of them routes through `audio/floor-control.ts`. That module already +owns what a second press means, what a release means, and when an untouched +microphone goes cold. Four surfaces re-deriving any of it would drift inside a +week. + +## The privacy invariant + +**While only the wake detector is running, no audio frame reaches a hosted +provider or leaves the process.** This holds whether the user picked Whisper or +OpenAI for speech-to-text, and it is not a preference. + +It is enforced structurally rather than by discipline: + +- `WakePhraseScorer.tier` is the string literal `'local'`, not a + `VoiceProviderTier`. A cloud provider's tier is `'cloud'`, so a hosted scorer + is not assignable and the mistake is a type error. +- `assertWakeScorerLocal()` re-checks at runtime, for anything that arrives + through a cast or across an IPC boundary. +- `WakeDetector` has exactly ONE outward edge, `onWake`. It never sees a + provider, never holds a socket, and cannot be handed one. +- `wake-detector.test.ts` feeds 200 frames past a hosted STT spy and a `fetch` + spy with neither called. + +## Wake word + +openWakeWord on `onnxruntime-node`, loaded through `native-loader.ts`. The chain +is melspectrogram -> embedding -> one small classifier per phrase, and the +detector's contract with the scorer is one 80 ms hop (1280 samples at 16 kHz) in, +a score per phrase id out. + +`createOnnxWakeScorer()` returns null when the runtime or the model files are +missing, which is the ordinary state of a machine that has not opted into +hands-free. The detector then runs INERT and says so; the capability gate is +where a missing model becomes a sentence the user can act on. + +Three orchestration properties the model does not provide: + +- **Per-phrase sensitivity.** A two-syllable agent name and "hey maestro" do not + false-fire at the same threshold. The score has to clear `1 - sensitivity`. +- **Debounce.** One spoken phrase clears the threshold over several consecutive + windows; without a debounce each would be a session. +- **Pre-roll.** The `WakeDetection` carries the audio around the phrase, drained + from the ring, so "Maestro, what's the status" does not reach the recogniser as + "...what's the status". Wire the detector to the AUDIO PIPELINE's ring so there + is one buffer rather than two. + +## Stop word versus barge-in + +The single most important distinction in this subsystem. + +| | Barge-in | Stop word | +| -------------- | -------------------------------------- | ------------- | +| Means | "stop talking, I am still here" | "we are done" | +| Speech | cancelled | cancelled | +| Floor | KEPT | released | +| Microphone | stays open | closed | +| Terminal state | `speaking -> interrupted -> listening` | `-> idle` | +| Event | `barge-in` | `stop-word` | + +They are separate modules, separate events, separate settings, and separate HUD +feedback, because every assistant that folded them together became one you cannot +get rid of. `StopWordController` is handed a session interface with `hardStop` +and deliberately WITHOUT `interrupt`, so it cannot reach barge-in even by +accident. + +The stop word runs on the local detector specifically because it must be heard +while TTS is speaking and while a cloud STT stream is open. It cannot depend on a +transcript coming back from a remote engine: the answer would arrive after the +thing it was meant to stop had finished. + +`armedPhrases(state, ...)` is the arming rule: wake phrases while the session is +cold, stop phrases in every active state. Never both, or a wake phrase spoken +mid-answer would stack a second session. + +## The global hotkey registry + +`src/main/global-hotkey-manager.ts` was one deliberate singleton for "show +Maestro". It is now a `GlobalHotkeyRegistry` keyed by id, because with three +hotkeys a shared failure path would mean losing "show Maestro" over a bad voice +combo. + +Failure kinds, distinguished because the user's next move differs: + +- `invalid-accelerator` - the combo has no non-modifier key. +- `maestro-conflict` - two Maestro hotkeys want the same combo. Detected here and + NAMED; left to Electron the second registration silently wins or loses by + platform. +- `os-conflict` - another application owns it. +- `register-error` - `globalShortcut.register` threw. + +Failures reach the renderer on `globalHotkey:registrationFailed`, now carrying +the whole `GlobalHotkeyStatus` (id included) rather than a bare key array. The +definitions - ids, labels, and defaults - live in `src/shared/global-hotkeys.ts`, +which both `DEFAULT_SHORTCUTS` and the main-process registry read, so a hotkey the +Settings list and the registry spelled differently cannot exist. + +## Tap versus hold + +Electron's `globalShortcut` fires on PRESS and never on release. `press-hold.ts` +turns one press callback into tap / hold-start / hold-end by polling a key-state +probe. + +**Today every platform returns null and the hotkeys report `tap-only`.** There is +no way to read live key state from Electron's own API: macOS needs +`CGEventSourceKeyState`, Windows `GetAsyncKeyState`, X11 `XQueryKeymap`, and all +three mean a native module Maestro does not ship. Auto-repeat timing is not a +substitute - the OS repeat delay is longer than any usable hold threshold. + +Rather than fake it, the detector says so: `describePressHoldCapability()` is +rendered in the Voice Controls settings. A push-to-talk key that silently behaves +like a toggle is a bug users blame themselves for. + +The seam is real, not decorative. `setKeyStateProbe()` is what a native module +plugs into, and it is how both branches are tested. Surfaces that DO have a real +release event - the HUD button, the Phase 10 phone button - never come through +here; they call `FloorController.press()`/`release()` directly. + +## Settings + +Everything except the two key bindings lives under `controls` in the one +`acappella` settings blob (`useVoiceControls`, read back by +`readVoiceControlSettings()` in main). The bindings live in the ordinary +`shortcuts` map, so the Shortcuts tab and the Voice Controls panel are two views +of one value rather than two values that can disagree. + +The Voice Controls panel shows each hotkey's REAL registration state inline, and +a Test button runs the wake detector with no session behind it so sensitivity can +be tuned by saying the phrase instead of by guessing. + +## Files + +| File | Owns | +| ------------------------------------------------------------------- | -------------------------------------------------- | +| `src/shared/global-hotkeys.ts` | Hotkey ids, labels, defaults, status shape | +| `src/shared/acappella/voice-controls.ts` | Shipped phrases and timing numbers, both processes | +| `src/main/global-hotkey-manager.ts` | `GlobalHotkeyRegistry` | +| `src/main/acappella/hotkeys/press-hold.ts` | Tap vs hold classification | +| `src/main/acappella/hotkeys/voice-hotkeys.ts` | The two hotkeys' semantics | +| `src/main/acappella/hotkeys/index.ts` | Electron / settings wiring | +| `src/main/acappella/wake/wake-detector.ts` | openWakeWord detector and the ONNX scorer | +| `src/main/acappella/wake/stop-word.ts` | Stop phrases, arming rule, teardown | +| `src/renderer/components/Settings/ACappella/VoiceControlsPanel.tsx` | The settings surface | diff --git a/docs/docs.json b/docs/docs.json index 22e1c7477d..a14fa0708e 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -80,6 +80,7 @@ "icon": "flask", "pages": [ "encore-features", + "voice-mode", "director-notes", "usage-dashboard", "symphony", diff --git a/docs/encore-features.md b/docs/encore-features.md index ee5c56e109..7d06da2621 100644 --- a/docs/encore-features.md +++ b/docs/encore-features.md @@ -18,6 +18,7 @@ Open **Settings** (`Cmd+,` / `Ctrl+,`) and navigate to the **Encore Features** t | Feature | Shortcut | Description | | ------------------------------------ | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------- | +| [A Cappella](./voice-mode) | `Cmd+Alt+V` / `Ctrl+Alt+V` | Voice interface (Beta): speak a request, have it routed to an agent, and hear the reply read back | | [Director's Notes](./director-notes) | `Cmd+Shift+O` / `Ctrl+Shift+O` | Unified timeline of all agent activity with AI-powered synopses | | [Usage Dashboard](./usage-dashboard) | `Opt+Cmd+U` / `Alt+Ctrl+U` | Comprehensive analytics for tracking AI usage patterns | | [Maestro Symphony](./symphony) | `Cmd+Shift+Y` / `Ctrl+Shift+Y` | Contribute to open source by donating AI tokens | diff --git a/docs/ios-client/app-store-review.md b/docs/ios-client/app-store-review.md new file mode 100644 index 0000000000..a1e87e3ef0 --- /dev/null +++ b/docs/ios-client/app-store-review.md @@ -0,0 +1,231 @@ +--- +type: specification +title: App Store Review +created: 2026-08-14 +tags: + - ios + - voice + - acappella +related: + - '[[overview]]' + - '[[background-and-entitlements]]' + - '[[connection-and-pairing]]' + - '[[project-structure]]' +--- + +# App Store Review + +Three things about this app make review harder than the code does: it asks for the microphone, it +asks for a restricted entitlement, and it does nothing at all without a Mac the reviewer does not +have. Each has a specific answer, and each answer has to exist before the first submission rather +than after the first rejection. + +## Usage description copy + +These strings are read by a human in a permission dialog at the moment they decide whether to +trust the app. Write them as answers to "why", not as restatements of "what". + +### Microphone + +``` +NSMicrophoneUsageDescription +``` + +> Maestro sends your voice to the computer you paired with, so you can talk to your agents from +> across the room. Audio is captured only while you are holding the talk button. + +Why each half is there: + +- **"the computer you paired with"** names the destination. A microphone prompt that does not say + where the audio goes is the prompt people deny. +- **"only while you are holding the talk button"** is a commitment the code keeps + (see [[audio-session]]) and the OS enforces on the Push to Talk path + (see [[background-and-entitlements]]). Do not write it unless both remain true. + +If the wake word ships enabled in a build, the second sentence has to change to include it, and +the wake-word toggle itself must carry the longer explanation. A usage string that describes a +narrower behaviour than the app performs is a 5.1.1 rejection and, worse, is a lie. + +### Camera + +``` +NSCameraUsageDescription +``` + +> Maestro scans the pairing code shown on your computer. The camera is used only for that scan. + +### Local network + +``` +NSLocalNetworkUsageDescription +``` + +> Maestro finds your computer on this network so you can pair without typing an address. + +The local-network prompt is the one users find most alarming, because iOS presents it in stark +terms. Two mitigations, both worth building: + +- **Do not trigger it at launch.** Ask only when the user taps "Find my Mac", so the prompt arrives + attached to an action they just took. +- **Offer the QR path first**, which needs no local-network permission at all. A user who scans a + code never sees the prompt. + +### Speech recognition + +**Not requested.** `NSSpeechRecognitionUsageDescription` must not appear in `Info.plist`. There is +no on-device Apple speech recognition in this app: the desktop transcribes. Requesting a permission +the app does not use is both a rejection risk and a privacy claim we do not need to make. + +## Privacy nutrition labels + +The honest position, which also happens to be the simplest label: + +| Data type | Collected? | Notes | +| --------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Audio Data | **No** | Audio is sent peer to peer to the user's own paired computer over an encrypted WebRTC connection. It is not sent to Maestro's servers and it is not retained anywhere by this app. | +| Identifiers | **No** | The device name and a device identifier are sent to the paired computer only, and stored only on that computer. | +| Usage Data, Diagnostics | **No** | v1 ships with no analytics SDK and no crash reporter. See below. | +| Contacts, Location, Photos, Health, Financial | **No** | Not accessed. | + +Two places this needs care: + +**The TURN relay.** If a build ships with a default TURN server operated by us, encrypted media +packets transit it when a direct path cannot be established. That is a transient relay of +end-to-end encrypted data that is not retained and not readable by the operator, which does not +meet Apple's definition of collection (transmitting off device in a way that allows access for +longer than necessary to service the request in real time). It is still worth a sentence in the +privacy policy, and the app already tells the user when it is happening: the connection indicator +says "Relayed" in words (see [[connection-and-pairing]]). + +**The desktop's providers.** A user can configure the desktop to transcribe with OpenAI or speak +with ElevenLabs. In that configuration the user's audio does reach a third party. That happens on +the user's own machine, under settings the user set there, and the desktop states it in one +sentence computed from the live configuration (`mic-state.egressStatement`, never hard-coded copy). +The iOS app must: + +- **Display that sentence verbatim** on its paired-device sheet, so a phone user can see where + their audio goes without walking to the desk. +- **Not claim in its own privacy label that it sends audio to third parties**, because it does not. + It sends audio to one computer. What that computer does next is that computer's disclosure, and + it makes it. + +**Analytics.** Ship v1 with none. It turns the entire label into "Data Not Collected", which is +worth more than the funnel data would be, and it removes a whole class of review questions. If a +crash reporter is added later, it becomes "Diagnostics, not linked to identity, app functionality", +and this document gets updated in the same commit. + +## The reviewer problem + +**The app is useless without a paired desktop.** A reviewer who installs it and opens it sees a +pairing screen and can go no further. Left unaddressed, that is a Guideline 2.1 rejection +(App Completeness) and possibly a 4.2 one (Minimum Functionality). + +Four things, in the order they matter. + +### 1. Make the unpaired state a real screen + +The first launch experience must be worth reading even for someone with no Mac: + +- One screen explaining what the app is: a remote microphone for Maestro on your computer. +- The three ways to connect, with the QR path first. +- A link to the Maestro documentation and to the desktop download. +- **No dead ends.** Never a spinner, never a blank list, never a "connect to continue" wall with + nothing behind it. + +An app that explains itself clearly to someone who cannot use it is not an incomplete app. It is a +companion app, which is a category Apple accepts, and the screen is what makes that legible. + +### 2. Provide a live demo desktop + +The best outcome is a reviewer who actually uses the app. Provide it: + +- Run a Mac with Maestro and A Cappella enabled, reachable over the internet through the TURN + relay, with two or three agents in the roster doing recognisable work. +- In App Review Information, include a **pairing QR image** and the six-character code, plus the + host and port for manual entry as a fallback. +- The demo desktop runs with pairing auto-approval enabled for the review window, because the + approval step needs a human at the desk and there will not be one at 03:00 in Cupertino. +- **Disclose the auto-approval in the review notes.** It is a configuration of the demo machine, + not a hidden feature of the app, and saying so is what keeps it from looking like one under + Guideline 2.3.1. +- Rotate the credentials and turn the demo machine off after the review. + +### 3. Provide a demo video regardless + +Networks fail, review happens at odd hours, and a demo desktop that is asleep is worse than none. +Attach a video, 60 to 90 seconds, screen recording of the phone with the Mac visible: + +1. Scan the QR code, approve on the Mac, connected. +2. The project wheel populates with real agents. +3. Hold the button, say "open a tab on the backend agent about the auth refactor". +4. The tab appears on the Mac. The reply is spoken on the phone. +5. Talk over the reply; it stops mid-sentence. +6. Say the stop word; the session ends. +7. Lock the phone, transmit from the Push to Talk system control, show it still works. + +Step 7 exists for the entitlement reviewer specifically, and it is the one to lead with in the +notes when the Push to Talk entitlement is under review. + +### 4. Write the review notes as if the reviewer has five minutes + +``` +WHAT THIS APP IS +Maestro is a desktop app for macOS and Windows that runs AI coding agents. This iOS app is a +remote microphone and speaker for it: you talk to your computer from across the room. It does +not run any AI on the phone and it requires a paired computer. + +HOW TO TEST IT (demo computer provided) +1. Open the app, tap Scan. +2. Scan the attached QR image (also: code 7K2MBX, host , port ). +3. The demo computer auto-approves during this review window (a configuration of our demo + machine, not a feature of the app; a normal user approves each device by hand). +4. Hold the large Talk button and say: "what are you working on". +5. The reply is spoken back on the phone. Tap the button while it speaks to interrupt. + +PERMISSIONS +Microphone: audio is sent over an encrypted WebRTC connection to the paired computer only. +Camera: pairing QR scan only. +Local network: optional Bonjour discovery, only after the user taps "Find my Mac". + +PUSH TO TALK ENTITLEMENT +Used for its designed purpose. The user joins a channel for their paired computer, transmits +while holding the button or the system control, and the microphone is closed at all other times. +See the attached video from 0:52. +``` + +## Other review surface + +- **Guideline 4.2 (Minimum Functionality).** The counter-argument is that this is a companion app + to a shipping desktop product with real users, in the same category as a remote or a companion + for hardware. The unpaired-state screen from step 1 is the evidence. Have a link to + https://maestro.sh and the docs in the notes. +- **Guideline 2.5.4 (background audio).** Only relevant if a build declares `UIBackgroundModes: +audio`. It should not. See [[background-and-entitlements]]. +- **Guideline 5.1.5 (location).** Not applicable; no location APIs. +- **Age rating.** 4+. No user-generated content is displayed by the app itself, and the agent + replies are the user's own machine talking to them. +- **Account requirement.** None. Guideline 5.1.1(v) is satisfied trivially: there is no sign-in, no + email collected, and no account to delete. Say so in the notes so nobody looks for one. +- **Export compliance.** The app uses DTLS-SRTP through WebRTC and TLS for signaling. Confirm the + exemption applies to that use before setting `ITSAppUsesNonExemptEncryption` to `false`, and file + the self-classification report if it does not. Do not copy the flag from another project. +- **Privacy policy URL.** Required because a permission-gated microphone is involved. It must + contain the same two claims this document makes: audio goes to the user's paired computer, and + onward only to the provider the user configured on that computer. + +## Submission checklist + +- [ ] `NSMicrophoneUsageDescription`, `NSCameraUsageDescription`, `NSLocalNetworkUsageDescription` + present, and each matches the copy above. +- [ ] `NSSpeechRecognitionUsageDescription` **absent**. +- [ ] `NSBonjourServices` contains `_maestro._tcp`. +- [ ] `UIBackgroundModes` contains `push-to-talk` and **not** `audio`. +- [ ] `com.apple.developer.push-to-talk` granted and present in the provisioning profile. +- [ ] Privacy nutrition labels filled in as "Data Not Collected", matching the analytics decision + actually shipped in this build. +- [ ] Privacy policy URL live and containing the two audio-destination claims. +- [ ] Demo desktop running, auto-approval on, credentials fresh, disclosed in the notes. +- [ ] Demo video attached, including the locked-screen Push to Talk segment. +- [ ] Review notes pasted, with the QR image attached. +- [ ] Unpaired first-launch screen reachable with no network at all, and useful there. +- [ ] Export compliance answered deliberately rather than copied. diff --git a/docs/ios-client/audio-session.md b/docs/ios-client/audio-session.md new file mode 100644 index 0000000000..f2561ccee3 --- /dev/null +++ b/docs/ios-client/audio-session.md @@ -0,0 +1,224 @@ +--- +type: specification +title: Audio Session and WebRTC Configuration +created: 2026-08-14 +tags: + - ios + - voice + - acappella +related: + - '[[overview]]' + - '[[connection-and-pairing]]' + - '[[interaction-model]]' + - '[[background-and-entitlements]]' + - '[[../architecture/acappella/latency-baseline]]' +--- + +# Audio Session and WebRTC Configuration + +This is the file that decides whether the app is usable. Everything else can be rebuilt from a +screenshot; an audio session configured wrong produces echo, a microphone that dies when AirPods +connect, or a call that silently kills the session, and none of those look like configuration +problems from the outside. + +## The one thing to get right + +**`AVAudioSession` in `.playAndRecord` category with `.voiceChat` mode.** + +That mode is what routes capture through Apple's Voice-Processing I/O audio unit, which gives +hardware acoustic echo cancellation, noise suppression, and automatic gain control tuned by people +with access to the microphone geometry of every iPhone ever shipped. It is the reason a phone +placed on a desk with its speaker playing a synthesised reply does not send that reply straight +back as a new utterance. + +The desktop cannot do this for us. It asks for it (`requestRemoteEchoCancellation: true` in the +`audio` config from `authenticated`) because the reference signal for cancelling the phone's echo +is the phone's own speaker output, and only the phone has it. The desktop applies its own AEC to +its own microphone, where it works. + +## Session configuration + +```swift +let session = AVAudioSession.sharedInstance() + +try session.setCategory( + .playAndRecord, + mode: .voiceChat, + options: [.allowBluetooth, .defaultToSpeaker, .allowBluetoothA2DP] +) +try session.setPreferredSampleRate(48_000) // Opus native rate +try session.setPreferredIOBufferDuration(0.02) // one 20 ms Opus frame +try session.setActive(true) +``` + +Each choice, and what breaks without it: + +| Setting | Reason | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.playAndRecord` | Duplex. `.record` gives no playback, `.playback` gives no microphone. | +| `.voiceChat` mode | The whole point. Enables VPIO, hence hardware AEC. Without it the phone hears itself. | +| `.allowBluetooth` | HFP, which is the only Bluetooth profile with an input path. Without it, connecting AirPods loses the microphone and the user cannot tell why. | +| `.defaultToSpeaker` | Otherwise `.playAndRecord` routes output to the earpiece receiver, and a phone on a desk sounds like it is broken. | +| `.allowBluetoothA2DP` | Lets output-only Bluetooth devices carry playback at full quality when nothing needs the HFP input path. | +| `setPreferredSampleRate(48000)` | Opus is native at 48 kHz. A resample on the way in is latency and quality thrown away. The system may not honour it (VPIO frequently runs at 24 kHz); read back `session.sampleRate` and do not assume. | +| `setPreferredIOBufferDuration(0.02)` | Matches the Opus frame size, so a capture callback fills exactly one packet. Lower buffers cost CPU and gain nothing over the network jitter budget in [[../architecture/acappella/latency-baseline]]. | + +**Do not** set `.mixWithOthers`. A voice assistant that ducks under a podcast rather than taking +the audio route is one you cannot hear when it matters. + +## WebRTC.framework configuration + +libwebrtc on iOS owns its own audio unit and will fight `AVAudioSession` if both are configured +independently. `RTCAudioSession` is the shared lock that stops that. + +```swift +let rtc = RTCAudioSession.sharedInstance() +rtc.lockForConfiguration() +defer { rtc.unlockForConfiguration() } + +// We decide when the microphone runs, not the peer connection lifecycle. +rtc.useManualAudio = true +rtc.isAudioEnabled = false // flipped true only while the floor is open + +let config = RTCAudioSessionConfiguration.webRTC() +config.category = AVAudioSession.Category.playAndRecord.rawValue +config.mode = AVAudioSession.Mode.voiceChat.rawValue +config.categoryOptions = [.allowBluetooth, .defaultToSpeaker, .allowBluetoothA2DP] +RTCAudioSessionConfiguration.setWebRTC(config) +``` + +Three requirements: + +- **`useManualAudio = true` is mandatory**, not an optimisation. It is what lets the microphone be + off while the peer connection is up, which is the entire privacy story: a paired phone sitting + in a pocket has a live connection and a cold microphone. Without it, libwebrtc starts capture as + soon as a sending transceiver exists. +- **Never call `setActive` on `AVAudioSession` directly while `RTCAudioSession` holds the lock.** + Route through `RTCAudioSession`'s own `setActive`/`setCategory` so the two agree on state. +- **Do not enable libwebrtc's software echo canceller.** On iOS the VPIO unit already cancelled; + running the software APM on top of it produces gating artifacts on the far end that sound like a + bad connection. Verify with the double-talk test below rather than trusting either default. + +### Opus parameters + +The desktop sends a `RemoteAudioConfig` in the `authenticated` message: + +```ts +{ fec: true, dtx: true, maxAverageBitrate: 24000, requestRemoteEchoCancellation: true } +``` + +Apply them to the outgoing `opus/48000/2` `fmtp` line in the local offer: + +| Config field | SDP / API | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `fec: true` | `useinbandfec=1`. This is what survives 5 percent loss on cellular. | +| `dtx: true` | `usedtx=1`. Stops sending during silence, which is most of a session, and lets the radio sleep. | +| `maxAverageBitrate: 24000` | `maxaveragebitrate=24000`. Speech at 24 kbps in Opus is transparent enough to route on. | +| `requestRemoteEchoCancellation` | Already satisfied by `.voiceChat`. If it ever arrives `false`, still keep `.voiceChat`: the desktop is asking us to skip processing, not telling us to send it echo. | + +Prefer mono (`stereo=0`, `sprop-stereo=0`). A speech pipeline downmixes anyway, and the second +channel is bitrate spent on nothing. + +**Never send more than one audio track.** The desktop gates exactly one remote microphone into the +capture pipeline at a time (`set-floor-holder`); a second track is received and discarded. + +## Route changes + +Subscribe to `AVAudioSession.routeChangeNotification` and act on the reason, not on the fact that +something changed: + +| `AVAudioSession.RouteChangeReason` | Situation | Behaviour | +| ---------------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.newDeviceAvailable` | AirPods or a headset connected | Let it take the route. Update the output label in the UI. If the floor is open, keep it open: the user connected headphones mid-sentence, they did not ask to stop. | +| `.oldDeviceUnavailable` | AirPods removed, cable unplugged | Apple's convention is to pause. **Close the floor** and send `floor: release`. A microphone that follows a yanked headset back to the phone's built-in mic without telling anyone is a privacy failure. | +| `.categoryChange` | Another app or the system changed the category | Re-apply our configuration. If re-application fails, surface it: the session is no longer what we think it is. | +| `.override` | Speaker/receiver override | Update the UI label only. | +| `.routeConfigurationChange` | Same route, different config | Read back `sampleRate` and `ioBufferDuration`; log if they moved. | + +Specific routes worth naming: + +- **AirPods and Bluetooth headsets** use HFP for duplex, which is 16 kHz wideband on current + firmware and 8 kHz narrowband on older devices. Both are below Opus's native rate. This is + acceptable and must not be treated as an error, but the quality indicator's tooltip should be + able to say the input is a Bluetooth headset when a user asks why transcription got worse. +- **CarPlay** presents as a route like any other. The app must work there, but the push-to-talk + button is not reachable while driving, so CarPlay is precisely the case that needs the wake word + and the Push to Talk framework path in [[background-and-entitlements]]. Do **not** ship a CarPlay + UI target in the first version; it is on the non-goal list in [[project-structure]]. +- **Wired headsets** behave like the built-in route with a different microphone. Nothing special. + +## Interruptions + +`AVAudioSession.interruptionNotification` is the one that decides whether an incoming phone call +leaves the app in a sane state. + +``` +.began + -> close the floor immediately (the OS already took the microphone) + -> send `floor: release` if we held it, and `interrupt: stop-word` is NOT sent: + the user did not end the conversation, the phone did + -> keep the peer connection and the data channel alive; the roster and transcript + stay on screen so returning from the call resumes a conversation rather than + restarting one + -> mark the mic button "Interrupted" + +.ended + -> if options contains .shouldResume: re-activate the session and re-apply the + RTCAudioSessionConfiguration, then return the button to its idle state + -> do NOT re-open the floor automatically. A microphone that opens itself after + a phone call is the worst possible failure mode for a microphone app. +``` + +Also handle: + +- **`AVAudioSession.mediaServicesWereResetNotification`.** The audio server died. Every + `AVAudioSession`, `AVAudioEngine`, and audio unit reference is now invalid. Tear down the local + capture graph and the `RTCAudioSession` configuration and rebuild both from scratch. This is + rare and it is unrecoverable if handled by anything less than a full rebuild. +- **`.mediaServicesWereLostNotification`.** Stop everything and wait for the reset notification. +- **Siri.** Arrives as an ordinary interruption. Nothing special beyond the above. + +## Microphone gating + +Two independent gates, because they answer two different questions. + +**Gate 1: is the peer connection carrying audio?** `RTCAudioSession.isAudioEnabled`, plus +`sender.track?.isEnabled`. Both off until the floor is open. This is the gate that satisfies +"no audio leaves this device before the floor is open". + +**Gate 2: is anything capturing at all?** With `useManualAudio = true` and audio disabled, the +WebRTC audio unit is not running. If the app also runs on-device wake-word detection, that +capture is a **separate, local-only** `AVAudioEngine` tap whose buffers never reach an encoder and +never leave the process. It exists so the phone can hear "hey maestro"; it is the one capture that +runs with the floor closed, and it is why the orange microphone indicator can be lit while nothing +is being transmitted. The UI must say which of the two states it is in, in words, in the HUD. See +[[interaction-model]]. + +## Playback + +The desktop's TTS arrives as a remote audio track. Requirements: + +- **Attenuate, do not stop, on local VAD.** When the on-device detector hears speech while the + remote track is playing, duck playback to roughly 20 percent within 20 ms and send + `interrupt: barge-in`. The authoritative cancellation comes back as a `barge-in` voice event + once the desktop has actually stopped generating; restore or stop playback then. Ducking first + is what makes barge-in feel instant even though the round trip is not. +- **Do not buffer ahead.** WebRTC's jitter buffer is the buffer. A second one on top of it adds + latency to a system whose entire budget is documented in + [[../architecture/acappella/latency-baseline]]. +- **`speak-end` is the end of speech, not the end of the track.** Do not tear down the audio unit + between replies; the reconfiguration cost lands on the front of the next one. + +## Verification + +An audio session cannot be reviewed by reading it. Before any release: + +1. **Double-talk test.** Phone on a desk, speaker at 80 percent, desktop reading a long reply. + Talk over it. The desktop must receive your words and must not receive its own voice. If the + transcript contains fragments of the reply, AEC is not engaged and the mode is wrong. +2. **AirPods mid-sentence.** Open the floor, speak, connect AirPods. Capture must survive. +3. **AirPods removed mid-sentence.** Floor must close and the desktop must be told. +4. **Incoming call during a reply.** Session survives, floor closes, nothing resumes by itself. +5. **CarPlay connect and disconnect** while the floor is open. +6. **Airplane mode toggle** while speaking, to exercise the reconnect path in + [[connection-and-pairing]] against a live audio session. diff --git a/docs/ios-client/background-and-entitlements.md b/docs/ios-client/background-and-entitlements.md new file mode 100644 index 0000000000..d3a3f97b31 --- /dev/null +++ b/docs/ios-client/background-and-entitlements.md @@ -0,0 +1,194 @@ +--- +type: specification +title: Background Audio and Entitlements +created: 2026-08-14 +tags: + - ios + - voice + - acappella +related: + - '[[overview]]' + - '[[audio-session]]' + - '[[app-store-review]]' + - '[[project-structure]]' +--- + +# Background Audio and Entitlements + +The problem this document solves: a user puts the phone on the desk, walks to the whiteboard, and +talks. The screen locks. On iOS, that is the end of the microphone unless the app has told the +system, in advance and in a way Apple sanctions, what it is. + +There are exactly two sanctioned answers and one tempting wrong one. + +## The clean answer: the Push to Talk framework (iOS 16+) + +`PushToTalk` was built for walkie-talkie apps, and A Cappella is one. The phone joins a channel, +the system shows a persistent indicator and its own transmit affordance, and the app is allowed to +capture audio in the background **while transmitting and only while transmitting**. + +That restriction is not an obstacle. It is the same rule the app already enforces: the floor is +open or it is not, and no audio leaves the device when it is not. The framework enforces in the OS +what [[audio-session]] enforces in our code, which means the two cannot drift. + +### What it requires + +| Requirement | Detail | +| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Minimum deployment target | **iOS 16.0.** This is the single constraint that sets the app's floor. See [[project-structure]]. | +| Entitlement | `com.apple.developer.push-to-talk`. **Restricted**: it must be requested from Apple through the developer account and granted before it can be added to a provisioning profile. Assume weeks, not hours. | +| Background mode | `UIBackgroundModes` must contain `push-to-talk`. | +| Push credentials | An APNs key configured for the `pushtotalk` push type, if the desktop should be able to wake a backgrounded phone. | +| Framework ownership of the audio session | The app must **not** activate `AVAudioSession` itself while in a channel. | + +### What it grants + +- **Background microphone access while transmitting.** Screen locked, app not foreground, floor + open: audio flows. +- **A system UI the user cannot lose.** iOS shows the channel in the status bar and Dynamic Island + with a transmit control, so the user can talk and stop talking without unlocking the phone. This + is the actual feature; our in-app button becomes the secondary path, not the primary one. +- **A wake path from the desktop.** A `pushtotalk` PushKit notification can bring the app back into + a live channel state without the user touching anything. +- **Battery behaviour the system understands**, rather than an app fighting the scheduler. + +### What Apple expects in return + +Read this as a contract, because review treats it as one: + +1. **The app is genuinely push-to-talk.** There is a transmit gesture, a clear start and end, and + the microphone is closed the rest of the time. A Cappella is exactly this; do not add a + continuous-listening mode to the framework path. +2. **The channel is joined in response to a user action** and left when the user is done. Joining + at launch and never leaving is the pattern Apple rejects. +3. **The framework owns the audio session.** Implement + `channelManager(_:didActivate:)` and `channelManager(_:didDeactivate:)` and do the WebRTC + configuration from [[audio-session]] inside those callbacks. Calling `setActive(true)` yourself + while in a channel is undefined behaviour and, in practice, silence. +4. **The system UI is authoritative.** When the user transmits from the status bar control, + `channelManager(_:didBeginTransmittingFrom:)` fires and we send `floor: press`. When they stop, + we send `floor: release`. The in-app button and the system control drive the same code path. +5. **Leaving the channel means leaving.** `leaveChannel` on user request, on revocation, and on a + terminal disconnect. A stale channel indicator on a user's status bar for an app that is not + connected to anything is the complaint that gets an entitlement pulled. + +### Mapping onto our protocol + +``` +PTChannelManagerDelegate A Cappella +------------------------------------------------ --------------------------------- +didActivate audioSession -> configure RTCAudioSession, isAudioEnabled = true +didBeginTransmittingFrom: .unknown/.userRequest-> send { type: 'floor', action: 'press', scope } +didEndTransmittingFrom: -> send { type: 'floor', action: 'release' } +didDeactivate audioSession -> isAudioEnabled = false, teardown capture +channelDescriptor -> name: the paired desktop's name + image: the selected agent's glyph +incomingPushResult -> reconnect signaling, re-auth, re-offer +``` + +`floor-state` from the desktop still drives the UI. If another device takes the floor while we are +transmitting, stop the transmission through `stopTransmitting(channelUUID:)` so the system UI +agrees with the app. + +### What it does not grant + +**Not a background wake word.** The framework gives the microphone during transmission, and the +wake word by definition runs before transmission. There is no supported way to run continuous +background capture for keyword spotting, and there should not be one. + +So: **the wake word works while the app is foregrounded, and only then.** Say that in the app, next +to the wake-word toggle, in one sentence. A user who thinks the phone is listening for "hey +maestro" in their pocket and finds it is not will conclude the feature is broken rather than +constrained, and they will be right to. + +## The alternative: foregrounded with a dimmed screen + +For iOS 15, for a user who does not want the system channel indicator, and as the fallback while +the entitlement request is pending: + +```swift +UIApplication.shared.isIdleTimerDisabled = true +// plus a large dark UI, and an explicit "Keep awake" toggle the user controls +``` + +The app stays foreground, the audio session stays active, and everything in [[audio-session]] +works unchanged. It requires no entitlement and no review conversation. + +The cost is real and must be stated in the UI rather than discovered: + +| Draw | Rough order of magnitude, to be measured before shipping | +| ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Opus at 24 kbps with DTX, over WiFi | Low. The radio is the smaller half of this. | +| Cellular instead of WiFi | Noticeably worse, because DTX lets the radio sleep less often than the periodic data channel traffic allows. | +| Screen on at minimum brightness | **Dominates everything else.** On an OLED device a near-black UI is much cheaper than a light one, which is why the keep-awake screen is specified black. | + +Those are shapes, not measurements. Before shipping, run the device for a measured hour in each of +three states (idle connected, floor open on WiFi, floor open on cellular) with Xcode's Energy +gauge, and put the real numbers in this table. Shipping an estimate as if it were a measurement is +how a battery complaint becomes a surprise. + +Mitigations that are worth building on this path: + +- A dedicated **keep-awake screen**: pure black, the talk button, the level meter, nothing else. +- **Auto-release the idle timer** whenever the floor has been closed for 5 minutes, and say so. +- **Disable it on Low Power Mode** (`ProcessInfo.processInfo.isLowPowerModeEnabled`), with a + visible explanation, and re-enable when it clears. + +## The tempting wrong answer: `UIBackgroundModes: audio` + +Declaring the `audio` background mode and keeping a `.playAndRecord` session alive does keep the +microphone running with the screen off. Do not ship it as the primary path. + +- **It is not what the mode is for.** The `audio` mode is for playback and for recording apps whose + recording is the user-visible product. Review reads a background `audio` declaration on a + microphone app as continuous background recording and asks hard questions, and the answers are + worse to give than the Push to Talk entitlement is to request. +- **It gives the OS no reason to protect the app.** Under memory pressure a background audio + session is a candidate for termination in a way a PTT channel is not. +- **It removes the OS-level guarantee** that the microphone is closed when the floor is closed. The + same guarantee then rests entirely on our own gating, with no second enforcement. + +There is one legitimate use: a **debug build** flag, so protocol work can happen before the +entitlement arrives. Keep it out of any configuration that can be archived for distribution. + +Also on the do-not list: **CallKit and VoIP PushKit**. Reporting an A Cappella session as a call to +keep the microphone alive misrepresents the app to the OS and to the user's call history, and a +VoIP push that does not report a call to CallKit terminates the app by design. + +## `Info.plist` and entitlements summary + +```xml + +NSMicrophoneUsageDescription +Maestro sends your voice to the computer you paired with, so you can talk to your agents from across the room. Audio is captured only while you are holding the talk button. + +NSCameraUsageDescription +Maestro scans the pairing code shown on your computer. The camera is used only for that scan. + +NSLocalNetworkUsageDescription +Maestro finds your computer on this network so you can pair without typing an address. + +NSBonjourServices +_maestro._tcp + +UIBackgroundModes +push-to-talk +``` + +```xml + +com.apple.developer.push-to-talk + +aps-environment +production +``` + +The exact copy for the usage descriptions, and why each sentence is worded the way it is, is in +[[app-store-review]]. + +## Decision, stated once + +**Ship the Push to Talk framework path, minimum iOS 16, with the dimmed-screen keep-awake mode as +a user-selectable alternative on the same build.** Request the entitlement on day one of the Swift +effort, because it is the longest-lead item in the whole project and everything else can be built +against the keep-awake path while it is pending. diff --git a/docs/ios-client/connection-and-pairing.md b/docs/ios-client/connection-and-pairing.md new file mode 100644 index 0000000000..242b13b13f --- /dev/null +++ b/docs/ios-client/connection-and-pairing.md @@ -0,0 +1,276 @@ +--- +type: specification +title: Connection and Pairing +created: 2026-08-14 +tags: + - ios + - voice + - acappella +related: + - '[[overview]]' + - '[[protocol-conformance]]' + - '[[audio-session]]' + - '[[../architecture/acappella/transport-and-pairing]]' +--- + +# Connection and Pairing + +The desktop side of everything here is specified in +[[../architecture/acappella/transport-and-pairing]] and implemented in +`src/main/acappella/pairing/pairing-service.ts` and +`src/main/acappella/transport/signaling.ts`. This document is the client half. + +## The security model the client must not undermine + +Four properties the desktop enforces. The client's job is to never work around any of them, and +to explain them honestly when they bite: + +1. **Knowing the pairing code is not enough.** A code is a pointer to a request, not an + authorisation. Pairing completes only when a human presses Approve on the desktop, looking at + the name and platform this app reported. There is no client-side path that skips that, and the + app must not imply there is. +2. **The window is short and one-shot.** A code lives 120 seconds and is consumed by the first + claim, approved or denied. Expiry is normal, not an error state to apologise for: show + "Ask the desktop for a new code" and a countdown. +3. **The token is never recoverable from the desktop.** The desktop stores only a salted SHA-256. + If the phone loses its Keychain item, the only path back is a fresh pairing. Never build a + "recover my token" flow, because the desktop cannot serve one. +4. **Revocation is immediate.** A `revoked` message on the data channel, or an `auth-failed` on + the signaling socket, is final. The client must delete its Keychain item and return to the + unpaired state on either. Retrying a revoked token is the one thing that turns a clean + revocation into a support ticket. + +## Transport summary + +| Leg | Carries | Where | +| --------- | ----------------------------------------- | ------------------------------------------- | +| Signaling | Pairing, auth, SDP, ICE candidates | `ws://://ws` | +| Media | Opus, both directions | WebRTC peer connection | +| State | Roster, tabs, floor, errors, revocation | `RTCDataChannel` labelled `acappella-state` | +| Realtime | Levels, partials, press/release, barge-in | `RTCDataChannel` labelled `acappella-live` | + +Every signaling frame is a JSON object `{ "type": "acappella_signal", "payload": }`. The +`payload` shapes are `SignalingClientMessage` and `SignalingServerMessage` in +`src/main/acappella/transport/signaling.ts`; they are enumerated exhaustively in +[[protocol-conformance]]. + +## Finding a desktop + +Three paths, in the order the UI should offer them. + +### 1. QR code (the primary path) + +The desktop's Settings panel (Encore Features -> A Cappella -> Paired Devices) renders a QR code. +Its payload is a JSON object: + +```json +{ + "kind": "maestro-acappella", + "v": 1, + "hosts": ["192.168.1.42", "100.83.11.9"], + "port": 17173, + "token": "", + "code": "K7QMBX", + "expiresAt": 1786820745000, + "fingerprint": "9F3A" +} +``` + +Client requirements: + +- **Reject anything whose `kind` is not `maestro-acappella`.** A QR scanner that tries to make + sense of an arbitrary payload is an attack surface. +- **Reject a `v` this build does not implement**, with the sentence "This Maestro is newer than + this app. Update the app." Do not attempt a best-effort parse. +- **Treat `expiresAt` as authoritative.** A payload already past it must not be sent to the + desktop; say the code expired and ask for a new one. +- **Try every entry in `hosts`, in order, in parallel, and take the first socket that opens.** + `hosts` is ordered with the desktop's primary interface first, and it can contain overlay + addresses (Tailscale allocates from `100.64.0.0/10`) that work from anywhere. A phone that only + tries `hosts[0]` fails on any Mac with more than one interface, which is most of them. +- **Show the `fingerprint` on the phone** after connecting, next to the same four characters shown + on the desktop. It is derived from the server token, so a matching pair means the phone is + talking to the machine whose screen the user is looking at. This is the only man-in-the-middle + check the user has; do not hide it behind a details view. + +Camera permission uses `NSCameraUsageDescription`: "Maestro scans the pairing code shown on your +computer. The camera is used only for that scan." The scan is `AVCaptureMetadataOutput` with +`.qr`, no image is written to disk, and no photo library access is requested. + +### 2. Bonjour LAN discovery + +The desktop advertises `_maestro._tcp` when discovery is enabled. Browse with `NWBrowser` +(`NWBrowser.Descriptor.bonjourWithTXTRecord(type: "_maestro._tcp", domain: nil)`) so the TXT +record arrives with the result. + +TXT keys, all public by construction: + +| Key | Meaning | +| ------------- | ----------------------------------------------------------- | +| `version` | Maestro app version | +| `proto` | A Cappella device protocol version | +| `fingerprint` | Pairing fingerprint, for the same four-character comparison | +| `host` | Human name of the desktop | + +Client requirements: + +- **Discovery never carries a credential.** There is no token and no pairing code in the TXT + record. A discovered desktop still needs a code typed or scanned, and the UI must make that + obvious rather than implying a discovered machine is a paired one. +- **Grey out, do not hide, a discovered desktop whose `proto` this build cannot speak**, with the + version sentence from [[protocol-conformance]]. +- Requires `NSLocalNetworkUsageDescription` and an `NSBonjourServices` array containing + `_maestro._tcp` in `Info.plist`. Without the latter the browse silently returns nothing on iOS + 14 and later, which reads as "my Mac is not discoverable" and sends users to the wrong problem. +- **An empty browse is not an error.** The desktop may have discovery switched off deliberately, + or mDNS may be unavailable. Fall through to manual entry with "Cannot see your Mac? Enter its + address" rather than a failure dialog. + +### 3. Manual host entry + +Always available, never buried. Host and port, with the port defaulting to the value the desktop +shows in its manual-entry hint. This path also covers the reverse-proxy and VPN cases that +discovery cannot reach. + +The server token still has to come from somewhere. On this path the user types the pairing code +and the desktop's URL including the token segment, which is what the desktop's manual hint +displays verbatim. + +## The pairing exchange + +``` +phone desktop + |-- {op:'pair-claim', code, name, | + | platform:'ios', appVersion} --------->| code checked, consumed, request created + |<- {op:'pair-pending', requestId, | human sees an Approve/Deny row + | expiresAt} --------------------------| + | | + |-- {op:'pair-poll', requestId} ---------->| (poll until resolved or expiresAt) + |<- {op:'pair-approved', deviceId, token} -| 32-byte token, minted once + | or {op:'pair-denied'} | + | or {op:'pair-rejected', reason, | + | message} | +``` + +- `platform` must be the literal `ios`. It is displayed verbatim in the desktop's device list and + in the approval row, so a human decides against something recognisable. +- `name` defaults to `UIDevice.current.name` and must be user-editable before the claim is sent. +- Poll on a 1 second interval, and stop at `expiresAt` rather than polling forever. +- `pair-approved` is the **only** time the token exists in the clear. Write it to the Keychain + before updating any UI state; a crash between "token received" and "token stored" leaves an + approved device that can never authenticate, and there is no recovery path. +- `pair-rejected` carries a `message` written for a human. Show it verbatim. Do not paraphrase and + do not append a generic "Try again". + +## Storing the token + +```swift +// Keychain item, one per paired desktop. +kSecClass: kSecClassGenericPassword +kSecAttrService: "sh.maestro.acappella.device" +kSecAttrAccount: deviceId // from pair-approved +kSecAttrAccessible: kSecAttrAccessibleAfterFirstUnlock +kSecAttrSynchronizable: false +kSecValueData: token // UTF-8 +``` + +Requirements, each with a reason: + +- **`kSecAttrAccessibleAfterFirstUnlock`, not `WhenUnlocked`.** The app has to reconnect from the + background with the screen locked (see [[background-and-entitlements]]), and `WhenUnlocked` + makes the token unreadable in exactly that state. +- **Never `ThisDeviceOnly` plus iCloud sync.** Synchronizable is off outright: a device token + identifies one physical device to one desktop, and syncing it to an iPad produces two devices + claiming one identity, which the desktop will treat as a stolen credential. +- **The server token and host list are stored alongside it**, in the same Keychain item's generic + attribute or a second item. They are not secrets of the same weight, but the server token is + still a credential and does not belong in `UserDefaults`. +- **Delete on revocation, delete on `auth-failed`, delete on user "Forget this Mac".** Three + paths, one function. + +## Authenticating and connecting + +``` + |-- {op:'auth', deviceId, token, protocolVersion} --->| + |<- {op:'authenticated', deviceId, protocolVersion, | + | iceServers, iceTransportPolicy, audio} ----------| + | or {op:'auth-failed', reason, message} | + | | + |-- {op:'offer', sdp} ------------------------------->| rate limited: 6 per 60s + |<- {op:'answer', sdp} -------------------------------| + |<=== {op:'ice-candidate'} both directions ==========>| +``` + +- **Version is negotiated at `auth`, before the credential is even checked.** An old client is + told to update the app; a client from the future is told to update the desktop. Handling is + specified in [[protocol-conformance]]. +- **Use the `iceServers` and `iceTransportPolicy` the desktop sent.** Do not carry a hard-coded + STUN server in the app. The desktop's settings are the single source of truth, including a + `relay`-only policy for a user who does not want their IP addresses exchanged. +- **Apply the `audio` config** (`fec`, `dtx`, `maxAverageBitrate`, `requestRemoteEchoCancellation`) + to the outgoing Opus encoder. See [[audio-session]] for how each maps onto WebRTC.framework. +- **Five failed `auth` attempts closes the socket.** Do not retry in a tight loop. One attempt per + reconnect, with backoff between reconnects. +- **Offers are limited to 6 per 60 seconds.** That budget is sized for an initial offer plus + renegotiation on network changes. A client that rebuilds its peer connection on every ICE + hiccup will exhaust it and be rate limited during exactly the handover it was trying to survive. + +## Reconnection + +The rules, in order of how often they matter: + +| Event | Client behaviour | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `RTCPeerConnectionState.disconnected` | **Do nothing for 5 seconds.** This is what an ordinary WiFi-to-LTE handover looks like, and tearing down here is how you break the case you were trying to handle. Show "Reconnecting" in the quality indicator, keep the UI live. | +| Still `disconnected` after 5s, or `failed` | Restart ICE and send a fresh `offer` on the same authenticated socket. The desktop applies it to the existing peer, so the media leg survives. | +| Signaling socket closed | Reconnect the socket with backoff, re-`auth`, then re-`offer`. The desktop never inherits an authenticated state across sockets, so `auth` is mandatory every time. | +| `closed`, or the desktop sent `revoked` | Terminal. Do not reconnect. | +| `auth-failed` | Terminal. Delete the Keychain item and go to the unpaired state. | +| App backgrounded without the Push to Talk framework | Expect the connection to die. Reconnect on foreground; see [[background-and-entitlements]]. | + +Backoff: 1s, 2s, 4s, 8s, 15s, 30s, then 30s steady, each with up to 30 percent jitter. Reset the +schedule on any successful `authenticated`. Never reconnect while the app is in the background +without an active Push to Talk session; a phone in a pocket retrying every 30 seconds all night is +a battery complaint and a one-star review. + +**The floor does not survive a reconnect.** On reconnect the client's mic button starts closed and +waits for a `floor-state` message. Assuming the floor is still held is how a phone ends up with a +hot microphone the desktop does not know about. + +## The connection quality indicator + +Driven by `link-quality` messages on the lossy channel, which both ends emit from throttled +`getStats()` readings: + +```ts +{ type: 'link-quality', rttMs: number | null, jitterMs: number | null, + packetLoss: number, candidateType: 'lan' | 'stun' | 'relay' | 'unknown' } +``` + +The indicator has two halves, and both are needed: + +**Quality**, from `rttMs`, `jitterMs`, and `packetLoss`: + +| Bars | Condition | +| ---- | ---------------------------------------------- | +| 3 | rtt < 80 ms, jitter < 20 ms, loss < 2 percent | +| 2 | rtt < 200 ms, jitter < 50 ms, loss < 5 percent | +| 1 | connected, anything worse | +| 0 | not connected | + +**Path**, from `candidateType`, shown as a word, not a colour: + +| Value | Label | What it means to the user | +| --------- | ------------------ | -------------------------------------------------------------------------- | +| `lan` | "Direct" | Host candidate. Same network or an overlay. No infrastructure in the path. | +| `stun` | "Direct (via NAT)" | Both ends punched through. Media is still peer to peer. | +| `relay` | "Relayed" | A TURN server is forwarding every packet. Works everywhere, costs latency. | +| `unknown` | "Connecting" | No candidate pair selected yet. | + +Showing the path in words is deliberate. "Why is it slow" and "where does my audio go" are the +same question for a relayed connection, and a user who can see the word "Relayed" can answer it +without a support thread. + +The client emits its own `link-quality` at most once per 2 seconds. It is throttled at the sender +because it rides the lossy channel alongside the audio meter, and a chatty stats message competes +with the thing the user actually hears. diff --git a/docs/ios-client/interaction-model.md b/docs/ios-client/interaction-model.md new file mode 100644 index 0000000000..a56f5fe1e8 --- /dev/null +++ b/docs/ios-client/interaction-model.md @@ -0,0 +1,229 @@ +--- +type: specification +title: Interaction Model +created: 2026-08-14 +tags: + - ios + - voice + - acappella +related: + - '[[overview]]' + - '[[audio-session]]' + - '[[protocol-conformance]]' + - '[[../architecture/acappella/wake-and-hotkeys]]' +--- + +# Interaction Model + +One screen. A wheel at the top, a button in the middle, a transcript you pull up from the bottom. +Everything else is a sheet reached from a corner. + +``` + +--------------------------------------------------+ + | (( Direct )) 3 bars Pedram's Mac ... | status strip + +--------------------------------------------------+ + | | + | < [ Conductor ] [ backend ] [ docs ] [ api ] > | project wheel + | | + | | + | +----------+ | + | | | | + | | TALK | | push-to-talk + | | | | + | +----------+ | + | | + | "open a tab on the backend" | live partial + | | + +--------------------------------------------------+ + | ^ Transcript | drag up for the sheet + +--------------------------------------------------+ +``` + +## The push-to-talk button + +The single control that matters. It supports **both** gestures, and which one happened is decided +by how long the finger was down, against exactly the threshold the desktop uses. + +``` +touchDown + | + +-- send `floor: press` IMMEDIATELY (do not wait to classify) + | haptic: .impact(.medium) + | + +-- start a timer at holdThresholdMs (default 300 ms, clamped 100 to 2000) + | +touchUp before threshold -> TAP: the floor stays open. The button is now latched. + A second tap sends `floor: release`. +touchUp after threshold -> HOLD: send `floor: release` on the lift. + haptic: .impact(.light) +``` + +Requirements, each earned: + +- **Send `press` on touch-down, not on classification.** Waiting 300 ms to find out whether this + is a tap or a hold puts 300 ms in front of every utterance, and the desktop's `press` is + idempotent, so there is nothing to be gained by waiting. The classification only decides what + happens on the lift. +- **Use the desktop's threshold.** It arrives with the voice settings; do not carry a second + constant. `DEFAULT_HOLD_THRESHOLD_MS = 300` in `src/shared/acappella/voice-controls.ts` is the + default, `resolveHoldThresholdMs()` is the clamp. A phone that classifies at 250 ms while the + desktop hotkey classifies at 300 ms produces two devices that disagree about what a tap is. +- **Cancel on drag-off.** A finger that slides off the button before lifting sends `floor: release` + and shows the cancelled state. A push-to-talk button you cannot back out of is a button people + are afraid to press. +- **The button reflects `floor-state`, not the local gesture.** The gesture is a request. The + authoritative state arrives as `{ type: 'floor-state', holder, isSelf, takenOverBy }`, and the + button renders that. If another device takes the floor, the button snaps back with + `takenOverBy` shown ("Taken by Pedram's iPad"), because a button that lies about holding a + microphone is worse than one that flickers. +- **Never assume the floor across a reconnect.** Start closed, wait for `floor-state`. + +### Button states + +| State | Appearance | Reached by | +| ------------- | --------------------------------------- | ------------------------------------------------- | +| Idle | Outline, "Talk" | Floor closed | +| Pressed | Filled, growing ring, level meter | `floor-state.isSelf === true` | +| Latched (tap) | Filled with a lock glyph, "Tap to stop" | Tap classified, floor still open | +| Held by other | Dimmed, "Pedram's iPad is talking" | `floor-state.holder != null && !isSelf` | +| Thinking | Pulsing, no meter | `dispatch` seen, no `speak-start` yet | +| Speaking | Waveform, "Tap to interrupt" | Between `speak-start` and `speak-end` | +| Interrupted | Outline, brief flash | System audio interruption (see [[audio-session]]) | +| Disconnected | Greyed, not tappable, reason underneath | Peer or signaling down | + +Tapping during **Speaking** sends `interrupt: barge-in`, not a floor press. That is the one +overloaded gesture in the app and it is worth it: reaching for a separate stop button while +something is talking at you is exactly when a user cannot aim. + +## Haptics + +Haptics are the app's only feedback channel when the screen is off or in a pocket, so they are +specified rather than decorative: + +| Moment | Feedback | +| ------------------------------------ | -------------------------------------------------- | +| Floor opens (any cause) | `UIImpactFeedbackGenerator(style: .medium)` | +| Floor closes (any cause) | `UIImpactFeedbackGenerator(style: .light)` | +| Floor taken by another device | `UINotificationFeedbackGenerator(.warning)` | +| Wake word fired | `.medium`, identical to a press, because it is one | +| Stop word fired | `UINotificationFeedbackGenerator(.success)` | +| Barge-in accepted (`barge-in` event) | `UISelectionFeedbackGenerator` | +| Session error | `UINotificationFeedbackGenerator(.error)` | +| Roster changed | Nothing. Background state changes must not buzz. | + +Prepare the generators ahead of the gesture (`prepare()` on touch-down) or the first haptic of a +session arrives late enough to feel like a different event. + +## The project wheel + +A horizontally scrolling row of agents, driven **entirely** by the `agent-roster` voice event: + +```ts +{ type: 'agent-roster', agents: RosterAgent[] } +// RosterAgent: { sessionId, name, agentType, cwd, tabs, status?, recentWork? } +``` + +Rules: + +- **The first item is always Conductor**, which is not an agent. Selecting it sets the scope of + the next floor press to `{ kind: 'conductor' }`, which lets the desktop route the utterance + itself. Every other item sets `{ kind: 'agent', sessionId }`. +- **The roster is a snapshot, not a diff.** Replace the list on every `agent-roster` event. Do not + merge; the desktop sends a whole roster precisely so the phone cannot accumulate agents that no + longer exist. +- **`status` colours the item** using the same vocabulary as the desktop Left Bar: `idle` green, + `busy` yellow, `error` red. Absent means unknown, which renders neutral rather than green. +- **`recentWork` is the subtitle.** It is the synopsis the desktop's history manager already wrote, + and it is what makes the wheel scannable when four agents have similar names. +- **Selection is a client-side preference only.** It changes the `scope` on the next `floor: press` + and nothing else. It does not tell the desktop to focus anything, and it does not survive a + roster that no longer contains the selection: fall back to Conductor and say so. +- **`tab-state` events update the selected agent's tab count** in the item's accessory. A phone + that shows tab state is a phone that can answer "did it actually open the tab" without the user + walking to the desk. + +Scrolling snaps to items. Selecting one is a single tap, with `UISelectionFeedbackGenerator`. + +## The live transcript sheet + +A `UISheetPresentationController` with detents `[.height(120), .medium(), .large()]`, presented +non-modally so the talk button stays reachable at the small detent. + +Content, in arrival order, built from voice events: + +| Event | Row | +| -------------------- | --------------------------------------------------------------------------------------------------------------- | +| `partial-transcript` | Replaces the in-flight user row. Lossy channel: rows may skip, never reorder. Key off `seq`. | +| `final-transcript` | Commits the user row. | +| `route-decision` | A quiet caption on the user row: "-> backend agent". Includes the confidence when the desktop was unsure. | +| `dispatch` | A caption: "Opened tab: auth refactor". | +| `route-correction` | Rewrites the caption in place, with a corrected marker. Do not append a second row; the user said one sentence. | +| `agent-reply` | Starts the assistant row. | +| `speak-sentence` | Appends to the assistant row, sentence by sentence, so the text tracks the audio. | +| `speak-end` | Closes the assistant row. | +| `barge-in` | Marks the assistant row truncated. Do not delete what was already said. | +| `stop-word` | A divider. The conversation is over. | +| `session-error` | An inline error row with the desktop's own message, verbatim. | + +Requirements: + +- **The desktop's text is the truth.** Never re-render a transcript from local audio, and never + "clean up" the desktop's copy. +- **Order by `seq`, not by arrival.** Two channels means the reliable and lossy streams interleave, + and a partial can arrive after the final that superseded it. +- **The transcript is per-session and not persisted.** A conversation history that lives on the + phone is a second brain, and this app does not have one. See [[overview]]. +- **Copy is available, sharing is not.** Long-press copies a row. There is no export, no cloud + sync, no attachment sheet. + +## The status strip + +Three elements, left to right: + +1. **The connection quality indicator**, exactly as specified in [[connection-and-pairing]]: bars + plus a path word (Direct, Direct via NAT, Relayed, Connecting). +2. **The desktop name and fingerprint**, tappable to show the paired-device sheet with the four + characters to compare. +3. **A microphone-state pill**, which is the honesty control for the two capture gates in + [[audio-session]]. It has three states and each says something different: + - "Mic off" - nothing is capturing. + - "Listening for wake word" - a local-only tap is running; nothing is transmitted. + - "Sending" - the floor is open and audio is going to the desktop. + +That third element is not optional. iOS lights an orange indicator whenever any capture is +running, including the local wake-word tap, and a user who sees it with nothing on screen +explaining why will assume the worst thing. + +## Wake word and stop word + +Both run **on the phone**, for the reasons in [[../architecture/acappella/wake-and-hotkeys]] and +[[overview]]: the wake word cannot be detected remotely without sending the audio it is meant to +gate, and the stop word must be heard while the desktop is speaking. + +Behaviour: + +- A wake-word hit is exactly a `floor: press` with the currently selected scope, plus the same + haptic a physical press produces. From the desktop's point of view there is no difference, and + there must not be one. +- A stop-word hit sends `interrupt: { kind: 'stop-word' }`. The floor closes and the session ends. +- **Barge-in and stop word are different things and stay different.** Barge-in means "stop talking, + I am still here": speech cancelled, floor kept, microphone open. Stop word means "we are done": + speech cancelled, floor released, microphone closed. Every assistant that merged them became one + you cannot get rid of. +- Arming follows the desktop's rule: wake phrases only while the session is cold, stop phrases in + every active state, never both. Otherwise a wake phrase spoken mid-answer stacks a second + session. +- Wake word is **off by default** and its toggle sits on the main screen, one tap away. It is the + setting most likely to be turned off in a meeting. + +## Accessibility + +- The talk button is a single large target well past the 44 pt minimum, and it is the only control + needed to use the app. +- VoiceOver: the button announces its state, not its label ("Talk, idle", "Talk, sending", + "Talk, held by Pedram's iPad"). The transcript sheet is a standard list and reads normally. +- Every state distinguished by colour is also distinguished by a word or a glyph. The agent status + dots pair with the status word in the item's accessibility label. +- Dynamic Type is honoured everywhere except the talk button's own label. +- Reduce Motion removes the pulsing and waveform animations; the states stay distinguishable by + fill and label. diff --git a/docs/ios-client/overview.md b/docs/ios-client/overview.md new file mode 100644 index 0000000000..3e3d5ee9de --- /dev/null +++ b/docs/ios-client/overview.md @@ -0,0 +1,124 @@ +--- +type: specification +title: A Cappella iOS Client Overview +created: 2026-08-14 +tags: + - ios + - voice + - acappella +related: + - '[[../architecture/acappella/system-overview]]' + - '[[connection-and-pairing]]' + - '[[audio-session]]' + - '[[interaction-model]]' + - '[[background-and-entitlements]]' + - '[[app-store-review]]' + - '[[protocol-conformance]]' + - '[[project-structure]]' +--- + +# A Cappella iOS Client Overview + +## What this app is + +**A remote microphone and speaker for a Maestro desktop.** Nothing more, and the "nothing more" +is the load-bearing part of the design. + +The phone captures audio, sends it to a paired desktop over WebRTC, and plays back what the +desktop sends home. Every decision about that audio - which speech recogniser transcribes it, +which agent it is routed to, what tab gets opened, which voice reads the answer - is made on the +desktop by the code described in [[../architecture/acappella/system-overview]]. The phone +contributes exactly three things the desktop cannot do for itself: a microphone in another room, +a speaker in that same room, and a screen to press. + +## What this app is not + +It is **not a second brain**. It does not: + +- run a speech recogniser, a router, or a text-to-speech engine; +- hold its own conversation state, its own transcript history, or its own agent list; +- talk to Anthropic, OpenAI, ElevenLabs, or any other provider directly; +- work at all without a paired desktop that is awake and running Maestro. + +If the phone ever needs to make a decision the desktop already makes, that is a bug in this +specification, not a feature request for the phone. Two implementations of routing will disagree, +and the one the user is not looking at will be the one that is wrong. + +There are exactly **three** exceptions, and each exists because latency or privacy makes a round +trip to the desktop unacceptable. They are specified in detail in [[protocol-conformance]]: + +| Local behaviour | Why it cannot live on the desktop | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Wake word | No audio may leave the device before the wake phrase fires. Detecting it remotely would require sending audio first, which is the thing the rule forbids. | +| Stop word | It must be heard while the desktop is speaking and a recogniser stream is open. A stop that arrives after the thing it was meant to stop has finished is not a stop. | +| VAD ducking | Barge-in has to attenuate playback within roughly 20 ms. A round trip to the desktop is 30 to 120 ms before the desktop has even decided anything. | + +Everything else the phone does is: press a button, draw what the data channel says, play what the +audio track carries. + +## The shape of a session + +``` + iPhone Maestro desktop + | | + | user presses talk (or says wake word) | + |------ floor: press --------------------> FloorController.press() + | | same object the desktop hotkey drives + |<----- floor-state: isSelf=true --------- + | | + |======= Opus audio, phone -> desktop ===> STT, routing, dispatch + | | + |<----- voice-event: partial-transcript -- (lossy channel) + |<----- voice-event: route-decision ------ + |<----- voice-event: dispatch ------------ + |<====== Opus audio, desktop -> phone ==== TTS + |<----- voice-event: speak-sentence ------ + | | + | user talks over the reply | + |------ interrupt: barge-in -------------> session.interrupt() + |<----- voice-event: barge-in ------------ authoritative, speech actually cancelled +``` + +The phone never invents a `voice-event`. It **requests** (`floor`, `interrupt`) and it +**renders** (`voice-event`, `floor-state`). Requests are hopeful; events are the truth. + +## Why WebRTC and not a WebSocket + +Decided in [[../architecture/acappella/decisions/adr-001-webrtc-transport]] and restated here +because it is the first question any iOS developer will ask: + +- **Opus with in-band FEC and DTX** survives the loss profile of a phone on cellular. A raw PCM + stream over a WebSocket does not, and a phone leaving WiFi mid-sentence is the normal case, not + the edge case. +- **The audio path is peer to peer** when the network allows it, so a phone on the same LAN as the + desktop pays no server hop at all. See the `lan`/`stun`/`relay` distinction in + [[connection-and-pairing]]. +- **`AVAudioSession` in `.voiceChat` mode** gives us Apple's hardware voice-processing echo + canceller for free, and that mode is designed around a duplex real-time call. See + [[audio-session]]. +- **Renegotiation is a first-class path.** A phone walking from WiFi to LTE re-offers on the same + authenticated socket and the media leg survives the handover. A WebSocket audio stream would be + torn down and rebuilt, mid-sentence. + +Signaling still rides Maestro's existing authenticated WebSocket, so there is no second port and +no second authentication surface. + +## Reading order + +| Document | Answers | +| ------------------------------- | ----------------------------------------------------------------------- | +| [[connection-and-pairing]] | How the phone finds a desktop, earns a token, keeps it, and reconnects. | +| [[audio-session]] | How to configure `AVAudioSession` and WebRTC so the audio is usable. | +| [[interaction-model]] | What is on screen and what every gesture does. | +| [[background-and-entitlements]] | How the app keeps a microphone alive when the screen is off. | +| [[app-store-review]] | How to get an app that is useless without a paired Mac through review. | +| [[protocol-conformance]] | Exactly what to send and handle, checkable item by item. | +| [[project-structure]] | Xcode layout, dependencies, signing, and the non-goal list. | + +## The reference client + +Before any Swift exists, there is a browser reference client at +`src/web-desktop/acappella-client/` that speaks this identical protocol. It is not a demo. It is +the second endpoint the desktop is regression-tested against, and it is the executable answer to +"what does the wire actually look like" for anyone writing the Swift. When this specification and +the reference client disagree, the reference client is right and this document has a bug. diff --git a/docs/ios-client/project-structure.md b/docs/ios-client/project-structure.md new file mode 100644 index 0000000000..1c268b628f --- /dev/null +++ b/docs/ios-client/project-structure.md @@ -0,0 +1,354 @@ +--- +type: specification +title: A Cappella iOS Project Structure +created: 2026-08-14 +tags: + - ios + - voice + - acappella +related: + - '[[overview]]' + - '[[connection-and-pairing]]' + - '[[audio-session]]' + - '[[interaction-model]]' + - '[[background-and-entitlements]]' + - '[[app-store-review]]' + - '[[protocol-conformance]]' +--- + +# iOS Project Structure + +The layout, the dependencies, the signing chain, and the list of things the first Swift effort is +not allowed to build. Everything here exists to keep one app small enough that one person can +finish it, and to keep the protocol in it from drifting away from the desktop that defines it. + +## Where the code lives + +**In this repository, at `ios/`.** Not a second repository. + +The argument for a separate repo is that an Xcode project in a TypeScript monorepo is noise, and +that is true. The argument against it is stronger: the wire protocol is defined by +`src/shared/acappella/device-protocol.ts`, the reference implementation of the client half is at +`src/web-desktop/acappella-client/`, and the conformance suite that judges both is at +`src/__tests__/acappella/conformance/`. A protocol change touches all of them, and it should be +able to touch the Swift in the same commit and the same review. Two repositories means a change +lands in one of them first, and the window between the two is exactly where the phone goes silent +in a way nobody notices until a device is in a hand. + +Cost, paid deliberately: macOS runners are expensive, so the iOS job is a separate workflow gated +on `paths: ['ios/**', 'src/shared/acappella/**']` rather than a fourth leg of the existing +`test` matrix in `.github/workflows/ci.yml`. The Node CI legs must never wait on Xcode. + +## Deployment target and toolchain + +| Choice | Value | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Minimum iOS | **16.0** | +| Devices | iPhone, and iPad unmodified (universal, no iPad-specific layout in v1) | +| Language | Swift, strict concurrency on, no Objective-C beyond what WebRTC's headers bring in | +| UI | SwiftUI for everything except the talk button's gesture handling, which is a `UIViewRepresentable` so touch-down can be handled without SwiftUI's gesture-recognition delay | +| Xcode | Pinned in `ios/.xcode-version` and read by CI. Do not track "latest"; a toolchain bump is a commit like any other. | +| Dependency manager | Swift Package Manager only. No CocoaPods, no Carthage. | + +**iOS 16 is set by one thing and one thing only:** the Push to Talk framework, which is the +sanctioned way to hold a microphone with the screen off and is therefore the whole background +story. The reasoning, and the two alternatives that were rejected, are in +[[background-and-entitlements]]. If that decision is ever revisited, the floor moves with it; no +other API in this app needs more than iOS 15. + +Everything in [[audio-session]], [[connection-and-pairing]], and [[protocol-conformance]] runs on +iOS 15. The keep-awake path exists partly so the app is usable before the Push to Talk entitlement +is granted, and it should not be quietly deleted once it is. + +## Repository layout + +``` +ios/ +├── .xcode-version Toolchain pin, read by CI. +├── README.md How to build, how to pair against a dev desktop. +├── ACappella.xcodeproj/ One project. No workspace: SPM needs no .xcworkspace. +├── ACappella/ The app target. UI and platform glue only. +│ ├── ACappellaApp.swift @main, scene setup, deep-link and PTT restoration entry. +│ ├── Info.plist Usage strings and NSBonjourServices; see below. +│ ├── ACappella.entitlements push-to-talk, aps-environment. +│ ├── Screens/ +│ │ ├── UnpairedScreen.swift The real screen a reviewer sees first (app-store-review). +│ │ ├── PairingScreen.swift QR scan, Bonjour list, manual host entry. +│ │ ├── SessionScreen.swift Status strip, project wheel, talk button, live partial. +│ │ ├── TranscriptSheet.swift The three-detent sheet. +│ │ └── KeepAwakeScreen.swift Pure-black fallback background mode. +│ ├── Components/ +│ │ ├── TalkButton.swift Gesture down/up, states, VoiceOver announcements. +│ │ ├── ProjectWheel.swift Snap-scrolling roster row. +│ │ ├── QualityIndicator.swift Bars plus the path word ("Direct", "Relayed"). +│ │ └── MicPill.swift Off / Listening for wake word / Sending. Three states. +│ ├── Haptics/Haptics.swift One place that owns every feedback generator. +│ └── Assets.xcassets/ +├── Packages/ +│ ├── ACappellaKit/ The protocol. No UIKit, no AVFoundation, no WebRTC. +│ │ ├── Package.swift +│ │ ├── Sources/ACappellaKit/ +│ │ │ ├── Generated/ProtocolConstants.swift GENERATED. Do not hand-edit. +│ │ │ ├── Signaling/SignalingMessage.swift Codable client and server messages. +│ │ │ ├── Signaling/SignalingSocket.swift Protocol (the Swift kind). Injected. +│ │ │ ├── Device/DeviceMessage.swift The ten data-channel messages. +│ │ │ ├── Device/DeviceCodec.swift encode/decode, the `v` stamp, seq gaps. +│ │ │ ├── Device/SessionEvent.swift The twenty voice events. +│ │ │ ├── Session/ClientState.swift Phase, floor view, roster, transcript. +│ │ │ ├── Session/ACappellaClient.swift The state machine. Everything injected. +│ │ │ ├── Session/Reconnection.swift Backoff, terminal versus retryable. +│ │ │ └── Ports/ PeerPort, MicrophonePort, Clock, TokenStore. +│ │ └── Tests/ACappellaKitTests/ +│ │ ├── Fixtures/ GENERATED golden frames. See below. +│ │ └── Conformance/ Test names carry their C-nn. +│ ├── ACappellaAudio/ The adapters. WebRTC and AVFoundation live only here. +│ │ ├── Sources/ACappellaAudio/ +│ │ │ ├── WebRTCPeer.swift PeerPort over RTCPeerConnection. +│ │ │ ├── AudioSessionController.swift RTCAudioSession, category, mode, manual audio. +│ │ │ ├── RouteObserver.swift routeChange and interruption handling. +│ │ │ ├── OpusPreferences.swift fmtp munging from RemoteAudioConfig. +│ │ │ ├── LevelMeter.swift audio-level source, floor-gated. +│ │ │ └── Ducking.swift Local VAD and the 20 ms attenuation. +│ │ └── Tests/ACappellaAudioTests/ +│ ├── ACappellaTransport/ The other adapters: socket, discovery, Keychain. +│ │ ├── Sources/ACappellaTransport/ +│ │ │ ├── URLSessionSignalingSocket.swift +│ │ │ ├── BonjourBrowser.swift NWBrowser over _maestro._tcp. +│ │ │ ├── PairingPayload.swift QR JSON parse and validation. +│ │ │ └── KeychainPairingStore.swift TokenStore over kSecClassGenericPassword. +│ │ └── Tests/ACappellaTransportTests/ +│ └── ACappellaWake/ The one on-device model. Isolated so it can be removed. +│ ├── Sources/ACappellaWake/ +│ │ ├── WakeWordDetector.swift +│ │ └── StopWordDetector.swift +│ └── Tests/ACappellaWakeTests/ +└── Fastlane/ (optional) Only if TestFlight uploads become manual toil. +``` + +Three rules the layout is enforcing, each of which is a lesson from the reference client: + +- **`ACappellaKit` imports nothing.** Not UIKit, not AVFoundation, not WebRTC, not Network. It is + the direct analogue of `client.ts`, which has no DOM in it, and it exists for the same reason: + the protocol has to be testable on a machine with no microphone, no camera, and no phone. If a + Swift file in `ACappellaKit` needs `import AVFoundation`, the thing it is doing belongs in + `ACappellaAudio` behind a port. +- **Ports are protocols, and the app target owns the wiring.** `ACappellaClient` receives a + `PeerPort`, a `MicrophonePort`, a `Clock`, and a `TokenStore`. In the app they are the WebRTC, + AVFoundation, system, and Keychain implementations. In tests they are fakes. Nothing in + `ACappellaKit` ever constructs one. +- **`ACappellaWake` is a separate package specifically so it can be cut.** It is the only thing in + the app with a model file in it, it is the only thing that opens a microphone the desktop does + not know about, and it is the most likely thing to be replaced. Isolating it means the answer to + "what happens if the wake word ships later" is "remove one dependency", not "unpick it from the + session code". + +## Dependencies + +| Dependency | How | Why, and what happens without it | +| -------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **WebRTC** | Binary XCFramework via SPM, exact version pinned, checksum recorded | The entire media path. There is no first-party Apple or Google SPM package: Google's `GoogleWebRTC` CocoaPod has been unmaintained for years, so the practical choice is a maintained community XCFramework build. | +| **Push to Talk** | System framework (`import PushToTalk`) | Background microphone. Entitlement-gated; see the signing section. | +| **AVFoundation** | System | Audio session, and `AVCaptureMetadataOutput` for the QR scan. | +| **Network** | System | `NWBrowser` for `_maestro._tcp` discovery. | +| **Security** | System | Keychain. See [[connection-and-pairing]] for the exact item attributes. | +| **Wake-word engine** | Decided in the Swift effort, isolated in `ACappellaWake` | Keyword spotting only. Whatever is chosen must run offline, be small, and never transmit. This is the one place a licence or a per-device fee can enter the project, so decide it with eyes open. | + +**Explicit non-dependencies.** Each of these is a package someone will reach for, and each has a +reason not to be here: + +- **No QR-scanning library.** `AVCaptureMetadataOutput` with `.qr` is about forty lines and reads + the payload in [[connection-and-pairing]] directly. A scanning SDK is a camera permission story + and a privacy manifest for something the OS already does. +- **No networking library.** Signaling is one WebSocket. `URLSessionWebSocketTask` is enough, and + it is the only thing in the app that has to survive an OS behaviour change on backgrounding. +- **No analytics, crash, or attribution SDK in v1.** A microphone app whose privacy label says + "no data collected" must be able to keep saying it. If crash reporting is added later, it goes + through Apple's own organiser first, and the privacy label changes with it, deliberately. +- **No dependency injection, logging, or reactive framework.** Four packages and a state machine + do not need one, and the reference client proves the protocol fits in one file with no framework + at all. + +**Third-party binary obligations.** Before the first submission, check the WebRTC XCFramework +against Apple's current rules for third-party SDKs: whether it appears on the commonly-used-SDK +list (which forces a privacy manifest and a signature), what required-reason APIs it touches, and +whether its own bundled dependencies are separately listed. Do not assume it is exempt because it +is a media library. If the shipped binary lacks what is required, the escape hatch is building +libwebrtc from source with a generated privacy manifest, which is a week of work and needs to be +discovered before the submission, not during it. + +## Generated protocol constants, and generated fixtures + +Two files in the tree above are marked GENERATED. Both exist because a hand-copied constant is a +protocol split into two disagreeing halves, and the disagreement is silent on the wire. + +**`Generated/ProtocolConstants.swift`** is emitted by `scripts/generate-ios-protocol-constants.mjs` +from the TypeScript that already owns each value: + +| Swift constant | Source | +| ------------------------ | ---------------------------------------------------------------------------- | +| `deviceProtocolVersion` | `DEVICE_PROTOCOL_VERSION` in `src/shared/acappella/device-protocol.ts` | +| `minSupportedVersion` | `MIN_SUPPORTED_DEVICE_PROTOCOL_VERSION`, same file | +| `reliableChannelLabel` | `RELIABLE_CHANNEL_LABEL` (`acappella-state`) | +| `unreliableChannelLabel` | `UNRELIABLE_CHANNEL_LABEL` (`acappella-live`) | +| channel inits | `RELIABLE_CHANNEL_INIT`, `UNRELIABLE_CHANNEL_INIT` | +| `defaultHoldThresholdMs` | `DEFAULT_HOLD_THRESHOLD_MS` and its 100 to 2000 clamp in `voice-controls.ts` | +| `defaultWakePhrase` | `DEFAULT_WAKE_PHRASE`, `DEFAULT_STOP_PHRASE`, `FALLBACK_STOP_PHRASE` | +| Bonjour service type | `_maestro._tcp`, from `src/main/acappella/pairing/discovery.ts` | + +The generator runs in the iOS CI job and the job fails if the output differs from what is +committed. A drift is then a red build on the commit that caused it rather than a phone that +cannot open a data channel. + +Note the difference between a default and a live value. `defaultHoldThresholdMs` is what the client +uses before it has heard from a desktop. The desktop's own value arrives in the `authenticated` +message and wins; see [[interaction-model]]. The constant exists so the two are the same number on +the first press, not so the client can decide the threshold. + +**`Tests/ACappellaKitTests/Fixtures/`** holds golden frames exported by +`scripts/export-acappella-fixtures.mjs` from the conformance suite: one JSON file per message type, +byte-for-byte what the desktop encodes and what it accepts. The Swift tests decode them and encode +back. This is what catches an optional field that Swift's `Codable` silently drops, which is the +failure that looks exactly like a working client that does nothing. + +## Mapping to the reference client + +A Swift developer starting cold should read the reference client first and this table second. Every +row is a file that already exists and already runs. + +| Swift | Reference client | +| -------------------------------------- | -------------------------------------------- | +| `ACappellaKit` | `src/web-desktop/acappella-client/client.ts` | +| `ACappellaAudio`, `ACappellaTransport` | `src/web-desktop/acappella-client/main.ts` | +| `Screens/`, `Components/` | `src/web-desktop/acappella-client/ui.ts` | +| `ACappellaKitTests/Conformance/` | `src/__tests__/acappella/conformance/` | +| `ProtocolConstants.swift` | direct imports from `src/shared/acappella/` | + +Where the Swift and the reference client disagree, one of them is wrong, and +[[protocol-conformance]] decides which. There is exactly one sanctioned divergence, documented in +both places: the browser cannot keep the audio unit cold the way `RTCAudioSession.useManualAudio` +does, so it acquires and stops the track around the floor instead. + +## Build configurations + +| Configuration | Purpose | Push to Talk | Distribution | +| ------------- | -------------------------------- | -------------------------------------------------- | -------------- | +| `Debug` | Simulator and device development | Off. `UIBackgroundModes: audio` allowed here ONLY. | Never archived | +| `Beta` | TestFlight | On once the entitlement is granted | TestFlight | +| `Release` | App Store | On | App Store | + +The `Debug`-only background `audio` mode is the compromise in [[background-and-entitlements]]: it +lets protocol work happen with the screen off before the entitlement exists. It must be impossible +to archive. Put it in a `Debug`-only `Info.plist` fragment rather than behind an `#if DEBUG` in +Swift, because a plist key cannot be conditionally compiled and a reviewer reads the plist. + +## Signing and capabilities + +| Item | Value | +| ----------------- | --------------------------------------------------------------------------------------------- | +| Bundle identifier | `sh.maestro.acappella` (matches the Keychain `kSecAttrService` in [[connection-and-pairing]]) | +| Capabilities | Push to Talk, Push Notifications, Background Modes (`push-to-talk` only) | +| Entitlements file | `ACappella/ACappella.entitlements` | +| Signing | Automatic for `Debug`, manual with a committed profile name for `Beta` and `Release` | + +### The Push to Talk entitlement, and the ordering problem it creates + +`com.apple.developer.push-to-talk` is a **restricted entitlement**. It is requested through Apple's +form, it is reviewed by a human, and until it is granted the identifier cannot carry it, which +means the provisioning profile cannot carry it, which means a build that declares it will not sign. + +That is a schedule constraint, not a paperwork detail: + +1. **Request the entitlement on day one of the Swift effort.** It is the longest-lead item in the + project and everything else can be built while it is pending. +2. Build against the keep-awake path in the meantime. It needs no entitlement and no review + conversation, and it has to exist in the shipped app anyway. +3. When the grant arrives, add the capability to the identifier, regenerate the `Beta` and + `Release` profiles, and flip the configuration. Nothing else changes: the PTT channel manager + drives the same `floor` messages the button does. + +The request text should say what the app is in one sentence and then answer the question the +reviewer is actually asking, which is whether this is a walkie-talkie or a background recorder. The +answer is in [[app-store-review]] and the entitlement request should reuse its wording rather than +inventing a second description of the same app. + +Push Notifications is required because the Push to Talk framework delivers channel activity over +PushKit. `aps-environment` is `development` in `Debug` and `production` in `Beta` and `Release`. +The app sends no other push and must not acquire a notification permission prompt it does not need. + +### TestFlight + +- **Internal testers first**, which needs no beta review and is where the pairing flow gets its + real-network testing. A simulator cannot scan a QR code from a real screen and cannot join a + Bluetooth headset, and both of those are where this app breaks. +- **External testing needs beta app review**, and that review hits the same wall as the App Store + review does: the app does nothing without a paired Mac. Have the demo desktop and the demo video + from [[app-store-review]] ready before the first external build, not after it is rejected. +- **Build numbers are monotonic and generated**, not typed. Marketing version tracks the desktop's + minor version so a support conversation can compare them. +- **Export compliance is a real question, not a checkbox.** The app encrypts: DTLS-SRTP inside + WebRTC and TLS on the signaling socket. Whether that qualifies for the standard-algorithms + exemption depends on the third-party binary as much as on our own code. Decide it once, in + writing, with someone qualified to answer, and record the answer next to the `Info.plist` key + rather than re-guessing at each submission. + +## Continuous integration + +One workflow, `.github/workflows/ios.yml`, on a macOS runner, gated on `ios/**` and +`src/shared/acappella/**`: + +1. `node scripts/generate-ios-protocol-constants.mjs --check` and the fixture equivalent. Fail on + any diff against what is committed. +2. Build `ACappella` for the simulator. +3. `swift test` on `ACappellaKit`, `ACappellaTransport`, and `ACappellaWake`. These need no + simulator and no device, which is the payoff of the import rules above. +4. `xcodebuild test` for `ACappellaAudio`, which does need a simulator. + +The existing Node matrix in `.github/workflows/ci.yml` stays exactly as it is. An iOS build must +never be able to make `test (ubuntu-latest)` or `test (windows-latest)` slower or redder. + +## Non-goals + +The first Swift effort ships a remote microphone and speaker. These are the things it will be +tempted to build, and each one is a way to turn a two-month project into a year. + +**Not building, ever, because the desktop already decides it:** + +- **No on-device speech recognition, routing, or text-to-speech.** No `SFSpeechRecognizer`, no + `AVSpeechSynthesizer`, no LLM, no summariser. `NSSpeechRecognitionUsageDescription` must not + appear in `Info.plist`. The single exception is the wake and stop keyword spotter in + `ACappellaWake`, which classifies one phrase and transmits nothing; it is not a recogniser and + must not be allowed to become one. +- **No agent management.** No creating, deleting, renaming, or reconfiguring agents. No running + commands. No file browsing. The project wheel is a picker over a roster the desktop sent, and + selecting an item changes the scope of the next press and nothing else. +- **No settings that duplicate desktop settings.** Provider choice, model downloads, TTS voice and + rate, hold threshold, wake phrase text, idle timeout: all of these live on the desktop and arrive + over the wire. The phone's settings screen holds exactly what is about this phone: which desktop + is paired, whether the wake word is armed, whether keep-awake is on, and the output route. If a + setting can be answered by "what did the desktop say", it is not a phone setting. +- **No local conversation history.** The transcript sheet renders the current session from the + events that arrived and is gone when the session ends. History lives on the desktop, which is + where it can be searched, and where it is already backed up. + +**Not in v1, deferred deliberately:** + +- Multiple simultaneous desktops. Store several pairings, connect to one at a time. +- Apple Watch, macOS, visionOS, and CarPlay targets. CarPlay in particular is called out as a + do-not-ship in [[audio-session]] for reasons that outlast v1. +- Siri and App Intents. The wake word covers the same ground without an entitlement conversation. +- An iPad-specific layout, widgets, Live Activities, and Focus filters. +- Any account, sign-in, or server of our own. The pairing token is the only credential the app has + and there is nothing to log into. + +If a feature request cannot be phrased as "the microphone, the speaker, or the button", the answer +is that it belongs on the desktop. That is the whole thesis of [[overview]], and this list is what +it looks like when it is enforced. + +## Related + +- [[overview]] for what the app is and, more usefully, what it is not. +- [[protocol-conformance]] for the wire behaviour every one of these packages exists to implement. +- [[background-and-entitlements]] for why the deployment target is 16.0 and why the entitlement is + the critical-path item. +- [[app-store-review]] for the submission material the TestFlight external build needs first. +- `src/web-desktop/acappella-client/README.md` for the endpoint this project is a port of. diff --git a/docs/ios-client/protocol-conformance.md b/docs/ios-client/protocol-conformance.md new file mode 100644 index 0000000000..60026a33ae --- /dev/null +++ b/docs/ios-client/protocol-conformance.md @@ -0,0 +1,565 @@ +--- +type: specification +title: Protocol Conformance +created: 2026-08-14 +tags: + - ios + - voice + - acappella +related: + - '[[overview]]' + - '[[connection-and-pairing]]' + - '[[interaction-model]]' + - '[[audio-session]]' + - '[[background-and-entitlements]]' + - '[[project-structure]]' + - '[[../architecture/acappella/voice-session-protocol]]' + - '[[../architecture/acappella/transport-and-pairing]]' +--- + +# Protocol Conformance + +This is the contract. [[connection-and-pairing]] says how a phone finds a desktop and +[[interaction-model]] says what the screen does; this file says exactly what goes on the wire, in +what order, and what a client must do with every frame it receives. + +Everything here was read out of the implementation rather than designed alongside it. The source of +truth, in the order a message meets it: + +| Layer | File | +| -------------------------- | ----------------------------------------------------------------- | +| WebSocket envelope | `src/main/web-server/handlers/messageHandlers/acappellaSignal.ts` | +| Signaling messages | `src/shared/acappella/signaling-protocol.ts` | +| Signaling behaviour | `src/main/acappella/transport/signaling.ts` | +| Data-channel messages | `src/shared/acappella/device-protocol.ts` | +| Session events | `src/shared/acappella/protocol.ts` | +| Peer and channel behaviour | `src/renderer/acappella-audio/peer-connection.ts` | +| SDP tuning and link stats | `src/shared/acappella/peer-tuning.ts` | +| Floor rules | `src/main/acappella/transport/remote-session.ts` | + +**If this document and those files disagree, the files win and this document is a bug.** The +conformance suite at `src/__tests__/acappella/conformance/` exists to make that disagreement fail in +CI rather than at App Store review, and every checklist item at the end of this file carries an ID +the suite can name. + +**There is also a working client.** `src/web-desktop/acappella-client/` is a browser implementation +of this entire document: it pairs, authenticates, offers, opens both data channels, holds the floor, +and speaks every message defined below, in about 1,700 lines of framework-free TypeScript. Where this +document says what a client must do, that client is the runnable version, and where a comment there +cites a `C-nn` it is implementing the checklist item of that name. Read it alongside this file; the +desktop is served it at `/$TOKEN/acappella`. + +## The two layers + +``` + phone desktop + | | + | 1. WebSocket wss://host:port/$TOKEN/ws | + | {type:'acappella_signal', payload:{op:...}} | pairing, auth, SDP, ICE + |<-------------------------------------------------------->| + | | + | 2. WebRTC peer connection | + | audio track (Opus) | the microphone + | 'acappella-state' reliable, ordered | roster, floor, events + | 'acappella-live' unreliable, unordered | meter, partials, gestures + |<========================================================>| +``` + +Layer 1 exists to build layer 2 and then to tear it down. Once the peer is up, nothing that matters +to the user travels over the WebSocket: a phone whose socket dropped but whose peer is healthy keeps +working, and that is deliberate. + +--- + +## 1. Signaling messages + +### Envelope + +Every signaling frame is a WebSocket text frame carrying the app's ordinary client-message shape: + +```json +{ "type": "acappella_signal", "payload": { "op": "..." } } +``` + +`payload` is one `SignalingClientMessage` outbound, one `SignalingServerMessage` inbound. There is no +second port, no second token, and no separate handshake: the URL's `$TOKEN` is the server token from +the QR payload, and clearing it is what gets a frame looked at in the first place. + +A client that sends `acappella_signal` while A Cappella is switched off on the desktop receives +`{op:'error', code:'not-authenticated', message:'A Cappella is not running on this desktop. Turn it +on in Encore Features.'}`. Show that sentence. It is the difference between a feature that is off and +a network that ate the frame, and only one of those is worth retrying. + +### Client to desktop + +| `op` | Payload | Precondition | +| --------------- | ---------------------------------------------------------------------------------- | ------------------------- | +| `pair-claim` | `{ code: string, name: string, platform: string, appVersion?: string }` | Unpaired only | +| `pair-poll` | `{ requestId: string }` | After `pair-pending` | +| `auth` | `{ deviceId: string, token: string, protocolVersion: number }` | Have a stored token | +| `offer` | `{ sdp: { type: 'offer', sdp: string } }` | **After `authenticated`** | +| `ice-candidate` | `{ candidate: { candidate: string, sdpMid?, sdpMLineIndex?, usernameFragment? } }` | **After `authenticated`** | +| `bye` | `{}` | Any time | + +What the desktop's parser actually does with a malformed frame, because a client that relies on +coercion will break the day the parser tightens: + +- `pair-claim` without a string `code` is dropped as `malformed`. A non-string `name` or `platform` + is **silently coerced to the empty string**, which produces a nameless row in the approval sheet. + Send both, always, and send `platform` as the literal `ios`. +- `auth` without a string `deviceId` **and** a string `token` is dropped as `malformed`. +- `auth` whose `protocolVersion` is not a number is treated as version 0, which fails negotiation + with `client-too-old`. Absent is not "unversioned"; it is "too old". +- `offer` is accepted only when `sdp` is an object with a string `sdp` field. The `type` is forced to + `offer` regardless of what was sent, so a client cannot smuggle an answer through the offer path. +- `ice-candidate` requires a string `candidate`. The three optional fields default to `null` when + they are the wrong type, which is why an end-of-candidates marker must be sent as an empty-string + `candidate` rather than as a null one. +- Anything else, including an unknown `op`, produces one + `{op:'error', code:'malformed', message:'Unrecognised signaling message.'}` and no state change. + +### Desktop to client + +| `op` | Payload | Meaning | +| --------------- | ---------------------------------------------------------------------------------------- | ------------------------------------------- | +| `pair-pending` | `{ requestId: string, expiresAt: number }` | A human is being asked | +| `pair-approved` | `{ deviceId: string, token: string }` | The only time the token exists in the clear | +| `pair-denied` | `{}` | A human said no | +| `pair-rejected` | `{ reason: string, message: string }` | Bad, used, busy, or expired code | +| `authenticated` | `{ deviceId, protocolVersion, iceServers, iceTransportPolicy: 'all' \| 'relay', audio }` | Signaling session open | +| `auth-failed` | `{ reason: 'unauthorized', message: string }` | Unknown, wrong, or revoked | +| `answer` | `{ sdp: { type: 'answer', sdp: string } }` | Apply as the remote description | +| `ice-candidate` | `{ candidate: IceCandidatePayload }` | Trickled from the desktop | +| `closed` | `{ reason: string }` | This signaling session is over | +| `error` | `{ code: SignalingErrorCode, message: string }` | See below | + +`audio` is a `RemoteAudioConfig`: +`{ fec: boolean, dtx: boolean, maxAverageBitrate: number, requestRemoteEchoCancellation: boolean }`, +defaulting to `{ true, true, 24000, true }`. Apply it to the outgoing encoder; see [[audio-session]]. + +Three shapes that catch every first implementation: + +1. **`pair-pending` in response to a `pair-poll` carries `expiresAt: 0`.** Only the first + `pair-pending`, the one answering `pair-claim`, carries the real deadline. Keep that value; do not + overwrite it from a poll response, or the pairing screen's countdown jumps to 1970. +2. **`auth-failed` is deliberately vague.** Unknown device, wrong token, and revoked device all + produce `reason: 'unauthorized'` with one sentence. Do not try to tell them apart; the desktop is + refusing to be an enumeration oracle. Every one of them is terminal for the stored token. +3. **A version failure is an `error`, not an `auth-failed`.** See section 3. + +### Error codes + +| `code` | When | Client behaviour | +| ------------------- | -------------------------------------------------------- | ---------------------------------------------------------------- | +| `not-authenticated` | `offer` or `ice-candidate` before `authenticated` | Bug. Fix the ordering; never retry blind. | +| `rate-limited` | 7th offer in 60 s, or a 6th `auth` attempt on one socket | Stop. Back off. For auth, open a **new socket** before retrying. | +| `protocol-version` | `auth` with an unusable `protocolVersion` | Terminal. Show the message verbatim. See section 3. | +| `malformed` | Unparseable frame | Bug. Log it with the frame; do not retry the same bytes. | +| `peer-failed` | The desktop's peer connection failed | Tear down the peer, restart ICE, re-`offer` under the backoff. | + +### Ordering and limits + +- **`offer` and `ice-candidate` are refused until `auth` has succeeded on THIS socket.** An + authenticated state is never inherited across sockets. Reconnect means `auth` again, every time. +- **Offers: 6 per 60 seconds, sliding window.** Sized for an initial offer plus a renegotiation per + network change. A client that rebuilds its peer on every ICE hiccup exhausts the budget during + exactly the handover it was trying to survive. +- **Failed auths: 5 per socket.** The 6th and every one after it answers `rate-limited` on that + socket forever. The recovery is a new socket after backoff, not a tighter loop. +- **One live signaling session per device.** A second successful `auth` for the same `deviceId` + displaces the first, and the displaced socket receives + `{op:'closed', reason:'this device connected again from somewhere else'}`. That is a normal message + on a phone that changed networks, not an error to show. +- **`bye` before a deliberate teardown.** It closes the peer cleanly instead of leaving the desktop to + discover the loss from ICE, which takes seconds the user can hear. + +--- + +## 2. Data-channel messages + +### The channels + +The **client is the offerer**, so the client creates both data channels. The desktop binds them in +`ondatachannel` by label and **closes any channel whose label it does not recognise**, so a typo is a +silently dead channel rather than a warning. + +| Label | `RTCDataChannelInit` | Carries | +| ----------------- | --------------------------------------- | ------------------------------------- | +| `acappella-state` | `{ ordered: true }` | State the far end must not miss | +| `acappella-live` | `{ ordered: false, maxRetransmits: 0 }` | Traffic superseded within about 50 ms | + +Create both before the offer, so the negotiated SDP includes the SCTP association from the start. + +### Encoding + +One JSON object per message, sent as a string. Every frame carries `v`, the negotiated protocol +version, stamped by the sender. + +```json +{ "type": "floor", "action": "press", "scope": { "kind": "conductor" }, "v": 1 } +``` + +The desktop's decoder returns "that message did not exist" rather than throwing, for anything that +fails any of these: + +- not a string, or not parseable JSON, or not a non-array object; +- `type` is not a string, or is not one of the ten known types; +- **`v` is not a number.** A frame without `v` is dropped in silence. This is the single most common + way a first client appears to be connected and does nothing at all. +- `voice-event` without an `event` object carrying a string `type`; +- `floor` whose `action` is neither `press` nor `release`. + +Malformed frames are dropped individually. They never close the channel, and there is no negative +acknowledgement, so a client cannot detect this by waiting for a complaint. + +### Client to desktop + +Exactly five types. The desktop's `DEVICE_ORIGINATED_MESSAGES` list is +`['hello', 'floor', 'interrupt', 'audio-level', 'link-quality']`; anything else arriving from a device +is not the client's to send. + +| Type | Payload | Channel | When | +| -------------- | --------------------------------------------------------- | ------- | ------------------------------------------------ | +| `hello` | `{ identity: { deviceId, name, platform, appVersion? } }` | `state` | First frame after the state channel opens | +| `floor` | `{ action: 'press' \| 'release', scope?: VoiceScope }` | `live` | Push-to-talk, and a wake-word hit | +| `interrupt` | `{ kind: 'barge-in' \| 'stop-word' }` | `live` | Talking over the reply, or the stop word | +| `audio-level` | `{ level: number, speech: boolean }` | `live` | ~20/s **while the floor is open, and only then** | +| `link-quality` | `{ rttMs, jitterMs, packetLoss, candidateType }` | `live` | Every ~2 s from a throttled `getStats()` | + +`VoiceScope` is `{ kind: 'conductor' }` or `{ kind: 'agent', sessionId: string }`, where `sessionId` +is an **agent** id from the roster, never a voice session id. Omitting `scope` means conductor. + +**Push-to-talk deliberately rides the lossy channel.** A dropped release cannot leave a hot +microphone: the desktop's floor has an idle timeout, the next press is idempotent, and the +authoritative `floor-state` comes back either way. + +**A client must not send a `voice-event`.** The Phase 01 protocol marks `wake`, `final-transcript`, +`barge-in`, and `stop-word` as client-originable, but not over this transport: the device channel +expresses them as `floor` and `interrupt`, and a wrapped `voice-event` from a device is dropped by +the coordinator's switch. There is no error, so this fails as silence. + +### Desktop to client + +| Type | Payload | Channel | +| ------------------ | -------------------------------------------------------------------- | --------- | +| `welcome` | `{ version: number, appVersion: string, sessionId: string \| null }` | `state` | +| `version-rejected` | `{ reason, message, desktopVersion, minimumVersion }` | `state` | +| `voice-event` | `{ event: VoiceEvent }` | see below | +| `floor-state` | `{ holder: string \| null, isSelf: boolean, takenOverBy?: string }` | `state` | +| `revoked` | `{ message: string }` | `state` | +| `link-quality` | `{ rttMs, jitterMs, packetLoss, candidateType }` | `live` | + +- `floor-state.holder` is a device id, the literal `'local'` for the desktop's own microphone, or + `null` when nobody holds the floor. `isSelf` saves the client an id comparison; trust it. +- `takenOverBy` is a **display name**, already resolved, and is set only on the message sent to the + device that just lost the floor. Show it as written. It is also **momentary**: it rides its own + frame, and the ordinary `floor-state` broadcast that follows a takeover carries no name at all. So + react to the frame (a banner, a haptic) rather than rendering the field out of stored state, or the + notice will erase itself a few milliseconds after it appears. +- `revoked` is the last frame before teardown, sent for revocation and for any deliberate close. Its + `message` is the reason and is written for a human. + +### Channel routing + +The split is total and stated in one place, `deviceChannelForMessage()`. A client that guesses will +eventually put a `revoked` on the lossy channel, which is a device that keeps its microphone. + +| Message | Channel | +| ---------------------------------------------------------------- | ------------ | +| `hello`, `welcome`, `version-rejected`, `floor-state`, `revoked` | `reliable` | +| `floor`, `interrupt`, `audio-level`, `link-quality` | `unreliable` | +| `voice-event` with `audio-level` or `partial-transcript` | `unreliable` | +| `voice-event`, every other event type | `reliable` | + +The desktop falls back to the reliable channel when a lossy message needs to go out before +`acappella-live` is open. A client should do the same for the first `hello`-adjacent traffic and stop +once both channels report open. + +### The session-event catalogue + +Every `voice-event` carries `sessionId` (the **voice** session, not an agent), `seq` (monotonic from +1 per session), and `ts` (epoch ms). All twenty types, and what a conforming client does with each: + +| Event | Client duty | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `wake` | Show arming. Carries `source` and `origin`; a remote origin naming another device is not this phone. | +| `listen-start` | The floor is genuinely open. Start the meter. `sttProviderId` names the engine; show it on demand. | +| `listen-stop` | The floor closed. `reason` is `endpoint`, `stopped`, `interrupted`, or `error`. Clears a remote holder. | +| `partial-transcript` | Replace the in-flight user row. Lossy: may skip, never reorder. `text` is the full hypothesis, not a delta. | +| `final-transcript` | Commit the user row. | +| `route-decision` | Caption the user row with the target and the latency. | +| `dispatch` | Caption with the agent, tab, and whether the prompt was sent. This is the "it actually landed" signal. | +| `route-correction` | **Rewrite the existing caption in place.** Do not append a row: the user said one sentence. | +| `agent-reply` | Start the assistant row. Show `text`; `spokenText` is what is being said aloud. | +| `speak-start` | Speech begins. `sentenceCount` is a **lower bound while `streaming` is true**, so indices will run past it. Never clamp. | +| `speak-sentence` | Append. Drop any sentence whose `utteranceId` is not the current run. | +| `speak-end` | `complete`, `cancelled`, or `error`. Return the button to idle or latched. | +| `barge-in` | The authoritative confirmation of an interrupt. Selection haptic. The floor is **kept**. | +| `stop-word` | The session ended. Floor released, mic closed, success haptic. | +| `session-error` | Show `message` verbatim. `code` decides the affordance; `recoverable` decides whether to offer a retry. | +| `audio-level` | Meter, when the desktop's microphone is the open one. | +| `mic-state` | The **desktop's** microphone, not the phone's. Never drive the local mic pill from this. | +| `provider-state` | Powers the "where does my audio go" sheet. Show `egressStatement` verbatim and honour `audioLeavesMachine`. | +| `tab-state` | Update the selected agent's tab accessory. | +| `agent-roster` | **Replace** the project wheel. A snapshot, never a diff. | + +`session-error` codes, all of which must render a sentence rather than a spinner: +`provider-unavailable`, `provider-auth-failed`, `provider-quota-exceeded`, `provider-network-error`, +`no-agent-matched`, `dispatch-failed`, `audio-capture-failed`. + +**Sequence handling.** `seq` is contiguous per voice session on the reliable channel; a gap there +means frames were lost and the client should treat its transcript as suspect rather than silently +stitching. Gaps in `audio-level` and `partial-transcript` are expected and carry no meaning: they are +the two events that ride the lossy channel by design. A `sessionId` change resets `seq` to 1, which +is not a gap. + +**Unknown is not fatal.** A client must ignore an unrecognised `voice-event.type` and an +unrecognised field, and must keep processing the stream. That rule is what lets the desktop add an +event without breaking every shipped phone, and it is the reason the version below only moves for +changes that are genuinely breaking. + +--- + +## 3. Version handshake + +Two checks, at two layers, and they answer different questions. + +### At `auth`, before the credential is looked at + +`negotiateProtocolVersion()` runs first, on purpose: a client that cannot be talked to correctly +should be told THAT, rather than being authenticated into a session where it will misbehave in +silence. The current window is `MIN_SUPPORTED_DEVICE_PROTOCOL_VERSION = 1` to +`DEVICE_PROTOCOL_VERSION = 1`. + +| Outcome | Condition | Desktop sends | +| ---------------- | ----------------------------------- | -------------------------------------------------------------------- | +| Accepted | `min <= v <= max` | `{op:'authenticated', protocolVersion: , ...}` | +| `client-too-old` | `v < min` | `{op:'error', code:'protocol-version', message}` | +| `client-too-new` | `v > max` | `{op:'error', code:'protocol-version', message}` | +| `malformed` | Not an integer, or `< 1`, or absent | `{op:'error', code:'protocol-version', message}` | + +The `message` already names the end that has to update. Client-too-old says "Update the app on the +device"; client-too-new says "Update Maestro on the desktop". **Show it verbatim.** Telling a user to +update the wrong end of the pair is worse than saying nothing. + +What the client must do: + +- **Treat it as terminal.** No reconnect, no backoff, no silent retry. The state is a full-screen + message with the desktop's sentence and one action, which is not "Retry". +- **Do not delete the Keychain item.** This is not an authentication failure; the pairing is still + valid and will work the moment one end is updated. Deleting the token here turns a five-minute + update into a re-pair. +- **Use `authenticated.protocolVersion`, not your own constant,** in every subsequent `v`. It is the + lower of the two ends and it may be below the version the client compiled against. +- **Never fall back a version you do not implement.** If the negotiated version is one the client + does not fully speak, say so and stop. Half-speaking a protocol is the failure mode the two-number + window exists to prevent. + +### On the data channel + +`welcome` and `version-rejected` are the peer-connection-level equivalents: +`welcome` carries the agreed `version`, the desktop's `appVersion`, and the live `sessionId` or +`null`; `version-rejected` carries the reason, a human sentence, and both `desktopVersion` and +`minimumVersion` so the client can say which end is behind. + +**Desktop v1 does not emit either one.** The desktop settles the version at `auth`, so the +data-channel handshake is currently one-sided: the client sends `hello`, the desktop records the name +for its device list, and nothing comes back. A conforming client therefore: + +- sends `hello` as the first frame on `acappella-state` and **does not block on `welcome`**. Do not + gate the UI, the offer, or the floor button on a reply that will not arrive; +- handles `welcome` correctly if it does arrive, because it will once the desktop side closes this + gap, and a client that treats it as unknown would drop the `sessionId` it carries; +- handles `version-rejected` by closing the peer and showing its `message`, exactly as for the + signaling-layer rejection above. + +This gap is tracked by conformance items C-31 and C-32 and is the one place where this document +describes a protocol the desktop has not finished implementing. It is stated rather than quietly +omitted because the shapes are already frozen in `device-protocol.ts` and a Swift client will +otherwise implement a handshake that appears to hang. + +### What a version bump is for + +`DEVICE_PROTOCOL_VERSION` moves only for a **breaking** change: a removed message type, a removed or +retyped field, a changed channel assignment, or a changed meaning for an existing value. Adding a +message type, an optional field, or a new `VoiceEvent` is not breaking, because both ends are +required to ignore what they do not recognise. A client that crashes on an unknown field has turned +an additive change into a breaking one on its own. + +--- + +## 4. Required local behaviours + +These are not UI suggestions. They are the client's half of guarantees the desktop makes to the user, +and a client that skips them breaks a promise made in the desktop's own settings copy. + +### No capture before the floor is open + +**Nothing is captured, encoded, or transmitted before the floor opens.** The rule falls out of the +architecture rather than being enforced by the desktop: the phone's microphone is not sent anywhere +until the phone opens the floor, so a client that streams early is unobservable from the desktop and +must be caught here. + +Concretely, with `RTCAudioSession.useManualAudio = true` (see [[audio-session]]): + +- The WebRTC audio unit stays **off** while the peer connection is up and the floor is closed. +- The outbound audio track exists but is disabled; the desktop's `set-floor-holder` gating is a second + line of defence, not the first. +- `audio-level` messages are sent only while the floor is open. A meter running with the floor closed + is a client measuring a microphone it should not have open. + +### Wake word and stop word, on the device + +Both run locally, for the reason in [[../architecture/acappella/wake-and-hotkeys]]: a wake word cannot +be detected remotely without sending the audio it exists to gate, and a stop word must be heard while +the desktop is speaking. + +- A wake-word hit is **exactly** a `floor: press` with the selected scope. Not a `wake` voice event, + not a new message type. From the desktop's side there is no difference and there must not be one. +- A stop-word hit is `interrupt: { kind: 'stop-word' }`. +- Arming follows the desktop's rule: wake phrases only while the session is cold, stop phrases in + every active state, never both. Otherwise a wake phrase spoken mid-answer stacks a second session. +- The wake-word tap is a **separate capture gate** from the WebRTC one. It runs locally, transmits + nothing, and still lights the system recording indicator, which is why the microphone pill in + [[interaction-model]] has three states rather than two. +- Defaults come from the desktop (`DEFAULT_WAKE_PHRASE = 'hey maestro'`, + `DEFAULT_STOP_PHRASE = 'maestro stop'`, with `'nevermind'` always armed and not editable). Do not + carry a second copy of these strings as client constants. + +### Barge-in ducking within 20 ms + +While TTS is playing, a local VAD watches the microphone. On detected speech: + +1. **Duck local playback within 20 ms**, locally, before anything goes on the wire. One frame of + audio, not one round trip. A phone on a relayed path is 150 ms from the desktop and back, and a + user who has started talking over the reply has already decided the reply is wrong. +2. Send `interrupt: { kind: 'barge-in' }` in the same turn of the run loop. +3. Restore the level only on the authoritative `barge-in` **or** `speak-end` voice event. If neither + arrives within 500 ms, restore the level and keep the floor: a duck that never lifts is a session + that appears to have died. + +Barge-in keeps the floor. Stop word releases it. Every assistant that merged those two became one you +cannot get rid of, and the desktop enforces the distinction: `barge-in` calls `interrupt()`, +`stop-word` calls `hardStop()`. + +Both are **refused outright from a device that is not holding the floor**, with no error frame. A +client that shows a stop button while another device holds the floor is showing a button that does +nothing. + +### Floor state is never assumed + +- Send `press` on touch-down, not after classifying tap versus hold. The desktop's press is + idempotent and waiting 300 ms puts 300 ms in front of every utterance. +- Render the button from `floor-state`, never from the local gesture. The gesture is a request. +- Start closed after any reconnect and wait for `floor-state`. A phone that assumes it still holds + the floor is a hot microphone the desktop does not know about. +- A release from a device that is not the current holder is discarded, by design: without that rule a + device that just lost the floor would shut the microphone of the device that just took it. + +--- + +## 5. Conformance checklist + +Each item is independently testable and is named by the suite at +`src/__tests__/acappella/conformance/`. An implementation is conformant when every item passes, and +each item is written so that "passes" is observable from outside the client. + +The suite runs in `npm run test`, on a harness (`harness.ts`) that assembles the real desktop stack - +`ACappellaTransport` over a real `PeerRegistry` - and drives it with the real browser reference +client, so a frame really is encoded, crosses a loopback data channel, and comes back out of +`decodeDeviceMessage()` on the far side. Four files, split by what fails: + +| File | Items | What it proves | +| ------------------------------------- | -------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `signaling.conformance.test.ts` | C-01 to C-15 | Pairing, auth, the ordering rules, both rate limits, and what `authenticated` carries. | +| `data-channel.conformance.test.ts` | C-16 to C-30, C-37 to C-40 | Channel labels and inits, `v`, routing, malformed frames, the event catalogue, and the microphone gate. | +| `failure-paths.conformance.test.ts` | C-12 to C-15, C-33 to C-49 | Version mismatch, mid-session revocation, a network drop and reconnect, floor takeover, and the stop word. | +| `../../web-desktop/acappella-client/` | C-24 to C-28, C-31 to C-36, C-44, C-47 | The client-side half, in `src/__tests__/web-desktop/acappella-client/` where the DOM and the gestures live. | + +A rule the desktop enforces is asserted from a raw signaling socket rather than from the reference +client, because a limit a conforming client never reaches is a limit nobody has tested. + +### Signaling + +| ID | Requirement | +| ---- | ------------------------------------------------------------------------------------------------------------------ | +| C-01 | Every frame is `{type:'acappella_signal', payload:{op}}` over `/$TOKEN/ws`. No second socket, no second token. | +| C-02 | `pair-claim` sends a non-empty `name` and the literal `platform: 'ios'`. | +| C-03 | `pair-poll` runs on a 1 s interval and stops at the `expiresAt` from the **first** `pair-pending`. | +| C-04 | `pair-approved` writes the token to the Keychain **before** any UI state changes. | +| C-05 | `pair-rejected` and `auth-failed` messages are shown verbatim, with nothing appended. | +| C-06 | `auth` always carries an integer `protocolVersion >= 1`. | +| C-07 | No `offer` or `ice-candidate` is sent before `authenticated` arrives on the same socket. | +| C-08 | Fewer than 6 offers are sent in any 60 s window under normal network churn. | +| C-09 | At most one `auth` attempt per socket; a failure opens a new socket after backoff. | +| C-10 | `authenticated.iceServers` and `iceTransportPolicy` are used as sent. No hard-coded STUN server exists in the app. | +| C-11 | `authenticated.audio` is applied to the outgoing encoder (`fec`, `dtx`, bitrate, remote AEC request). | +| C-12 | `closed` and `revoked` are terminal. No reconnect follows either. | +| C-13 | `auth-failed` deletes the Keychain item and returns the app to the unpaired state. | +| C-14 | `bye` is sent before any deliberate teardown, including backgrounding without a PTT session. | +| C-15 | `error: peer-failed` restarts ICE and re-offers under the backoff schedule rather than re-pairing. | + +### Data channel + +| ID | Requirement | +| ---- | ------------------------------------------------------------------------------------------------------------------------ | +| C-16 | Both channels are created by the client, before the offer, with the exact labels `acappella-state` and `acappella-live`. | +| C-17 | `acappella-live` is created with `{ordered:false, maxRetransmits:0}`; `acappella-state` with `{ordered:true}`. | +| C-18 | Every outbound frame carries a numeric `v` equal to the negotiated version. | +| C-19 | `hello` is the first frame on `acappella-state` and carries a complete `identity`. | +| C-20 | Only the five device-originated types are ever sent. No `voice-event`, `floor-state`, `welcome`, or `revoked`. | +| C-21 | Each message goes out on the channel the routing table names, with no exceptions. | +| C-22 | Malformed or unknown inbound frames are ignored without closing the channel or the peer. | +| C-23 | An unknown `voice-event.type` and an unknown field are both ignored, and processing continues. | +| C-24 | `agent-roster` replaces the wheel wholesale; no merge, no accumulation of stale agents. | +| C-25 | `route-correction` rewrites the existing caption in place and never appends a second row. | +| C-26 | `speak-start.sentenceCount` is treated as a lower bound while `streaming` is true; indices past it are not clamped. | +| C-27 | `speak-sentence` frames whose `utteranceId` is not the current run are dropped. | +| C-28 | `mic-state` drives only the desktop indicator, never the local microphone pill. | +| C-29 | A `seq` gap on the reliable channel is surfaced as a suspect transcript, not stitched over. | +| C-30 | `provider-state.egressStatement` is shown verbatim wherever the app answers "where does my audio go". | + +### Version handshake + +| ID | Requirement | +| ---- | ---------------------------------------------------------------------------------------------------------- | +| C-31 | `hello` is sent without blocking on `welcome`; the UI, the offer, and the floor button do not wait for it. | +| C-32 | `welcome` and `version-rejected` are both handled correctly if received. | +| C-33 | `error: protocol-version` is terminal, shows the desktop's sentence verbatim, and offers no Retry. | +| C-34 | A version rejection does **not** delete the Keychain item. | +| C-35 | `authenticated.protocolVersion` is used for `v`, not the client's own constant. | +| C-36 | A negotiated version the client does not fully implement is refused loudly rather than half-spoken. | + +### Local behaviour + +| ID | Requirement | +| ---- | --------------------------------------------------------------------------------------------------- | +| C-37 | No audio is captured, encoded, or transmitted before `floor-state.isSelf` is true. | +| C-38 | `RTCAudioSession.useManualAudio` is true and the audio unit is started only for an open floor. | +| C-39 | `audio-level` is sent at roughly 20/s while the floor is open, and never while it is closed. | +| C-40 | `link-quality` is sent from a throttled `getStats()` at roughly 2 s intervals. | +| C-41 | The wake word runs on the device and produces a plain `floor: press` with the selected scope. | +| C-42 | The stop word runs on the device and produces `interrupt: {kind:'stop-word'}`. | +| C-43 | Wake phrases arm only while the session is cold; stop phrases arm in every active state. | +| C-44 | Local VAD ducks playback within 20 ms of detected speech, before the `interrupt` frame is sent. | +| C-45 | The duck lifts on `barge-in` or `speak-end`, or after 500 ms with the floor kept. | +| C-46 | Barge-in keeps the floor and the microphone; only the stop word releases them. | +| C-47 | `floor: press` is sent on touch-down, before tap-versus-hold is classified. | +| C-48 | The button renders `floor-state`, not the local gesture, including `takenOverBy` after a takeover. | +| C-49 | The floor starts closed after every reconnect and waits for `floor-state`. | +| C-50 | The microphone pill distinguishes "Mic off", "Listening for wake word", and "Sending" at all times. | + +--- + +## What to read next + +- `src/web-desktop/acappella-client/README.md` for the browser reference client: this document, but + running, and the endpoint the desktop is regression-tested against. +- [[connection-and-pairing]] for the discovery, pairing, Keychain, and reconnection behaviour these + messages sit inside. +- [[audio-session]] for how `RemoteAudioConfig` and the two capture gates map onto + `AVAudioSession` and WebRTC.framework. +- [[interaction-model]] for what each of these events does to the screen. +- [[project-structure]] for where the code implementing this lives. +- [[../architecture/acappella/voice-session-protocol]] for the desktop-side narrative behind the + session events. diff --git a/docs/screenshots/acappella-models.png b/docs/screenshots/acappella-models.png new file mode 100644 index 0000000000..0bfdb479d9 Binary files /dev/null and b/docs/screenshots/acappella-models.png differ diff --git a/docs/screenshots/acappella-paired-devices.png b/docs/screenshots/acappella-paired-devices.png new file mode 100644 index 0000000000..3bfc310fd2 Binary files /dev/null and b/docs/screenshots/acappella-paired-devices.png differ diff --git a/docs/screenshots/acappella-stop-word-hotkeys.png b/docs/screenshots/acappella-stop-word-hotkeys.png new file mode 100644 index 0000000000..2e1e6b2d98 Binary files /dev/null and b/docs/screenshots/acappella-stop-word-hotkeys.png differ diff --git a/docs/screenshots/acappella-voice-providers.png b/docs/screenshots/acappella-voice-providers.png new file mode 100644 index 0000000000..431c83f5df Binary files /dev/null and b/docs/screenshots/acappella-voice-providers.png differ diff --git a/docs/screenshots/acappella-voice-setup.png b/docs/screenshots/acappella-voice-setup.png new file mode 100644 index 0000000000..412afda577 Binary files /dev/null and b/docs/screenshots/acappella-voice-setup.png differ diff --git a/docs/screenshots/acappella-wake-word.png b/docs/screenshots/acappella-wake-word.png new file mode 100644 index 0000000000..9fd0467e3f Binary files /dev/null and b/docs/screenshots/acappella-wake-word.png differ diff --git a/docs/voice-mode.md b/docs/voice-mode.md new file mode 100644 index 0000000000..75e525b5e4 --- /dev/null +++ b/docs/voice-mode.md @@ -0,0 +1,256 @@ +--- +title: A Cappella (Voice Mode) +description: Talk to Maestro. Speak a request, have it routed to the right agent and tab, and hear the reply read back. +icon: microphone +--- + +A Cappella is Maestro's voice interface. You say something, Maestro works out which agent and which tab you meant, sends it there, and reads the reply back to you. You can drive it with a wake word, with a global hotkey from inside another application, or from a paired phone across the room. + +It is a **front end for the agents you already have**, not a second assistant. It does not think about your code, it does not answer questions itself, and it never invents an agent to talk to. Every utterance ends up in an ordinary Maestro tab that you can scroll back through afterwards. + + + A Cappella is an **Encore Feature** and it is in **Beta**. It is off by default. While it is off, no + microphone is opened, no model is downloaded, no hotkey is registered, no Bonjour advert is + published, and no voice UI renders anywhere in the app. + + +## Turning it on + +1. Open **Settings** (`Cmd+,` / `Ctrl+,`) and go to the **Plugins** tab. +2. Find the **A Cappella** card and enable it. A confirmation sheet lists what the feature reads before anything is switched on. +3. Select the card to open its detail pane. The **Settings** sub-tab holds every voice panel: Voice Setup, Voice Providers, Voice Controls, Voice and Speed, Paired Devices, and Models. + +Enabling the feature only makes those panels reachable. **Nothing is downloaded and no device is opened until you ask for it**: models arrive when you press Download, and the microphone opens at your first real session. + +![A Cappella settings and Voice Setup](./screenshots/acappella-voice-setup.png) + +## Where your audio goes + +This is the only question that really matters when you configure a voice assistant, so Maestro computes the answer from your actual selection rather than writing it into help text. The sentence at the top of **Voice Providers** and **Voice and Speed** is derived from the providers you picked, and it updates the moment you change a slot. + +There are three independent slots plus one alternative pipeline shape: + +| Slot | What it does | Default | +| ------------------- | --------------------------------------------------------- | ------- | +| **Speech-to-Text** | Turns what you said into words | Local | +| **Text-to-Speech** | Turns the reply into sound | Local | +| **Conductor Brain** | Decides which agent and tab you meant, and shapes replies | Local | + +![Voice Providers panel with the audio destination statement](./screenshots/acappella-voice-providers.png) + +Each slot is configured and validated on its own. A missing Whisper model does not stop you using a hosted Brain, and **no path anywhere silently substitutes one provider for another**. If a slot cannot run, the session refuses with a specific reason instead of quietly routing your microphone somewhere you did not choose. + +### The providers, and what leaves your machine + +| Provider | Slot | Leaves this machine | +| -------------------------------------- | --------------- | ----------------------------------------- | +| Whisper (local) | Speech-to-Text | Nothing | +| OpenAI (hosted) | Speech-to-Text | **Your audio**, to OpenAI | +| Kokoro (local) | Text-to-Speech | Nothing | +| ElevenLabs (hosted) | Text-to-Speech | The reply text, to ElevenLabs | +| Qwen3 1.7B (local) | Conductor Brain | Nothing | +| OpenAI (hosted) | Conductor Brain | Your transcripts, to OpenAI | +| Anthropic (hosted) | Conductor Brain | Your transcripts, to Anthropic | +| Conductor agent | Conductor Brain | Nothing new (it runs an agent you set up) | +| Mock providers | Any | Nothing (no microphone, no model) | +| **OpenAI Realtime** (speech-to-speech) | All three | **Your audio**, to OpenAI | + +**Realtime is a pipeline shape, not a fourth slot.** Choosing it replaces all three slots with one speech-to-speech API: the lowest latency available, in exchange for your audio going to OpenAI and the assistant speaking in that provider's voice. + +Hosted providers need an API key, entered in **Voice Providers**. Keys are stored per service and validated when you add them. + + + The **wake word is always local and never optional**. While only the wake detector is running, no + audio frame reaches a hosted provider, whichever speech-to-text engine you picked. That is enforced + in the type system and re-checked at runtime, not left to discipline. + + +### The Conductor agent option + +The Conductor Brain can also be a real Maestro agent instead of a model. It is slower than the other options, and in exchange it can reason about your actual projects when deciding where an utterance belongs. Nothing new leaves your machine: it runs an agent you already configured, wherever you already configured it to run. + +## Downloading the models + +Local providers need model files. **Voice Setup** lists every one with its exact size, license, source repository, and install path before it downloads anything, and mounting the panel makes zero network calls. + + + **The local speech engines are not in this build yet.** The models below install and verify fine, + but the runtimes that read them (whisper.cpp, ONNX Runtime, and llama.cpp) ship in a later + release. Until then Voice Setup says so against each affected slot and a session refuses to start + rather than half-opening, so you can see it before spending the download. Use a hosted provider, + or wait for the runtimes. + + +| Model | Role | Size | License | +| ---------------------------- | --------------- | -------- | ---------- | +| Whisper Base (English) | Speech-to-Text | 141.1 MB | MIT | +| openWakeWord Base | Wake word | 2.3 MB | Apache-2.0 | +| Kokoro 82M | Text-to-Speech | 310.9 MB | Apache-2.0 | +| Qwen3 1.7B Instruct (Q4_K_M) | Conductor Brain | 1.0 GB | Apache-2.0 | + +Voice Setup offers them as two bundles, and the button always shows the total of what is still **missing** rather than the size of the whole set: + +- **Hands-free (local)** - Whisper, openWakeWord, and Kokoro. **454.4 MB.** Everything the microphone touches stays on this machine. +- **Fully local** - the above plus the Conductor Brain. **1.5 GB.** Routing and spoken replies never call an API either. + +Every file is downloaded from a pinned revision and checked against a SHA-256 recorded in the app, so a model that was tampered with in transit fails to install rather than quietly running. + +### Disk usage and getting the space back + +Models install under `models/acappella` inside Maestro's user data directory. The **Models** page shows the total footprint, each model's size, when it was installed, and when it was last verified, with **Remove** and **Re-verify** next to each one. + +When you switch A Cappella **off**, the Models page stays visible and offers to reclaim the disk. It deletes only the A Cappella model directory, and it confirms first. + +![The Models page with the disk footprint and the runtime self-test](./screenshots/acappella-models.png) + +## Wake word and stop word + +Both live in **Voice Controls**. + +### Wake word + +- Default phrase: **"hey maestro"**. Say it and a Conductor session opens without stealing focus from whatever you are working in. +- **Sensitivity** is a single slider. Higher fires more easily. +- **Test** runs the detector with no session behind it, so you can tune the sensitivity by saying the phrase instead of guessing, restarting, and guessing again. A test run closes the microphone on its own after 15 seconds. +- **Per-agent wake phrases.** Any agent can be given its own phrase, and saying it opens a session bound directly to that agent, skipping routing entirely. + +The wake word needs the openWakeWord model (2.3 MB). Without it the detector runs inert and says so rather than pretending to listen. + +![Wake word settings, with a slot reporting exactly which model it is missing](./screenshots/acappella-wake-word.png) + +### Stop word + +The stop word is a separate control in its own card, because it is a different action from interrupting. + +| | Barge-in (just start talking) | Stop word | +| ----------- | ------------------------------- | ------------- | +| Means | "stop talking, I am still here" | "we are done" | +| Speech | Cancelled | Cancelled | +| Microphone | Stays open | Closes | +| The session | Keeps going | Ends | + +- Default stop phrase: **"maestro stop"**. +- **"nevermind" is always armed and cannot be edited.** A stop word you have to remember is not a stop word, so this one is the same in every install and works even if you have never opened the settings. + +Wake phrases are only listened for while the session is cold, and stop phrases only while it is running, so saying the wake word mid-answer can never stack a second session on top of the first. + +## The two hotkeys + +Both are system-wide, both ship bound, and both only register while A Cappella is on. + +| Hotkey | Default | What it does | +| ------------------------- | -------------------------- | --------------------------------------------------------------------------------------------- | +| **Talk to Maestro** | `Cmd+Alt+V` / `Ctrl+Alt+V` | Opens a Conductor session **without stealing focus**, so you can talk while working elsewhere | +| **Talk to Current Agent** | `Cmd+Alt+A` / `Ctrl+Alt+A` | Brings Maestro forward and opens a session bound to the agent you are looking at | + +Rebind either one in **Voice Controls** or in the **Shortcuts** tab; they are two views of the same binding, not two settings. Voice Controls also shows each hotkey's **real registration state**, which matters because a combination another application already owns is the commonest way a global shortcut silently does nothing. + + + **The hotkeys are tap-to-toggle on every platform today.** Press once to open the floor, press again + to close it. Electron reports key presses but not key releases, so true press-and-hold on a global + hotkey would need a native module Maestro does not ship. Rather than fake it, Voice Controls says + which behaviour you are getting. The HUD's talk button and a paired phone's button do have a real + release event, so press-and-hold works there. + + +![Stop word, hotkeys, and the tap-versus-hold threshold](./screenshots/acappella-stop-word-hotkeys.png) + +You can also start a session from the microphone under the **Send button** in the composer, the **command palette** (`Cmd+K` / `Ctrl+K`, then "Talk to..."), or by right-clicking an agent in the **Left Bar**. + +## While a session is running + +A small draggable HUD appears. It remembers where you put it. + +The indicator has five states, told apart by shape and motion as well as colour: **idle**, **listening** (with a live input level), **thinking**, **speaking**, and **error**. Under `prefers-reduced-motion` all of it goes static. + +- **Minimize collapses the HUD and leaves the session running. Close ends the session.** They are deliberately different: an open microphone with no visible surface is exactly what the close button exists to prevent. +- The **transcript** panel shows what was heard, what was sent, and which agent and tab each turn landed in. Route chips jump you straight to that tab, reopening it if you had closed or snoozed it. +- A session that hears nothing goes cold on its own. The idle timeout defaults to **60 seconds** and is adjustable. + +**Voice and Speed** controls what the assistant sounds like: voice, speaking rate (0.7x to 1.4x), and its own volume, separate from the system volume. Every one of them applies to the **next spoken sentence**, so you can audition a change without restarting a conversation. + +**Background announcements** decide whether an agent finishing in the background gets spoken about. The default is **Auto**: yes in a Conductor session, where you are supervising a fleet and a finished agent is the news you are waiting for; no inside a session bound to one agent, where another agent talking over your conversation is an interruption you did not ask for. + +## Microphone permission + +Maestro asks for the microphone at your **first real session**, never at launch and never when you switch the Encore Feature on. + +| Platform | Behaviour | +| -------- | --------------------------------------------------------------------------------------------------------- | +| macOS | A real permission prompt. Once denied, the recovery is System Settings, and Maestro links there directly. | +| Windows | Maestro can read the permission state but cannot prompt. The OS privacy setting is the only recovery. | +| Linux | No permission system to query. A failure is reported when the capture attempt fails. | + +A denied microphone is reported as a denied microphone, with its own reason and its own fix. It is never collapsed into a generic "voice unavailable" next to 1.5 GB of perfectly good models. + +## Pairing a phone + +A paired device becomes a remote microphone and speaker for this computer. It is not a second brain: the session, the routing, and the agents all stay on the desktop. + + + The iOS app is specified but not yet built. What ships today is the **browser reference client**, a + small dependency-free page that speaks the identical protocol. See + [the client specification](https://github.com/RunMaestro/Maestro/tree/main/docs/ios-client) for the + native app design. + + +### The flow + +1. In **Paired Devices**, press **Show pairing code**. Maestro displays a QR code and a 6-character pairing code, good for **two minutes** and spent by the first device that claims it. +2. Scan it from the device. For the browser reference client, open `http://://acappella` and paste the payload behind the QR code. +3. **Approve the request on the desktop.** Knowing the code is not enough. It buys the device a row in a dialog showing its name and platform, and nothing else. +4. Compare the four-character fingerprint shown on both screens. Matching fingerprints mean nothing got in between. + +A paired device can hold this computer's microphone, hear replies in your configured voice, and dispatch spoken prompts to your agents. It cannot read your files or change your settings. The device's token is stored as a salted hash, never in plain text. + +**Revoke** is per-device and takes effect on a live connection immediately, tearing down the audio and closing the voice session rather than waiting for the next connect. There is also one control that drops every device at once. + +![Paired Devices, discovery, and the connection settings](./screenshots/acappella-paired-devices.png) + +### Discovery + +Maestro can advertise itself over Bonjour so a device on the same network finds it without anyone typing an address. It is a convenience and never the connection itself: the QR code carries the addresses directly, and manual host entry always works. The advert carries the app version, protocol version, machine name, and pairing fingerprint, and never a token or a pairing code. You can switch it off. + +### Whether it will actually connect + +Each connected device shows the connection path it actually won, in plain words. + +| Path | Needs | Works | +| ----------------------- | ------------------------------------ | --------------------------------------------------- | +| Direct (LAN or overlay) | Nothing | Same WiFi or wire, or over Tailscale-style overlays | +| Direct (through NAT) | A STUN server | Most home networks | +| Relayed (TURN) | **A TURN server you run or pay for** | Cellular, hotel WiFi, corporate networks | + +If you already run an overlay network such as Tailscale, that is the whole answer: both machines have a routable address for each other and the connection is direct from anywhere. The pairing QR carries every local address the desktop has, overlay addresses included. + + + **A phone on a mobile network needs a TURN server.** Carrier-grade NAT does not support the + hole punching that STUN relies on, so no amount of STUN gets through it. And **the Cloudflare quick + tunnel that serves Maestro's browser interface cannot carry this audio**: it is an HTTPS reverse + proxy, while the media leg is a direct UDP association between the two devices. Signaling goes + through the tunnel fine, which is why "the tunnel is up" tells you nothing about whether audio will + flow. + + +## Troubleshooting + +| Symptom | Likely cause | +| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| The hotkey does nothing | Another application owns the combination. Voice Controls shows the real registration state and names the conflict. | +| The wake word never fires | The openWakeWord model is not installed, or the sensitivity is too low. Use **Test** to tune it by voice. | +| A session refuses to start | Open Voice Providers. Each slot reports its own specific reason: a missing model, a missing key, or a denied microphone. | +| Speech works but replies are silent | Check the assistant volume in Voice and Speed. It is separate from the system volume and has its own floor. | +| A phone pairs but no audio arrives | The media path is separate from signaling. Check the connection path shown on the device row; cellular needs TURN. | +| Voice was disabled but disk is still in use | The Models page stays available when the feature is off, and offers to reclaim the model directory. | + +Two tools on the **Models** page are worth reaching for before filing a bug, and both are safe to run at any time: + +- **Run voice self-test** loads each speech runtime and runs a trivial operation against it. No model is loaded and no microphone is opened. Include the result when you report a voice problem. +- **Read last turn** shows where the last spoken turn actually spent its time, from the moment the detector heard you stop talking. Include this when you report that voice feels slow. + +## Related + +- [Encore Features](./encore-features) - how optional features are gated +- [Keyboard Shortcuts](./keyboard-shortcuts) - the full shortcut and global hotkey list +- [Remote Control](./remote-control) - the browser interface and the Cloudflare tunnel +- [Configuration](./configuration) - where Maestro stores its data diff --git a/package-lock.json b/package-lock.json index acee79bf9c..d2dede6403 100644 --- a/package-lock.json +++ b/package-lock.json @@ -95,6 +95,7 @@ "remark-math": "^6.0.0", "semver": "^7.7.4", "shiki": "^4.0.2", + "tar": "^7.5.17", "ws": "^8.16.0", "zustand": "^5.0.11" }, @@ -2589,7 +2590,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^7.0.4" @@ -19078,7 +19078,6 @@ "version": "7.5.17", "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.17.tgz", "integrity": "sha512-wPEBwzapC+2PaTYPH6e2L+cNOEE227S47wUYFqlegcs8zlLLmeb9Fcff1HVZY4Fwku/1Eyv38n7GYwB2aaS71g==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", @@ -19169,7 +19168,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" @@ -19179,7 +19177,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", - "dev": true, "license": "MIT", "dependencies": { "minipass": "^7.1.2" @@ -19192,7 +19189,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" diff --git a/package.json b/package.json index 1607a9885e..0b5dd356c4 100644 --- a/package.json +++ b/package.json @@ -35,16 +35,17 @@ "build:maestro-p": "node scripts/build-maestro-p.mjs", "build:permission-relay-bridge": "node scripts/build-permission-relay-bridge.mjs", "build:renderer": "vite build", - "package": "node scripts/set-version.mjs npm run build && node scripts/set-version.mjs electron-builder --mac --win --linux", - "package:mac": "node scripts/set-version.mjs npm run build && node scripts/set-version.mjs electron-builder --mac", - "package:win": "node scripts/set-version.mjs npm run build && node scripts/set-version.mjs electron-builder --win", - "package:linux": "node scripts/set-version.mjs npm run build && node scripts/set-version.mjs electron-builder --linux", + "package": "node scripts/set-version.mjs npm run build && node scripts/set-version.mjs electron-builder --mac --win --linux && npm run verify:native-packaging", + "package:mac": "node scripts/set-version.mjs npm run build && node scripts/set-version.mjs electron-builder --mac && npm run verify:native-packaging", + "package:win": "node scripts/set-version.mjs npm run build && node scripts/set-version.mjs electron-builder --win && npm run verify:native-packaging", + "package:linux": "node scripts/set-version.mjs npm run build && node scripts/set-version.mjs electron-builder --linux && npm run verify:native-packaging", + "verify:native-packaging": "node scripts/verify-native-packaging.mjs", "start": "electron .", "clean": "rm -rf dist release node_modules/.vite", "preinstall": "node scripts/check-python.mjs", "prepare": "node scripts/setup-git-hooks.mjs", "postinstall": "patch-package && node scripts/ensure-electron.mjs && electron-rebuild -f -w node-pty,better-sqlite3", - "lint": "tsc -p tsconfig.lint.json && tsc -p tsconfig.main.json --noEmit && tsc -p tsconfig.cli.json --noEmit", + "lint": "tsc -p tsconfig.lint.json && tsc -p tsconfig.main.json --noEmit && tsc -p tsconfig.cli.json --noEmit && tsc -p tsconfig.scripts.json", "lint:eslint": "eslint src/ && eslint -c eslint.dashes.config.mjs src/__tests__ scripts", "format": "prettier --write \"src/**/*.{ts,tsx}\"", "format:all": "prettier --write .", @@ -57,6 +58,8 @@ "test:e2e": "bun run build:main && bun run build:renderer && playwright test", "test:e2e:ui": "bun run build:main && bun run build:renderer && playwright test --ui", "test:e2e:headed": "bun run build:main && bun run build:renderer && playwright test --headed", + "acappella:eval": "node scripts/acappella-routing-eval.mjs", + "acappella:latency": "node scripts/acappella-speech-latency.mjs", "test:integration": "vitest run --config vitest.integration.config.ts", "test:integration:watch": "vitest --config vitest.integration.config.ts", "test:performance": "vitest run --config vitest.performance.config.mts", @@ -95,7 +98,11 @@ "node_modules/@napi-rs/keyring/**/*", "node_modules/@napi-rs/keyring-*/*", "node_modules/bindings/**/*", - "node_modules/file-uri-to-path/**/*" + "node_modules/file-uri-to-path/**/*", + "node_modules/node-llama-cpp/**/*", + "node_modules/@node-llama-cpp/**/*", + "node_modules/smart-whisper/build/**/*", + "node_modules/onnxruntime-node/bin/**/*" ], "mac": { "category": "public.app-category.developer-tools", @@ -104,6 +111,9 @@ "notarize": false, "entitlements": "build/entitlements.mac.plist", "entitlementsInherit": "build/entitlements.mac.plist", + "extendInfo": { + "NSMicrophoneUsageDescription": "Maestro uses the microphone for A Cappella, its voice control feature, so you can speak to your agents. Your audio is processed on this machine when you select local speech providers, and is sent to a hosted provider only if you choose one." + }, "target": [ "dmg", "zip" @@ -375,6 +385,7 @@ "remark-math": "^6.0.0", "semver": "^7.7.4", "shiki": "^4.0.2", + "tar": "^7.5.17", "ws": "^8.16.0", "zustand": "^5.0.11" }, diff --git a/scripts/acappella-routing-eval.mjs b/scripts/acappella-routing-eval.mjs new file mode 100644 index 0000000000..47c6f43a2b --- /dev/null +++ b/scripts/acappella-routing-eval.mjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node +/** + * Runner for the A Cappella routing evaluation harness. + * + * The harness imports main-process modules (the router, the prompt manager, the + * routing log), so it has to be bundled before it can run outside Electron. The + * only thing standing in the way is `electron` itself, which two of those modules + * import for `app.getPath()`; it is stubbed with the three answers they need. + * + * The output lands two directories below the repo root on purpose: + * `prompt-manager.ts` resolves the bundled prompts as + * `__dirname/../../src/prompts`, so that depth is what makes the harness read the + * real `src/prompts/acappella-router.md` rather than the built-in fallback. + * + * See scripts/acappella-routing-eval.ts for what it measures and why. + */ + +import { spawn } from 'child_process'; +import * as esbuild from 'esbuild'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.resolve(__dirname, '..'); +const outfile = path.join(rootDir, 'dist', 'acappella-eval', 'routing-eval.cjs'); + +/** Enough of `electron` for the prompt manager and the routing log to resolve paths. */ +const electronStub = { + name: 'electron-stub', + setup(build) { + build.onResolve({ filter: /^electron$/ }, () => ({ + path: 'electron', + namespace: 'electron-stub', + })); + build.onLoad({ filter: /.*/, namespace: 'electron-stub' }, () => ({ + contents: ` + const userData = ${JSON.stringify(path.join(os.tmpdir(), 'acappella-routing-eval'))}; + exports.app = { + isPackaged: false, + getAppPath: () => ${JSON.stringify(rootDir)}, + getPath: () => userData, + getVersion: () => '0.0.0-eval', + on: () => {}, + whenReady: () => Promise.resolve(), + }; + exports.ipcMain = { on: () => {}, handle: () => {} }; + exports.BrowserWindow = { getAllWindows: () => [] }; + `, + loader: 'js', + })); + }, +}; + +await esbuild.build({ + entryPoints: [path.join(__dirname, 'acappella-routing-eval.ts')], + bundle: true, + platform: 'node', + target: 'node20', + format: 'cjs', + outfile, + sourcemap: 'inline', + // Native modules the main process pulls in transitively. The harness never + // reaches them; node-llama-cpp is loaded dynamically by the local Brain, which + // reports its own absence. + external: [ + 'fsevents', + 'node-pty', + 'node-llama-cpp', + '@node-llama-cpp/*', + 'better-sqlite3', + // Ships a .node binding esbuild cannot inline. Required at runtime instead, + // which is harmless: the harness injects its own credential reader. + '@napi-rs/keyring', + ], + plugins: [electronStub], + logLevel: 'warning', +}); + +fs.mkdirSync(path.join(os.tmpdir(), 'acappella-routing-eval'), { recursive: true }); + +const child = spawn(process.execPath, [outfile, ...process.argv.slice(2)], { stdio: 'inherit' }); +child.on('exit', (code) => process.exit(code ?? 1)); diff --git a/scripts/acappella-routing-eval.ts b/scripts/acappella-routing-eval.ts new file mode 100644 index 0000000000..90c2627560 --- /dev/null +++ b/scripts/acappella-routing-eval.ts @@ -0,0 +1,841 @@ +/** + * A Cappella routing evaluation harness. + * + * The model-in-the-loop half of `docs/architecture/acappella/routing-evaluation.md`. + * The deterministic suites in `src/__tests__/main/acappella/router/` prove the + * router's RULES against a scripted Brain; nothing in CI proves that a real model, + * handed the real prompt and a realistic roster, picks the right agent and the + * right tab. That is the number this file produces. + * + * It was originally written down as a fifteen-minute microphone session with four + * live agents. That is a bad instrument for the thing being measured: routing + * takes a TRANSCRIPT and a ROSTER, both of which are data, so speech recognition + * and real agents add two uncontrolled variables and make the result unrepeatable. + * Everything here below the Brain is the shipping code - `createConductorRouter`, + * the real prompt from `src/prompts/acappella-router.md`, `parseRouteDecision`, + * the grammar validator, the recall ranker, and the routing log itself - so the + * only thing being varied is the model. + * + * The harness plays the part of the user who corrects a misroute: a decision that + * does not match the expectation is marked `corrected` in the routing log, which + * is exactly what the HUD's correction control does in the app. `routingQuality()` + * then reports the hit rate the same way it will report it in the field. + * + * Usage: + * + * npm run acappella:eval # the Conductor-agent Brain + * npm run acappella:eval -- --brain anthropic + * npm run acappella:eval -- --brain openai --agent-type codex + * npm run acappella:eval -- --brain local --model-path /path/to/qwen3.gguf + * + * Hosted Brains read their key from `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` rather + * than the keychain, so a run does not depend on a configured desktop install. + */ + +import { spawn as spawnChild, type ChildProcess } from 'child_process'; +import { EventEmitter } from 'events'; +import * as os from 'os'; +import * as path from 'path'; + +import { AgentDetector } from '../src/main/agents'; +import { + createConductorRouter, + isCorrectionUtterance, + planCorrection, +} from '../src/main/acappella/router/conductor-router'; +import { + ConductorAgentBrain, + type ConductorProcessManager, +} from '../src/main/acappella/router/conductor-agent'; +import { + serializeRoutingContext, + type RoutingContext, +} from '../src/main/acappella/router/routing-context'; +import { + noteRoutingOutcome, + readRoutingLog, + resetRoutingLog, + routingQuality, + setRoutingLogPath, +} from '../src/main/acappella/router/routing-log'; +import { AnthropicBrainProvider } from '../src/main/acappella/providers/hosted/anthropic-brain'; +import { OpenAiBrainProvider } from '../src/main/acappella/providers/hosted/openai-brain'; +import { LlamaBrainProvider } from '../src/main/acappella/providers/local/llama-brain'; +import { initializePrompts } from '../src/main/prompt-manager'; +import { logger } from '../src/main/utils/logger'; +import type { RosterAgent } from '../src/shared/acappella/protocol'; +import type { BrainProvider } from '../src/shared/acappella/providers'; +import { + isClarification, + routeTargetSessionId, + type RouteDecision, + type RouteTabAction, +} from '../src/shared/acappella/route-decision'; + +// --------------------------------------------------------------------------- +// The fixture roster +// --------------------------------------------------------------------------- + +const MINUTE = 60_000; + +/** + * Fixed so two runs are comparable: recency is part of the recall ranking, and a + * roster built from the wall clock would score differently every time. + */ +const BASE_TIME = Date.parse('2026-08-15T12:00:00Z'); + +const SESSIONS = { + backend: 'sess-backend', + api: 'sess-api', + frontend: 'sess-frontend', + infra: 'sess-infra', +} as const; + +/** The four agents and twelve tabs the evaluation doc specifies. */ +const ROSTER: RosterAgent[] = [ + { + sessionId: SESSIONS.backend, + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/payments-api', + status: 'idle', + recentWork: 'Split the auth middleware out of the request pipeline', + tabs: [ + { + id: 'tab-auth-refactor', + name: 'Auth Refactor', + lastActiveAt: BASE_TIME - 2 * MINUTE, + state: 'open', + topic: 'rewriting the auth middleware and its token checks', + }, + { + id: 'tab-db-migrations', + name: 'DB Migrations', + lastActiveAt: BASE_TIME - 90 * MINUTE, + state: 'open', + topic: 'the pending payment schema migrations', + }, + { + id: 'tab-rate-limit-spike', + name: 'Rate Limit Spike', + lastActiveAt: BASE_TIME - 26 * 60 * MINUTE, + state: 'snoozed', + topic: 'an abandoned experiment with per-key rate limiting', + }, + ], + }, + { + sessionId: SESSIONS.api, + name: 'API', + agentType: 'codex', + cwd: '/repo/gateway', + status: 'idle', + recentWork: 'Added retry backoff to the webhook dispatcher', + tabs: [ + { + id: 'tab-gateway-routing', + name: 'Gateway Routing', + lastActiveAt: BASE_TIME - 15 * MINUTE, + state: 'open', + topic: 'how requests are routed through the gateway', + }, + { + id: 'tab-webhook-retries', + name: 'Webhook Retries', + lastActiveAt: BASE_TIME - 4 * 60 * MINUTE, + state: 'open', + topic: 'what the retry policy for failed webhooks should be', + }, + { + id: 'tab-old-auth-spike', + name: 'Old Auth Spike', + lastActiveAt: BASE_TIME - 12 * 24 * 60 * MINUTE, + state: 'closed', + topic: 'an early attempt at gateway-side auth, abandoned', + }, + ], + }, + { + sessionId: SESSIONS.frontend, + name: 'Frontend', + agentType: 'claude-code', + cwd: '/repo/web', + status: 'busy', + recentWork: 'Made the sidebar collapse animation respect reduced motion', + tabs: [ + { + id: 'tab-sidebar-collapse', + name: 'Sidebar Collapse', + lastActiveAt: BASE_TIME - 8 * MINUTE, + state: 'open', + topic: 'the collapsing sidebar and its animation', + }, + { + id: 'tab-checkout-flow', + name: 'Checkout Flow', + lastActiveAt: BASE_TIME - 6 * 60 * MINUTE, + state: 'open', + topic: 'the multi-step checkout form and its validation', + }, + { + id: 'tab-dark-mode', + name: 'Dark Mode', + lastActiveAt: BASE_TIME - 3 * 24 * 60 * MINUTE, + state: 'open', + topic: 'the dark mode toggle and how the choice is remembered', + }, + ], + }, + { + sessionId: SESSIONS.infra, + name: 'Infra', + agentType: 'opencode', + cwd: '/repo/terraform', + status: 'idle', + recentWork: 'Pinned the node pool to the previous Kubernetes minor', + tabs: [ + { + id: 'tab-cluster-upgrade', + name: 'Cluster Upgrade', + lastActiveAt: BASE_TIME - 40 * MINUTE, + state: 'open', + topic: 'upgrading the Kubernetes cluster version', + }, + { + id: 'tab-cost-report', + name: 'Cost Report', + lastActiveAt: BASE_TIME - 2 * 24 * 60 * MINUTE, + state: 'open', + topic: 'the monthly cloud spend breakdown', + }, + { + id: 'tab-log-retention', + name: 'Log Retention', + lastActiveAt: BASE_TIME - 5 * 24 * 60 * MINUTE, + state: 'open', + topic: 'how long logs are kept before they are rolled off', + }, + ], + }, +]; + +/** The agent the user is looking at when the script starts. */ +const ACTIVE_AGENT = SESSIONS.backend; + +// --------------------------------------------------------------------------- +// The script +// --------------------------------------------------------------------------- + +interface Expectation { + /** + * A roster session id, or `conductor`. + * + * Omitted for a case whose correct answer is a question: when the router is + * right to be unsure, which target it leaned toward while asking is not part + * of being right, and scoring it would penalise the intended behaviour. + */ + target?: string; + action?: RouteTabAction; + tabId?: string; + /** True when the correct behaviour is a spoken question rather than a dispatch. */ + clarify?: boolean; +} + +interface EvalCase { + n: number; + utterance: string; + tests: string; + expect: Expectation; + /** + * A correction is not routed at all: it is recognised before the Brain is + * consulted, which is the property worth testing. + */ + correction?: boolean; + /** + * The answer to the question this utterance is expected to provoke, routed on + * the next turn with the original utterance attached. Reported separately: it + * measures the disambiguation round trip, not the fifteen-utterance hit rate. + */ + answer?: { text: string; target: string }; +} + +const CASES: EvalCase[] = [ + { + n: 1, + utterance: 'run the tests', + tests: 'same-topic continuation', + expect: { target: SESSIONS.backend, action: 'current' }, + }, + { + n: 2, + utterance: 'what broke', + tests: 'pronoun-free follow-up', + expect: { target: SESSIONS.backend, action: 'current' }, + }, + { + n: 3, + utterance: 'add a rate limiter to the public endpoints', + tests: 'topic switch', + expect: { target: SESSIONS.backend, action: 'new' }, + }, + { + n: 4, + utterance: 'ask the frontend agent about the checkout flow', + tests: 'explicit agent naming', + expect: { target: SESSIONS.frontend, action: 'recall', tabId: 'tab-checkout-flow' }, + }, + { + n: 5, + utterance: 'tell infra to bump the cluster version', + tests: 'explicit agent naming', + expect: { target: SESSIONS.infra, action: 'current' }, + }, + { + n: 6, + utterance: 'back to the auth thing', + tests: 'vague recall', + expect: { target: SESSIONS.backend, action: 'recall', tabId: 'tab-auth-refactor' }, + }, + { + n: 7, + utterance: 'what did we decide about webhook retries', + tests: 'recall by topic', + expect: { target: SESSIONS.api, action: 'recall', tabId: 'tab-webhook-retries' }, + }, + { + n: 8, + utterance: 'the gateway one', + tests: 'recall by project path', + expect: { target: SESSIONS.api }, + }, + { + n: 9, + utterance: 'pick up that rate limit spike again', + tests: 'snoozed-tab wake', + expect: { target: SESSIONS.backend, action: 'recall', tabId: 'tab-rate-limit-spike' }, + }, + { + n: 10, + utterance: 'go back to the old auth spike', + tests: 'closed-tab reopen offer', + // A closed tab is an offer, not a dispatch: `applyRecallPolicy` attaches the + // question, so the correct decision here CARRIES a clarify. + expect: { + target: SESSIONS.api, + action: 'recall', + tabId: 'tab-old-auth-spike', + clarify: true, + }, + }, + { + n: 11, + utterance: 'how many agents do I have running', + tests: 'Maestro-level question', + expect: { target: 'conductor' }, + }, + { + n: 12, + utterance: 'which one is busy right now', + tests: 'fleet-level question', + expect: { target: 'conductor' }, + }, + { + n: 13, + utterance: 'make the dark mode toggle stick', + tests: 'topic match over recency', + expect: { target: SESSIONS.frontend, action: 'recall', tabId: 'tab-dark-mode' }, + }, + { + n: 14, + utterance: 'do the auth one', + tests: 'low confidence disambiguation', + // "Auth Refactor" on Backend and "Old Auth Spike" on API both fit, so the + // only right answer is a question. Which one it leaned toward is not scored. + expect: { clarify: true }, + answer: { text: 'the backend one', target: SESSIONS.backend }, + }, + { + n: 15, + utterance: 'no, the other one', + tests: 'correction path', + expect: {}, + correction: true, + }, +]; + +// --------------------------------------------------------------------------- +// Brains +// --------------------------------------------------------------------------- + +type BrainKey = 'agent' | 'anthropic' | 'openai' | 'local'; + +interface Options { + brain: BrainKey; + agentType: string; + cwd: string; + modelPath?: string; + model?: string; + json: boolean; +} + +/** + * A `ConductorProcessManager` backed by `child_process`. + * + * The real `ProcessManager` cannot run here: it needs Electron, node-pty and the + * session machinery. The interface is structural for exactly this reason, and the + * two behaviours the Brain depends on are reproduced faithfully: + * + * - Prompt delivery follows the no-image branch of `ChildProcessSpawner.spawn()` + * (`promptArgs`, else `noPromptSeparator`, else a `--` separator). + * - The `data` event carries the RESULT text, not raw stdout, the way + * `ExitHandler.handleBatchModeExit()` emits it: parse the buffer as JSON and + * emit `.result`, falling back to the raw buffer when it is not JSON. + */ +class SpawnProcessManager extends EventEmitter implements ConductorProcessManager { + private readonly children = new Map(); + + spawn(config: Record): { pid: number; success: boolean } | null { + const sessionId = String(config.sessionId); + const command = String(config.command); + const args = [...((config.args as string[]) ?? [])]; + const prompt = String(config.prompt ?? ''); + const promptArgs = config.promptArgs as ((value: string) => string[]) | undefined; + + if (promptArgs) args.push(...promptArgs(prompt)); + else if (config.noPromptSeparator) args.push(prompt); + else args.push('--', prompt); + + const child = spawnChild(command, args, { + cwd: String(config.cwd), + env: { + ...childEnv(), + ...((config.customEnvVars as Record) ?? {}), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + this.children.set(sessionId, child); + + let buffer = ''; + child.stdout?.on('data', (chunk: Buffer) => { + buffer += chunk.toString('utf8'); + }); + child.stderr?.on('data', (chunk: Buffer) => { + process.stderr.write(chunk); + }); + child.on('error', (error) => { + this.children.delete(sessionId); + this.emit('data', sessionId, `[error] ${error.message}`); + this.emit('exit', sessionId, 1); + }); + child.on('close', () => { + this.children.delete(sessionId); + this.emit('data', sessionId, extractResult(buffer)); + this.emit('exit', sessionId, 0); + }); + + return { pid: child.pid ?? -1, success: true }; + } + + kill(sessionId: string): void { + this.children.get(sessionId)?.kill('SIGTERM'); + this.children.delete(sessionId); + } +} + +/** What `ExitHandler.handleBatchModeExit()` would have emitted for this buffer. */ +function extractResult(buffer: string): string { + try { + const parsed = JSON.parse(buffer) as { result?: unknown }; + if (typeof parsed.result === 'string') return parsed.result; + } catch { + /* not JSON: the raw buffer is the answer, same as the real handler */ + } + return buffer; +} + +/** + * The environment a spawned agent gets. + * + * The Claude session-identity markers are stripped for the same reason + * `sanitizeChildEnv()` in `src/maestro-p/index.ts` strips them: inherited from the + * shell an agent is running in, they make the child believe it is a nested session. + */ +function childEnv(): NodeJS.ProcessEnv { + const env = { ...process.env }; + for (const key of [ + 'CLAUDECODE', + 'CLAUDE_CODE_SESSION_ID', + 'CLAUDE_CODE_CHILD_SESSION', + 'CLAUDE_CODE_ENTRYPOINT', + ]) { + delete env[key]; + } + return env; +} + +async function createBrain(options: Options): Promise { + switch (options.brain) { + case 'anthropic': + return new AnthropicBrainProvider({ + model: options.model, + readCredential: () => process.env.ANTHROPIC_API_KEY?.trim() || null, + }); + + case 'openai': + return new OpenAiBrainProvider({ + model: options.model, + readCredential: () => process.env.OPENAI_API_KEY?.trim() || null, + }); + + case 'local': + return new LlamaBrainProvider({ modelPath: options.modelPath }); + + case 'agent': { + const detector = new AgentDetector(); + const real = await detector.getAgent(options.agentType); + if (!real || !real.available) { + throw new Error( + `The '${options.agentType}' agent is not installed. Install it, or pass --brain anthropic.` + ); + } + return new ConductorAgentBrain({ + processManager: new SpawnProcessManager(), + // A one-shot JSON batch run. The shipping default is stream-json, which + // this harness's process manager does not reassemble; `--output-format + // json` produces the single envelope `extractResult()` reads. + agentDetector: { + getAgent: async () => ({ ...real, args: batchArgsFor(real.args ?? []) }), + } as unknown as AgentDetector, + agentType: options.agentType, + cwd: options.cwd, + // A real model may think for a while; the shipping 20s deadline is a + // voice budget, not an evaluation one. + timeoutMs: 120_000, + modelId: options.model, + }); + } + } +} + +/** Swap a streaming output format for the single-envelope one. */ +function batchArgsFor(args: readonly string[]): string[] { + const out: string[] = []; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--output-format') { + out.push('--output-format', 'json'); + i++; + continue; + } + if (args[i] === '--verbose') continue; + out.push(args[i]); + } + if (!out.includes('--output-format')) out.push('--output-format', 'json'); + return out; +} + +// --------------------------------------------------------------------------- +// The run +// --------------------------------------------------------------------------- + +interface CaseResult { + n: number; + utterance: string; + tests: string; + expected: string; + actual: string; + hit: boolean; + confidence: number | null; + latencyMs: number | null; + note?: string; +} + +async function run(options: Options): Promise { + // Agent detection alone logs a screenful of PATH probes. The result table is + // the output of this script; a real problem still comes through as an error. + logger.setLogLevel('error'); + + // The evaluation must measure the SHIPPING prompt, including any local edit, + // not the built-in fallback constant. + await initializePrompts(); + + const brain = await createBrain(options); + await probeBrain(brain); + + // A file, not a stub: the harness reports through the same instrument the field + // reports through, so a hit rate here and a hit rate in the app mean one thing. + setRoutingLogPath(path.join(os.tmpdir(), `acappella-routing-eval-${process.pid}.json`)); + resetRoutingLog(); + + const recentUtterances: string[] = []; + const context: RoutingContext = { + agents: ROSTER, + activeAgentSessionId: ACTIVE_AGENT, + recentUtterances, + droppedTabs: 0, + serializedChars: 0, + }; + context.serializedChars = serializeRoutingContext(context).length; + + const router = createConductorRouter({ + brain, + loadContext: async (recent) => ({ ...context, recentUtterances: [...recent] }), + }); + + const results: CaseResult[] = []; + let lastDispatchTarget: string | null = null; + + for (const testCase of CASES) { + if (testCase.correction) { + results.push(evaluateCorrection(testCase, lastDispatchTarget)); + continue; + } + + const startedAt = Date.now(); + let decision: RouteDecision; + try { + decision = await router.route(testCase.utterance, { + roster: ROSTER, + scope: { kind: 'conductor' }, + activeAgentSessionId: ACTIVE_AGENT, + recentUtterances: [...recentUtterances], + }); + } catch (error) { + results.push({ + n: testCase.n, + utterance: testCase.utterance, + tests: testCase.tests, + expected: describeExpectation(testCase.expect), + actual: `error: ${(error as Error).message}`, + hit: false, + confidence: null, + latencyMs: Date.now() - startedAt, + }); + continue; + } + + const hit = matches(decision, testCase.expect); + const turnId = router.lastTurnId(); + if (turnId && !hit) { + // Exactly what the HUD's correction control does, so the log's hit rate is + // the number this harness reports rather than a second tally beside it. + noteRoutingOutcome(turnId, 'corrected', `expected ${describeExpectation(testCase.expect)}`); + } + + const entry = readRoutingLog().find((candidate) => candidate.id === turnId); + results.push({ + n: testCase.n, + utterance: testCase.utterance, + tests: testCase.tests, + expected: describeExpectation(testCase.expect), + actual: describeDecision(decision), + hit, + confidence: decision.confidence, + latencyMs: entry?.latencyMs ?? Date.now() - startedAt, + }); + + recentUtterances.push(testCase.utterance); + if (!isClarification(decision)) { + lastDispatchTarget = routeTargetSessionId(decision.target); + } + + if (testCase.answer && isClarification(decision)) { + results.push(await evaluateAnswer(router, testCase, decision, recentUtterances)); + } + } + + report(options, brain, results); +} + +/** + * One throwaway decision before the script starts. + * + * A missing key or an uninstalled runtime fails identically on all fifteen + * utterances, and fifteen copies of "no API key is configured" printed inside a + * results table reads like a routing result. This turns it back into what it is: + * the Brain could not be reached, so there is nothing to measure. + */ +async function probeBrain(brain: BrainProvider): Promise { + try { + await brain.route('hello', { + roster: ROSTER, + scope: { kind: 'conductor' }, + activeAgentSessionId: ACTIVE_AGENT, + }); + } catch (error) { + throw new Error(`${brain.label} is not usable here: ${(error as Error).message}`); + } +} + +/** + * The turn after a disambiguation. + * + * Routed with `clarification` set, which is what stops "the backend one" from + * being treated as a request and becoming a tab called "the backend one". + */ +async function evaluateAnswer( + router: ReturnType, + testCase: EvalCase, + question: RouteDecision, + recentUtterances: string[] +): Promise { + const answer = testCase.answer!; + const startedAt = Date.now(); + const decision = await router.route(answer.text, { + roster: ROSTER, + scope: { kind: 'conductor' }, + activeAgentSessionId: ACTIVE_AGENT, + recentUtterances: [...recentUtterances], + clarification: { question: question.clarify!, utterance: testCase.utterance }, + }); + + const hit = matches(decision, { target: answer.target }); + const turnId = router.lastTurnId(); + if (turnId && !hit) noteRoutingOutcome(turnId, 'corrected', `expected ${answer.target}`); + + return { + n: testCase.n, + utterance: answer.text, + tests: 'disambiguation answer', + expected: answer.target, + actual: describeDecision(decision), + hit, + confidence: decision.confidence, + latencyMs: Date.now() - startedAt, + note: 'follow-up, excluded from the fifteen', + }; +} + +/** + * A correction never reaches the Brain. + * + * Recognised from the utterance alone and turned into a plan against the roster, + * so what is checked here is the recognition and the plan, not a decision. + */ +function evaluateCorrection(testCase: EvalCase, fromAgent: string | null): CaseResult { + const recognised = isCorrectionUtterance(testCase.utterance); + const plan = recognised ? planCorrection(ROSTER, fromAgent ?? '') : null; + // Four agents means "the other one" does not name anything, so asking is the + // only honest plan. A `move` here would mean the router guessed twice. + const hit = recognised && plan?.kind === 'ask'; + + return { + n: testCase.n, + utterance: testCase.utterance, + tests: testCase.tests, + expected: 'recognised as a correction, asks which target', + actual: recognised ? `correction -> ${plan?.kind}` : 'not recognised as a correction', + hit: Boolean(hit), + confidence: null, + latencyMs: null, + note: 'not routed, excluded from the hit rate', + }; +} + +function matches(decision: RouteDecision, expected: Expectation): boolean { + const target = routeTargetSessionId(decision.target) ?? 'conductor'; + if (expected.target && target !== expected.target) return false; + if (Boolean(expected.clarify) !== isClarification(decision)) return false; + if (expected.action && decision.tabAction !== expected.action) return false; + if (expected.tabId && decision.tabId !== expected.tabId) return false; + return true; +} + +function describeExpectation(expected: Expectation): string { + const parts = [expected.target ? nameOf(expected.target) : 'any target']; + if (expected.action) parts.push(expected.action); + if (expected.tabId) parts.push(expected.tabId); + if (expected.clarify) parts.push('clarify'); + return parts.join(' / '); +} + +function describeDecision(decision: RouteDecision): string { + const parts = [nameOf(routeTargetSessionId(decision.target) ?? 'conductor'), decision.tabAction]; + if (decision.tabId) parts.push(decision.tabId); + if (decision.tabName) parts.push(`"${decision.tabName}"`); + if (isClarification(decision)) parts.push('clarify'); + return parts.join(' / '); +} + +function nameOf(sessionId: string): string { + return ROSTER.find((agent) => agent.sessionId === sessionId)?.name ?? sessionId; +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +function report(options: Options, brain: BrainProvider, results: CaseResult[]): void { + const quality = routingQuality(); + + if (options.json) { + console.log(JSON.stringify({ brain: brain.id, results, quality }, null, 2)); + return; + } + + console.log(''); + console.log(`Brain: ${brain.label} (${brain.id})`); + console.log(''); + console.log('| # | Utterance | Expected | Actual | Conf | ms | Hit |'); + console.log('| --- | --------- | -------- | ------ | ---- | -- | --- |'); + for (const result of results) { + console.log( + `| ${result.n} | ${result.utterance} | ${result.expected} | ${result.actual} | ` + + `${result.confidence?.toFixed(2) ?? '-'} | ${result.latencyMs ?? '-'} | ` + + `${result.hit ? 'yes' : 'NO'} |` + ); + } + + const scored = results.filter((result) => !result.note); + const hits = scored.filter((result) => result.hit).length; + + console.log(''); + console.log( + `Script: ${hits}/${scored.length} routed utterances matched. ` + + 'The fifteenth is a correction and is never routed; the extra 14 is the answer to its question.' + ); + console.log( + `Routing log: ${quality.dispatched} dispatched, ${quality.corrected} corrected, ` + + `${quality.clarified} clarified, ${quality.failed} failed.` + ); + console.log( + `Hit rate: ${quality.hitRate === null ? '-' : `${(quality.hitRate * 100).toFixed(0)}%`}, ` + + `mean latency: ${quality.meanLatencyMs ?? '-'} ms.` + ); + console.log(''); + console.log('Paste the row into docs/architecture/acappella/routing-evaluation.md.'); +} + +// --------------------------------------------------------------------------- +// Entry +// --------------------------------------------------------------------------- + +function parseArgs(argv: string[]): Options { + const options: Options = { + brain: 'agent', + agentType: 'claude-code', + cwd: process.cwd(), + json: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--brain') options.brain = argv[++i] as BrainKey; + else if (arg === '--agent-type') options.agentType = argv[++i]; + else if (arg === '--cwd') options.cwd = argv[++i]; + else if (arg === '--model-path') options.modelPath = argv[++i]; + else if (arg === '--model') options.model = argv[++i]; + else if (arg === '--json') options.json = true; + else throw new Error(`Unknown option: ${arg}`); + } + + if (!['agent', 'anthropic', 'openai', 'local'].includes(options.brain)) { + throw new Error(`Unknown brain: ${options.brain}`); + } + return options; +} + +run(parseArgs(process.argv.slice(2))).catch((error: Error) => { + console.error(`Routing evaluation failed: ${error.message}`); + process.exit(1); +}); diff --git a/scripts/acappella-speech-latency.mjs b/scripts/acappella-speech-latency.mjs new file mode 100644 index 0000000000..25fa0916ae --- /dev/null +++ b/scripts/acappella-speech-latency.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node +/** + * Runner for the A Cappella speech latency harness. + * + * Same shape as `acappella-routing-eval.mjs` and for the same reason: the harness + * imports main-process modules, so it is bundled before it can run outside + * Electron, with `electron` itself stubbed down to the handful of answers those + * modules want for path resolution. + * + * See scripts/acappella-speech-latency.ts for what it measures and why. + */ + +import { spawn } from 'child_process'; +import * as esbuild from 'esbuild'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.resolve(__dirname, '..'); +const outfile = path.join(rootDir, 'dist', 'acappella-eval', 'speech-latency.cjs'); + +/** Enough of `electron` for the bundled main-process modules to resolve paths. */ +const electronStub = { + name: 'electron-stub', + setup(build) { + build.onResolve({ filter: /^electron$/ }, () => ({ + path: 'electron', + namespace: 'electron-stub', + })); + build.onLoad({ filter: /.*/, namespace: 'electron-stub' }, () => ({ + contents: ` + const userData = ${JSON.stringify(path.join(os.tmpdir(), 'acappella-speech-latency'))}; + exports.app = { + isPackaged: false, + getAppPath: () => ${JSON.stringify(rootDir)}, + getPath: () => userData, + getVersion: () => '0.0.0-latency', + on: () => {}, + whenReady: () => Promise.resolve(), + }; + exports.ipcMain = { on: () => {}, handle: () => {} }; + exports.BrowserWindow = { getAllWindows: () => [] }; + `, + loader: 'js', + })); + }, +}; + +await esbuild.build({ + entryPoints: [path.join(__dirname, 'acappella-speech-latency.ts')], + bundle: true, + platform: 'node', + target: 'node20', + format: 'cjs', + outfile, + sourcemap: 'inline', + external: [ + 'fsevents', + 'node-pty', + 'node-llama-cpp', + '@node-llama-cpp/*', + 'better-sqlite3', + '@napi-rs/keyring', + ], + plugins: [electronStub], + logLevel: 'warning', +}); + +fs.mkdirSync(path.join(os.tmpdir(), 'acappella-speech-latency'), { recursive: true }); + +const child = spawn(process.execPath, [outfile, ...process.argv.slice(2)], { stdio: 'inherit' }); +child.on('exit', (code) => process.exit(code ?? 1)); diff --git a/scripts/acappella-speech-latency.ts b/scripts/acappella-speech-latency.ts new file mode 100644 index 0000000000..58ce326547 --- /dev/null +++ b/scripts/acappella-speech-latency.ts @@ -0,0 +1,610 @@ +/** + * A Cappella speech latency harness. + * + * The measurement half of the "Time to first spoken word" section in + * `docs/architecture/acappella/latency-baseline.md`. That number is the one a + * user actually feels - how long they stand in silence before anything is said - + * and everything in `src/main/acappella/speech/` exists to shorten it. + * + * Measuring it with a microphone and a live agent measures four things at once: + * the decode, the model on the day, the network on the day, and the streaming + * layer. Only the last of those is ours, and it is the only one a regression can + * be attributed to. So the providers here are stubs with DECLARED costs and + * everything between them is the shipping code - `AgentOutputTap`, + * `ConversationalTranslator`, `SpeechScheduler`, and the one splitter in + * `src/shared/acappella/sentences.ts`. Vary the declared costs and the arms move + * together; vary the layer and only the streamed arm moves. That is the property + * that makes this a baseline rather than a benchmark. + * + * Two arms per fixture, and the comparison between them IS the result: + * + * - `streamed` - the shipped path. The tap cuts at a completed thought while + * the agent is still writing, the translator rewrites that piece alone, and + * the scheduler speaks it. + * - `buffered` - the counterfactual the layer replaced: wait for the whole + * reply, rewrite the whole thing, then speak. + * + * The zero point is the agent's first token, not the detector's speech end. That + * is deliberate and it matches the doc's own definition of the **First spoken + * sentence** span: STT and routing happen before this layer is involved, and + * folding them in would hide the thing being measured behind two hops the tap + * cannot affect. + * + * What it does NOT measure, and must not be read as measuring: the realtime + * pipeline. A speech-to-speech provider produces audio directly, so none of this + * code runs and its span is the provider's own. That row in the doc stays empty + * until someone records it with a key and a microphone. + * + * Usage: + * + * npm run acappella:latency # both cascade profiles + * npm run acappella:latency -- --profile hosted + * npm run acappella:latency -- --runs 3 --json + * + * A full run takes a few minutes, and that is the fixtures rather than the + * harness: a long agent reply takes a long time to write, and shortening it would + * shorten the very silence the layer exists to fill. + */ + +import { EventEmitter } from 'events'; + +import { createAgentOutputTap } from '../src/main/acappella/speech/agent-output-tap'; +import { ConversationalTranslator } from '../src/main/acappella/speech/conversational-translator'; +import { SpeechScheduler } from '../src/main/acappella/speech/speech-scheduler'; +import { buildProcessSessionId } from '../src/main/dispatch-callbacks/dispatch-callback-registry'; +import type { + BrainProvider, + TtsChunk, + TtsProvider, + VoiceConverseContext, +} from '../src/shared/acappella/providers'; +import type { RouteDecision } from '../src/shared/acappella/route-decision'; + +// --------------------------------------------------------------------------- +// Provider cost profiles +// --------------------------------------------------------------------------- + +/** + * What a hop costs. Round numbers on purpose: these are the harness's INPUT, not + * a claim about any provider on any day. They exist so the two arms are compared + * under the same conditions, and so a change in the layer shows up as a change in + * the gap between them rather than as noise. + */ +interface Profile { + key: string; + label: string; + /** Silence before the rewrite's first token. A local model load is not included. */ + brainFirstTokenMs: number; + /** Between rewrite tokens, once it has started. */ + brainTokenMs: number; + /** Synthesis of one sentence, before any of its audio exists. */ + ttsBaseMs: number; + /** Added per character of the sentence being synthesised. */ + ttsPerCharMs: number; + /** How fast the agent itself writes, in characters per second. */ + agentCharsPerSecond: number; +} + +const PROFILES: Profile[] = [ + { + key: 'local', + label: 'Fully local cascade (Qwen3 Brain, Kokoro TTS)', + brainFirstTokenMs: 320, + brainTokenMs: 12, + ttsBaseMs: 180, + ttsPerCharMs: 1.2, + agentCharsPerSecond: 220, + }, + { + key: 'hosted', + label: 'Fully hosted cascade (OpenAI Brain, ElevenLabs TTS)', + brainFirstTokenMs: 480, + brainTokenMs: 8, + ttsBaseMs: 260, + ttsPerCharMs: 0.6, + agentCharsPerSecond: 220, + }, +]; + +/** + * Speaking rate used to simulate playback offline, in characters per second. + * + * Roughly 150 words a minute, which is unhurried assistant speech. Only the + * inter-sentence gap depends on it: a gap exists when the next sentence's audio + * is not ready by the time the current one stops being audible. + */ +const SPEECH_CHARS_PER_SECOND = 14; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +interface Fixture { + key: string; + label: string; + text: string; +} + +/** A paragraph of the long summary, repeated with varying detail. */ +function summaryParagraph(n: number): string { + return [ + `Step ${n}: reworked the token check so a refresh that lands mid-request is not`, + `treated as an expiry. The old path compared the issued-at stamp against the`, + `request clock, which drifts, so a request that arrived during a refresh was`, + `rejected and the client retried into the same window. It now compares against`, + `the session's own high-water mark, which only moves forward.`, + ].join(' '); +} + +/** + * The three shapes an agent replies in, at the sizes they really are. + * + * Size is not decoration here: the whole point of the tap is that a long reply + * takes a long time to WRITE, so shrinking the fixtures to make the harness quick + * would shrink the effect being measured. These are trimmed to the smallest size + * that still takes tens of seconds to produce, which is what makes a run a few + * minutes rather than a coffee break. + */ +const FIXTURES: Fixture[] = [ + { + key: 'long-summary', + label: 'A long implementation summary (about 100 lines)', + text: [ + 'I fixed the authentication bug. It was a stale token check in the refresh path.', + '', + ...Array.from({ length: 20 }, (_, i) => [summaryParagraph(i + 1), '']).flat(), + 'All 214 tests pass and the lint is clean.', + ].join('\n'), + }, + { + key: 'diff-heavy', + label: 'A diff-heavy reply', + text: [ + 'Here is the change to the middleware.', + '', + '```diff', + '--- a/src/auth/middleware.ts', + '+++ b/src/auth/middleware.ts', + ...Array.from({ length: 40 }, (_, i) => `-\tconst stale${i} = issuedAt < now;`), + ...Array.from({ length: 40 }, (_, i) => `+\tconst stale${i} = issuedAt < highWater;`), + '```', + '', + 'The high-water mark only moves forward, so a refresh landing mid-request is no', + 'longer read as an expiry and the client stops retrying into the same window.', + ].join('\n'), + }, + { + key: 'confirmation', + label: 'A one-line confirmation', + text: 'Yes, the tests pass.', + }, +]; + +// --------------------------------------------------------------------------- +// Stub providers +// --------------------------------------------------------------------------- + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * A Brain that writes a plausible spoken rewrite at the profile's declared rate. + * + * The content is fixed rather than generated: this harness measures WHEN words + * arrive, and a model deciding what to say would make two runs incomparable. The + * shape is what matters - two short sentences, no markdown - because that is what + * the translator and the splitter downstream have to handle. + */ +function stubBrain(profile: Profile): BrainProvider { + async function* write(agentText: string, context: VoiceConverseContext): AsyncIterable { + await sleep(profile.brainFirstTokenMs); + const words = rewriteFor(agentText).split(' '); + for (const word of words) { + if (context.signal?.aborted) return; + yield `${word} `; + await sleep(profile.brainTokenMs); + } + } + + return { + id: 'stub-brain', + label: 'Stub Brain', + tier: 'mock', + route: (): Promise => { + throw new Error('the latency harness never routes'); + }, + converse: async (agentText, context) => { + let whole = ''; + for await (const delta of write(agentText, context)) whole += delta; + return whole.trim(); + }, + converseStream: write, + }; +} + +/** Two sentences and nothing markdown-shaped, sized from the source. */ +function rewriteFor(agentText: string): string { + if (agentText.length < 200) return 'Yes, everything passes.'; + return ( + 'Done, I fixed the auth bug and it turned out to be a stale token check. ' + + 'Everything passes now.' + ); +} + +/** Synthesis whose cost scales with the sentence, which is how real TTS behaves. */ +function stubTts(profile: Profile): TtsProvider { + let cancelled = false; + let index = 0; + + return { + id: 'stub-tts', + label: 'Stub TTS', + tier: 'mock', + cancel: () => { + cancelled = true; + }, + speak: async function* (text: string, options): AsyncIterable { + await sleep(profile.ttsBaseMs + text.length * profile.ttsPerCharMs); + if (cancelled) return; + yield { + utteranceId: options.utteranceId, + index: index++, + text, + format: 'none', + audio: null, + }; + }, + }; +} + +// --------------------------------------------------------------------------- +// One measured turn +// --------------------------------------------------------------------------- + +interface TurnResult { + /** Agent first token to the first sentence whose audio exists. */ + firstSpokenWordMs: number; + /** + * The same, ignoring the tap's own status lines. + * + * These are not the same number and conflating them flatters the result. A + * reply the layer cannot start speaking still produces sound at the twenty + * second mark, because the tap says the agent is still working rather than + * going silent. Counting that as the first spoken word would score the safety + * net as if it were the answer. + */ + firstAnswerWordMs: number; + /** Longest silence between two sentences, once playback is simulated. */ + maxGapMs: number; + meanGapMs: number; + sentences: number; + /** The rewrite reached the model, or was passed through untouched. */ + translated: boolean; + /** + * When each sentence's audio existed, in order. + * + * Reported under `--json` because an aggregate gap is not diagnosable: a long + * one can be the layer stalling or the agent writing something unspeakable, + * and only the stamps say which. + */ + stamps: { at: number; chars: number; status: boolean }[]; +} + +const AGENT_SESSION_ID = 'sess-backend'; +const TAB_ID = 'tab-auth-refactor'; + +/** + * Run one fixture through the real layer and record when each sentence's audio + * became available. + * + * `arm` decides only WHEN the agent's text reaches the tap: `streamed` writes it + * at the profile's rate, `buffered` withholds every character until the reply is + * finished. Nothing else differs, so the difference in the result is the tap. + */ +async function measureTurn( + profile: Profile, + fixture: Fixture, + arm: 'streamed' | 'buffered' +): Promise { + const source = new EventEmitter(); + const processSessionId = buildProcessSessionId(AGENT_SESSION_ID, TAB_ID); + const brain = stubBrain(profile); + const translator = new ConversationalTranslator({ brain }); + + /** When each sentence's audio existed, relative to the agent's first token. */ + const available: { text: string; at: number; status: boolean }[] = []; + /** + * Sentences the tap produced about itself rather than about the answer. + * + * A status chunk is passed through the translator verbatim, so its sentences + * arrive at the scheduler with the text they were written with and can be told + * apart by identity. Threading a kind through the scheduler instead would put + * a harness concern into the protocol. + */ + const statusSentences = new Set(); + let startedAt = 0; + + const scheduler = new SpeechScheduler({ + tts: stubTts(profile), + onStart: () => {}, + onSentence: (event) => + available.push({ + text: event.text, + at: Date.now() - startedAt, + status: statusSentences.has(event.text), + }), + onEnd: () => {}, + }); + + const translations: Promise[] = []; + const tap = createAgentOutputTap({ + source, + onChunk: (chunk) => { + translations.push( + (async () => { + for await (const sentence of translator.translate({ + agentSessionId: chunk.agentSessionId, + tabId: chunk.tabId, + text: chunk.text, + kind: chunk.kind, + })) { + if (chunk.kind === 'status') statusSentences.add(sentence); + scheduler.pushSentence(sentence); + } + })() + ); + }, + }); + + tap.watch({ agentSessionId: AGENT_SESSION_ID, tabId: TAB_ID }); + scheduler.begin(`utt-${fixture.key}-${arm}`); + startedAt = Date.now(); + + await writeAgentOutput(source, processSessionId, fixture.text, profile, arm); + source.emit('query-complete', processSessionId); + + // Every rewrite that the tap started has to finish before the run can be + // closed, or the scheduler would end on a gap in the translation rather than + // on the end of the reply. + await Promise.all(translations); + scheduler.close(); + await scheduler.drained(); + tap.dispose(); + + return summarise(available, translator.stats.translations > 0); +} + +/** + * Feed the agent's reply to the tap the way the process manager would. + * + * `buffered` is not "one event at the end of an instant reply": the agent takes + * just as long to write either way. Withholding the text until the reply is + * finished, at the same write rate, is what makes the arms comparable. + */ +async function writeAgentOutput( + source: EventEmitter, + processSessionId: string, + text: string, + profile: Profile, + arm: 'streamed' | 'buffered' +): Promise { + const perEvent = 120; + const delay = (perEvent / profile.agentCharsPerSecond) * 1000; + + if (arm === 'buffered') { + // The same write time as the streamed arm, spent in one silence instead of + // spread across events. Anything else would compare two different agents. + const writeMs = (Math.max(0, text.length - perEvent) / profile.agentCharsPerSecond) * 1000; + await sleep(writeMs); + source.emit('data', processSessionId, text); + return; + } + + for (let i = 0; i < text.length; i += perEvent) { + // Before the write, not after: a sleep following the LAST event would be + // charged to the layer as silence the agent had already stopped producing, + // and on a one-line reply that artefact is bigger than the thing measured. + if (i > 0) await sleep(delay); + source.emit('data', processSessionId, text.slice(i, i + perEvent)); + } +} + +/** + * Turn availability stamps into the two numbers the doc asks for. + * + * The gap is simulated rather than observed because the scheduler hands audio to + * a sink and does not wait for it to be heard. A sentence's audio being ready + * before the previous one stops being audible is exactly the no-gap property, and + * it is computable from the stamps: play serially at a speaking rate and see + * where the player runs out of material. + */ +function summarise( + available: { text: string; at: number; status: boolean }[], + translated: boolean +): TurnResult { + const stamps = available.map((entry) => ({ + at: entry.at, + chars: entry.text.length, + status: entry.status, + })); + if (available.length === 0) { + return { + firstSpokenWordMs: -1, + firstAnswerWordMs: -1, + maxGapMs: 0, + meanGapMs: 0, + sentences: 0, + translated, + stamps, + }; + } + + const gaps: number[] = []; + let playbackEnd = available[0].at; + for (let i = 0; i < available.length; i++) { + const start = i === 0 ? available[0].at : Math.max(playbackEnd, available[i].at); + if (i > 0) gaps.push(Math.max(0, available[i].at - playbackEnd)); + playbackEnd = start + (available[i].text.length / SPEECH_CHARS_PER_SECOND) * 1000; + } + + return { + firstSpokenWordMs: available[0].at, + firstAnswerWordMs: available.find((entry) => !entry.status)?.at ?? -1, + maxGapMs: gaps.length ? Math.max(...gaps) : 0, + meanGapMs: gaps.length ? gaps.reduce((sum, gap) => sum + gap, 0) / gaps.length : 0, + sentences: available.length, + translated, + stamps, + }; +} + +// --------------------------------------------------------------------------- +// The run +// --------------------------------------------------------------------------- + +interface Row { + profile: string; + fixture: string; + arm: 'streamed' | 'buffered'; + firstSpokenWordMs: number; + firstAnswerWordMs: number; + maxGapMs: number; + meanGapMs: number; + sentences: number; + translated: boolean; + /** First run's stamps, for diagnosing a gap rather than just reporting one. */ + stamps: { at: number; chars: number; status: boolean }[]; +} + +interface Options { + profiles: Profile[]; + runs: number; + json: boolean; +} + +async function run(options: Options): Promise { + const rows: Row[] = []; + + for (const profile of options.profiles) { + for (const fixture of FIXTURES) { + for (const arm of ['streamed', 'buffered'] as const) { + const results: TurnResult[] = []; + for (let i = 0; i < options.runs; i++) { + results.push(await measureTurn(profile, fixture, arm)); + } + rows.push({ + profile: profile.key, + fixture: fixture.key, + arm, + firstSpokenWordMs: median(results.map((r) => r.firstSpokenWordMs)), + firstAnswerWordMs: median(results.map((r) => r.firstAnswerWordMs)), + maxGapMs: median(results.map((r) => r.maxGapMs)), + meanGapMs: median(results.map((r) => r.meanGapMs)), + sentences: results[0].sentences, + translated: results[0].translated, + stamps: results[0].stamps, + }); + } + } + } + + report(options, rows); +} + +/** Median, because the first run of anything pays for a warm-up nobody hears twice. */ +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + const value = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : (sorted[mid] ?? 0); + return Math.round(value); +} + +function report(options: Options, rows: Row[]): void { + if (options.json) { + console.log(JSON.stringify({ rows, speechCharsPerSecond: SPEECH_CHARS_PER_SECOND }, null, 2)); + return; + } + + console.log(''); + console.log(`Runs per cell: ${options.runs} (median reported). Zero point: agent first token.`); + + for (const profile of options.profiles) { + console.log(''); + console.log(profile.label); + console.log(''); + console.log( + '| Fixture | Arm | First sound | First word of the answer | Max gap | Sentences | Rewrite |' + ); + console.log( + '| ------- | --- | ----------- | ------------------------ | ------- | --------- | ------- |' + ); + for (const row of rows.filter((candidate) => candidate.profile === profile.key)) { + console.log( + `| ${row.fixture} | ${row.arm} | ${row.firstSpokenWordMs} ms | ` + + `${row.firstAnswerWordMs} ms | ${row.maxGapMs} ms | ${row.sentences} | ` + + `${row.translated ? 'model' : 'passthrough'} |` + ); + } + } + + console.log(''); + for (const profile of options.profiles) { + for (const fixture of FIXTURES) { + const streamed = find(rows, profile.key, fixture.key, 'streamed'); + const buffered = find(rows, profile.key, fixture.key, 'buffered'); + if (!streamed || !buffered) continue; + // Against the answer, not against the first sound: the buffered arm's first + // sound on a long reply is the tap's twenty second hang notice, and scoring + // the layer against its own safety net would understate it by a factor of + // three. + const saved = buffered.firstAnswerWordMs - streamed.firstAnswerWordMs; + console.log( + `${profile.key}/${fixture.key}: the tap saves ${saved} ms of silence ` + + `(${buffered.firstAnswerWordMs} ms buffered, ${streamed.firstAnswerWordMs} ms streamed).` + ); + } + } + + console.log(''); + console.log('Paste the rows into docs/architecture/acappella/latency-baseline.md.'); + console.log('Realtime is not measured here: that provider speaks directly and this layer'); + console.log('never runs. Its row stays empty until it is recorded with a key and a mic.'); +} + +function find(rows: Row[], profile: string, fixture: string, arm: Row['arm']): Row | undefined { + return rows.find((row) => row.profile === profile && row.fixture === fixture && row.arm === arm); +} + +// --------------------------------------------------------------------------- +// Entry +// --------------------------------------------------------------------------- + +function parseArgs(argv: string[]): Options { + let profileKey = 'both'; + // One by default. A cell's cost is dominated by declared sleeps rather than by + // machine load, so repeats buy little, and a fixture that takes half a minute + // to write makes three of everything a five minute wait. + let runs = 1; + let json = false; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--profile') profileKey = argv[++i]; + else if (arg === '--runs') runs = Number(argv[++i]); + else if (arg === '--json') json = true; + else throw new Error(`Unknown option: ${arg}`); + } + + const profiles = + profileKey === 'both' ? PROFILES : PROFILES.filter((profile) => profile.key === profileKey); + if (profiles.length === 0) throw new Error(`Unknown profile: ${profileKey}`); + if (!Number.isFinite(runs) || runs < 1) throw new Error(`Runs must be at least 1`); + + return { profiles, runs, json }; +} + +run(parseArgs(process.argv.slice(2))).catch((error: Error) => { + console.error(`Speech latency harness failed: ${error.message}`); + process.exit(1); +}); diff --git a/scripts/verify-native-packaging.mjs b/scripts/verify-native-packaging.mjs new file mode 100644 index 0000000000..f7b78481ef --- /dev/null +++ b/scripts/verify-native-packaging.mjs @@ -0,0 +1,271 @@ +#!/usr/bin/env node +/** + * Packaging assertion for A Cappella's native runtimes. + * + * The bug this exists to catch does not appear in development. A native module + * left inside `app.asar`, or a per-platform prebuild that never got copied, + * works perfectly from source and fails only in the installed, signed app, on + * someone else's machine, after release. That is the most expensive kind of bug + * this codebase can ship, and it is entirely mechanical to detect. + * + * So: after packaging, walk the built app and assert, per runtime, + * + * 1. the package is present at all, + * 2. its platform binary exists at the path the registry promises, + * 3. that binary is UNPACKED (inside `app.asar.unpacked`, not `app.asar`), + * 4. on macOS, that every nested binary carries a signature, because + * notarization rejects a bundle containing an unsigned nested binary and + * the rejection arrives long after the build. + * + * A runtime whose package is not a dependency yet is reported and skipped: this + * script fails on things that are broken, not on work that has not started. + * Passing `--require-all` turns those skips into failures, which is what a + * release build should use once the providers land. + * + * The runtime facts come from `dist/shared/acappella/native-runtimes.js`, the + * compiled copy of the one registry the app itself reads, so this script cannot + * drift from what the loader expects. + * + * Usage: + * node scripts/verify-native-packaging.mjs [--app ] + * [--require-all] [--json] + */ + +import { createRequire } from 'node:module'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const require = createRequire(import.meta.url); +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const args = process.argv.slice(2); +const requireAll = args.includes('--require-all'); +const asJson = args.includes('--json'); +const appArgIndex = args.indexOf('--app'); +const appArg = appArgIndex >= 0 ? args[appArgIndex + 1] : null; + +/** Load the compiled runtime registry. Built by `npm run build:main`. */ +function loadRegistry() { + const compiled = path.join(repoRoot, 'dist', 'shared', 'acappella', 'native-runtimes.js'); + if (!fs.existsSync(compiled)) { + fail( + `Runtime registry not found at ${rel(compiled)}. Run "npm run build:main" before this script.` + ); + } + return require(compiled); +} + +function rel(target) { + return path.relative(repoRoot, target) || target; +} + +function fail(message) { + console.error(`\nverify-native-packaging: ${message}\n`); + process.exit(1); +} + +/** + * Find the packaged app's resources directory. + * + * Handles the three layouts electron-builder produces: a macOS .app bundle, a + * Windows/Linux unpacked directory, and the `release/` tree containing either. + */ +function resolveResourcesDir() { + const candidates = []; + if (appArg) candidates.push(path.resolve(appArg)); + + const releaseDir = path.join(repoRoot, 'release'); + if (fs.existsSync(releaseDir)) { + for (const entry of fs.readdirSync(releaseDir)) { + candidates.push(path.join(releaseDir, entry)); + } + } + + for (const candidate of candidates) { + if (!fs.existsSync(candidate)) continue; + + // macOS: .app/Contents/Resources + if (candidate.endsWith('.app')) { + const macResources = path.join(candidate, 'Contents', 'Resources'); + if (fs.existsSync(macResources)) return { resources: macResources, app: candidate }; + } + + // A directory holding a .app (release/mac-arm64/Maestro.app) + if (fs.statSync(candidate).isDirectory()) { + const nested = fs + .readdirSync(candidate) + .filter((entry) => entry.endsWith('.app')) + .map((entry) => path.join(candidate, entry, 'Contents', 'Resources')) + .find((entry) => fs.existsSync(entry)); + if (nested) return { resources: nested, app: path.dirname(path.dirname(nested)) }; + + // Windows/Linux: /resources + const resources = path.join(candidate, 'resources'); + if (fs.existsSync(resources)) return { resources, app: candidate }; + } + } + + return null; +} + +/** + * Whether a Mach-O binary carries a signature. + * + * `codesign --verify` on each nested binary rather than one `--deep` pass on the + * bundle: --deep reports the first failure and stops, and the useful output here + * is the full list of what is unsigned. + */ +function isSigned(binaryPath) { + try { + execFileSync('codesign', ['--verify', '--strict', binaryPath], { stdio: 'pipe' }); + return true; + } catch { + return false; + } +} + +function main() { + const { NATIVE_RUNTIMES, nativePlatformKey } = loadRegistry(); + const platformKey = nativePlatformKey(process.platform, process.arch); + if (!platformKey) { + fail(`No packaging matrix for ${process.platform}-${process.arch}.`); + } + + const located = resolveResourcesDir(); + if (!located) { + fail( + 'No packaged app found. Run an electron-builder target first, or pass --app .' + ); + } + + const unpackedRoot = path.join(located.resources, 'app.asar.unpacked'); + + // The package.json inside the bundle is what actually shipped, which is the + // only honest answer to "is this runtime a dependency of THIS build". + const shippedDeps = readShippedDependencies(located.resources); + + const results = []; + let failures = 0; + let skipped = 0; + + for (const runtime of NATIVE_RUNTIMES) { + const expected = runtime.packagedBinaries[platformKey] ?? []; + const declared = runtime.declared; + const shipped = shippedDeps ? shippedDeps.has(runtime.moduleId) : declared; + + // Not a dependency yet: nothing is broken, so this is a skip. `--require-all` + // is how a release build says "by now they should all be here". + if (!declared) { + // Counted as one or the other, never both: a line that reads "3 failed, 3 + // skipped" over three runtimes is a summary nobody can act on. + if (requireAll) failures += 1; + else skipped += 1; + results.push({ + runtime: runtime.id, + moduleId: runtime.moduleId, + status: requireAll ? 'fail' : 'skip', + reason: `${runtime.moduleId} is not a dependency of this build yet (registry declared=false).`, + }); + continue; + } + + // Declared but absent from what shipped is always a packaging bug: the app + // will try to load it and will not find it. + if (!shipped) { + failures += 1; + results.push({ + runtime: runtime.id, + moduleId: runtime.moduleId, + status: 'fail', + reason: `${runtime.moduleId} is a declared runtime but is missing from the packaged dependencies.`, + }); + continue; + } + + if (runtime.prebuilds[platformKey] === 'unavailable') { + results.push({ + runtime: runtime.id, + moduleId: runtime.moduleId, + status: 'skip', + reason: `No build of ${runtime.moduleId} exists for ${platformKey}.`, + }); + skipped += 1; + continue; + } + + const missing = []; + const unsigned = []; + + for (const relativePath of expected) { + const normalized = relativePath.split('/').join(path.sep); + const absolute = path.join(unpackedRoot, normalized); + if (!fs.existsSync(absolute)) { + missing.push(relativePath); + continue; + } + if (process.platform === 'darwin' && !isSigned(absolute)) unsigned.push(relativePath); + } + + const problems = []; + if (missing.length) { + // Both causes are named, because the evidence here cannot tell them apart + // without reading the asar index and the fixes differ: either the file + // never made it into the build, or it is inside app.asar with no + // asarUnpack entry, which works from source and dies once installed. + problems.push( + `not in app.asar.unpacked (absent from the build, or still packed inside app.asar): ${missing.join(', ')}` + ); + } + if (unsigned.length) problems.push(`unsigned: ${unsigned.join(', ')}`); + + if (problems.length) failures += 1; + results.push({ + runtime: runtime.id, + moduleId: runtime.moduleId, + status: problems.length ? 'fail' : 'pass', + reason: problems.join('; ') || `${expected.length} binaries present and signed.`, + }); + } + + if (asJson) { + console.log(JSON.stringify({ platformKey, app: located.app, results }, null, 2)); + } else { + console.log(`\nNative packaging check: ${located.app} (${platformKey})`); + for (const result of results) { + console.log( + ` ${result.status.toUpperCase().padEnd(5)} ${result.moduleId}: ${result.reason}` + ); + } + // Never silently truncate: a skipped runtime is stated, because "3 passed" + // over a list where two were skipped reads as full coverage. + console.log(` ${results.length} runtimes, ${failures} failed, ${skipped} skipped\n`); + } + + if (failures > 0) { + fail(`${failures} native runtime(s) are not correctly packaged. See the list above.`); + } +} + +/** Dependencies of the package.json that actually shipped inside the bundle. */ +function readShippedDependencies(resourcesDir) { + const candidates = [ + path.join(resourcesDir, 'app.asar.unpacked', 'package.json'), + path.join(resourcesDir, 'app', 'package.json'), + ]; + for (const candidate of candidates) { + if (!fs.existsSync(candidate)) continue; + try { + const parsed = JSON.parse(fs.readFileSync(candidate, 'utf8')); + return new Set(Object.keys(parsed.dependencies ?? {})); + } catch { + // A package.json we cannot read tells us nothing; fall through to the + // registry's own view rather than failing the build on a parse error. + return null; + } + } + return null; +} + +main(); diff --git a/src/__tests__/acappella/conformance/data-channel.conformance.test.ts b/src/__tests__/acappella/conformance/data-channel.conformance.test.ts new file mode 100644 index 0000000000..c3fda9bffa --- /dev/null +++ b/src/__tests__/acappella/conformance/data-channel.conformance.test.ts @@ -0,0 +1,368 @@ +/** + * Protocol conformance: the data channel, C-16 to C-30 and C-37 to C-40. + * + * Every frame here really crosses a channel: it is encoded by the reference + * client, decoded by `decodeDeviceMessage()` inside the desktop's own peer, and + * whatever comes back is decoded again on the way in. A message that the + * desktop drops is dropped in complete silence, which is exactly why these are + * asserted against `world.deviceMessages` (what the desktop DECODED) rather + * than against what the client believes it sent. + * + * The checklist items about drawing (C-24 to C-28) are asserted here as their + * wire half: that the event arrives intact and unclamped, so the client has + * what it needs. Their UI half lives in + * `src/__tests__/web-desktop/acappella-client/ui.test.tsx`. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + DEVICE_ORIGINATED_MESSAGES, + RELIABLE_CHANNEL_LABEL, + UNRELIABLE_CHANNEL_LABEL, + deviceChannelForMessage, + type DeviceMessage, +} from '../../../shared/acappella/device-protocol'; +import type { RosterAgent, VoiceEvent } from '../../../shared/acappella/protocol'; +import { + createConformanceWorld, + voiceEventsFrom, + type ConformanceClient, + type ConformanceWorld, +} from './harness'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock('../../../renderer/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +let world: ConformanceWorld; +let device: ConformanceClient; + +/** Take the floor properly: press, let the desktop grant it, wait for the mic. */ +async function openFloor(): Promise { + device.client.pressFloor(); + await world.advance(); + world.session.emit({ + type: 'listen-start', + scope: { kind: 'conductor' }, + sttProviderId: 'local', + }); + await world.advance(); +} + +beforeEach(async () => { + world = await createConformanceWorld(); + device = await world.connectClient(); +}); + +afterEach(async () => { + await world.dispose(); +}); + +describe('conformance: channels', () => { + it('creates both channels with the exact labels and inits, before the offer (C-16, C-17)', () => { + expect(device.peer().channel(RELIABLE_CHANNEL_LABEL).init).toEqual({ ordered: true }); + expect(device.peer().channel(UNRELIABLE_CHANNEL_LABEL).init).toEqual({ + ordered: false, + maxRetransmits: 0, + }); + // Both existed before the SDP crossed the wire, so the first offer already + // carried the SCTP association. + expect(device.peer().localDescription?.type).toBe('offer'); + expect(device.peer().channels).toHaveLength(2); + }); + + it('closes a channel whose label it does not recognise, without a word (C-16)', async () => { + const rogue = device.peer().createDataChannel('acappella-typo', { ordered: true }); + await world.advance(); + + expect(rogue.readyState).toBe('closed'); + // And the two real ones are untouched. + expect(device.peer().channel(RELIABLE_CHANNEL_LABEL).readyState).toBe('open'); + expect(device.peer().channel(UNRELIABLE_CHANNEL_LABEL).readyState).toBe('open'); + }); + + it('stamps every outbound frame with the negotiated version (C-18, C-35)', async () => { + await openFloor(); + device.client.requestBargeIn(); + await world.advance(); + + const negotiated = device.state().protocolVersion; + expect(negotiated).toBeGreaterThanOrEqual(1); + const frames = device.rawSentFrames(); + expect(frames.length).toBeGreaterThan(0); + for (const frame of frames) expect(frame.v).toBe(negotiated); + }); + + it('sends hello first on the state channel, with a complete identity (C-19)', () => { + const [first] = device.sentFrames('reliable'); + expect(first).toMatchObject({ + type: 'hello', + identity: { deviceId: device.deviceId, name: device.name, platform: 'ios' }, + }); + // And the desktop decoded it, which is what puts the name on the device row. + expect(world.deviceMessages[0]).toMatchObject({ + deviceId: device.deviceId, + message: { type: 'hello' }, + }); + }); + + it('sends only the five device-originated types, on the channels the table names (C-20, C-21)', async () => { + await openFloor(); + device.client.pressFloor({ kind: 'agent', sessionId: 'agent-7' }); + device.client.reportAudioLevel(0.4, true); + device.client.requestStop(); + await world.advance(2500); + + for (const { message } of world.deviceMessages) { + expect(DEVICE_ORIGINATED_MESSAGES).toContain(message.type); + } + const reliable = device.sentFrames('reliable'); + const unreliable = device.sentFrames('unreliable'); + for (const message of reliable) expect(deviceChannelForMessage(message)).toBe('reliable'); + for (const message of unreliable) expect(deviceChannelForMessage(message)).toBe('unreliable'); + expect(unreliable.map((message) => message.type)).toContain('floor'); + expect(unreliable.map((message) => message.type)).toContain('interrupt'); + }); + + it('puts each desktop message on the channel the table names too (C-21)', async () => { + world.session.emit({ type: 'partial-transcript', text: 'open the', stability: 0.4 }); + world.session.emit({ type: 'final-transcript', text: 'open the auth tab' }); + await world.advance(); + + const reliable = device.receivedFrames('reliable'); + const unreliable = device.receivedFrames('unreliable'); + for (const message of reliable) expect(deviceChannelForMessage(message)).toBe('reliable'); + for (const message of unreliable) expect(deviceChannelForMessage(message)).toBe('unreliable'); + expect(voiceEventsFrom(unreliable, 'partial-transcript')).toHaveLength(1); + expect(voiceEventsFrom(reliable, 'final-transcript')).toHaveLength(1); + }); +}); + +describe('conformance: malformed and unknown frames', () => { + it('drops a malformed frame from a device without closing anything (C-22)', async () => { + const live = device.peer().channel(UNRELIABLE_CHANNEL_LABEL); + live.send('not json'); + live.send(JSON.stringify({ type: 'floor', action: 'press' })); // no `v` + live.send(JSON.stringify({ type: 'invented', v: 1 })); + live.send(JSON.stringify({ type: 'floor', action: 'sideways', v: 1 })); + await world.advance(); + + expect(world.deviceMessages.filter((entry) => entry.message.type === 'floor')).toHaveLength(0); + expect(live.readyState).toBe('open'); + // The connection is still fully usable afterwards. + device.client.pressFloor(); + await world.advance(); + expect(world.deviceMessages.some((entry) => entry.message.type === 'floor')).toBe(true); + }); + + it('drops a malformed frame from the desktop without closing anything (C-22)', async () => { + const channel = device.desktopPeer().channel(RELIABLE_CHANNEL_LABEL); + channel.send('not json'); + channel.send(JSON.stringify({ type: 'floor-state', holder: 'someone', isSelf: true })); + await world.advance(); + + expect(device.state().phase).toBe('connected'); + expect(device.state().floor.isSelf).toBe(false); + expect(device.peer().channel(RELIABLE_CHANNEL_LABEL).readyState).toBe('open'); + }); + + it('ignores an unknown voice event and keeps processing the stream (C-23)', async () => { + world.session.emit({ type: 'invented-event' as VoiceEvent['type'] }); + world.session.emit({ type: 'listen-stop', reason: 'stopped' }); + await world.advance(); + + expect(device.state().phase).toBe('connected'); + expect(voiceEventsFrom(device.receivedFrames(), 'listen-stop')).toHaveLength(1); + }); + + it('flags a seq gap on the reliable channel rather than stitching over it (C-29)', async () => { + world.session.emit({ type: 'listen-stop', reason: 'stopped' }); + await world.advance(); + expect(device.state().transcriptSuspect).toBe(false); + + // Five events the client never saw. + world.session.seq += 5; + world.session.emit({ type: 'listen-stop', reason: 'stopped' }); + await world.advance(); + expect(device.state().transcriptSuspect).toBe(true); + }); +}); + +describe('conformance: the session-event catalogue round-trips', () => { + it('carries an agent roster the client can replace its wheel from (C-24)', async () => { + const agents: RosterAgent[] = [ + { + sessionId: 'agent-1', + name: 'acappella', + agentType: 'claude-code', + status: 'idle', + cwd: '/tmp/one', + tabs: [{ id: 'tab-1', name: 'Phase 11', lastActiveAt: 42, state: 'open' }], + }, + ]; + world.session.emit({ type: 'agent-roster', agents }); + await world.advance(); + + const [roster] = voiceEventsFrom(device.receivedFrames(), 'agent-roster'); + expect(roster).toMatchObject({ type: 'agent-roster', agents }); + }); + + it('carries a route correction as its own event, with both targets (C-25)', async () => { + world.session.emit({ + type: 'route-correction', + fromAgentSessionId: 'agent-1', + fromTabId: 'tab-1', + agentSessionId: 'agent-2', + agentName: 'the other one', + tabId: 'tab-9', + action: 'created', + promptSent: true, + source: 'voice', + }); + await world.advance(); + + // Both ends of the correction travel, which is what lets a client rewrite + // the caption it already drew instead of appending a second row. + const [correction] = voiceEventsFrom(device.receivedFrames(), 'route-correction'); + expect(correction).toMatchObject({ + fromAgentSessionId: 'agent-1', + fromTabId: 'tab-1', + agentSessionId: 'agent-2', + tabId: 'tab-9', + }); + }); + + it('carries a provisional sentence count and an index past it, unclamped (C-26, C-27)', async () => { + world.session.emit({ + type: 'speak-start', + utteranceId: 'utt-1', + sentenceCount: 2, + ttsProviderId: 'local', + streaming: true, + }); + world.session.emit({ type: 'speak-sentence', utteranceId: 'utt-1', index: 4, text: 'Fourth.' }); + await world.advance(); + + const [start] = voiceEventsFrom(device.receivedFrames(), 'speak-start'); + const [sentence] = voiceEventsFrom(device.receivedFrames(), 'speak-sentence'); + expect(start).toMatchObject({ sentenceCount: 2, streaming: true }); + // The index is delivered as sent. Clamping it to `sentenceCount` here would + // hide the streaming case from every client at once. + expect(sentence).toMatchObject({ utteranceId: 'utt-1', index: 4 }); + }); + + it('describes the DESKTOP microphone in mic-state and leaves the phone"s alone (C-28)', async () => { + world.session.emit({ + type: 'mic-state', + permission: 'granted', + capturing: true, + deviceId: 'builtin', + deviceLabel: 'MacBook Pro Microphone', + issue: null, + deviceChanged: false, + }); + await world.advance(); + + expect(voiceEventsFrom(device.receivedFrames(), 'mic-state')).toHaveLength(1); + // Nothing about the desktop's microphone opened this phone's. + expect(device.state().sending).toBe(false); + expect(device.client.microphone).toBeNull(); + }); + + it('carries the egress statement verbatim (C-30)', async () => { + const egressStatement = 'Audio stays on this machine. Nothing is sent to a provider.'; + world.session.emit({ + type: 'provider-state', + pipeline: 'cascade', + slots: [ + { role: 'stt', providerId: 'whisper-local', label: 'Whisper (local)', tier: 'local' }, + ], + audioLeavesMachine: false, + egressStatement, + }); + await world.advance(); + + const [state] = voiceEventsFrom(device.receivedFrames(), 'provider-state'); + expect(state).toMatchObject({ audioLeavesMachine: false, egressStatement }); + }); +}); + +describe('conformance: the microphone is gated on the floor', () => { + it('captures nothing before the desktop grants the floor (C-37, C-38)', async () => { + // Pressed, but the desktop has not answered yet. + device.client.pressFloor(); + expect(world.captured).toHaveLength(0); + expect(device.client.microphone).toBeNull(); + + await world.advance(); + expect(device.state().floor.isSelf).toBe(true); + expect(device.client.microphone).not.toBeNull(); + // And only now does a stream reach the desktop's capture pipeline. + expect(world.captured).toEqual([{ deviceId: device.deviceId }]); + }); + + it('stops capturing the moment the floor closes (C-37)', async () => { + await openFloor(); + expect(device.state().sending).toBe(true); + + world.session.emit({ type: 'listen-stop', reason: 'endpoint' }); + await world.advance(); + + expect(device.state().floor.isSelf).toBe(false); + expect(device.state().sending).toBe(false); + expect(device.micTrack.stop).toHaveBeenCalled(); + }); + + it('sends audio-level only while the floor is open, and throttled (C-39)', async () => { + device.client.reportAudioLevel(0.4, true); + await world.advance(); + expect( + world.deviceMessages.filter((entry) => entry.message.type === 'audio-level') + ).toHaveLength(0); + + await openFloor(); + device.client.reportAudioLevel(0.4, true); + device.client.reportAudioLevel(0.5, true); // Same millisecond: throttled away. + await world.advance(60); + device.client.reportAudioLevel(0.6, false); + await world.advance(); + + const levels = world.deviceMessages + .map((entry) => entry.message) + .filter((message): message is Extract => { + return message.type === 'audio-level'; + }); + expect(levels).toHaveLength(2); + expect(levels[1]).toMatchObject({ level: 0.6, speech: false }); + }); + + it('reports link quality from a throttled getStats (C-40)', async () => { + device.peer().statsReports = [ + { + type: 'candidate-pair', + selected: true, + currentRoundTripTime: 0.031, + localCandidateId: 'L', + remoteCandidateId: 'R', + }, + { type: 'local-candidate', id: 'L', candidateType: 'host' }, + { type: 'remote-candidate', id: 'R', candidateType: 'host' }, + { type: 'inbound-rtp', kind: 'audio', jitter: 0.004, packetsReceived: 99, packetsLost: 1 }, + ]; + await world.advance(2100); + + const quality = world.deviceMessages + .map((entry) => entry.message) + .filter((message): message is Extract => { + return message.type === 'link-quality'; + }); + expect(quality).toHaveLength(1); + expect(quality[0]).toMatchObject({ rttMs: 31, candidateType: 'lan' }); + // One measurement, one bar: the client shows what it sent. + expect(device.state().quality).toMatchObject({ rttMs: 31, candidateType: 'lan' }); + }); +}); diff --git a/src/__tests__/acappella/conformance/failure-paths.conformance.test.ts b/src/__tests__/acappella/conformance/failure-paths.conformance.test.ts new file mode 100644 index 0000000000..028e6636d7 --- /dev/null +++ b/src/__tests__/acappella/conformance/failure-paths.conformance.test.ts @@ -0,0 +1,452 @@ +/** + * Protocol conformance: the paths a phone actually hits, C-12 to C-15 and C-33 + * to C-49. + * + * The happy path is the easy half. What breaks an iOS client in the field is a + * version bump on the desktop, a pairing revoked from another room, a train + * going into a tunnel, a second device grabbing the floor, and a stop word + * spoken while the assistant is mid-sentence. Every one of those runs here + * against the real desktop stack, because every one of them is a place where + * "the desktop just stops answering" is the symptom a client sees and cannot + * diagnose. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + DEVICE_PROTOCOL_VERSION, + MIN_SUPPORTED_DEVICE_PROTOCOL_VERSION, + RELIABLE_CHANNEL_LABEL, + type DeviceMessage, +} from '../../../shared/acappella/device-protocol'; +import { createConformanceWorld, voiceEventsFrom, type ConformanceWorld } from './harness'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock('../../../renderer/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +let world: ConformanceWorld; + +beforeEach(async () => { + world = await createConformanceWorld(); +}); + +afterEach(async () => { + await world.dispose(); +}); + +/** Floor-state frames one device was sent, newest last. */ +function floorStates( + frames: DeviceMessage[] +): Array> { + return frames.filter( + (frame): frame is Extract => + frame.type === 'floor-state' + ); +} + +describe('conformance: version mismatch', () => { + it('refuses a version below the window before it looks at the credential (C-33)', async () => { + const raw = world.openRawDevice(); + await raw.send({ + op: 'auth', + deviceId: 'whoever', + token: 'whatever', + protocolVersion: MIN_SUPPORTED_DEVICE_PROTOCOL_VERSION - 1, + }); + + const [error] = raw.ofType('error'); + expect(error.code).toBe('protocol-version'); + // While the floor and the ceiling are both v1, everything below the floor + // is also below 1 and reads as unusable rather than as merely old. The + // `client-too-old` sentence itself is exercised through + // `negotiateProtocolVersion`'s range parameter in the device-protocol unit + // tests, which is what that parameter exists for. + expect(error.message).toBe('This device did not report a usable A Cappella protocol version.'); + // The credential was never looked at: a client that cannot be talked to + // correctly is told THAT rather than authenticated into silence. + expect(raw.ofType('auth-failed')).toHaveLength(0); + expect(raw.ofType('authenticated')).toHaveLength(0); + }); + + it('refuses a client from above the window, and blames the desktop (C-33)', async () => { + const raw = world.openRawDevice(); + await raw.send({ + op: 'auth', + deviceId: 'whoever', + token: 'whatever', + protocolVersion: DEVICE_PROTOCOL_VERSION + 1, + }); + + expect(raw.ofType('error')[0].message).toContain('Update Maestro on the desktop'); + }); + + it('treats a missing version as too old rather than as unversioned (C-06)', async () => { + const raw = world.openRawDevice(); + await raw.send({ op: 'auth', deviceId: 'whoever', token: 'whatever' }); + + expect(raw.ofType('error')[0].code).toBe('protocol-version'); + }); + + it('is terminal at the client, keeps the Keychain item, and offers no retry (C-33, C-34)', async () => { + const device = await world.connectClient(); + const raw = world.openRawDevice(); + await raw.send({ op: 'auth', deviceId: 'x', token: 'y', protocolVersion: 99 }); + const rejection = raw.ofType('error')[0]; + + // The desktop's own frame, given to a real client. + device.socket().handlers.onMessage(rejection); + await world.advance(); + + const state = device.state(); + expect(state.phase).toBe('terminal'); + expect(state.canRetry).toBe(false); + expect(state.message).toBe(rejection.message); + // The pairing is still valid; only one end is behind. Deleting the token + // here would turn a five-minute update into a re-pair. + expect(device.store.value).not.toBeNull(); + await world.advance(60_000); + expect(device.sockets).toHaveLength(1); + }); +}); + +describe('conformance: a revoked pairing mid-session', () => { + it('ends everything the device was holding, and does not reconnect (C-12)', async () => { + const device = await world.connectClient(); + device.client.pressFloor(); + await world.advance(); + expect(world.transport.signaling.isOnline(device.deviceId)).toBe(true); + + await world.transport.revokeDevice(device.deviceId); + await world.advance(); + + expect(device.state().phase).toBe('terminal'); + expect(world.transport.signaling.isOnline(device.deviceId)).toBe(false); + // The floor it was holding is not left open for a timeout to notice. + expect(world.session.interrupt).toHaveBeenCalled(); + expect(world.session.stopSession).toHaveBeenCalledWith('user'); + await world.advance(60_000); + expect(device.sockets).toHaveLength(1); + }); + + it('sends revoked down a data channel that is still up, and the client forgets the token (C-12, C-13)', async () => { + const device = await world.connectClient(); + // The socket died but the peer is healthy, which is the case where the + // device can still be told in words rather than by ICE noticing. + device.socket().drop(); + await world.advance(); + + const revoked = device + .receivedFrames('reliable') + .filter((frame) => frame.type === 'revoked') as Array< + Extract + >; + expect(revoked).toHaveLength(1); + expect(revoked[0].message).toBeTruthy(); + expect(device.state().phase).toBe('terminal'); + expect(device.store.value).toBeNull(); + }); +}); + +describe('conformance: the Encore Feature switched off mid-session', () => { + it('drops the connection a phone was holding, without forgetting the phone', async () => { + const device = await world.connectClient(); + device.client.pressFloor(); + await world.advance(); + expect(world.transport.signaling.isOnline(device.deviceId)).toBe(true); + + // Somebody at the keyboard unticks the box. Everything the phone was + // holding has to go with it, or the switch is a lie. + world.setACappellaEnabled(false); + world.transport.standDown(); + await world.advance(); + + expect(world.transport.signaling.isOnline(device.deviceId)).toBe(false); + expect(world.transport.discoveryStatus()).toEqual({ state: 'disabled' }); + + // But the pairing survives. "Stop" is not "forget my phone", and a device + // that had to be re-paired because a checkbox was toggled would teach people + // not to touch the checkbox. + const devices = await world.transport.listDevices(); + expect(devices.map((entry) => entry.id)).toContain(device.deviceId); + expect(devices.find((entry) => entry.id === device.deviceId)?.revokedAt).toBeNull(); + }); + + it('refuses a reconnect while the feature is off, in a sentence a phone can show', async () => { + const device = await world.connectClient(); + world.setACappellaEnabled(false); + world.transport.standDown(); + await world.advance(); + + // A phone whose socket died retries. The desktop that answers has to say + // which of "switched off" and "the network ate it" this is, because only one + // of them is worth retrying. + const raw = world.openRawDevice(); + await raw.send({ op: 'auth', deviceId: device.deviceId, token: 'anything', v: 1 }); + + expect(raw.ofType('error')).toHaveLength(1); + expect(raw.ofType('error')[0].message).toMatch(/Encore Features/); + }); + + it('serves the same device again once the feature comes back on', async () => { + world.setACappellaEnabled(false); + world.transport.standDown(); + await world.advance(); + world.setACappellaEnabled(true); + + // The transport was stood down, not disposed, so this needs no restart. + const device = await world.connectClient(); + expect(world.transport.signaling.isOnline(device.deviceId)).toBe(true); + }); +}); + +describe('conformance: a network drop and a reconnect', () => { + it('re-authenticates on a new socket and starts the floor closed (C-09, C-49)', async () => { + const device = await world.connectClient(); + device.client.pressFloor(); + await world.advance(); + expect(device.state().floor.isSelf).toBe(true); + + device.dropNetwork(); + await world.advance(); + // The desktop does not wait for a timeout to end a session whose microphone + // walked away. + expect(world.session.stopSession).toHaveBeenCalledWith('user'); + expect(device.state().floor.isSelf).toBe(false); + + // The backoff starts at a second, and a reconnect is a fresh `auth` on a + // fresh socket: an authenticated state is never inherited. + await world.advance(1200); + await world.advance(200); + expect(device.sockets.length).toBeGreaterThan(1); + expect(device.socket().ops()).toContain('auth'); + expect(device.socket().ops()).not.toContain('pair-claim'); + expect(device.state().phase).toBe('connected'); + expect(device.state().floor).toEqual({ holder: null, isSelf: false }); + expect(world.transport.signaling.isOnline(device.deviceId)).toBe(true); + }); + + it('speaks the protocol again on the new connection (C-19)', async () => { + const device = await world.connectClient(); + device.dropNetwork(); + await world.advance(1400); + + // A new peer, a new pair of channels, and `hello` first on the state one. + expect(device.peers.length).toBeGreaterThan(1); + const [first] = device.sentFrames('reliable'); + expect(first).toMatchObject({ type: 'hello', identity: { deviceId: device.deviceId } }); + + device.client.pressFloor(); + await world.advance(); + expect(device.state().floor.isSelf).toBe(true); + }); +}); + +describe('conformance: two devices, one floor', () => { + it('gives the floor to the device that pressed last and tells the other who took it (C-48)', async () => { + const phone = await world.connectClient({ name: 'Phone A' }); + const tablet = await world.connectClient({ name: 'Phone B' }); + + phone.client.pressFloor(); + await world.advance(); + expect(phone.state().floor.isSelf).toBe(true); + + tablet.client.pressFloor(); + await world.advance(); + + expect(tablet.state().floor.isSelf).toBe(true); + expect(phone.state().floor.isSelf).toBe(false); + // The displaced device was told BEFORE the new session started, so its + // button lets go while the takeover happens rather than after. + // + // The notice is momentary by design: `takenOverBy` rides its own frame and + // the ordinary `floor-state` broadcast that follows carries no name, so a + // client must react to the frame rather than render the field out of its + // stored state. + const displaced = floorStates(phone.receivedFrames('reliable')); + expect(displaced.some((frame) => frame.takenOverBy === 'Phone B')).toBe(true); + expect(displaced[displaced.length - 1]).toMatchObject({ holder: tablet.deviceId }); + }); + + it('ignores the stale release the displaced device sends a moment later', async () => { + const phone = await world.connectClient({ name: 'Phone A' }); + const tablet = await world.connectClient({ name: 'Phone B' }); + phone.client.pressFloor(); + await world.advance(); + tablet.client.pressFloor(); + await world.advance(); + + phone.client.releaseFloor(); + await world.advance(); + + // Acting on it would shut the microphone of the device that just took the + // floor. + expect(world.floor.releases).toHaveLength(0); + expect(tablet.state().floor.isSelf).toBe(true); + }); + + it('refuses an interrupt from a device that is not holding the floor', async () => { + const phone = await world.connectClient({ name: 'Phone A' }); + const tablet = await world.connectClient({ name: 'Phone B' }); + phone.client.pressFloor(); + await world.advance(); + + tablet.client.requestBargeIn(); + tablet.client.requestStop(); + await world.advance(); + + expect(world.session.interrupt).not.toHaveBeenCalled(); + expect(world.session.hardStop).not.toHaveBeenCalled(); + }); + + it('shows every device the whole session, whoever holds the microphone', async () => { + const phone = await world.connectClient({ name: 'Phone A' }); + const tablet = await world.connectClient({ name: 'Phone B' }); + phone.client.pressFloor(); + await world.advance(); + + world.session.emit({ type: 'final-transcript', text: 'open the auth tab' }); + await world.advance(); + + // Only the microphone is exclusive. A phone in a pocket still has to be + // able to show what the Mac is doing. + expect(voiceEventsFrom(tablet.receivedFrames(), 'final-transcript')).toHaveLength(1); + expect(voiceEventsFrom(phone.receivedFrames(), 'final-transcript')).toHaveLength(1); + }); +}); + +describe('conformance: a stop word while TTS is streaming', () => { + it('ends the session, releases the floor, and closes the microphone (C-42, C-46)', async () => { + const device = await world.connectClient(); + device.client.pressFloor(); + await world.advance(); + world.session.emit({ + type: 'speak-start', + utteranceId: 'utt-1', + sentenceCount: 1, + ttsProviderId: 'local', + streaming: true, + }); + world.session.emit({ + type: 'speak-sentence', + utteranceId: 'utt-1', + index: 0, + text: 'The auth', + }); + await world.advance(); + expect(device.state().sending).toBe(true); + + device.client.requestStop(); + await world.advance(); + + // A stop word is a hard stop, never an interrupt: the difference is whether + // the user can get rid of the assistant. + expect(world.session.hardStop).toHaveBeenCalledWith('client-button'); + expect(world.session.interrupt).not.toHaveBeenCalled(); + + // The session ending is what releases the floor, so the device holds it + // until the desktop says otherwise. + world.session.emit({ type: 'speak-end', utteranceId: 'utt-1', reason: 'cancelled' }); + world.session.emit({ type: 'stop-word', source: 'voice', phrase: 'maestro stop' }); + await world.advance(); + + expect(device.state().floor).toEqual({ holder: null, isSelf: false }); + expect(device.state().sending).toBe(false); + expect(device.micTrack.stop).toHaveBeenCalled(); + }); + + it('ducks locally before the frame goes out and lifts on the desktop"s answer (C-44, C-45)', async () => { + const device = await world.connectClient(); + device.client.pressFloor(); + await world.advance(); + world.session.emit({ + type: 'speak-start', + utteranceId: 'utt-2', + sentenceCount: 2, + ttsProviderId: 'local', + }); + await world.advance(); + device.events.length = 0; + + device.client.requestBargeIn(); + await world.advance(); + + // The duck happened locally, in the same turn, before the interrupt could + // possibly have reached the desktop. + expect(device.events[0]).toEqual({ type: 'duck', ducked: true }); + expect(world.session.interrupt).toHaveBeenCalledWith('client-button'); + // Barge-in KEEPS the floor. + expect(device.state().floor.isSelf).toBe(true); + + world.session.emit({ + type: 'barge-in', + source: 'client-button', + cancelledUtteranceId: 'utt-2', + }); + await world.advance(); + expect(device.events).toContainEqual({ type: 'duck', ducked: false }); + expect(device.state().floor.isSelf).toBe(true); + }); + + it('lifts a duck the desktop never answers, and keeps the floor (C-45)', async () => { + const device = await world.connectClient(); + device.client.pressFloor(); + await world.advance(); + device.events.length = 0; + + device.client.requestBargeIn(); + await world.advance(600); + + expect(device.events).toContainEqual({ type: 'duck', ducked: false }); + expect(device.state().floor.isSelf).toBe(true); + }); +}); + +describe('conformance: a wake word is an ordinary press', () => { + it('opens the same floor a hotkey opens, with the scope the wheel selected (C-41)', async () => { + const device = await world.connectClient(); + // What a wake-word hit produces on the device: a plain `floor: press`, not + // a `wake` event and not a message type of its own. + device.client.pressFloor({ kind: 'agent', sessionId: 'agent-7' }); + await world.advance(); + + expect(world.floor.presses).toHaveLength(1); + expect(world.floor.presses[0]).toEqual({ + scope: { kind: 'agent', sessionId: 'agent-7' }, + origin: { kind: 'remote', deviceId: device.deviceId, deviceName: device.name }, + }); + expect(world.floor.press).toHaveBeenCalledWith('remote-device'); + // And the desktop routes the phone's microphone into the one capture + // pipeline rather than a second one. + expect(world.captured).toEqual([{ deviceId: device.deviceId }]); + expect(device.sentFrames('unreliable')[0]).toMatchObject({ + type: 'floor', + action: 'press', + scope: { kind: 'agent', sessionId: 'agent-7' }, + }); + }); + + it('never lets a device-sent voice-event drive the session', async () => { + const device = await world.connectClient(); + // The Phase 01 protocol marks `wake` as client-originable, but not over + // this transport: the device channel expresses it as `floor`. + device + .peer() + .channel(RELIABLE_CHANNEL_LABEL) + .send( + JSON.stringify({ + type: 'voice-event', + v: DEVICE_PROTOCOL_VERSION, + event: { type: 'wake', sessionId: 's', seq: 1, ts: 0, scope: { kind: 'conductor' } }, + }) + ); + await world.advance(); + + // It decodes, and then goes nowhere: no floor, no session. + expect(world.floor.presses).toHaveLength(0); + expect(world.hostCommands.some((command) => command.kind === 'set-floor-holder')).toBe(false); + }); +}); diff --git a/src/__tests__/acappella/conformance/harness.ts b/src/__tests__/acappella/conformance/harness.ts new file mode 100644 index 0000000000..4711e949c0 --- /dev/null +++ b/src/__tests__/acappella/conformance/harness.ts @@ -0,0 +1,852 @@ +/** + * A desktop, a network, and a phone, in one process. + * + * This is the harness the protocol conformance suite runs on. It assembles the + * REAL desktop stack - `ACappellaTransport` (pairing, signaling, the remote + * session coordinator) wired to a real `PeerRegistry` through the same + * `applyWebRtcCommand` switch the hidden audio window uses - and drives it with + * the REAL browser reference client from `src/web-desktop/acappella-client/`. + * Nothing between them is stubbed except the two things a test cannot have: a + * socket and libwebrtc. + * + * That is the whole point. A conformance suite that mocked either end would + * assert that a mock agrees with itself. Here a message leaves + * `ACappellaReferenceClient.send()`, is JSON, crosses a loopback data channel, + * and is decoded by `decodeDeviceMessage()` inside the desktop's `DevicePeer` - + * so a desktop-side change that would break an iOS client fails here rather + * than at App Store review. + * + * What is faked, and how faithfully: + * + * - **The WebSocket** ({@link LoopbackSocket}) carries the real + * `{type:'acappella_signal', payload}` envelope over the real `/$TOKEN/ws` + * URL shape, mirroring the two lines in + * `main/web-server/handlers/messageHandlers/acappellaSignal.ts`. `onClose` + * fires in a microtask, as a browser's does, because firing it + * synchronously inside `close()` would let a teardown reconnect to itself. + * - **The peer connection** ({@link LoopbackPeer}) mirrors data channels + * between the two ends, delivers a replaced track to the far side's + * `ontrack`, and answers `getStats()`. A channel closed by one end closes + * on the other, which is what makes "the desktop closes a label it does not + * recognise" observable from the client. + * + * Timers are faked by {@link createConformanceWorld} and released by + * `dispose()`, because the client's pairing poll is a 1 s interval and its + * reconnect backoff starts at 1 s. + */ + +import * as fs from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; + +import { vi, type Mock } from 'vitest'; + +import { ACappellaTransport } from '../../../main/acappella/transport'; +import type { + RemoteFloor, + RemoteVoiceSession, +} from '../../../main/acappella/transport/remote-session'; +import { + PeerRegistry, + applyWebRtcCommand, + type PeerAudioBinding, +} from '../../../renderer/acappella-audio/peer-connection'; +import { + decodeDeviceMessage, + type DeviceChannelKind, + type DeviceMessage, + RELIABLE_CHANNEL_LABEL, + UNRELIABLE_CHANNEL_LABEL, +} from '../../../shared/acappella/device-protocol'; +import type { VoiceEvent, VoiceOrigin, VoiceScope } from '../../../shared/acappella/protocol'; +import { + ACAPPELLA_SIGNAL_MESSAGE, + type SignalingClientMessage, + type SignalingServerMessage, +} from '../../../shared/acappella/signaling-protocol'; +import type { + IceCandidatePayload, + WebRtcHostCommand, + WebRtcHostEvent, +} from '../../../shared/acappella/webrtc-host'; +import { + ACappellaReferenceClient, + PAIR_POLL_INTERVAL_MS, + type ClientEvent, + type ClientState, + type PairingStore, + type SignalingSocket, + type SignalingSocketHandlers, + type StoredPairing, +} from '../../../web-desktop/acappella-client/client'; + +/** The web server's security token, which is what gets a frame looked at at all. */ +export const SERVER_TOKEN = 'conformance-server-token'; +export const SERVER_PORT = 4123; +export const SERVER_HOST = '192.168.1.5'; +export const DESKTOP_APP_VERSION = '1.2.3'; + +/** + * An SDP with one Opus m-line, which is all `applyOpusPreferences()` reads. + * + * Deliberately carries `useinbandfec=0` so an assertion that the tuning was + * applied cannot pass by accident on an SDP that already said the right thing. + */ +export const OPUS_SDP = [ + 'v=0', + 'o=- 0 0 IN IP4 127.0.0.1', + 's=-', + 'm=audio 9 UDP/TLS/RTP/SAVPF 111', + 'a=rtpmap:111 opus/48000/2', + 'a=fmtp:111 minptime=10;useinbandfec=0', + '', +].join('\r\n'); + +// --------------------------------------------------------------------------- +// The loopback peer connection +// --------------------------------------------------------------------------- + +export class LoopbackChannel { + readyState: RTCDataChannelState = 'connecting'; + onopen: (() => void) | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + onclose: (() => void) | null = null; + /** Every frame this end put on the wire, as sent. */ + readonly sent: string[] = []; + /** The channel at the other end of the SCTP stream. */ + remote: LoopbackChannel | null = null; + + constructor( + readonly label: string, + readonly init: RTCDataChannelInit | undefined + ) {} + + open(): void { + if (this.readyState !== 'connecting') return; + this.readyState = 'open'; + this.onopen?.(); + } + + send(data: string): void { + this.sent.push(data); + const remote = this.remote; + if (!remote || remote.readyState !== 'open') return; + remote.onmessage?.({ data }); + } + + close(): void { + if (this.readyState === 'closed') return; + this.readyState = 'closed'; + const remote = this.remote; + this.remote = null; + this.onclose?.(); + if (remote) { + remote.remote = null; + remote.close(); + } + } + + /** Everything this end sent, decoded through the real protocol decoder. */ + messages(): DeviceMessage[] { + return this.sent + .map((raw) => decodeDeviceMessage(raw)) + .filter((message): message is DeviceMessage => message !== null); + } + + /** Everything this end sent, as raw objects, so a missing `v` is visible. */ + frames(): Array> { + return this.sent.map((raw) => JSON.parse(raw) as Record); + } +} + +interface LoopbackSender { + track: MediaStreamTrack | null; + replaceTrack(track: MediaStreamTrack | null): Promise; + getParameters(): { encodings?: Array> }; + setParameters(parameters: unknown): Promise; +} + +/** Enough `RTCPeerConnection` for both ends of this protocol, and no more. */ +export class LoopbackPeer { + connectionState: RTCPeerConnectionState = 'new'; + onicecandidate: ((event: { candidate: RTCIceCandidate | null }) => void) | null = null; + ontrack: ((event: { streams: MediaStream[]; track: MediaStreamTrack }) => void) | null = null; + onconnectionstatechange: (() => void) | null = null; + ondatachannel: ((event: { channel: LoopbackChannel }) => void) | null = null; + + readonly channels: LoopbackChannel[] = []; + readonly senders: LoopbackSender[] = []; + readonly addedTracks: MediaStreamTrack[] = []; + readonly candidates: IceCandidatePayload[] = []; + /** What `getStats()` answers. Tests fill this in when they care. */ + statsReports: Array> = []; + localDescription: { type: string; sdp?: string } | null = null; + remoteDescription: { type: string; sdp?: string } | null = null; + closed = false; + remote: LoopbackPeer | null = null; + + constructor( + readonly config: RTCConfiguration, + readonly role: 'client' | 'desktop' + ) {} + + createDataChannel(label: string, init?: RTCDataChannelInit): LoopbackChannel { + const channel = new LoopbackChannel(label, init); + this.channels.push(channel); + // A channel created after the peers were linked still has to reach the far + // end, which is how the "unrecognised label" path is exercised. + if (this.remote) this.remote.acceptChannel(channel); + return channel; + } + + /** Mirror a channel the far end created, and open both halves. */ + acceptChannel(theirs: LoopbackChannel): void { + const mirror = new LoopbackChannel(theirs.label, theirs.init); + mirror.remote = theirs; + theirs.remote = mirror; + this.channels.push(mirror); + this.ondatachannel?.({ channel: mirror }); + // The far end may have closed it on sight, and a closed channel must not be + // reopened by the link step. + if (mirror.readyState === 'closed' || theirs.readyState === 'closed') return; + mirror.open(); + theirs.open(); + } + + addTransceiver(_kind: string, _init: { direction: string }): { sender: LoopbackSender } { + return { sender: this.createSender() }; + } + + addTrack(track: MediaStreamTrack): LoopbackSender { + this.addedTracks.push(track); + const sender = this.createSender(); + void sender.replaceTrack(track); + return sender; + } + + getSenders(): LoopbackSender[] { + return this.senders; + } + + createOffer(_options?: { iceRestart?: boolean }): Promise<{ type: string; sdp: string }> { + return Promise.resolve({ type: 'offer', sdp: OPUS_SDP }); + } + + createAnswer(): Promise<{ type: string; sdp: string }> { + // Echo whatever was offered, so an assertion about the answer's tuning is + // about the desktop's shaping rather than about this fake's constant. + return Promise.resolve({ type: 'answer', sdp: this.remoteDescription?.sdp ?? OPUS_SDP }); + } + + setLocalDescription(description: { type: string; sdp?: string }): Promise { + this.localDescription = description; + return Promise.resolve(); + } + + setRemoteDescription(description: { type: string; sdp?: string }): Promise { + this.remoteDescription = description; + return Promise.resolve(); + } + + addIceCandidate(candidate: IceCandidatePayload): Promise { + this.candidates.push(candidate); + return Promise.resolve(); + } + + getStats(): Promise<{ forEach: (fn: (value: unknown) => void) => void }> { + const reports = this.statsReports; + return Promise.resolve({ forEach: (fn) => reports.forEach(fn) }); + } + + close(): void { + if (this.closed) return; + this.closed = true; + for (const channel of this.channels) channel.close(); + } + + /** One gathered candidate, trickled out the way ICE does. */ + gatherCandidate(candidate: string): void { + this.onicecandidate?.({ + candidate: { + candidate, + sdpMid: '0', + sdpMLineIndex: 0, + usernameFragment: 'ufrag', + } as unknown as RTCIceCandidate, + }); + } + + setConnectionState(state: RTCPeerConnectionState): void { + this.connectionState = state; + this.onconnectionstatechange?.(); + } + + channel(label: string): LoopbackChannel { + const found = this.channels.find((entry) => entry.label === label); + if (!found) throw new Error(`No channel '${label}' on the ${this.role} peer`); + return found; + } + + private createSender(): LoopbackSender { + const peer = this; + const sender: LoopbackSender = { + track: null, + replaceTrack(track: MediaStreamTrack | null): Promise { + sender.track = track; + // A track attached at this end arrives at the other end's `ontrack`, + // which is what makes "nothing before the floor opens" observable from + // the desktop rather than only from the client. + if (track) peer.remote?.deliverTrack(track); + return Promise.resolve(); + }, + getParameters: () => ({ encodings: [{}] }), + setParameters: () => Promise.resolve(), + }; + this.senders.push(sender); + return sender; + } + + private deliverTrack(track: MediaStreamTrack): void { + const stream = { + id: `stream-${this.role}`, + getTracks: () => [track], + } as unknown as MediaStream; + this.ontrack?.({ streams: [stream], track }); + } +} + +/** Join two peers and open every channel either of them has already created. */ +export function linkPeers(client: LoopbackPeer, desktop: LoopbackPeer): void { + client.remote = desktop; + desktop.remote = client; + for (const channel of [...client.channels]) desktop.acceptChannel(channel); +} + +// --------------------------------------------------------------------------- +// The loopback socket +// --------------------------------------------------------------------------- + +/** One `acappella_signal` envelope, as it would appear on the WebSocket. */ +export interface SignalEnvelope { + type: string; + payload: SignalingClientMessage | SignalingServerMessage; +} + +export class LoopbackSocket implements SignalingSocket { + /** Envelopes this socket put on the wire, outermost shape included. C-01. */ + readonly outbound: SignalEnvelope[] = []; + readonly inbound: SignalingServerMessage[] = []; + closed = false; + + constructor( + readonly clientId: string, + readonly url: string, + readonly handlers: SignalingSocketHandlers, + private readonly deliver: (payload: unknown) => void, + private readonly onGone: () => void + ) {} + + send(message: SignalingClientMessage): void { + if (this.closed) return; + const envelope: SignalEnvelope = { type: ACAPPELLA_SIGNAL_MESSAGE, payload: message }; + this.outbound.push(envelope); + // The web-server handler unwraps `payload` and hands it over untouched. + this.deliver(envelope.payload); + } + + /** The desktop wrote back. */ + receive(message: SignalingServerMessage): void { + if (this.closed) return; + this.inbound.push(message); + this.handlers.onMessage(message); + } + + close(): void { + this.drop(); + } + + /** The socket went away, for any reason. */ + drop(): void { + if (this.closed) return; + this.closed = true; + this.onGone(); + // A browser fires `close` in a later task. Firing it inline would run the + // client's reconnect logic in the middle of its own teardown. + queueMicrotask(() => this.handlers.onClose()); + } + + ops(): string[] { + return this.outbound.map((envelope) => String((envelope.payload as { op: string }).op)); + } +} + +// --------------------------------------------------------------------------- +// Desktop-side fakes +// --------------------------------------------------------------------------- + +/** The voice session, reduced to what a remote device can observe and drive. */ +export interface FakeVoiceSession extends RemoteVoiceSession { + emit(event: Partial & { type: VoiceEvent['type'] }): void; + interrupt: Mock; + hardStop: Mock; + stopSession: Mock; + /** Session id and monotonic seq, so a broadcast stream is contiguous. */ + sessionId: string; + seq: number; +} + +export interface FakeFloor extends RemoteFloor { + readonly presses: Array<{ scope: VoiceScope; origin: VoiceOrigin }>; + readonly releases: string[]; +} + +function createFakeSession(): FakeVoiceSession { + let listeners: Array<(event: VoiceEvent) => void> = []; + const session = { + sessionId: 'voice-1', + seq: 0, + subscribe(listener: (event: VoiceEvent) => void) { + listeners.push(listener); + return () => { + listeners = listeners.filter((entry) => entry !== listener); + }; + }, + interrupt: vi.fn(() => true), + hardStop: vi.fn(async () => {}), + stopSession: vi.fn(async () => {}), + getState: () => 'listening', + emit(event: Partial & { type: VoiceEvent['type'] }) { + session.seq += 1; + const full = { + sessionId: session.sessionId, + seq: session.seq, + ts: 0, + ...event, + } as VoiceEvent; + for (const listener of [...listeners]) listener(full); + }, + } as unknown as FakeVoiceSession; + return session; +} + +// --------------------------------------------------------------------------- +// The world +// --------------------------------------------------------------------------- + +export interface ConformanceClient { + readonly client: ACappellaReferenceClient; + readonly deviceId: string; + readonly name: string; + readonly store: PairingStore & { value: StoredPairing | null }; + readonly events: ClientEvent[]; + readonly sockets: LoopbackSocket[]; + readonly peers: LoopbackPeer[]; + /** The live signaling socket. */ + socket(): LoopbackSocket; + /** The client's end of the peer connection. */ + peer(): LoopbackPeer; + /** The desktop's end of the same connection. */ + desktopPeer(): LoopbackPeer; + state(): ClientState; + /** Frames this client put on the wire, on one channel or both. */ + sentFrames(kind?: DeviceChannelKind): DeviceMessage[]; + /** Frames the desktop sent to this client, on one channel or both. */ + receivedFrames(kind?: DeviceChannelKind): DeviceMessage[]; + /** Raw objects this client sent, so a missing `v` is visible. */ + rawSentFrames(): Array>; + /** The network went away: media dead, socket dead. */ + dropNetwork(): void; + /** The microphone stream `openMicrophone()` hands out. */ + readonly micTrack: MediaStreamTrack; +} + +/** A device with no client behind it, for the frames a conforming client never sends. */ +export interface RawDevice { + socket: LoopbackSocket; + send(message: SignalingClientMessage | Record): Promise; + received: SignalingServerMessage[]; + last(): SignalingServerMessage | undefined; + ofType( + op: T + ): Array>; +} + +export interface ConformanceWorld { + readonly transport: ACappellaTransport; + readonly peers: PeerRegistry; + readonly session: FakeVoiceSession; + readonly floor: FakeFloor; + /** Every command the transport sent to the audio host, in order. */ + readonly hostCommands: WebRtcHostCommand[]; + /** + * Every frame the desktop actually DECODED off a data channel. + * + * The proof that a client's message round-tripped: it left the client as a + * string, crossed the channel, and came back out of `decodeDeviceMessage()` + * inside the desktop's own peer. A frame the desktop dropped never appears + * here, and dropping is silent by design. + */ + readonly deviceMessages: Array<{ deviceId: string; message: DeviceMessage }>; + /** Streams the desktop routed into the capture pipeline, and when. */ + readonly captured: Array<{ deviceId: string }>; + readonly detached: string[]; + /** Pair, authenticate, and connect one reference client. */ + connectClient(options?: { name?: string; platform?: string }): Promise; + /** A socket that speaks signaling by hand, with no client behind it. */ + openRawDevice(): RawDevice; + /** + * Flip the Encore Feature. Off is a state a CONNECTED phone can be put into by + * somebody at the keyboard, so it is a wire behaviour and not just a settings + * concern. + */ + setACappellaEnabled(enabled: boolean): void; + /** Run every pending microtask and timer up to `ms`. */ + advance(ms?: number): Promise; + dispose(): Promise; +} + +export async function createConformanceWorld(): Promise { + const userDataPath = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-conformance-')); + vi.useFakeTimers(); + + const session = createFakeSession(); + const presses: Array<{ scope: VoiceScope; origin: VoiceOrigin }> = []; + const releases: string[] = []; + const floor = { + presses, + releases, + press: vi.fn(async (source?: string) => { + void source; + }), + release: vi.fn(async (source?: string) => { + releases.push(source ?? 'unknown'); + }), + close: vi.fn(async () => {}), + isFloorOpen: false, + } as unknown as FakeFloor; + + const captured: Array<{ deviceId: string }> = []; + const detached: string[] = []; + const audio: PeerAudioBinding = { + attachRemoteStream: (_stream, deviceId) => captured.push({ deviceId }), + detachRemoteStream: (deviceId) => detached.push(deviceId), + getOutboundTrack: () => ({ kind: 'audio', id: 'assistant-voice' }) as MediaStreamTrack, + }; + + const clientsByDevice = new Map(); + const desktopPeers = new Map(); + const hostCommands: WebRtcHostCommand[] = []; + const deviceMessages: Array<{ deviceId: string; message: DeviceMessage }> = []; + let pendingDeviceId: string | null = null; + let transport!: ACappellaTransport; + + const peers = new PeerRegistry({ + audio, + callbacks: { + onAnswer: (deviceId, answer) => + transport.handleHostEvent({ kind: 'answer', deviceId, answer }), + onIceCandidate: (deviceId, candidate) => + transport.handleHostEvent({ kind: 'ice-candidate', deviceId, candidate }), + onConnectionState: (deviceId, state) => + transport.handleHostEvent({ kind: 'connection-state', deviceId, state }), + onStats: (stats) => transport.handleHostEvent({ kind: 'stats', stats }), + onMessage: (deviceId, message) => { + deviceMessages.push({ deviceId, message }); + transport.handleHostEvent({ kind: 'message', deviceId, message }); + }, + onError: (deviceId, message) => + transport.handleHostEvent({ kind: 'peer-error', deviceId, message }), + }, + createPeerConnection: (config) => { + const peer = new LoopbackPeer(config, 'desktop'); + const deviceId = pendingDeviceId; + pendingDeviceId = null; + if (deviceId) { + desktopPeers.set(deviceId, peer); + // Linked a microtask later: the `DevicePeer` constructor has not run + // yet, so `ondatachannel` is not bound at the moment this factory is + // called and mirroring here would drop both channels on the floor. + queueMicrotask(() => { + const client = clientsByDevice.get(deviceId); + if (!client) return; + linkPeers(client.peer(), peer); + peer.setConnectionState('connected'); + client.peer().setConnectionState('connected'); + }); + } + return peer as unknown as RTCPeerConnection; + }, + // Long enough that the desktop's own stats poll never fires by surprise; + // tests that care drive `getStats()` by advancing deliberately. + statsIntervalMs: 10 * 60_000, + }); + + /** + * Settings, mutable, with the Encore Feature on. + * + * On because a conformance run is by definition a run with A Cappella + * enabled, and mutable because switching it off is itself a failure path a + * connected phone will hit. + */ + const settings: Record = { encoreFeatures: { aCappella: true } }; + + transport = new ACappellaTransport({ + settingsStore: { + get: (key: string, defaultValue?: unknown) => settings[key] ?? defaultValue ?? {}, + }, + userDataPath, + sendToAudioHost: (command: WebRtcHostCommand) => { + hostCommands.push(command); + if (command.kind === 'accept-offer') pendingDeviceId = command.deviceId; + applyWebRtcCommand(peers, command, (event: WebRtcHostEvent) => + transport.handleHostEvent(event) + ); + }, + acquireFloor: (scope: VoiceScope, origin?: VoiceOrigin) => { + presses.push({ scope, origin: origin ?? { kind: 'local' } }); + return floor; + }, + getSession: () => session, + getServerToken: () => SERVER_TOKEN, + getServerPort: () => SERVER_PORT, + getAppVersion: () => DESKTOP_APP_VERSION, + getMachineName: () => 'Conformance Desktop', + }); + + let socketSeq = 0; + function openSocket(url: string, handlers: SignalingSocketHandlers): LoopbackSocket { + socketSeq += 1; + const clientId = `socket-${socketSeq}`; + const socket: LoopbackSocket = new LoopbackSocket( + clientId, + url, + handlers, + (payload) => { + // The Encore gate, exactly as `acappellaSignal.ts` applies it: the + // transport outlives the flag on purpose, so its existence is not + // permission to serve. + if (!transport.featureEnabled()) { + socket.receive({ + op: 'error', + code: 'not-authenticated', + message: 'A Cappella is not running on this desktop. Turn it on in Encore Features.', + }); + return; + } + // Lazy, idempotent registration on the first message, exactly as the + // WebSocket route does it. + transport.registerClient({ + clientId, + send: (message: SignalingServerMessage) => socket.receive(message), + remoteAddress: '192.168.1.44', + }); + void transport.handleSignalMessage(clientId, payload); + }, + () => transport.handleClientDisconnect(clientId) + ); + queueMicrotask(() => { + if (!socket.closed) handlers.onOpen(); + }); + return socket; + } + + async function advance(ms = 0): Promise { + await vi.advanceTimersByTimeAsync(ms); + // A couple of extra turns for the promise chains the client runs between + // `authenticated` and its first offer. + for (let i = 0; i < 8; i += 1) await Promise.resolve(); + } + + async function connectClient( + options: { name?: string; platform?: string } = {} + ): Promise { + const name = options.name ?? `Reference ${clientsByDevice.size + 1}`; + const platform = options.platform ?? 'ios'; + const offer = transport.startPairing(); + if (!offer) throw new Error('The desktop refused to open a pairing window'); + + const sockets: LoopbackSocket[] = []; + const clientPeers: LoopbackPeer[] = []; + const events: ClientEvent[] = []; + const micTrack = { kind: 'audio', id: 'mic', stop: vi.fn() } as unknown as MediaStreamTrack; + const store: PairingStore & { value: StoredPairing | null } = { + value: null, + read() { + return this.value; + }, + write(pairing: StoredPairing) { + this.value = pairing; + }, + clear() { + this.value = null; + }, + }; + + const client = new ACappellaReferenceClient({ + identity: { name, platform, appVersion: '9.9.9' }, + store, + openSocket: (url, handlers) => { + const socket = openSocket(url, handlers); + sockets.push(socket); + return socket; + }, + createPeerConnection: (config) => { + const peer = new LoopbackPeer(config, 'client'); + clientPeers.push(peer); + return peer as unknown as RTCPeerConnection; + }, + openMicrophone: () => + Promise.resolve({ + getTracks: () => [micTrack], + getAudioTracks: () => [micTrack], + } as unknown as MediaStream), + }); + + const entry: ConformanceClient = { + client, + deviceId: '', + name, + store, + events, + sockets, + peers: clientPeers, + micTrack, + socket: () => sockets[sockets.length - 1], + peer: () => clientPeers[clientPeers.length - 1], + desktopPeer: () => { + const peer = desktopPeers.get(entry.deviceId); + if (!peer) throw new Error(`No desktop peer for ${entry.deviceId}`); + return peer; + }, + state: () => client.snapshot(), + sentFrames: (kind?: DeviceChannelKind) => channelFrames(entry.peer(), kind), + receivedFrames: (kind?: DeviceChannelKind) => channelFrames(entry.desktopPeer(), kind), + rawSentFrames: () => [ + ...entry.peer().channel(RELIABLE_CHANNEL_LABEL).frames(), + ...entry.peer().channel(UNRELIABLE_CHANNEL_LABEL).frames(), + ], + dropNetwork: () => { + // A network drop takes the media with it. Dropping only the socket + // would leave a live data channel the desktop then writes `revoked` + // down, which is a different failure with a different meaning. + const peer = entry.peer(); + const desktop = desktopPeers.get(entry.deviceId); + desktop?.setConnectionState('failed'); + peer.setConnectionState('failed'); + for (const channel of peer.channels) channel.close(); + entry.socket().drop(); + }, + }; + client.subscribe((event) => events.push(event)); + + client.connect({ host: SERVER_HOST, port: SERVER_PORT, token: SERVER_TOKEN, code: offer.code }); + await advance(); + + // A human approves, which is the only thing a pairing code ever buys. + const request = transport.pairing.pendingRequest(); + if (!request) throw new Error('The desktop never saw the pairing claim'); + const device = await transport.pairing.approve(request.requestId); + if (!device) throw new Error('The desktop refused to approve the pairing request'); + + // The client collects the token on its next poll, authenticates, offers, + // and the peers link. + await advance(PAIR_POLL_INTERVAL_MS + 50); + await advance(50); + + const deviceId = client.snapshot().deviceId; + if (!deviceId) throw new Error(`Client '${name}' never authenticated`); + (entry as { deviceId: string }).deviceId = deviceId; + clientsByDevice.set(deviceId, entry); + // The peer for this device was created before the id was known here, so the + // link runs now rather than in the factory's microtask. + const desktopPeer = desktopPeers.get(deviceId); + if (desktopPeer && !desktopPeer.remote) { + linkPeers(entry.peer(), desktopPeer); + desktopPeer.setConnectionState('connected'); + entry.peer().setConnectionState('connected'); + } + await advance(50); + return entry; + } + + function openRawDevice(): RawDevice { + const received: SignalingServerMessage[] = []; + const socket = openSocket(`ws://${SERVER_HOST}:${SERVER_PORT}/${SERVER_TOKEN}/ws`, { + onOpen: () => {}, + onMessage: (message) => received.push(message), + onClose: () => {}, + }); + return { + socket, + received, + send: async (message) => { + socket.send(message as SignalingClientMessage); + await advance(); + }, + last: () => received[received.length - 1], + ofType: (op: T) => + received.filter( + (message): message is Extract => message.op === op + ), + }; + } + + return { + transport, + peers, + session, + floor, + hostCommands, + deviceMessages, + captured, + detached, + connectClient, + openRawDevice, + setACappellaEnabled: (enabled: boolean) => { + settings.encoreFeatures = { aCappella: enabled }; + }, + advance, + dispose: async () => { + transport.dispose(); + peers.closeAll('the conformance world was torn down'); + vi.useRealTimers(); + // `noteConnected()` persists off a peer connecting and nobody awaits it, + // so a `devices.json` write can still be in flight here, and one can even + // be enqueued after a plain drain. Closing settles what is queued and + // drops what comes later, so removing the directory cannot rename into + // nothing. + await transport.pairing.close(); + await fs.rm(userDataPath, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 20, + }); + }, + }; +} + +/** Decoded frames one peer put on the wire, on one channel or across both. */ +function channelFrames(peer: LoopbackPeer, kind?: DeviceChannelKind): DeviceMessage[] { + const labels = + kind === 'reliable' + ? [RELIABLE_CHANNEL_LABEL] + : kind === 'unreliable' + ? [UNRELIABLE_CHANNEL_LABEL] + : [RELIABLE_CHANNEL_LABEL, UNRELIABLE_CHANNEL_LABEL]; + const frames: DeviceMessage[] = []; + for (const label of labels) { + const channel = peer.channels.find((entry) => entry.label === label); + if (channel) frames.push(...channel.messages()); + } + return frames; +} + +/** Every voice event of one type a client received, unwrapped. */ +export function voiceEventsFrom(frames: DeviceMessage[], type?: VoiceEvent['type']): VoiceEvent[] { + return frames + .filter((frame): frame is Extract => { + return frame.type === 'voice-event' && (!type || frame.event.type === type); + }) + .map((frame) => frame.event); +} diff --git a/src/__tests__/acappella/conformance/signaling.conformance.test.ts b/src/__tests__/acappella/conformance/signaling.conformance.test.ts new file mode 100644 index 0000000000..472b267fb7 --- /dev/null +++ b/src/__tests__/acappella/conformance/signaling.conformance.test.ts @@ -0,0 +1,264 @@ +/** + * Protocol conformance: the signaling layer, C-01 to C-15. + * + * Every assertion here is one row of the checklist in + * `docs/ios-client/protocol-conformance.md`, run against the real desktop stack + * rather than against a description of it. Where an item is a rule about what a + * CLIENT does, the reference client is the subject; where it is a rule the + * desktop enforces, a raw device sends the frame a conforming client never + * would, because a limit nobody has hit is a limit nobody has tested. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { DEFAULT_STUN_URLS } from '../../../main/acappella/transport/ice-config'; +import { AUTH_ATTEMPT_LIMIT, OFFER_RATE_LIMIT } from '../../../main/acappella/transport/signaling'; +import { ACAPPELLA_SIGNAL_MESSAGE } from '../../../shared/acappella/signaling-protocol'; +import { DEFAULT_REMOTE_AUDIO_CONFIG } from '../../../shared/acappella/webrtc-host'; +import { + SERVER_HOST, + SERVER_PORT, + SERVER_TOKEN, + createConformanceWorld, + type ConformanceWorld, +} from './harness'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock('../../../renderer/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +let world: ConformanceWorld; + +beforeEach(async () => { + world = await createConformanceWorld(); +}); + +afterEach(async () => { + await world.dispose(); +}); + +describe('conformance: pairing and authentication', () => { + it('carries every frame in the acappella_signal envelope on /$TOKEN/ws (C-01)', async () => { + const device = await world.connectClient(); + + // One socket, one token, no second port. + expect(device.socket().url).toBe(`ws://${SERVER_HOST}:${SERVER_PORT}/${SERVER_TOKEN}/ws`); + expect(device.sockets).toHaveLength(1); + for (const envelope of device.sockets[0].outbound) { + expect(envelope.type).toBe(ACAPPELLA_SIGNAL_MESSAGE); + expect(typeof (envelope.payload as { op: string }).op).toBe('string'); + } + }); + + it('claims with a non-empty name and platform, and the desktop shows both (C-02)', async () => { + const offer = world.transport.startPairing(); + const raw = world.openRawDevice(); + await raw.send({ + op: 'pair-claim', + code: offer?.code, + name: "Pedram's iPhone", + platform: 'ios', + }); + + const request = world.transport.pairing.pendingRequest(); + expect(request).toMatchObject({ name: "Pedram's iPhone", platform: 'ios' }); + // A non-string is coerced to the empty string rather than refused, which is + // how a nameless row appears in the approval sheet. + expect(raw.ofType('pair-pending')).toHaveLength(1); + }); + + it('polls on an interval and keeps the deadline from the FIRST pair-pending (C-03)', async () => { + const offer = world.transport.startPairing(); + const raw = world.openRawDevice(); + await raw.send({ op: 'pair-claim', code: offer?.code, name: 'Phone', platform: 'ios' }); + const first = raw.ofType('pair-pending')[0]; + expect(first.expiresAt).toBeGreaterThan(Date.now()); + + await raw.send({ op: 'pair-poll', requestId: first.requestId }); + // The answer to a poll carries no deadline at all. A client that takes it + // collapses its own countdown to 1970. + expect(raw.ofType('pair-pending')[1].expiresAt).toBe(0); + }); + + it('stores the token the desktop issued, then authenticates with it (C-04, C-06)', async () => { + const device = await world.connectClient(); + + expect(device.store.value).toMatchObject({ deviceId: device.deviceId }); + expect(device.store.value?.token).toBeTruthy(); + const auth = device.sockets[0].outbound + .map((envelope) => envelope.payload as Record) + .find((payload) => payload.op === 'auth'); + expect(auth).toMatchObject({ deviceId: device.deviceId }); + expect(Number.isInteger(auth?.protocolVersion as number)).toBe(true); + expect(auth?.protocolVersion as number).toBeGreaterThanOrEqual(1); + }); + + it('shows the desktop"s rejection sentence verbatim (C-05)', async () => { + world.transport.startPairing(); + const raw = world.openRawDevice(); + await raw.send({ op: 'pair-claim', code: 'WRONG1', name: 'Phone', platform: 'ios' }); + + expect(raw.last()).toEqual({ + op: 'pair-rejected', + reason: 'unknown-code', + message: 'That pairing code does not match the one on the desktop.', + }); + }); + + it('refuses an offer and a candidate before auth on the same socket (C-07)', async () => { + const raw = world.openRawDevice(); + await raw.send({ op: 'offer', sdp: { type: 'offer', sdp: 'v=0' } }); + await raw.send({ + op: 'ice-candidate', + candidate: { candidate: 'candidate:1 1 udp 1 h 1 typ host' }, + }); + + expect(raw.ofType('error').map((error) => error.code)).toEqual([ + 'not-authenticated', + 'not-authenticated', + ]); + expect(world.hostCommands).toHaveLength(0); + }); + + it('never inherits an authenticated state across sockets (C-07, C-09)', async () => { + const device = await world.connectClient(); + const stored = device.store.value; + expect(stored).not.toBeNull(); + + // A second socket for the same device starts from nothing. + const raw = world.openRawDevice(); + await raw.send({ op: 'offer', sdp: { type: 'offer', sdp: 'v=0' } }); + expect(raw.ofType('error')[0].code).toBe('not-authenticated'); + }); + + it('rate limits offers past the budget the client stays under (C-08)', async () => { + const device = await world.connectClient(); + const raw = world.openRawDevice(); + await raw.send({ + op: 'auth', + deviceId: device.deviceId, + token: device.store.value?.token, + protocolVersion: 1, + }); + expect(raw.ofType('authenticated')).toHaveLength(1); + + for (let i = 0; i < OFFER_RATE_LIMIT; i += 1) { + await raw.send({ op: 'offer', sdp: { type: 'offer', sdp: 'v=0' } }); + } + expect(raw.ofType('error')).toHaveLength(0); + + await raw.send({ op: 'offer', sdp: { type: 'offer', sdp: 'v=0' } }); + expect(raw.ofType('error')[0].code).toBe('rate-limited'); + }); + + it('cuts a socket off after too many failed auths (C-09)', async () => { + const raw = world.openRawDevice(); + for (let i = 0; i < AUTH_ATTEMPT_LIMIT; i += 1) { + await raw.send({ op: 'auth', deviceId: 'nope', token: 'nope', protocolVersion: 1 }); + } + expect(raw.ofType('auth-failed')).toHaveLength(AUTH_ATTEMPT_LIMIT); + + await raw.send({ op: 'auth', deviceId: 'nope', token: 'nope', protocolVersion: 1 }); + expect(raw.ofType('error')[0].code).toBe('rate-limited'); + // The recovery is a new socket, and a new socket really does start clean. + const second = world.openRawDevice(); + await second.send({ op: 'auth', deviceId: 'nope', token: 'nope', protocolVersion: 1 }); + expect(second.ofType('auth-failed')).toHaveLength(1); + }); + + it('sends exactly one auth per socket (C-09)', async () => { + const device = await world.connectClient(); + expect(device.sockets[0].ops().filter((op) => op === 'auth')).toHaveLength(1); + }); +}); + +describe('conformance: what authenticated carries', () => { + it('builds the peer from the ICE servers the desktop sent, and hard-codes none (C-10)', async () => { + const device = await world.connectClient(); + + expect(device.peer().config.iceServers).toEqual([{ urls: [...DEFAULT_STUN_URLS] }]); + expect(device.peer().config.iceTransportPolicy).toBe('all'); + }); + + it('applies the desktop audio config to the offer, and gets it back in the answer (C-11)', async () => { + const device = await world.connectClient(); + + const offer = device.sockets[0].outbound + .map((envelope) => envelope.payload as { op: string; sdp?: { sdp: string } }) + .find((payload) => payload.op === 'offer'); + expect(offer?.sdp?.sdp).toContain('useinbandfec=1'); + expect(offer?.sdp?.sdp).toContain('usedtx=1'); + expect(offer?.sdp?.sdp).toContain( + `maxaveragebitrate=${DEFAULT_REMOTE_AUDIO_CONFIG.maxAverageBitrate}` + ); + // And the desktop answered with the same shaping rather than the client"s. + expect(device.desktopPeer().localDescription?.sdp).toContain('useinbandfec=1'); + }); + + it('trickles candidates both ways once, and only once, authenticated (C-07)', async () => { + const device = await world.connectClient(); + device.peer().gatherCandidate('candidate:1 1 udp 1 10.0.0.2 1 typ host'); + await world.advance(); + + expect(device.desktopPeer().candidates).toHaveLength(1); + expect(device.desktopPeer().candidates[0].candidate).toContain('typ host'); + }); +}); + +describe('conformance: teardown', () => { + it('sends bye before a deliberate teardown, and the desktop lets go (C-14)', async () => { + const device = await world.connectClient(); + device.client.disconnect(); + await world.advance(); + + expect(device.socket().ops()).toContain('bye'); + expect(world.transport.signaling.isOnline(device.deviceId)).toBe(false); + }); + + it('treats a closed session as terminal and does not reconnect (C-12)', async () => { + const device = await world.connectClient(); + world.transport.disconnectAll('the desktop disconnected all devices'); + await world.advance(); + + expect(device.state().phase).toBe('terminal'); + await world.advance(60_000); + expect(device.sockets).toHaveLength(1); + }); + + it('restarts ICE and re-offers on peer-failed rather than re-pairing (C-15)', async () => { + const device = await world.connectClient(); + const before = device + .socket() + .ops() + .filter((op) => op === 'offer').length; + const claimsBefore = device + .socket() + .ops() + .filter((op) => op === 'pair-claim').length; + + // The desktop's peer died, which is what the audio host reports. + world.transport.handleHostEvent({ + kind: 'peer-error', + deviceId: device.deviceId, + message: 'The connection to this device failed.', + }); + await world.advance(); + + const after = device + .socket() + .ops() + .filter((op) => op === 'offer').length; + expect(after).toBe(before + 1); + // Still the same pairing: nothing claimed a second code. + expect( + device + .socket() + .ops() + .filter((op) => op === 'pair-claim') + ).toHaveLength(claimsBefore); + expect(device.store.value).not.toBeNull(); + }); +}); diff --git a/src/__tests__/helpers/index.ts b/src/__tests__/helpers/index.ts index ec2927904d..600f3ce176 100644 --- a/src/__tests__/helpers/index.ts +++ b/src/__tests__/helpers/index.ts @@ -8,4 +8,12 @@ export { createMockAITab, createMockFileTab } from './mockTab'; export { createMockSession } from './mockSession'; export { installLocalStorageMock } from './mockLocalStorage'; +export { + createFakeAudioBuffer, + createFakeAudioContext, + createFakeGainNode, + createFakeMediaStream, + installAudioWorkletNodeMock, + installMediaDevicesMock, +} from './mockWebAudio'; export { ALL_RENDERER_STORES, resetAllStores, resetStore, resetStores } from './resetStores'; diff --git a/src/__tests__/helpers/mockWebAudio.ts b/src/__tests__/helpers/mockWebAudio.ts new file mode 100644 index 0000000000..bdade4fa0f --- /dev/null +++ b/src/__tests__/helpers/mockWebAudio.ts @@ -0,0 +1,376 @@ +/** + * Shared Web Audio test doubles. + * + * jsdom implements none of the Web Audio API, and A Cappella's audio host is + * built entirely on it. Rather than let each suite hand-roll another half of an + * `AudioContext`, this is the one fake: a controllable clock, a node graph that + * records its own connections, and a `getUserMedia` that can be told to fail + * with any DOM exception name. + * + * The fakes record rather than assert. Suites check what they care about + * (connections made, sources stopped, gain ramps scheduled) off the recorded + * state. + * + * Usage: + * + * import { createFakeAudioContext, installMediaDevicesMock } from '/helpers/mockWebAudio'; + * + * const context = createFakeAudioContext(); + * const media = installMediaDevicesMock(); + * media.failWith('NotAllowedError'); + */ + +import { vi } from 'vitest'; + +// --------------------------------------------------------------------------- +// Node graph +// --------------------------------------------------------------------------- + +export interface FakeAudioNode { + connect: ReturnType; + disconnect: ReturnType; + /** Everything this node has been connected to, in order. */ + connectedTo: unknown[]; + disconnectCount: number; +} + +function createNode(): FakeAudioNode { + const node: FakeAudioNode = { + connectedTo: [], + disconnectCount: 0, + connect: vi.fn((target: unknown) => { + node.connectedTo.push(target); + return target; + }), + disconnect: vi.fn(() => { + node.disconnectCount += 1; + }), + }; + return node; +} + +/** One scheduled automation on a fake `AudioParam`. */ +export interface GainAutomation { + kind: 'cancel' | 'set' | 'ramp'; + value: number; + time: number; +} + +export interface FakeGainNode extends FakeAudioNode { + gain: { + value: number; + cancelScheduledValues(time: number): void; + setValueAtTime(value: number, time: number): void; + linearRampToValueAtTime(value: number, time: number): void; + }; + automations: GainAutomation[]; +} + +export function createFakeGainNode(): FakeGainNode { + const automations: GainAutomation[] = []; + const node = createNode() as FakeGainNode; + node.automations = automations; + node.gain = { + value: 1, + cancelScheduledValues(time: number) { + automations.push({ kind: 'cancel', value: node.gain.value, time }); + }, + setValueAtTime(value: number, time: number) { + automations.push({ kind: 'set', value, time }); + node.gain.value = value; + }, + linearRampToValueAtTime(value: number, time: number) { + automations.push({ kind: 'ramp', value, time }); + node.gain.value = value; + }, + }; + return node; +} + +export interface FakeBufferSourceNode extends FakeAudioNode { + buffer: FakeAudioBuffer | null; + onended: (() => void) | null; + startedAt: number | null; + stopped: boolean; + start(when?: number): void; + stop(when?: number): void; + /** Test-only: run the `onended` callback as the renderer would. */ + finish(): void; +} + +export interface FakeAudioBuffer { + numberOfChannels: number; + length: number; + sampleRate: number; + duration: number; + getChannelData(channel: number): Float32Array; +} + +export function createFakeAudioBuffer( + length: number, + sampleRate: number, + channels = 1 +): FakeAudioBuffer { + const data = Array.from({ length: channels }, () => new Float32Array(length)); + return { + numberOfChannels: channels, + length, + sampleRate, + duration: length / sampleRate, + getChannelData: (channel: number) => data[channel], + }; +} + +// --------------------------------------------------------------------------- +// Context +// --------------------------------------------------------------------------- + +export interface FakeAudioContext { + state: AudioContextState; + sampleRate: number; + currentTime: number; + destination: FakeAudioNode; + /** Every buffer source the context has handed out. */ + sources: FakeBufferSourceNode[]; + gains: FakeGainNode[]; + addedModules: string[]; + closed: boolean; + resume: ReturnType; + close: ReturnType; + audioWorklet: { addModule: ReturnType }; + createGain(): FakeGainNode; + createBufferSource(): FakeBufferSourceNode; + createMediaStreamSource(stream: unknown): FakeAudioNode & { mediaStream: unknown }; + /** The outbound tap the WebRTC leg connects playback into. */ + createMediaStreamDestination(): FakeAudioNode & { stream: { getAudioTracks(): unknown[] } }; + createBuffer(channels: number, length: number, sampleRate: number): FakeAudioBuffer; + decodeAudioData: ReturnType; + /** Test-only: advance the audio clock. */ + advance(seconds: number): void; +} + +export interface FakeAudioContextOptions { + sampleRate?: number; + state?: AudioContextState; + /** Make `audioWorklet.addModule` reject, as a malformed worklet chunk would. */ + addModuleError?: Error; +} + +export function createFakeAudioContext(options: FakeAudioContextOptions = {}): FakeAudioContext { + const context: FakeAudioContext = { + state: options.state ?? 'running', + sampleRate: options.sampleRate ?? 48000, + currentTime: 0, + destination: createNode(), + sources: [], + gains: [], + addedModules: [], + closed: false, + resume: vi.fn(async () => { + context.state = 'running'; + }), + close: vi.fn(async () => { + context.closed = true; + context.state = 'closed'; + }), + audioWorklet: { + addModule: vi.fn(async (url: string) => { + if (options.addModuleError) throw options.addModuleError; + context.addedModules.push(url); + }), + }, + createGain: () => { + const gain = createFakeGainNode(); + context.gains.push(gain); + return gain; + }, + createBufferSource: () => { + const source = createNode() as FakeBufferSourceNode; + source.buffer = null; + source.onended = null; + source.startedAt = null; + source.stopped = false; + source.start = (when = 0) => { + source.startedAt = when; + }; + source.stop = () => { + source.stopped = true; + }; + source.finish = () => { + source.onended?.(); + }; + context.sources.push(source); + return source; + }, + createMediaStreamSource: (stream: unknown) => + Object.assign(createNode(), { mediaStream: stream }), + createMediaStreamDestination: () => { + const track = { kind: 'audio', id: 'outbound' }; + return Object.assign(createNode(), { + stream: { getAudioTracks: () => [track] }, + }); + }, + createBuffer: (channels: number, length: number, sampleRate: number) => + createFakeAudioBuffer(length, sampleRate, channels), + decodeAudioData: vi.fn(async (data: ArrayBuffer) => + createFakeAudioBuffer(data.byteLength / 2, 24000) + ), + advance: (seconds: number) => { + context.currentTime += seconds; + }, + }; + return context; +} + +// --------------------------------------------------------------------------- +// Media devices +// --------------------------------------------------------------------------- + +export interface FakeMediaStreamTrack { + kind: 'audio'; + label: string; + readyState: 'live' | 'ended'; + stop: ReturnType; + getSettings(): { deviceId: string }; + addEventListener(type: string, listener: () => void): void; + removeEventListener(type: string, listener: () => void): void; + /** Test-only: fire `ended`, as unplugging a headset does. */ + end(): void; +} + +export interface FakeMediaStream { + getTracks(): FakeMediaStreamTrack[]; + getAudioTracks(): FakeMediaStreamTrack[]; + track: FakeMediaStreamTrack; +} + +export function createFakeMediaStream(label = 'MacBook Pro Microphone'): FakeMediaStream { + const listeners = new Map void>>(); + const track: FakeMediaStreamTrack = { + kind: 'audio', + label, + readyState: 'live', + stop: vi.fn(() => { + track.readyState = 'ended'; + }), + getSettings: () => ({ deviceId: 'default' }), + addEventListener: (type, listener) => { + if (!listeners.has(type)) listeners.set(type, new Set()); + listeners.get(type)!.add(listener); + }, + removeEventListener: (type, listener) => { + listeners.get(type)?.delete(listener); + }, + end: () => { + track.readyState = 'ended'; + listeners.get('ended')?.forEach((listener) => listener()); + }, + }; + return { + track, + getTracks: () => [track], + getAudioTracks: () => [track], + }; +} + +export interface MediaDevicesMock { + getUserMedia: ReturnType; + stream: FakeMediaStream; + /** Reject the next `getUserMedia` with a DOMException of this name. */ + failWith(name: string, message?: string): void; + /** Test-only: fire `devicechange`. */ + emitDeviceChange(): void; + listenerCount(): number; + restore(): void; +} + +/** + * Install `navigator.mediaDevices` on the jsdom window. Returns a handle for + * driving it, plus a `restore` that puts the original descriptor back. + */ +export function installMediaDevicesMock(): MediaDevicesMock { + const stream = createFakeMediaStream(); + const listeners = new Set<() => void>(); + let failure: Error | null = null; + + const mediaDevices = { + getUserMedia: vi.fn(async () => { + if (failure) { + const error = failure; + failure = null; + throw error; + } + return stream; + }), + addEventListener: (_type: string, listener: () => void) => { + listeners.add(listener); + }, + removeEventListener: (_type: string, listener: () => void) => { + listeners.delete(listener); + }, + }; + + const original = Object.getOwnPropertyDescriptor(navigator, 'mediaDevices'); + Object.defineProperty(navigator, 'mediaDevices', { + value: mediaDevices, + configurable: true, + writable: true, + }); + + return { + getUserMedia: mediaDevices.getUserMedia, + stream, + failWith: (name: string, message = name) => { + const error = new Error(message); + error.name = name; + failure = error; + }, + emitDeviceChange: () => listeners.forEach((listener) => listener()), + listenerCount: () => listeners.size, + restore: () => { + if (original) Object.defineProperty(navigator, 'mediaDevices', original); + else delete (navigator as unknown as Record).mediaDevices; + }, + }; +} + +/** + * Stub the global `AudioWorkletNode` constructor. Returns the list of nodes it + * built so a suite can drive `port.onmessage` the way the worklet would. + */ +export interface FakeAudioWorkletNode extends FakeAudioNode { + name: string; + options: unknown; + port: { onmessage: ((event: { data: unknown }) => void) | null }; + /** Test-only: deliver a message from the audio thread. */ + emit(data: unknown): void; +} + +export function installAudioWorkletNodeMock(): { + nodes: FakeAudioWorkletNode[]; + restore(): void; +} { + const nodes: FakeAudioWorkletNode[] = []; + + class MockAudioWorkletNode { + constructor(_context: unknown, name: string, options?: unknown) { + const node = createNode() as FakeAudioWorkletNode; + node.name = name; + node.options = options; + node.port = { onmessage: null }; + node.emit = (data: unknown) => node.port.onmessage?.({ data }); + nodes.push(node); + return node as unknown as MockAudioWorkletNode; + } + } + + const original = (globalThis as Record).AudioWorkletNode; + (globalThis as Record).AudioWorkletNode = MockAudioWorkletNode; + + return { + nodes, + restore: () => { + (globalThis as Record).AudioWorkletNode = original; + }, + }; +} diff --git a/src/__tests__/main/acappella/audio-host-window.test.ts b/src/__tests__/main/acappella/audio-host-window.test.ts new file mode 100644 index 0000000000..c4c25036fd --- /dev/null +++ b/src/__tests__/main/acappella/audio-host-window.test.ts @@ -0,0 +1,250 @@ +/** + * A Cappella hidden audio host window. + * + * The window itself is untestable without an Electron runtime, so the + * `BrowserWindow` constructor is mocked and the assertions are about the + * contract the rest of the app depends on: it is never visible, never + * throttled, registered as a feature kind (so the multi-window machinery skips + * it), created at most once, and fully deregistered when it closes. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { WindowRegistry } from '../../../main/window-registry'; + +interface MockWebContents { + setWindowOpenHandler: ReturnType; + on: ReturnType; +} + +class MockBrowserWindow { + static instances: MockBrowserWindow[] = []; + static lastOptions: Record | null = null; + + options: Record; + destroyed = false; + excludedFromShownWindowsMenu = false; + loadedUrls: string[] = []; + handlers = new Map void>>(); + webContents: MockWebContents; + webContentsHandlers = new Map void>>(); + + constructor(options: Record) { + this.options = options; + MockBrowserWindow.lastOptions = options; + MockBrowserWindow.instances.push(this); + this.webContents = { + setWindowOpenHandler: vi.fn(), + on: vi.fn((event: string, handler: (...args: any[]) => void) => { + const list = this.webContentsHandlers.get(event) ?? []; + list.push(handler); + this.webContentsHandlers.set(event, list); + }), + }; + } + + isDestroyed(): boolean { + return this.destroyed; + } + + loadURL(url: string): Promise { + this.loadedUrls.push(url); + return Promise.resolve(); + } + + on(event: string, handler: (...args: any[]) => void): this { + const list = this.handlers.get(event) ?? []; + list.push(handler); + this.handlers.set(event, list); + return this; + } + + close(): void { + this.destroyed = true; + for (const handler of this.handlers.get('closed') ?? []) handler(); + } + + emitWebContents(event: string, ...args: any[]): void { + for (const handler of this.webContentsHandlers.get(event) ?? []) handler(...args); + } +} + +// A GETTER, not `BrowserWindow: MockBrowserWindow`. `vi.mock` is hoisted, so +// this factory runs the moment any imported module pulls in `electron` - which +// happens while this file's own body (and therefore the class binding below) is +// still in its temporal dead zone. Reading the class at property-access time +// instead of factory time makes the mock independent of which import happens to +// reach electron first. +vi.mock('electron', () => ({ + get BrowserWindow() { + return MockBrowserWindow; + }, +})); + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +const PROD_DEPS_BASE = { + preloadPath: '/app/dist/main/preload.js', + rendererProductionUrl: 'maestro://renderer/index.html', + devServerUrl: 'http://localhost:17173', +}; + +async function loadModule() { + return import('../../../main/acappella/audio-host-window'); +} + +describe('main/acappella/audio-host-window', () => { + let registry: WindowRegistry; + const originalPlatform = process.platform; + + beforeEach(() => { + vi.resetModules(); + MockBrowserWindow.instances = []; + MockBrowserWindow.lastOptions = null; + registry = new WindowRegistry(); + }); + + afterEach(() => { + // Module state is per-test (resetModules above), so only the platform + // override has to be undone. + Object.defineProperty(process, 'platform', { + value: originalPlatform, + configurable: true, + }); + }); + + const deps = (overrides: Partial<{ isDevelopment: boolean }> = {}) => ({ + ...PROD_DEPS_BASE, + isDevelopment: false, + windowRegistry: registry, + ...overrides, + }); + + it('creates a hidden, untracked-by-the-OS window that is never background throttled', async () => { + const { ensureAcappellaAudioHostWindow } = await loadModule(); + ensureAcappellaAudioHostWindow(deps()); + + const options = MockBrowserWindow.lastOptions!; + expect(options.show).toBe(false); + expect(options.skipTaskbar).toBe(true); + expect(options.focusable).toBe(false); + expect(options.paintWhenInitiallyHidden).toBe(false); + // A throttled timer in the audio path is an audible dropout. + expect(options.webPreferences.backgroundThrottling).toBe(false); + expect(options.webPreferences.contextIsolation).toBe(true); + expect(options.webPreferences.nodeIntegration).toBe(false); + expect(options.webPreferences.preload).toBe(PROD_DEPS_BASE.preloadPath); + }); + + it('boots the renderer bundle into audio-host mode in both dev and production', async () => { + const { ensureAcappellaAudioHostWindow, closeAcappellaAudioHostWindow } = await loadModule(); + + ensureAcappellaAudioHostWindow(deps()); + expect(MockBrowserWindow.instances[0].loadedUrls).toEqual([ + 'maestro://renderer/index.html?acappellaAudio', + ]); + + closeAcappellaAudioHostWindow(); + ensureAcappellaAudioHostWindow(deps({ isDevelopment: true })); + expect(MockBrowserWindow.instances[1].loadedUrls).toEqual([ + 'http://localhost:17173?acappellaAudio', + ]); + }); + + it('registers as an acappella-audio feature window the multi-window machinery skips', async () => { + const { ensureAcappellaAudioHostWindow } = await loadModule(); + ensureAcappellaAudioHostWindow(deps()); + + const entry = registry.getByKind('acappella-audio'); + expect(entry).toBeDefined(); + expect(entry!.sessionIds).toEqual([]); + expect(entry!.isMain).toBe(false); + // "Move to Window", persistence, auto-close and telemetry all read + // getAppWindows(); the audio host must not be offered to any of them. + expect(registry.getAppWindows()).toHaveLength(0); + }); + + it('is created once no matter how many sessions start', async () => { + const { ensureAcappellaAudioHostWindow, getAcappellaAudioHostWindow } = await loadModule(); + + const first = ensureAcappellaAudioHostWindow(deps()); + const second = ensureAcappellaAudioHostWindow(deps()); + + expect(second).toBe(first); + expect(MockBrowserWindow.instances).toHaveLength(1); + expect(getAcappellaAudioHostWindow()).toBe(first); + }); + + it('deregisters and forgets the window when it closes, and rebuilds on the next start', async () => { + const { + ensureAcappellaAudioHostWindow, + getAcappellaAudioHostWindow, + closeAcappellaAudioHostWindow, + } = await loadModule(); + + ensureAcappellaAudioHostWindow(deps()); + closeAcappellaAudioHostWindow(); + + expect(getAcappellaAudioHostWindow()).toBeNull(); + expect(registry.getAll()).toHaveLength(0); + + ensureAcappellaAudioHostWindow(deps()); + expect(MockBrowserWindow.instances).toHaveLength(2); + expect(registry.getByKind('acappella-audio')).toBeDefined(); + }); + + it('closing when nothing is open is a no-op', async () => { + const { closeAcappellaAudioHostWindow } = await loadModule(); + expect(() => closeAcappellaAudioHostWindow()).not.toThrow(); + expect(MockBrowserWindow.instances).toHaveLength(0); + }); + + it('denies popups and any navigation away from its own entry url', async () => { + const { ensureAcappellaAudioHostWindow } = await loadModule(); + ensureAcappellaAudioHostWindow(deps()); + const win = MockBrowserWindow.instances[0]; + + const openHandler = win.webContents.setWindowOpenHandler.mock.calls[0][0] as () => { + action: string; + }; + expect(openHandler()).toEqual({ action: 'deny' }); + + const sameUrl = { preventDefault: vi.fn() }; + win.emitWebContents('will-navigate', sameUrl, 'maestro://renderer/index.html?acappellaAudio'); + expect(sameUrl.preventDefault).not.toHaveBeenCalled(); + + const elsewhere = { preventDefault: vi.fn() }; + win.emitWebContents('will-navigate', elsewhere, 'https://example.com'); + expect(elsewhere.preventDefault).toHaveBeenCalled(); + }); + + it('grants the microphone only to its own web contents', async () => { + const { ensureAcappellaAudioHostWindow, isAcappellaAudioHostContents } = await loadModule(); + + expect(isAcappellaAudioHostContents({} as never)).toBe(false); + + ensureAcappellaAudioHostWindow(deps()); + const win = MockBrowserWindow.instances[0]; + + expect(isAcappellaAudioHostContents(win.webContents as never)).toBe(true); + expect(isAcappellaAudioHostContents({} as never)).toBe(false); + expect(isAcappellaAudioHostContents(null)).toBe(false); + }); + + it('hides itself from the macOS Window menu, and touches nothing elsewhere', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }); + const macModule = await loadModule(); + macModule.ensureAcappellaAudioHostWindow(deps()); + expect(MockBrowserWindow.instances[0].excludedFromShownWindowsMenu).toBe(true); + macModule.closeAcappellaAudioHostWindow(); + + vi.resetModules(); + MockBrowserWindow.instances = []; + Object.defineProperty(process, 'platform', { value: 'win32', configurable: true }); + const winModule = await loadModule(); + winModule.ensureAcappellaAudioHostWindow(deps()); + expect(MockBrowserWindow.instances[0].excludedFromShownWindowsMenu).toBe(false); + }); +}); diff --git a/src/__tests__/main/acappella/audio/audio-bridge.test.ts b/src/__tests__/main/acappella/audio/audio-bridge.test.ts new file mode 100644 index 0000000000..d5544d0b5e --- /dev/null +++ b/src/__tests__/main/acappella/audio/audio-bridge.test.ts @@ -0,0 +1,580 @@ +/** + * @file audio-bridge.test.ts + * + * The composition root where capture, the meter, the microphone projection, and + * playback meet the session. + * + * The bridge is where "who opens the microphone, and when" is decided, so these + * tests are mostly about edges the individual modules cannot see: a text-in + * provider that must never cost the user a permission prompt, a host renderer + * that boots after the session already asked for capture, and a speech run that + * was cut off rather than finished. + * + * No Electron and no audio device: the session is a fake and the command sink is + * an array. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/sentry', () => ({ captureException: vi.fn() })); +vi.mock('../../../../main/utils/logger', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + VoiceAudioBridge, + createVoiceAudioBridge, +} from '../../../../main/acappella/audio/audio-bridge'; +import type { AudioBridgeSession } from '../../../../main/acappella/audio/audio-bridge'; +import { + ACAPPELLA_AUDIO_FRAME_SAMPLES, + ACAPPELLA_AUDIO_SAMPLE_RATE, + type AudioFrame, + type AudioHostCommand, + type AudioHostErrorCode, +} from '../../../../shared/acappella/audio-host'; +import type { MicState, VoiceEvent } from '../../../../shared/acappella/protocol'; +import type { SttCallbacks, SttProvider, TtsChunk } from '../../../../shared/acappella/providers'; +import type { VoiceSessionState } from '../../../../shared/acappella/session-state'; + +// --------------------------------------------------------------------------- +// Signal generators +// --------------------------------------------------------------------------- + +function build(fill: (index: number) => number): Int16Array { + const samples = new Int16Array(ACAPPELLA_AUDIO_FRAME_SAMPLES); + for (let i = 0; i < ACAPPELLA_AUDIO_FRAME_SAMPLES; i++) { + const value = Math.max(-1, Math.min(1, fill(i))); + samples[i] = value < 0 ? value * 0x8000 : value * 0x7fff; + } + return samples; +} + +const silence = (): Int16Array => build(() => 0); + +const tone = (amplitude = 0.4): Int16Array => + build((i) => amplitude * Math.sin((2 * Math.PI * 200 * i) / ACAPPELLA_AUDIO_SAMPLE_RATE)); + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +class FakeStt implements SttProvider { + readonly id = 'fake-stt'; + readonly label = 'Fake STT'; + readonly tier = 'mock' as const; + readonly sampleRate = ACAPPELLA_AUDIO_SAMPLE_RATE; + + readonly fed: Int16Array[] = []; + flushes = 0; + flushError: Error | null = null; + + constructor(readonly acceptsAudio: boolean) {} + + async start(_callbacks: SttCallbacks): Promise {} + feed(pcm: Int16Array): void { + this.fed.push(pcm); + } + async flush(): Promise { + this.flushes += 1; + if (this.flushError) throw this.flushError; + } + async stop(): Promise {} +} + +class FakeSession implements AudioBridgeSession { + state: VoiceSessionState = 'idle'; + stt: FakeStt | null = null; + interrupts = 0; + readonly levels: Array<{ level: number; speech: boolean }> = []; + readonly micStates: MicState[] = []; + readonly failures: Array<{ code: AudioHostErrorCode; message: string }> = []; + + private readonly listeners = new Set<(event: VoiceEvent) => void>(); + + getState(): VoiceSessionState { + return this.state; + } + interrupt(): boolean { + if (this.state !== 'speaking') return false; + this.interrupts += 1; + this.state = 'listening'; + return true; + } + subscribe(listener: (event: VoiceEvent) => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + getActiveStt(): SttProvider | null { + return this.state === 'idle' ? null : this.stt; + } + publishAudioLevel(level: number, speech: boolean): void { + this.levels.push({ level, speech }); + } + publishMicState(state: MicState): void { + this.micStates.push(state); + } + reportAudioCaptureFailure(code: AudioHostErrorCode, message: string): void { + this.failures.push({ code, message }); + } + + /** Emit a protocol event the way the real service fans out to its subscribers. */ + emit(event: Partial & { type: VoiceEvent['type'] }): void { + const full = { sessionId: 'voice-1', seq: 1, ts: 0, ...event } as VoiceEvent; + for (const listener of [...this.listeners]) listener(full); + } + + /** Open the floor the way `startSession` does: state first, then the event. */ + listen(): void { + this.state = 'listening'; + this.emit({ type: 'listen-start', scope: { kind: 'conductor' }, sttProviderId: 'fake-stt' }); + } +} + +interface Harness { + bridge: VoiceAudioBridge; + session: FakeSession; + stt: FakeStt; + commands: AudioHostCommand[]; + kinds: () => string[]; + push(samples: Int16Array, count?: number): void; +} + +let sequence = 0; + +function frame(samples: Int16Array): AudioFrame { + sequence += 1; + return { + seq: sequence, + capturedAt: 1_000 + sequence * 20, + rms: 0, + pcm: samples.buffer.slice(0) as ArrayBuffer, + }; +} + +function harness(options: { acceptsAudio?: boolean } = {}): Harness { + sequence = 0; + const session = new FakeSession(); + const stt = new FakeStt(options.acceptsAudio ?? true); + session.stt = stt; + const commands: AudioHostCommand[] = []; + + const bridge = createVoiceAudioBridge({ + session, + sendCommand: (command) => commands.push(command), + }); + + return { + bridge, + session, + stt, + commands, + kinds: () => commands.map((command) => command.kind), + push: (samples, count = 1) => { + for (let i = 0; i < count; i++) bridge.handleFrame(frame(samples)); + }, + }; +} + +/** Boot the host and open the floor, then forget the commands that took. */ +function ready(h: Harness): void { + h.bridge.handleStatus({ kind: 'ready' }); + h.session.listen(); + h.bridge.handleStatus({ + kind: 'capture-start', + device: { deviceId: 'default', label: 'Built-in Microphone' }, + contextSampleRate: 48_000, + }); + h.commands.length = 0; + h.session.micStates.length = 0; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +// --------------------------------------------------------------------------- + +describe('VoiceAudioBridge capture lifecycle', () => { + it('opens the microphone when the floor opens', () => { + const h = harness(); + h.bridge.handleStatus({ kind: 'ready' }); + + h.session.listen(); + + expect(h.kinds()).toContain('start-capture'); + }); + + it('never opens a device for a recogniser that cannot hear', () => { + const h = harness({ acceptsAudio: false }); + h.bridge.handleStatus({ kind: 'ready' }); + + h.session.listen(); + + // The mock tier is text-in by construction. A permission prompt on its behalf + // buys a level meter over a transcript that is never coming. + expect(h.kinds()).not.toContain('start-capture'); + }); + + it('closes the microphone when the session ends', () => { + const h = harness(); + ready(h); + + h.session.state = 'idle'; + h.session.emit({ type: 'listen-stop', reason: 'stopped' }); + + expect(h.kinds()).toEqual(['stop-capture', 'flush']); + }); + + it('re-requests capture when the host renderer boots after the session did', () => { + const h = harness(); + + // The window is created on the first session start, so the renderer is still + // loading its bundle when the floor opens: the first request is lost. + h.session.listen(); + h.commands.length = 0; + + h.bridge.handleStatus({ kind: 'ready' }); + + expect(h.kinds()).toEqual(['start-capture']); + }); + + it('does not request capture on a ready that no session is waiting for', () => { + const h = harness(); + + h.bridge.handleStatus({ kind: 'ready' }); + + expect(h.commands).toEqual([]); + }); +}); + +describe('VoiceAudioBridge frame routing', () => { + it('feeds captured audio to the recogniser while listening', () => { + const h = harness(); + ready(h); + + h.push(tone(), 5); + + expect(h.stt.fed.length).toBeGreaterThan(0); + expect(h.bridge.getStats().framesReceived).toBe(5); + }); + + it('counts frames that arrive with nowhere to go rather than queueing them', () => { + const h = harness(); + ready(h); + h.session.state = 'dispatching'; + + h.push(tone(), 30); + + expect(h.stt.fed).toEqual([]); + expect(h.bridge.getStats().framesDropped).toBe(30); + }); + + it('ignores frames once disposed', () => { + const h = harness(); + ready(h); + + h.bridge.dispose(); + h.push(tone(), 5); + + expect(h.stt.fed).toEqual([]); + }); +}); + +describe('VoiceAudioBridge level meter', () => { + it('publishes a downsampled level rather than one update per frame', () => { + const h = harness(); + ready(h); + + h.push(tone(), 30); + + // The meter's window is what decides the rate; the bridge only publishes what + // comes back out of it. + expect(h.session.levels.length).toBeGreaterThan(0); + expect(h.session.levels.length).toBeLessThan(30); + expect(h.session.levels.at(-1)!.level).toBeGreaterThan(0); + }); + + it('marks the windows the detector held the floor open for', () => { + const h = harness(); + ready(h); + + h.push(tone(), 30); + + // A meter that shows a loud room and one that shows a person talking are + // different things, and only this flag can tell them apart. + expect(h.session.levels.some((update) => update.speech)).toBe(true); + }); + + it('stops republishing an unchanging silence', () => { + const h = harness(); + ready(h); + + h.push(silence(), 60); + + // An open microphone in a quiet room is most of a voice session, not an edge + // case: the meter falls to zero once and then says nothing. + expect(h.session.levels).toHaveLength(1); + expect(h.session.levels[0].level).toBe(0); + }); +}); + +describe('VoiceAudioBridge microphone state', () => { + it('publishes the device once capture proves the permission', () => { + const h = harness(); + h.bridge.handleStatus({ kind: 'ready' }); + h.session.listen(); + + h.bridge.handleStatus({ + kind: 'capture-start', + device: { deviceId: 'default', label: 'Built-in Microphone' }, + contextSampleRate: 48_000, + }); + + expect(h.session.micStates.at(-1)).toMatchObject({ + permission: 'granted', + capturing: true, + deviceLabel: 'Built-in Microphone', + }); + }); + + it('turns a capture failure into both a microphone state and a session error', () => { + const h = harness(); + ready(h); + + h.bridge.handleStatus({ + kind: 'mic-error', + code: 'permission-denied', + message: 'Permission denied', + }); + + // A dead microphone must never present as a session that is merely quiet, so + // it produces the state a HUD draws AND the error a client can act on. + expect(h.session.micStates.at(-1)).toMatchObject({ + permission: 'denied', + issue: 'permission-denied', + capturing: false, + }); + expect(h.session.failures).toEqual([ + { code: 'permission-denied', message: 'Permission denied' }, + ]); + }); + + it('leaves the meter at rest when the device goes away', () => { + const h = harness(); + ready(h); + h.push(tone(), 30); + h.session.levels.length = 0; + + h.bridge.handleStatus({ kind: 'capture-stop', reason: 'device-lost' }); + h.session.state = 'listening'; + h.push(silence(), 10); + + // A bar left standing over a device nobody is reading is the same lie as a + // listening indicator over a denied one, so the next run republishes from + // scratch instead of inheriting the last level. + expect(h.session.levels.length).toBeGreaterThan(0); + expect(h.session.levels[0].level).toBe(0); + }); + + it('says nothing about statuses that say nothing about the microphone', () => { + const h = harness(); + ready(h); + + h.bridge.handleStatus({ + kind: 'playback-state', + playing: true, + utteranceId: 'u1', + queuedMs: 40, + }); + + expect(h.session.micStates).toEqual([]); + }); +}); + +describe('VoiceAudioBridge barge-in', () => { + it('cuts the assistant off when the user talks over it', () => { + const h = harness(); + ready(h); + h.session.state = 'speaking'; + + h.push(tone(), 12); + + expect(h.session.interrupts).toBe(1); + expect(h.kinds()).toContain('flush'); + }); + + it('ducks on suspicion before it is sure', () => { + const h = harness(); + ready(h); + h.session.state = 'speaking'; + + h.push(tone(), 1); + + // 80 ms before a `speech-start` can be confirmed, the user hears themselves + // win the room. + expect(h.commands[0]).toMatchObject({ kind: 'duck' }); + expect(h.session.interrupts).toBe(0); + }); +}); + +describe('VoiceAudioBridge playback', () => { + function chunk(overrides: Partial = {}): TtsChunk { + return { + utteranceId: 'u1', + index: 0, + text: 'All done.', + format: 'pcm16', + sampleRate: 24_000, + audio: new Uint8Array([1, 2, 3, 4]), + ...overrides, + }; + } + + it('plays a chunk through the same host that captures', () => { + const h = harness(); + ready(h); + + h.bridge.handleSpeechChunk(chunk()); + + expect(h.commands).toEqual([ + expect.objectContaining({ + kind: 'play', + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 24_000, + }), + ]); + const command = h.commands[0] as Extract; + expect(new Uint8Array(command.data)).toEqual(new Uint8Array([1, 2, 3, 4])); + }); + + it('sends a container format on to be decoded', () => { + const h = harness(); + ready(h); + + h.bridge.handleSpeechChunk(chunk({ format: 'mp3', sampleRate: undefined })); + + expect(h.commands[0]).toMatchObject({ kind: 'play', format: 'encoded' }); + }); + + it('drops a silent chunk instead of playing nothing', () => { + const h = harness(); + ready(h); + + h.bridge.handleSpeechChunk(chunk({ format: 'none', audio: null })); + + expect(h.commands).toEqual([]); + }); + + it('refuses raw samples with no sample rate rather than guessing one', () => { + const h = harness(); + ready(h); + + h.bridge.handleSpeechChunk(chunk({ sampleRate: undefined })); + + // Guessing is how a voice ends up an octave out. + expect(h.commands).toEqual([]); + }); + + it('drains the queue when a speech run finishes on its own', () => { + const h = harness(); + ready(h); + + h.session.emit({ type: 'speak-end', utteranceId: 'u1', reason: 'complete' }); + + expect(h.commands).toEqual([{ kind: 'end-utterance', utteranceId: 'u1' }]); + }); + + it('throws the queue away when a run was cut off', () => { + const h = harness(); + ready(h); + + h.session.emit({ type: 'speak-end', utteranceId: 'u1', reason: 'cancelled' }); + + // What is queued is audio the user has already talked over. + expect(h.commands).toEqual([{ kind: 'flush' }]); + }); + + it('drops playback aimed at a host that has not booted yet', () => { + const h = harness(); + h.session.listen(); + h.commands.length = 0; + + h.bridge.handleSpeechChunk(chunk()); + + expect(h.commands).toEqual([]); + }); +}); + +describe('VoiceAudioBridge endpointing', () => { + it('forces the recogniser to endpoint on demand', () => { + const h = harness(); + ready(h); + + h.bridge.endUtterance(); + + expect(h.stt.flushes).toBe(1); + }); + + it('is a no-op when there is no session to endpoint', () => { + const h = harness(); + + expect(() => h.bridge.endUtterance()).not.toThrow(); + expect(h.stt.flushes).toBe(0); + }); + + it('survives a recogniser that cannot take the hint', async () => { + const h = harness(); + ready(h); + h.stt.flushError = new Error('no endpointing here'); + + h.bridge.endUtterance(); + await Promise.resolve(); + + // Endpointing is a hint; the recogniser still has the audio. + expect(h.stt.flushes).toBe(1); + }); +}); + +describe('VoiceAudioBridge teardown', () => { + it('closes the microphone on dispose rather than after it', () => { + const h = harness(); + ready(h); + + h.bridge.dispose(); + + // The stop command has to leave BEFORE the bridge marks itself dead, or the + // device stays open with nothing left to close it. + expect(h.kinds()).toContain('stop-capture'); + }); + + it('is safe to dispose twice', () => { + const h = harness(); + ready(h); + + h.bridge.dispose(); + h.commands.length = 0; + h.bridge.dispose(); + + expect(h.commands).toEqual([]); + }); + + it('stops following the session it was disposed for', () => { + const h = harness(); + ready(h); + h.bridge.dispose(); + h.commands.length = 0; + + h.session.listen(); + + expect(h.commands).toEqual([]); + }); +}); + +describe('VoiceAudioBridge', () => { + it('is constructible through its factory', () => { + const h = harness(); + expect(h.bridge).toBeInstanceOf(VoiceAudioBridge); + }); +}); diff --git a/src/__tests__/main/acappella/audio/audio-pipeline.test.ts b/src/__tests__/main/acappella/audio/audio-pipeline.test.ts new file mode 100644 index 0000000000..afdb1b5136 --- /dev/null +++ b/src/__tests__/main/acappella/audio/audio-pipeline.test.ts @@ -0,0 +1,627 @@ +/** + * @file audio-pipeline.test.ts + * + * The duplex audio pipeline: what reaches the recogniser, what is dropped, what + * is held in the pre-roll, and what happens when the user talks over the + * assistant. + * + * Everything here is generated PCM against a fake session and a fake recogniser. + * No `AudioContext`, no audio host window, no timers - the pipeline is injected + * with its three seams (session, provider, command sink) precisely so a test can + * drive a full barge-in in a few microseconds. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/sentry', () => ({ captureException: vi.fn() })); +vi.mock('../../../../main/utils/logger', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { captureException } from '../../../../main/utils/sentry'; +import { + AudioFrameRing, + AudioPipeline, + DEFAULT_PRE_ROLL_MS, + MAX_PRE_ROLL_MS, + createAudioPipeline, + type AudioPipelineOptions, + type AudioPipelineSession, +} from '../../../../main/acappella/audio/audio-pipeline'; +import { + ACAPPELLA_AUDIO_FRAME_MS, + ACAPPELLA_AUDIO_FRAME_SAMPLES, + ACAPPELLA_AUDIO_SAMPLE_RATE, + type AudioFrame, + type AudioHostCommand, +} from '../../../../shared/acappella/audio-host'; +import type { SttCallbacks, SttProvider } from '../../../../shared/acappella/providers'; +import type { VoiceSessionState } from '../../../../shared/acappella/session-state'; + +// --------------------------------------------------------------------------- +// Signal generators +// --------------------------------------------------------------------------- + +function build(fill: (index: number) => number): Int16Array { + const samples = new Int16Array(ACAPPELLA_AUDIO_FRAME_SAMPLES); + for (let i = 0; i < ACAPPELLA_AUDIO_FRAME_SAMPLES; i++) { + const value = Math.max(-1, Math.min(1, fill(i))); + samples[i] = value < 0 ? value * 0x8000 : value * 0x7fff; + } + return samples; +} + +const silence = (): Int16Array => build(() => 0); + +/** Voiced speech stand-in: 200 Hz sits squarely inside the VAD's zero-crossing band. */ +const tone = (amplitude = 0.4): Int16Array => + build((i) => amplitude * Math.sin((2 * Math.PI * 200 * i) / ACAPPELLA_AUDIO_SAMPLE_RATE)); + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +class FakeStt implements SttProvider { + readonly id = 'fake-stt'; + readonly label = 'Fake STT'; + readonly tier = 'mock' as const; + readonly sampleRate = ACAPPELLA_AUDIO_SAMPLE_RATE; + readonly acceptsAudio = true; + + readonly fed: Int16Array[] = []; + flushes = 0; + feedError: Error | null = null; + flushError: Error | null = null; + + async start(_callbacks: SttCallbacks): Promise {} + + feed(pcm: Int16Array): void { + if (this.feedError) throw this.feedError; + this.fed.push(pcm); + } + + async flush(): Promise { + this.flushes += 1; + if (this.flushError) throw this.flushError; + } + + async stop(): Promise {} +} + +class FakeSession implements AudioPipelineSession { + state: VoiceSessionState = 'idle'; + interrupts = 0; + + getState(): VoiceSessionState { + return this.state; + } + + /** Mirrors the real service: barge-in only means something while speaking. */ + interrupt(): boolean { + if (this.state !== 'speaking') return false; + this.interrupts += 1; + this.state = 'listening'; + return true; + } +} + +interface Harness { + pipeline: AudioPipeline; + session: FakeSession; + stt: FakeStt; + commands: AudioHostCommand[]; + bargeIns: number; + /** Push `count` copies of one frame through the pipeline. */ + push(samples: Int16Array, count?: number): void; +} + +function harness(overrides: Partial = {}): Harness { + const session = new FakeSession(); + const stt = new FakeStt(); + const commands: AudioHostCommand[] = []; + let seq = 0; + + const state = { bargeIns: 0 }; + const pipeline = createAudioPipeline({ + session, + getStt: () => stt, + sendCommand: (command) => commands.push(command), + onBargeIn: () => { + state.bargeIns += 1; + }, + ...overrides, + }); + + return { + pipeline, + session, + stt, + commands, + get bargeIns() { + return state.bargeIns; + }, + push(samples, count = 1) { + for (let i = 0; i < count; i++) { + seq += 1; + pipeline.handleFrame(frameOf(samples, seq)); + } + }, + }; +} + +function frameOf(samples: Int16Array, seq: number): AudioFrame { + // A copy per frame, because the pipeline keeps references in the pre-roll and a + // shared buffer would make every buffered frame the last one pushed. + const copy = new Int16Array(samples); + return { + seq, + capturedAt: 1_700_000_000_000 + seq * ACAPPELLA_AUDIO_FRAME_MS, + rms: 0, + pcm: copy.buffer, + }; +} + +function commandsOfKind( + commands: AudioHostCommand[], + kind: K +): Extract[] { + return commands.filter((c): c is Extract => c.kind === kind); +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +// --------------------------------------------------------------------------- + +describe('AudioFrameRing', () => { + it('evicts the oldest frame once it is full', () => { + const ring = new AudioFrameRing(2); + const first = new Int16Array([1]); + const second = new Int16Array([2]); + const third = new Int16Array([3]); + ring.push(first); + ring.push(second); + ring.push(third); + + expect(ring.size).toBe(2); + expect(ring.drain()).toEqual([second, third]); + expect(ring.size).toBe(0); + }); + + it('holds nothing at zero capacity', () => { + const ring = new AudioFrameRing(0); + ring.push(new Int16Array([1])); + expect(ring.size).toBe(0); + }); +}); + +describe('capture lifecycle', () => { + it('opens the microphone on start and closes it on stop', () => { + const h = harness(); + h.pipeline.start(); + expect(h.commands).toEqual([{ kind: 'start-capture' }]); + + h.pipeline.stop(); + // The flush is not optional: a session that ends mid-sentence must not keep + // talking into a room whose microphone it just released. + expect(h.commands).toEqual([ + { kind: 'start-capture' }, + { kind: 'stop-capture' }, + { kind: 'flush' }, + ]); + }); + + it('is idempotent in both directions', () => { + const h = harness(); + h.pipeline.start(); + h.pipeline.start(); + h.pipeline.stop(); + h.pipeline.stop(); + expect(commandsOfKind(h.commands, 'start-capture')).toHaveLength(1); + expect(commandsOfKind(h.commands, 'stop-capture')).toHaveLength(1); + }); + + it('ignores frames that arrive before start or after stop', () => { + const h = harness(); + h.session.state = 'listening'; + + h.push(tone(), 3); + expect(h.stt.fed).toHaveLength(0); + + h.pipeline.start(); + h.push(tone(), 3); + expect(h.stt.fed).toHaveLength(3); + + h.pipeline.stop(); + h.push(tone(), 3); + expect(h.stt.fed).toHaveLength(3); + }); + + it('clears the counters between runs', () => { + const h = harness(); + h.pipeline.start(); + h.push(tone(), 5); + expect(h.pipeline.getStats().framesReceived).toBe(5); + + h.pipeline.stop(); + h.pipeline.start(); + expect(h.pipeline.getStats().framesReceived).toBe(0); + }); +}); + +describe('routing frames to the recogniser', () => { + it('feeds every frame while the session is listening', () => { + const h = harness(); + h.session.state = 'listening'; + h.pipeline.start(); + + h.push(tone(), 10); + + expect(h.stt.fed).toHaveLength(10); + expect(h.pipeline.getStats()).toMatchObject({ + framesReceived: 10, + framesDelivered: 10, + framesDropped: 0, + }); + }); + + it('drops frames rather than queueing them when the session is not listening', () => { + const h = harness(); + h.session.state = 'dispatching'; + h.pipeline.start(); + + h.push(tone(), 200); + + expect(h.stt.fed).toHaveLength(0); + expect(h.pipeline.getStats().framesDropped).toBe(200); + }); + + it('drops frames when no recogniser is running', () => { + const h = harness({ getStt: () => null }); + h.session.state = 'listening'; + h.pipeline.start(); + + h.push(tone(), 4); + + expect(h.pipeline.getStats()).toMatchObject({ framesDelivered: 0, framesDropped: 4 }); + }); + + it('forwards the VAD endpoint to the recogniser as a flush', () => { + const h = harness({ vad: { adaptiveNoiseFloor: false, endpointSilenceMs: 100 } }); + h.session.state = 'listening'; + h.pipeline.start(); + + h.push(tone(), 6); + expect(h.stt.flushes).toBe(0); + + // 100 ms of silence is five frames; the fifth endpoints. + h.push(silence(), 5); + expect(h.stt.flushes).toBe(1); + }); + + it('counts frames the transport lost instead of hiding the gap', () => { + const h = harness(); + h.session.state = 'listening'; + h.pipeline.start(); + + h.pipeline.handleFrame(frameOf(tone(), 1)); + h.pipeline.handleFrame(frameOf(tone(), 5)); + + expect(h.pipeline.getStats().sequenceGaps).toBe(3); + }); +}); + +describe('pre-roll', () => { + const preRollFrames = DEFAULT_PRE_ROLL_MS / ACAPPELLA_AUDIO_FRAME_MS; + + it('sizes itself from the configured window', () => { + expect(harness().pipeline.preRollCapacity).toBe(preRollFrames); + expect(harness({ preRollMs: 200 }).pipeline.preRollCapacity).toBe(10); + expect(harness({ preRollMs: 0 }).pipeline.preRollCapacity).toBe(0); + }); + + it('clamps an absurd window rather than trusting the setting', () => { + expect(harness({ preRollMs: 60_000 }).pipeline.preRollCapacity).toBe( + MAX_PRE_ROLL_MS / ACAPPELLA_AUDIO_FRAME_MS + ); + expect(harness({ preRollMs: Number.NaN }).pipeline.preRollCapacity).toBe(preRollFrames); + }); + + it('replays the audio spoken just before the floor opened', () => { + const h = harness(); + h.pipeline.start(); + + // The user starts talking before the wake word has opened the session. + h.push(tone(), 3); + expect(h.stt.fed).toHaveLength(0); + + h.session.state = 'listening'; + h.push(tone(), 1); + + // Three pre-roll frames plus the live one: the first syllable survives. + expect(h.stt.fed).toHaveLength(4); + expect(h.pipeline.getStats().preRollFramesDelivered).toBe(3); + }); + + it('is bounded: a long idle stretch replays only the last window', () => { + const h = harness(); + h.pipeline.start(); + + h.push(tone(), 1000); + h.session.state = 'listening'; + h.push(tone(), 1); + + expect(h.pipeline.getStats().preRollFramesDelivered).toBe(preRollFrames); + expect(h.stt.fed).toHaveLength(preRollFrames + 1); + expect(h.pipeline.getStats().framesDropped).toBe(1000); + }); + + it('does not re-deliver audio the recogniser already has', () => { + const h = harness(); + h.session.state = 'listening'; + h.pipeline.start(); + + h.push(tone(), 5); + h.session.state = 'dispatching'; + h.push(silence(), 2); + h.session.state = 'listening'; + h.push(tone(), 1); + + // Five live frames, the two dropped ones from the pre-roll, one more live. + expect(h.stt.fed).toHaveLength(8); + expect(h.pipeline.getStats().preRollFramesDelivered).toBe(2); + }); + + it('drops the pre-roll when the capture device goes away', () => { + const h = harness(); + h.pipeline.start(); + h.push(tone(), 5); + + h.pipeline.handleStatus({ kind: 'mic-error', code: 'device-lost', message: 'gone' }); + h.session.state = 'listening'; + h.push(tone(), 1); + + // Audio from a device that no longer exists is not context, it is confusion. + expect(h.pipeline.getStats().preRollFramesDelivered).toBe(0); + expect(h.stt.fed).toHaveLength(1); + }); + + it('keeps the pre-roll when a device is merely plugged in', () => { + const h = harness(); + h.pipeline.start(); + h.push(tone(), 5); + + // `device-change` says the device SET moved, not that ours did. Plugging in + // headphones mid-sentence must not throw away the words already spoken. + h.pipeline.handleStatus({ kind: 'device-change' }); + h.pipeline.handleStatus({ kind: 'ready' }); + h.pipeline.handleStatus({ + kind: 'playback-state', + playing: false, + utteranceId: null, + queuedMs: 0, + }); + h.session.state = 'listening'; + h.push(tone(), 1); + + expect(h.pipeline.getStats().preRollFramesDelivered).toBe(5); + expect(h.stt.fed).toHaveLength(6); + }); +}); + +describe('barge-in', () => { + it('flushes playback, cancels the speech run, and takes the floor back', () => { + const h = harness(); + h.session.state = 'speaking'; + h.pipeline.start(); + + h.push(tone(), 6); + + expect(commandsOfKind(h.commands, 'flush')).toHaveLength(1); + expect(h.session.interrupts).toBe(1); + expect(h.session.state).toBe('listening'); + expect(h.bargeIns).toBe(1); + expect(h.pipeline.getStats().bargeIns).toBe(1); + }); + + it('delivers the audio the user interrupted with', () => { + const h = harness(); + h.session.state = 'speaking'; + h.pipeline.start(); + + h.push(tone(), 6); + + // The interrupting syllables were in the pre-roll when the floor opened, so + // the recogniser hears the whole word rather than its tail. + expect(h.stt.fed.length).toBeGreaterThanOrEqual(6); + expect(h.pipeline.getStats().preRollFramesDelivered).toBeGreaterThan(0); + }); + + it('ducks output on suspicion, before the interrupt is confirmed', () => { + const h = harness(); + h.session.state = 'speaking'; + h.pipeline.start(); + + h.push(tone(), 1); + + // One frame in: too little evidence to cancel a speech run, enough to get out + // of the user's way. + expect(commandsOfKind(h.commands, 'duck')).toEqual([{ kind: 'duck', gain: 0.2, ms: 60 }]); + expect(commandsOfKind(h.commands, 'flush')).toHaveLength(0); + expect(h.session.interrupts).toBe(0); + }); + + it('restores the gain when the suspicion does not become speech', () => { + const h = harness(); + h.session.state = 'speaking'; + h.pipeline.start(); + + h.push(tone(), 2); + h.push(silence(), 2); + + const ducks = commandsOfKind(h.commands, 'duck'); + expect(ducks).toHaveLength(2); + expect(ducks[1]).toEqual({ kind: 'duck', gain: 1, ms: 60 }); + expect(h.session.interrupts).toBe(0); + }); + + it('restores the gain when playback ends on its own', () => { + const h = harness(); + h.session.state = 'speaking'; + h.pipeline.start(); + + h.push(tone(), 1); + h.session.state = 'listening'; + h.push(silence(), 1); + + const ducks = commandsOfKind(h.commands, 'duck'); + expect(ducks[ducks.length - 1]).toEqual({ kind: 'duck', gain: 1, ms: 60 }); + }); + + it('does not interrupt while the session is listening', () => { + const h = harness(); + h.session.state = 'listening'; + h.pipeline.start(); + + h.push(tone(), 20); + + expect(commandsOfKind(h.commands, 'flush')).toHaveLength(0); + expect(commandsOfKind(h.commands, 'duck')).toHaveLength(0); + expect(h.session.interrupts).toBe(0); + }); + + it('needs sustained speech, not one loud frame', () => { + const h = harness({ vad: { adaptiveNoiseFloor: false, enterFrames: 4 } }); + h.session.state = 'speaking'; + h.pipeline.start(); + + h.push(tone(), 3); + expect(h.session.interrupts).toBe(0); + + h.push(tone(), 1); + expect(h.session.interrupts).toBe(1); + }); + + it('starts barge-in detection from a closed floor when playback begins', () => { + const h = harness(); + h.session.state = 'listening'; + h.pipeline.start(); + + // The user is mid-utterance, so the detector is open when the reply starts. + h.push(tone(), 10); + h.session.state = 'speaking'; + + // A single frame of leftover speech must not read as an interruption; the + // evidence has to be gathered again against the assistant's own voice. + h.push(tone(), 1); + expect(h.session.interrupts).toBe(0); + h.push(tone(), 3); + expect(h.session.interrupts).toBe(1); + }); + + it('still flushes when the speech run ended between the frame and the interrupt', () => { + const session = new FakeSession(); + const h = harness({ session }); + session.state = 'speaking'; + h.pipeline.start(); + + // The service moved on by itself: `interrupt()` reports nothing to cancel. + session.interrupt = () => false; + h.push(tone(), 6); + + expect(commandsOfKind(h.commands, 'flush')).toHaveLength(1); + expect(h.pipeline.getStats().bargeIns).toBe(0); + expect(h.bargeIns).toBe(0); + }); +}); + +describe('failure handling', () => { + it('counts a throwing feed once and keeps the run alive', () => { + const h = harness(); + h.session.state = 'listening'; + h.pipeline.start(); + h.stt.feedError = new Error('provider died'); + + expect(() => h.push(tone(), 50)).not.toThrow(); + + expect(h.pipeline.getStats().feedErrors).toBe(50); + expect(h.pipeline.getStats().framesDelivered).toBe(0); + // 50 identical reports a second is how a Sentry project becomes unreadable. + expect(captureException).toHaveBeenCalledTimes(1); + }); + + it('reports a rejected endpoint without failing the frame', async () => { + const h = harness({ vad: { adaptiveNoiseFloor: false, endpointSilenceMs: 100 } }); + h.session.state = 'listening'; + h.pipeline.start(); + h.stt.flushError = new Error('no endpoint'); + + h.push(tone(), 6); + h.push(silence(), 5); + await Promise.resolve(); + + expect(captureException).toHaveBeenCalledTimes(1); + expect(h.pipeline.getStats().framesDelivered).toBe(11); + }); +}); + +describe('observability', () => { + it('reports every frame with its verdict and whether it was delivered', () => { + const seen: { delivered: boolean; rms: number }[] = []; + const h = harness({ + onFrame: ({ result, delivered }) => seen.push({ delivered, rms: result.rms }), + }); + h.session.state = 'idle'; + h.pipeline.start(); + + h.push(tone(), 2); + h.session.state = 'listening'; + h.push(tone(), 2); + + expect(seen).toHaveLength(4); + expect(seen.map((s) => s.delivered)).toEqual([false, false, true, true]); + // The level meter in Phase 02's HUD task reads exactly this. + expect(seen[0].rms).toBeGreaterThan(0); + }); + + it('resets its counters when capture restarts', () => { + const h = harness(); + h.session.state = 'listening'; + h.pipeline.start(); + h.push(tone(), 4); + + h.pipeline.handleStatus({ + kind: 'capture-start', + device: { deviceId: 'default', label: 'Mic' }, + contextSampleRate: 48_000, + }); + + expect(h.pipeline.getStats().framesReceived).toBe(0); + }); + + it('stops and releases everything on dispose', () => { + const h = harness(); + h.session.state = 'listening'; + h.pipeline.start(); + h.push(tone(), 4); + + h.pipeline.dispose(); + + expect(h.pipeline.isRunning).toBe(false); + expect(commandsOfKind(h.commands, 'stop-capture')).toHaveLength(1); + }); +}); + +describe('AudioPipeline construction', () => { + it('accepts a partial VAD config and exposes what it resolved', () => { + const pipeline = new AudioPipeline({ + session: new FakeSession(), + getStt: () => null, + sendCommand: () => {}, + vad: { endpointSilenceMs: 400 }, + }); + expect(pipeline.isRunning).toBe(false); + expect(pipeline.preRollCapacity).toBe(DEFAULT_PRE_ROLL_MS / ACAPPELLA_AUDIO_FRAME_MS); + }); +}); diff --git a/src/__tests__/main/acappella/audio/floor-control.test.ts b/src/__tests__/main/acappella/audio/floor-control.test.ts new file mode 100644 index 0000000000..4272e9c830 --- /dev/null +++ b/src/__tests__/main/acappella/audio/floor-control.test.ts @@ -0,0 +1,656 @@ +/** + * @file floor-control.test.ts + * + * Who holds the microphone: tap versus hold semantics, the idle timeout, and the + * ways the floor can change without anyone pressing anything. + * + * The controller is injected with a fake session, so every case here runs + * without Electron, a hotkey, a phone, or an audio device. Time is faked because + * the idle timeout is the one part of A Cappella's audio path that legitimately + * uses a wall clock: it is measuring a human who walked away, not audio. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +vi.mock('../../../../main/utils/sentry', () => ({ captureException: vi.fn() })); +vi.mock('../../../../main/utils/logger', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { captureException } from '../../../../main/utils/sentry'; +import { + DEFAULT_FLOOR_MODE, + DEFAULT_IDLE_TIMEOUT_MS, + FloorController, + MAX_IDLE_TIMEOUT_MS, + MIN_IDLE_TIMEOUT_MS, + createFloorController, + resolveFloorControlConfig, + type FloorCloseReason, + type FloorControlOptions, + type FloorControlSession, + type FloorOpenReason, + type FloorSessionStopReason, +} from '../../../../main/acappella/audio/floor-control'; +import type { VoiceEvent, VoiceScope, WakeSource } from '../../../../shared/acappella/protocol'; +import type { VoiceSessionState } from '../../../../shared/acappella/session-state'; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +class FakeSession implements FloorControlSession { + state: VoiceSessionState = 'idle'; + readonly starts: { scope: VoiceScope; source?: WakeSource }[] = []; + readonly stops: FloorSessionStopReason[] = []; + interrupts = 0; + startError: Error | null = null; + stopError: Error | null = null; + /** Set by a test to hold `startSession` open until it resolves. */ + startGate: Promise | null = null; + + getState(): VoiceSessionState { + return this.state; + } + + async startSession(params: { scope: VoiceScope; source?: WakeSource }): Promise { + if (this.startGate) await this.startGate; + if (this.startError) throw this.startError; + this.starts.push(params); + this.state = 'listening'; + return { state: this.state }; + } + + async stopSession(reason: FloorSessionStopReason): Promise { + this.stops.push(reason); + if (this.stopError) throw this.stopError; + this.state = 'idle'; + } + + interrupt(): boolean { + if (this.state !== 'speaking') return false; + this.interrupts += 1; + this.state = 'listening'; + return true; + } +} + +interface Harness { + floor: FloorController; + session: FakeSession; + changes: { open: boolean; reason: FloorOpenReason | FloorCloseReason }[]; + endUtterances: number; + errors: Error[]; +} + +function harness(overrides: Partial = {}): Harness { + const session = new FakeSession(); + const changes: Harness['changes'] = []; + const errors: Error[] = []; + const state = { endUtterances: 0 }; + + const floor = createFloorController({ + session, + onFloorChange: (open, reason) => changes.push({ open, reason }), + onError: (error) => errors.push(error), + endUtterance: () => { + state.endUtterances += 1; + }, + ...overrides, + }); + + return { + floor, + session, + changes, + errors, + get endUtterances() { + return state.endUtterances; + }, + }; +} + +/** A protocol event with the envelope filled in. Only `type` matters here. */ +function event(body: Partial & { type: VoiceEvent['type'] }): VoiceEvent { + return { sessionId: 'voice-1', seq: 1, ts: 0, ...body } as unknown as VoiceEvent; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +describe('resolveFloorControlConfig', () => { + it('defaults to hands-free with the default idle timeout', () => { + expect(resolveFloorControlConfig()).toEqual({ + mode: DEFAULT_FLOOR_MODE, + idleTimeoutMs: DEFAULT_IDLE_TIMEOUT_MS, + }); + }); + + it('clamps an idle timeout that would hang up on an ordinary pause', () => { + expect(resolveFloorControlConfig({ idleTimeoutMs: 200 }).idleTimeoutMs).toBe( + MIN_IDLE_TIMEOUT_MS + ); + }); + + it('clamps an idle timeout that would leave a microphone open all day', () => { + expect(resolveFloorControlConfig({ idleTimeoutMs: 60 * 60_000 }).idleTimeoutMs).toBe( + MAX_IDLE_TIMEOUT_MS + ); + }); + + it('treats zero as explicitly disabled rather than clamping it up', () => { + expect(resolveFloorControlConfig({ idleTimeoutMs: 0 }).idleTimeoutMs).toBe(0); + }); + + it('falls back to the default for a nonsense timeout', () => { + expect(resolveFloorControlConfig({ idleTimeoutMs: Number.NaN }).idleTimeoutMs).toBe( + DEFAULT_IDLE_TIMEOUT_MS + ); + expect(resolveFloorControlConfig({ idleTimeoutMs: -5 }).idleTimeoutMs).toBe( + DEFAULT_IDLE_TIMEOUT_MS + ); + }); + + it('falls back to the default for an unknown mode', () => { + expect(resolveFloorControlConfig({ mode: 'push' as never }).mode).toBe(DEFAULT_FLOOR_MODE); + }); +}); + +// --------------------------------------------------------------------------- +// Tap to toggle +// --------------------------------------------------------------------------- + +describe('tap-to-toggle', () => { + it('opens a conductor-scoped session on the first press', async () => { + const h = harness(); + + await h.floor.press('hotkey'); + + expect(h.session.starts).toEqual([{ scope: { kind: 'conductor' }, source: 'hotkey' }]); + expect(h.floor.isFloorOpen).toBe(true); + expect(h.changes).toEqual([{ open: true, reason: 'press' }]); + }); + + it('opens the scope the caller supplies', async () => { + const scope: VoiceScope = { kind: 'agent', sessionId: 'agent-7' }; + const h = harness({ getScope: () => scope }); + + await h.floor.press(); + + expect(h.session.starts[0].scope).toEqual(scope); + }); + + it('closes the session on the next press', async () => { + const h = harness(); + + await h.floor.press(); + await h.floor.release(); + await h.floor.press(); + + expect(h.session.stops).toEqual(['user']); + expect(h.floor.isFloorOpen).toBe(false); + expect(h.changes.at(-1)).toEqual({ open: false, reason: 'toggle' }); + }); + + it('ignores a held key repeating, so a repeat cannot toggle the floor', async () => { + const h = harness(); + + await h.floor.press(); + await h.floor.press(); + await h.floor.press(); + + expect(h.session.starts).toHaveLength(1); + expect(h.session.stops).toEqual([]); + expect(h.floor.isFloorOpen).toBe(true); + }); + + it('ignores a release: a tap and a long press are the same gesture', async () => { + const h = harness(); + + await h.floor.press(); + await h.floor.release(); + + expect(h.floor.isFloorOpen).toBe(true); + expect(h.session.stops).toEqual([]); + expect(h.endUtterances).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// Hold to talk +// --------------------------------------------------------------------------- + +describe('hold-to-talk', () => { + it('opens the floor on press', async () => { + const h = harness({ mode: 'hold-to-talk' }); + + await h.floor.press('hotkey'); + + expect(h.floor.isFloorOpen).toBe(true); + expect(h.floor.isHeld).toBe(true); + expect(h.session.starts).toHaveLength(1); + }); + + it('endpoints the utterance on release, bypassing VAD silence', async () => { + const h = harness({ mode: 'hold-to-talk' }); + + await h.floor.press(); + await h.floor.release(); + + expect(h.endUtterances).toBe(1); + expect(h.floor.isFloorOpen).toBe(false); + expect(h.changes.at(-1)).toEqual({ open: false, reason: 'release' }); + }); + + it('keeps the session alive after release so the reply can arrive', async () => { + const h = harness({ mode: 'hold-to-talk' }); + + await h.floor.press(); + await h.floor.release(); + + expect(h.session.stops).toEqual([]); + expect(h.session.state).toBe('listening'); + }); + + it('ignores a release with no matching press', async () => { + const h = harness({ mode: 'hold-to-talk' }); + + await h.floor.release(); + + expect(h.endUtterances).toBe(0); + expect(h.changes).toEqual([]); + }); + + it('serialises a release that lands while the session is still starting', async () => { + const h = harness({ mode: 'hold-to-talk' }); + let openTheGate!: () => void; + h.session.startGate = new Promise((resolve) => { + openTheGate = resolve; + }); + + const pressed = h.floor.press(); + const released = h.floor.release(); + await Promise.resolve(); + // The release cannot be allowed to run against a session that does not exist + // yet, so it waits behind the press rather than seeing a closed floor. + expect(h.endUtterances).toBe(0); + + openTheGate(); + await pressed; + await released; + + expect(h.session.starts).toHaveLength(1); + expect(h.endUtterances).toBe(1); + expect(h.floor.isFloorOpen).toBe(false); + }); + + it('closes the floor even when endpointing fails', async () => { + const error = new Error('provider gone'); + const h = harness({ + mode: 'hold-to-talk', + endUtterance: () => { + throw error; + }, + }); + + await h.floor.press(); + await h.floor.release(); + + expect(h.floor.isFloorOpen).toBe(false); + expect(h.errors).toEqual([error]); + expect(captureException).toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// Interruption +// --------------------------------------------------------------------------- + +describe('press over active speech', () => { + it('interrupts rather than ending the session', async () => { + const h = harness(); + await h.floor.press(); + await h.floor.release(); + h.session.state = 'speaking'; + h.floor.handleEvent(event({ type: 'speak-start' })); + + await h.floor.press(); + + expect(h.session.interrupts).toBe(1); + expect(h.session.stops).toEqual([]); + expect(h.floor.isFloorOpen).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Idle timeout +// --------------------------------------------------------------------------- + +describe('idle timeout', () => { + it('closes a listening session that hears nothing', async () => { + const h = harness({ idleTimeoutMs: 10_000 }); + await h.floor.press(); + + await vi.advanceTimersByTimeAsync(10_000); + await h.floor.whenSettled(); + + expect(h.session.stops).toEqual(['timeout']); + expect(h.floor.isFloorOpen).toBe(false); + expect(h.changes.at(-1)).toEqual({ open: false, reason: 'idle-timeout' }); + }); + + it('restarts on speech so a request in progress is never cut off', async () => { + const h = harness({ idleTimeoutMs: 10_000 }); + await h.floor.press(); + + await vi.advanceTimersByTimeAsync(9_000); + h.floor.noteActivity(); + await vi.advanceTimersByTimeAsync(9_000); + + expect(h.session.stops).toEqual([]); + + await vi.advanceTimersByTimeAsync(1_000); + await h.floor.whenSettled(); + expect(h.session.stops).toEqual(['timeout']); + }); + + it('does not run while the session is working on the user behalf', async () => { + const h = harness({ idleTimeoutMs: 10_000 }); + await h.floor.press(); + + h.session.state = 'dispatching'; + h.floor.handleEvent(event({ type: 'dispatch' })); + await vi.advanceTimersByTimeAsync(60_000); + + expect(h.session.stops).toEqual([]); + expect(h.floor.isFloorOpen).toBe(true); + }); + + it('rearms when the floor comes back to listening', async () => { + const h = harness({ idleTimeoutMs: 10_000 }); + await h.floor.press(); + h.session.state = 'speaking'; + h.floor.handleEvent(event({ type: 'speak-start' })); + await vi.advanceTimersByTimeAsync(60_000); + + h.session.state = 'listening'; + h.floor.handleEvent(event({ type: 'listen-start' })); + await vi.advanceTimersByTimeAsync(10_000); + await h.floor.whenSettled(); + + expect(h.session.stops).toEqual(['timeout']); + }); + + it('closes a hold-to-talk session whose key was already released', async () => { + const h = harness({ mode: 'hold-to-talk', idleTimeoutMs: 10_000 }); + await h.floor.press(); + await h.floor.release(); + + await vi.advanceTimersByTimeAsync(10_000); + await h.floor.whenSettled(); + + expect(h.session.stops).toEqual(['timeout']); + }); + + it('is disabled by a zero timeout', async () => { + const h = harness({ idleTimeoutMs: 0 }); + await h.floor.press(); + + await vi.advanceTimersByTimeAsync(MAX_IDLE_TIMEOUT_MS); + + expect(h.session.stops).toEqual([]); + expect(h.floor.isFloorOpen).toBe(true); + }); + + it('is not kept alive by the app talking to itself', async () => { + const h = harness({ idleTimeoutMs: 10_000 }); + await h.floor.press(); + + // A roster push every few seconds is background noise, not a human in the + // room: it must not hold the floor open indefinitely. + for (let i = 0; i < 5; i++) { + await vi.advanceTimersByTimeAsync(2_000); + h.floor.handleEvent(event({ type: 'agent-roster', agents: [] })); + } + await h.floor.whenSettled(); + + expect(h.session.stops).toEqual(['timeout']); + }); + + it('is restarted by a transcript', async () => { + const h = harness({ idleTimeoutMs: 10_000 }); + await h.floor.press(); + + await vi.advanceTimersByTimeAsync(9_000); + h.floor.handleEvent(event({ type: 'partial-transcript', text: 'hey', stability: 0.4 })); + await vi.advanceTimersByTimeAsync(9_000); + + expect(h.session.stops).toEqual([]); + }); + + it('applies a new timeout immediately rather than after the armed one', async () => { + const h = harness({ idleTimeoutMs: 60_000 }); + await h.floor.press(); + + h.floor.configure({ idleTimeoutMs: 10_000 }); + await vi.advanceTimersByTimeAsync(10_000); + await h.floor.whenSettled(); + + expect(h.session.stops).toEqual(['timeout']); + }); +}); + +// --------------------------------------------------------------------------- +// Following the session +// --------------------------------------------------------------------------- + +describe('session events', () => { + it('adopts a floor opened by the wake word', () => { + const h = harness(); + h.session.state = 'listening'; + + h.floor.handleEvent(event({ type: 'listen-start' })); + + expect(h.floor.isFloorOpen).toBe(true); + expect(h.changes).toEqual([{ open: true, reason: 'session-started' }]); + }); + + it('releases the floor when the stop word ends the session', async () => { + const h = harness(); + await h.floor.press(); + + h.session.state = 'idle'; + h.floor.handleEvent(event({ type: 'listen-stop', reason: 'stopped' })); + + expect(h.floor.isFloorOpen).toBe(false); + expect(h.floor.isHeld).toBe(false); + // The session ended on its own; closing it again would be a second stop. + expect(h.session.stops).toEqual([]); + }); + + it('releases the floor on an unrecoverable failure', async () => { + const h = harness(); + await h.floor.press(); + + h.session.state = 'error'; + h.floor.handleEvent( + event({ + type: 'session-error', + code: 'provider-unavailable', + message: 'no stt', + recoverable: false, + }) + ); + + expect(h.floor.isFloorOpen).toBe(false); + }); + + it('keeps the floor through a recoverable failure', async () => { + const h = harness(); + await h.floor.press(); + + h.floor.handleEvent( + event({ + type: 'session-error', + code: 'no-agent-matched', + message: 'nobody home', + recoverable: true, + }) + ); + + expect(h.floor.isFloorOpen).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Mode changes, failures, shutdown +// --------------------------------------------------------------------------- + +describe('configure', () => { + it('keeps an open floor when switching to tap while held', async () => { + const h = harness({ mode: 'hold-to-talk' }); + await h.floor.press(); + + h.floor.configure({ mode: 'tap-to-toggle' }); + await h.floor.release(); + + expect(h.floor.isFloorOpen).toBe(true); + expect(h.endUtterances).toBe(0); + }); + + it('reports the resolved mode and timeout', () => { + const h = harness(); + + h.floor.configure({ mode: 'hold-to-talk', idleTimeoutMs: 1 }); + + expect(h.floor.mode).toBe('hold-to-talk'); + expect(h.floor.idleTimeoutMs).toBe(MIN_IDLE_TIMEOUT_MS); + }); +}); + +describe('failures', () => { + it('leaves the floor closed when the session cannot start', async () => { + const h = harness(); + h.session.startError = new Error('no provider'); + + await h.floor.press(); + + expect(h.floor.isFloorOpen).toBe(false); + expect(h.floor.isHeld).toBe(false); + expect(h.errors).toHaveLength(1); + expect(captureException).toHaveBeenCalled(); + }); + + it('accepts a press after a failed start rather than wedging', async () => { + const h = harness(); + h.session.startError = new Error('no provider'); + await h.floor.press(); + + h.session.startError = null; + await h.floor.press(); + + expect(h.floor.isFloorOpen).toBe(true); + expect(h.session.starts).toHaveLength(1); + }); + + it('reports a throwing floor subscriber without abandoning the rest of the action', async () => { + const session = new FakeSession(); + const errors: Error[] = []; + const floor = createFloorController({ + session, + onError: (error) => errors.push(error), + // The capture gate is this seam and it sends IPC, so a window destroyed + // between the press and the notify throws right here. + onFloorChange: () => { + throw new Error('audio host is gone'); + }, + }); + + await floor.press(); + expect(errors).toHaveLength(1); + expect(captureException).toHaveBeenCalled(); + // The session did open; only the notification failed. + expect(session.starts).toHaveLength(1); + + await floor.release(); + await floor.press(); + + // The close notify throws too, and the session is still stopped: a listener + // that cannot be told must not leave a live session behind a shut floor. + expect(errors).toHaveLength(2); + expect(session.stops).toEqual(['user']); + expect(floor.isFloorOpen).toBe(false); + }); + + it('reports a failing stop without leaving the floor open', async () => { + const h = harness(); + await h.floor.press(); + await h.floor.release(); + h.session.stopError = new Error('teardown exploded'); + + await h.floor.press(); + + expect(h.floor.isFloorOpen).toBe(false); + expect(h.errors).toHaveLength(1); + }); +}); + +describe('dispose', () => { + it('closes the floor and the session', async () => { + const h = harness(); + await h.floor.press(); + + await h.floor.dispose(); + + expect(h.floor.isFloorOpen).toBe(false); + expect(h.session.stops).toEqual(['shutdown']); + expect(h.changes.at(-1)).toEqual({ open: false, reason: 'shutdown' }); + }); + + it('stops a hold-to-talk session whose floor was already closed', async () => { + const h = harness({ mode: 'hold-to-talk' }); + await h.floor.press(); + await h.floor.release(); + + await h.floor.dispose(); + + expect(h.session.stops).toEqual(['shutdown']); + }); + + it('ignores input afterwards and is safe to repeat', async () => { + const h = harness(); + await h.floor.press(); + await h.floor.dispose(); + await h.floor.dispose(); + + await h.floor.press(); + + expect(h.session.starts).toHaveLength(1); + expect(h.session.stops).toEqual(['shutdown']); + expect(h.floor.isFloorOpen).toBe(false); + }); +}); + +describe('createFloorController', () => { + it('builds a controller with the resolved config', () => { + const controller = createFloorController({ + session: new FakeSession(), + mode: 'hold-to-talk', + idleTimeoutMs: 99, + }); + + expect(controller).toBeInstanceOf(FloorController); + expect(controller.mode).toBe('hold-to-talk'); + expect(controller.idleTimeoutMs).toBe(MIN_IDLE_TIMEOUT_MS); + }); +}); diff --git a/src/__tests__/main/acappella/audio/level-meter.test.ts b/src/__tests__/main/acappella/audio/level-meter.test.ts new file mode 100644 index 0000000000..ffe62bb648 --- /dev/null +++ b/src/__tests__/main/acappella/audio/level-meter.test.ts @@ -0,0 +1,178 @@ +/** + * @file level-meter.test.ts + * + * The A Cappella input level meter. + * + * Contracts defended: + * - The window is counted in frames, so the update rate is a property of the + * audio quantum rather than of how busy the main thread was. + * - What comes out is the true RMS of the window, not a mean of RMSs. + * - Speech in any frame of the window survives into the update; a meter that + * averaged the flag away could not tell a loud room from a talking person. + * - Silence stops republishing itself after the meter has visibly fallen, and a + * reset makes the next run publish again. + * - Bad configuration is clamped, never thrown: these numbers arrive from user + * settings and this runs inside the audio path. + */ + +import { describe, it, expect } from 'vitest'; +import { + AudioLevelMeter, + DEFAULT_LEVEL_METER_CONFIG, + DEFAULT_LEVEL_SILENCE, + createAudioLevelMeter, + resolveLevelMeterConfig, +} from '../../../../main/acappella/audio/level-meter'; +import type { AudioLevelUpdate } from '../../../../main/acappella/audio/level-meter'; + +/** Push `count` frames at one level and collect whatever was published. */ +function push( + meter: AudioLevelMeter, + count: number, + rms: number, + speech = false +): AudioLevelUpdate[] { + const updates: AudioLevelUpdate[] = []; + for (let i = 0; i < count; i += 1) { + const update = meter.push(rms, speech); + if (update) updates.push(update); + } + return updates; +} + +describe('resolveLevelMeterConfig', () => { + it('defaults to a 20 ms frame and a 20 Hz target', () => { + expect(resolveLevelMeterConfig()).toEqual(DEFAULT_LEVEL_METER_CONFIG); + expect(DEFAULT_LEVEL_METER_CONFIG.frameMs).toBe(20); + expect(DEFAULT_LEVEL_METER_CONFIG.updateHz).toBe(20); + }); + + it('falls back rather than throwing on nonsense', () => { + const config = resolveLevelMeterConfig({ + frameMs: 0, + updateHz: Number.NaN, + silenceLevel: -1, + }); + expect(config.frameMs).toBe(DEFAULT_LEVEL_METER_CONFIG.frameMs); + expect(config.updateHz).toBe(DEFAULT_LEVEL_METER_CONFIG.updateHz); + expect(config.silenceLevel).toBe(0); + }); +}); + +describe('AudioLevelMeter windowing', () => { + it('publishes once per window, not once per frame', () => { + const meter = new AudioLevelMeter(); + expect(meter.windowFrames).toBe(3); + + expect(meter.push(0.1, true)).toBeNull(); + expect(meter.push(0.1, true)).toBeNull(); + expect(meter.push(0.1, true)).not.toBeNull(); + }); + + it('lands within a couple of Hz of the 20 per second target', () => { + // The frame quantum decides the achievable rate: 2 frames is 25/s and 3 is + // 16.7/s, and neither is 20. What must hold is that it is close, and that + // the number reported is the realised rate rather than the requested one. + const meter = new AudioLevelMeter(); + expect(meter.updateHz).toBeCloseTo(16.7, 1); + expect(Math.abs(meter.updateHz - 20)).toBeLessThan(4); + }); + + it('never degenerates to zero frames per window on an absurd rate', () => { + expect(new AudioLevelMeter({ updateHz: 5000 }).windowFrames).toBe(1); + }); + + it('honours a wider window when a client asks for fewer updates', () => { + const meter = new AudioLevelMeter({ updateHz: 5 }); + expect(meter.windowFrames).toBe(10); + expect(push(meter, 9, 0.1)).toHaveLength(0); + expect(push(meter, 1, 0.1)).toHaveLength(1); + }); +}); + +describe('AudioLevelMeter measurement', () => { + it('reports the root mean square of the window, not the mean', () => { + const meter = new AudioLevelMeter({ updateHz: 1000 / (2 * 20) }); + expect(meter.windowFrames).toBe(2); + + meter.push(0.3, false); + const update = meter.push(0.1, false); + // sqrt((0.09 + 0.01) / 2) = 0.2236, above the arithmetic mean of 0.2. + expect(update?.level).toBeCloseTo(Math.sqrt(0.05), 6); + expect(update?.level).toBeGreaterThan(0.2); + }); + + it('clamps a frame outside 0 to 1 instead of publishing it', () => { + const meter = new AudioLevelMeter({ updateHz: 50 }); + expect(meter.windowFrames).toBe(1); + expect(meter.push(4, false)?.level).toBe(1); + expect(meter.push(-1, false)?.level).toBe(0); + // Still at rest, so the second silent window says nothing new. + expect(meter.push(Number.NaN, false)).toBeNull(); + }); + + it('carries speech out of the window if any frame in it was speech', () => { + const meter = new AudioLevelMeter(); + meter.push(0.001, false); + meter.push(0.001, true); + expect(meter.push(0.001, false)?.speech).toBe(true); + }); + + it('starts the next window clean', () => { + const meter = new AudioLevelMeter(); + expect(push(meter, 3, 0.2, true)[0].speech).toBe(true); + expect(push(meter, 3, 0.2, false)[0].speech).toBe(false); + }); +}); + +describe('AudioLevelMeter rest suppression', () => { + it('publishes the fall to silence once and then goes quiet', () => { + const meter = new AudioLevelMeter(); + expect(push(meter, 3, 0.2)).toHaveLength(1); + + // The first silent window has to reach the client, or the meter freezes at + // whatever the last loud frame left on screen. + const first = push(meter, 3, 0); + expect(first).toHaveLength(1); + expect(first[0].level).toBe(0); + + // 60 more windows of an open microphone in a quiet room: nothing to say. + expect(push(meter, 180, 0)).toHaveLength(0); + }); + + it('publishes again the moment the room moves', () => { + const meter = new AudioLevelMeter(); + push(meter, 30, 0); + expect(push(meter, 3, 0.2)).toHaveLength(1); + }); + + it('does not suppress a quiet window the detector called speech', () => { + const meter = new AudioLevelMeter(); + push(meter, 3, 0); + push(meter, 3, 0); + // Below the silence level, but the floor is open: a whisper still moves the + // meter, and a client that stopped hearing updates would show it as closed. + expect(push(meter, 3, DEFAULT_LEVEL_SILENCE / 2, true)).toHaveLength(1); + }); + + it('republishes silence after a reset, so a new run is never born frozen', () => { + const meter = createAudioLevelMeter(); + push(meter, 6, 0); + expect(push(meter, 3, 0)).toHaveLength(0); + + meter.reset(); + expect(push(meter, 3, 0)).toHaveLength(1); + }); + + it('drops the partial window on reset', () => { + const meter = new AudioLevelMeter(); + meter.push(0.5, false); + meter.push(0.5, false); + meter.reset(); + + expect(meter.push(0.1, false)).toBeNull(); + expect(meter.push(0.1, false)).toBeNull(); + // A window built only from the frames pushed after the reset. + expect(meter.push(0.1, false)?.level).toBeCloseTo(0.1, 6); + }); +}); diff --git a/src/__tests__/main/acappella/audio/mic-state.test.ts b/src/__tests__/main/acappella/audio/mic-state.test.ts new file mode 100644 index 0000000000..c8cbca4f7e --- /dev/null +++ b/src/__tests__/main/acappella/audio/mic-state.test.ts @@ -0,0 +1,170 @@ +/** + * @file mic-state.test.ts + * + * The A Cappella microphone state projection. + * + * Contracts defended: + * - A live capture is proof of permission, and the device label it carries + * outlives the capture run rather than blanking on the next status. + * - A denied permission and a missing device are different facts with different + * recoveries, so they never collapse into one another. + * - Nothing observable changed means nothing is published, except a device + * change, which is news even when our own capture is unaffected. + * - `deviceChanged` is a flag on an event and never part of the state, so it + * cannot stick. + */ + +import { describe, it, expect } from 'vitest'; +import { + INITIAL_MIC_STATE, + createMicStateTracker, +} from '../../../../main/acappella/audio/mic-state'; +import type { AudioHostStatus } from '../../../../shared/acappella/audio-host'; + +const CAPTURE_START: AudioHostStatus = { + kind: 'capture-start', + device: { deviceId: 'default', label: 'MacBook Pro Microphone' }, + contextSampleRate: 48000, +}; + +describe('MicStateTracker', () => { + it('starts knowing nothing rather than assuming the best', () => { + const tracker = createMicStateTracker(); + expect(tracker.state).toEqual(INITIAL_MIC_STATE); + expect(tracker.state.permission).toBe('unknown'); + }); + + it('ignores the statuses that say nothing about the microphone', () => { + const tracker = createMicStateTracker(); + expect(tracker.apply({ kind: 'ready' })).toBeNull(); + expect( + tracker.apply({ kind: 'playback-state', playing: true, utteranceId: 'u1', queuedMs: 40 }) + ).toBeNull(); + expect(tracker.state).toEqual(INITIAL_MIC_STATE); + }); + + it('treats a live capture as proof of permission and names the device', () => { + const tracker = createMicStateTracker(); + const state = tracker.apply(CAPTURE_START); + + expect(state).toEqual({ + permission: 'granted', + capturing: true, + deviceId: 'default', + deviceLabel: 'MacBook Pro Microphone', + issue: null, + deviceChanged: false, + }); + }); + + it('publishes nothing when the same capture is reported twice', () => { + const tracker = createMicStateTracker(); + expect(tracker.apply(CAPTURE_START)).not.toBeNull(); + expect(tracker.apply(CAPTURE_START)).toBeNull(); + }); + + it('keeps permission and the device name after a requested stop', () => { + const tracker = createMicStateTracker(); + tracker.apply(CAPTURE_START); + const state = tracker.apply({ kind: 'capture-stop', reason: 'requested' }); + + expect(state?.capturing).toBe(false); + expect(state?.issue).toBeNull(); + // A session ending is not the microphone becoming unknown again. + expect(state?.permission).toBe('granted'); + expect(state?.deviceLabel).toBe('MacBook Pro Microphone'); + }); + + it('reports a device that was taken away as a fault, and forgets it', () => { + const tracker = createMicStateTracker(); + tracker.apply(CAPTURE_START); + const state = tracker.apply({ kind: 'capture-stop', reason: 'device-lost' }); + + expect(state?.issue).toBe('device-lost'); + expect(state?.deviceId).toBeNull(); + expect(state?.deviceLabel).toBeNull(); + // The user granted access; the microphone is what left. + expect(state?.permission).toBe('granted'); + }); + + it('records a denial as a permission fact, not a device fact', () => { + const tracker = createMicStateTracker(); + const state = tracker.apply({ + kind: 'mic-error', + code: 'permission-denied', + message: 'Permission dismissed', + }); + + expect(state?.permission).toBe('denied'); + expect(state?.issue).toBe('permission-denied'); + expect(state?.capturing).toBe(false); + }); + + it('keeps a missing device separate from a denied one', () => { + const tracker = createMicStateTracker(); + const state = tracker.apply({ + kind: 'mic-error', + code: 'no-device', + message: 'No input found', + }); + + expect(state?.issue).toBe('no-device'); + // Nothing was denied - there is simply nothing plugged in. + expect(state?.permission).toBe('unknown'); + }); + + it('collapses the two environment failures into the one with no user recovery', () => { + for (const code of ['unsupported', 'audio-init-failed'] as const) { + const tracker = createMicStateTracker(); + const state = tracker.apply({ kind: 'mic-error', code, message: 'nope' }); + expect(state?.issue).toBe('unavailable'); + } + }); + + it('clears the fault when capture succeeds after a failure', () => { + const tracker = createMicStateTracker(); + tracker.apply({ kind: 'mic-error', code: 'permission-denied', message: 'denied' }); + const state = tracker.apply(CAPTURE_START); + + expect(state?.issue).toBeNull(); + expect(state?.permission).toBe('granted'); + }); + + it('publishes a device change even when our own capture is unaffected', () => { + const tracker = createMicStateTracker(); + tracker.apply(CAPTURE_START); + + const state = tracker.apply({ kind: 'device-change' }); + expect(state?.deviceChanged).toBe(true); + expect(state?.capturing).toBe(true); + expect(state?.deviceLabel).toBe('MacBook Pro Microphone'); + }); + + it('never lets deviceChanged stick to the state', () => { + const tracker = createMicStateTracker(); + tracker.apply(CAPTURE_START); + tracker.apply({ kind: 'device-change' }); + + expect(tracker.state.deviceChanged).toBe(false); + // And the next real transition is not mislabelled as a device change. + expect(tracker.apply({ kind: 'capture-stop', reason: 'requested' })?.deviceChanged).toBe(false); + }); + + it('hands back a copy, so a client cannot mutate the tracker', () => { + const tracker = createMicStateTracker(); + tracker.apply(CAPTURE_START); + + tracker.state.deviceLabel = 'something else'; + expect(tracker.state.deviceLabel).toBe('MacBook Pro Microphone'); + }); + + it('forgets everything on reset', () => { + const tracker = createMicStateTracker(); + tracker.apply(CAPTURE_START); + tracker.reset(); + + expect(tracker.state).toEqual(INITIAL_MIC_STATE); + // A fresh tracker has to publish the first status it sees. + expect(tracker.apply(CAPTURE_START)).not.toBeNull(); + }); +}); diff --git a/src/__tests__/main/acappella/audio/vad.test.ts b/src/__tests__/main/acappella/audio/vad.test.ts new file mode 100644 index 0000000000..5985981792 --- /dev/null +++ b/src/__tests__/main/acappella/audio/vad.test.ts @@ -0,0 +1,478 @@ +/** + * A Cappella voice activity detection. + * + * The detector is pure and synchronous over frames, so every case here is + * generated PCM: sine tones for voiced speech, alternating samples for hiss, a + * sub-audio partial cycle for rumble, zeros for silence. No audio device, no + * fake timers, no `AudioContext` - if any of those were needed the VAD would + * have the wrong shape. + * + * Cases that assert exact thresholds run with `adaptiveNoiseFloor: false` so the + * configured absolute numbers are the operative ones. Adaptation gets its own + * block. + */ + +import { describe, expect, it } from 'vitest'; + +import { + ACAPPELLA_AUDIO_FRAME_SAMPLES, + ACAPPELLA_AUDIO_SAMPLE_RATE, +} from '../../../../shared/acappella/audio-host'; +import type { AudioFrame } from '../../../../shared/acappella/audio-host'; +import { + DEFAULT_VAD_CONFIG, + VoiceActivityDetector, + createVoiceActivityDetector, + measure, + resolveVadConfig, +} from '../../../../main/acappella/audio/vad'; +import type { VadConfig, VadEvent, VadFrameResult } from '../../../../main/acappella/audio/vad'; + +// --------------------------------------------------------------------------- +// Signal generators +// --------------------------------------------------------------------------- + +const FRAME = ACAPPELLA_AUDIO_FRAME_SAMPLES; + +function build(fill: (index: number) => number): Int16Array { + const samples = new Int16Array(FRAME); + for (let i = 0; i < FRAME; i++) { + const value = Math.max(-1, Math.min(1, fill(i))); + samples[i] = value < 0 ? value * 0x8000 : value * 0x7fff; + } + return samples; +} + +const silence = (): Int16Array => build(() => 0); + +/** Voiced speech stand-in: 200 Hz puts the zero-crossing rate squarely in band. */ +const tone = (amplitude: number, frequency = 200): Int16Array => + build((i) => amplitude * Math.sin((2 * Math.PI * frequency * i) / ACAPPELLA_AUDIO_SAMPLE_RATE)); + +/** Broadband transient / hiss: a sign flip every sample is the maximum possible ZCR. */ +const hiss = (amplitude: number): Int16Array => + build((i) => (i % 2 === 0 ? amplitude : -amplitude)); + +/** Rumble: 15 Hz never completes a cycle inside a 20 ms frame, so it never crosses zero. */ +const rumble = (amplitude: number): Int16Array => tone(amplitude, 15); + +function feed(detector: VoiceActivityDetector, frame: Int16Array, count: number): VadFrameResult[] { + const results: VadFrameResult[] = []; + for (let i = 0; i < count; i++) results.push(detector.process(frame)); + return results; +} + +function events(results: VadFrameResult[]): VadEvent[] { + return results.map((r) => r.event).filter((e): e is VadEvent => e !== null); +} + +/** Deterministic thresholds: adaptation is exercised separately. */ +const fixed = (overrides: Partial = {}): VoiceActivityDetector => + createVoiceActivityDetector({ adaptiveNoiseFloor: false, ...overrides }); + +// --------------------------------------------------------------------------- + +describe('measure', () => { + it('reports the RMS of a full-scale tone as roughly 1/sqrt(2)', () => { + const { rms } = measure(tone(1)); + expect(rms).toBeCloseTo(Math.SQRT1_2, 2); + }); + + it('reports zero for an empty frame rather than NaN', () => { + expect(measure(new Int16Array(0))).toEqual({ rms: 0, zeroCrossingRate: 0 }); + }); + + it('reports a real level for a one-sample frame without dividing by zero', () => { + // A truncated frame is a transport artefact, not a signal. The level is + // still measurable; the crossing rate needs two samples to have a meaning, + // and a NaN here would poison every threshold comparison downstream. + const { rms, zeroCrossingRate } = measure(new Int16Array([16384])); + expect(rms).toBeCloseTo(0.5, 2); + expect(zeroCrossingRate).toBe(0); + }); + + it('puts a 200 Hz tone inside the default zero-crossing band', () => { + const { zeroCrossingRate } = measure(tone(0.5)); + expect(zeroCrossingRate).toBeGreaterThan(DEFAULT_VAD_CONFIG.minZeroCrossingRate); + expect(zeroCrossingRate).toBeLessThan(DEFAULT_VAD_CONFIG.maxZeroCrossingRate); + }); + + it('puts hiss above the band and rumble below it', () => { + expect(measure(hiss(0.5)).zeroCrossingRate).toBeGreaterThan( + DEFAULT_VAD_CONFIG.maxZeroCrossingRate + ); + expect(measure(rumble(0.5)).zeroCrossingRate).toBeLessThan( + DEFAULT_VAD_CONFIG.minZeroCrossingRate + ); + }); +}); + +describe('VoiceActivityDetector - onset', () => { + it('stays silent through silence', () => { + const vad = fixed(); + const results = feed(vad, silence(), 100); + expect(events(results)).toEqual([]); + expect(vad.state).toBe('silence'); + expect(results.every((r) => !r.active)).toBe(true); + }); + + it('opens on exactly the configured number of consecutive speech frames', () => { + const vad = fixed({ enterFrames: 4 }); + const results = feed(vad, tone(0.2), 4); + + expect(events(results.slice(0, 3))).toEqual([]); + expect(results[3].event).toEqual({ type: 'speech-start', atMs: 0 }); + expect(vad.state).toBe('speech'); + expect(results[3].active).toBe(true); + }); + + it('backdates speech-start to the first qualifying frame, not the deciding one', () => { + const vad = fixed({ enterFrames: 3, frameMs: 20 }); + feed(vad, silence(), 5); + const results = feed(vad, tone(0.2), 3); + + // Frames 0-4 were silence, so the onset begins at frame 5 => 100 ms. + expect(results[2].event).toEqual({ type: 'speech-start', atMs: 100 }); + // The decision itself landed two frames later, at the 160 ms mark. + expect(results[2].elapsedMs).toBe(160); + }); + + it('does not open on a transient shorter than enterFrames', () => { + const vad = fixed({ enterFrames: 4 }); + // Three loud frames, a gap, three more: the run counter resets, so neither + // burst is ever sustained enough to take the floor. + const results = [ + ...feed(vad, tone(0.4), 3), + ...feed(vad, silence(), 2), + ...feed(vad, tone(0.4), 3), + ]; + + expect(events(results)).toEqual([]); + expect(vad.state).toBe('silence'); + }); + + it('rejects loud hiss: energy alone is not speech', () => { + const vad = fixed(); + expect(events(feed(vad, hiss(0.9), 50))).toEqual([]); + expect(vad.state).toBe('silence'); + }); + + it('rejects loud rumble below the zero-crossing band', () => { + const vad = fixed(); + expect(events(feed(vad, rumble(0.9), 50))).toEqual([]); + expect(vad.state).toBe('silence'); + }); + + it('ignores the zero-crossing band once speech is open, so a trailing fricative sustains it', () => { + const vad = fixed({ enterFrames: 2, endpointSilenceMs: 200 }); + feed(vad, tone(0.3), 2); + expect(vad.state).toBe('speech'); + + // Hiss would never have opened the floor, but it is loud, so it holds it. + const results = feed(vad, hiss(0.3), 20); + expect(events(results)).toEqual([]); + expect(vad.state).toBe('speech'); + }); +}); + +describe('VoiceActivityDetector - hysteresis', () => { + it('sustains speech on levels that were too quiet to open it', () => { + const vad = fixed({ enterRms: 0.05, exitRms: 0.01, enterFrames: 2, endpointSilenceMs: 200 }); + vad.processMeasurement(0.1, 0.05); + vad.processMeasurement(0.1, 0.05); + expect(vad.state).toBe('speech'); + + // Between the two thresholds: too quiet to have entered, loud enough to stay. + const held = Array.from({ length: 30 }, () => vad.processMeasurement(0.03, 0.05)); + expect(events(held)).toEqual([]); + expect(vad.state).toBe('speech'); + expect(held.every((r) => r.active)).toBe(true); + }); + + it('does not open on levels between the two thresholds', () => { + const vad = fixed({ enterRms: 0.05, exitRms: 0.01, enterFrames: 2 }); + const results = Array.from({ length: 30 }, () => vad.processMeasurement(0.03, 0.05)); + expect(events(results)).toEqual([]); + expect(vad.state).toBe('silence'); + }); +}); + +describe('VoiceActivityDetector - candidate frames', () => { + it('flags a voice-like frame long before it would open the floor', () => { + const vad = fixed({ enterFrames: 4 }); + const results = feed(vad, tone(0.3), 3); + + // Every frame looks like voice; none of them is enough evidence yet. This is + // the 80 ms head start the pipeline ducks TTS output on. + expect(results.map((r) => r.candidate)).toEqual([true, true, true]); + expect(events(results)).toEqual([]); + }); + + it('does not flag rumble or hiss, which is what makes the duck safe', () => { + const vad = fixed({ enterFrames: 4 }); + expect(vad.process(rumble(0.3)).candidate).toBe(false); + expect(vad.process(hiss(0.3)).candidate).toBe(false); + expect(vad.process(silence()).candidate).toBe(false); + }); + + it('tracks energy alone once the floor is open', () => { + const vad = fixed({ enterRms: 0.05, exitRms: 0.01, enterFrames: 2 }); + vad.processMeasurement(0.1, 0.05); + vad.processMeasurement(0.1, 0.05); + + // A trailing fricative is high-ZCR and still carries the utterance, so the + // entry band does not apply on the way out. + expect(vad.process(hiss(0.3)).candidate).toBe(true); + expect(vad.process(silence()).candidate).toBe(false); + }); +}); + +describe('VoiceActivityDetector - hangover', () => { + it('keeps frames active through the hangover and drops them after', () => { + const vad = fixed({ enterFrames: 2, hangoverFrames: 5, endpointSilenceMs: 700 }); + feed(vad, tone(0.3), 2); + + const quiet = feed(vad, silence(), 8); + // Silent frames 1..5 are hangover: still part of the utterance. + expect(quiet.slice(0, 5).map((r) => r.active)).toEqual([true, true, true, true, true]); + // From frame 6 the audio stops being fed onward, but the utterance is not + // over: the endpoint decision is a separate, longer clock. + expect(quiet.slice(5).map((r) => r.active)).toEqual([false, false, false]); + expect(quiet.map((r) => r.state)).toEqual(Array(8).fill('speech')); + }); + + it('resets the hangover when speech resumes', () => { + const vad = fixed({ enterFrames: 2, hangoverFrames: 3, endpointSilenceMs: 700 }); + feed(vad, tone(0.3), 2); + feed(vad, silence(), 3); + const resumed = feed(vad, tone(0.3), 1); + + expect(resumed[0].active).toBe(true); + expect(resumed[0].silenceMs).toBe(0); + }); +}); + +describe('VoiceActivityDetector - endpointing', () => { + it('ends the utterance after the configured silence and reports its span', () => { + const vad = fixed({ enterFrames: 2, endpointSilenceMs: 700, frameMs: 20 }); + feed(vad, tone(0.3), 10); // 200 ms of speech + const quiet = feed(vad, silence(), 40); + + const emitted = events(quiet); + expect(emitted).toEqual([ + { + type: 'speech-end', + atMs: 200, + startedAtMs: 0, + durationMs: 200, + trailingSilenceMs: 700, + }, + ]); + // 700 ms is 35 frames, so the decision lands on the 35th silent frame. + expect(quiet.findIndex((r) => r.event !== null)).toBe(34); + expect(vad.state).toBe('silence'); + }); + + it('defaults the endpoint to 700 ms', () => { + expect(DEFAULT_VAD_CONFIG.endpointSilenceMs).toBe(700); + }); + + it('honours a shorter endpoint setting', () => { + const vad = fixed({ enterFrames: 2, endpointSilenceMs: 300, frameMs: 20 }); + feed(vad, tone(0.3), 5); + const quiet = feed(vad, silence(), 20); + + expect(quiet.findIndex((r) => r.event !== null)).toBe(14); // 15 frames = 300 ms + expect(events(quiet)[0]).toMatchObject({ type: 'speech-end', trailingSilenceMs: 300 }); + }); + + it('does not split an utterance on a pause shorter than the endpoint', () => { + const vad = fixed({ enterFrames: 2, endpointSilenceMs: 700, frameMs: 20 }); + feed(vad, tone(0.3), 5); + const pause = feed(vad, silence(), 20); // 400 ms: a person thinking + const second = feed(vad, tone(0.3), 5); + + expect(events(pause)).toEqual([]); + expect(events(second)).toEqual([]); + expect(vad.state).toBe('speech'); + }); + + it('dates speech-end to the last voiced frame, excluding the endpoint silence', () => { + const vad = fixed({ enterFrames: 2, endpointSilenceMs: 200, frameMs: 20 }); + feed(vad, tone(0.3), 5); // ends at 100 ms + feed(vad, silence(), 3); // a 60 ms gap, well short of the endpoint + feed(vad, tone(0.3), 2); // ends at 200 ms + const quiet = feed(vad, silence(), 10); + + expect(events(quiet)[0]).toMatchObject({ atMs: 200, durationMs: 200 }); + }); + + it('opens a second utterance after the first ends', () => { + const vad = fixed({ enterFrames: 2, endpointSilenceMs: 200, frameMs: 20 }); + const all = [ + ...feed(vad, tone(0.3), 5), + ...feed(vad, silence(), 12), + ...feed(vad, tone(0.3), 5), + ]; + + expect(events(all).map((e) => e.type)).toEqual(['speech-start', 'speech-end', 'speech-start']); + }); +}); + +describe('VoiceActivityDetector - noise floor', () => { + it('rejects steady room noise that a fixed threshold would treat as speech', () => { + // ~0.028 RMS: above the 0.02 absolute enter threshold, and in the ZCR band. + const noise = tone(0.04); + expect(measure(noise).rms).toBeGreaterThan(DEFAULT_VAD_CONFIG.enterRms); + + const withoutAdaptation = fixed(); + expect(events(feed(withoutAdaptation, noise, 60)).map((e) => e.type)).toEqual(['speech-start']); + + const withAdaptation = createVoiceActivityDetector(); + expect(events(feed(withAdaptation, noise, 60))).toEqual([]); + expect(withAdaptation.state).toBe('silence'); + }); + + it('still opens on speech that clears the raised threshold', () => { + const vad = createVoiceActivityDetector(); + feed(vad, tone(0.04), 60); // let the floor settle on the room + expect(vad.noiseFloor).toBeGreaterThan(0.01); + + expect(events(feed(vad, tone(0.5), 10)).map((e) => e.type)).toEqual(['speech-start']); + }); + + it('clamps the floor so noise can never adapt the microphone into deafness', () => { + const vad = createVoiceActivityDetector(); + // Hiss is loud but out of band, so it trains the floor without ever opening. + feed(vad, hiss(0.9), 500); + expect(vad.noiseFloor).toBe(DEFAULT_VAD_CONFIG.maxNoiseFloor); + expect(vad.state).toBe('silence'); + }); + + it('follows a room back down quickly when the noise stops', () => { + const vad = createVoiceActivityDetector(); + feed(vad, hiss(0.9), 500); + feed(vad, silence(), 20); + expect(vad.noiseFloor).toBeLessThan(0.001); + }); + + it('freezes the estimate while an utterance is open', () => { + const vad = createVoiceActivityDetector({ enterFrames: 2, endpointSilenceMs: 10_000 }); + feed(vad, tone(0.5), 2); + const floorAtOnset = vad.noiseFloor; + + feed(vad, tone(0.5), 200); + expect(vad.noiseFloor).toBe(floorAtOnset); + }); + + it('does not delay onset for someone who speaks the instant the mic opens', () => { + // Calibration trains on speech here, which is exactly the case the ceiling + // exists for: the floor saturates and real speech still clears it. + const vad = createVoiceActivityDetector({ enterFrames: 4 }); + const results = feed(vad, tone(0.5), 4); + expect(results[3].event).toEqual({ type: 'speech-start', atMs: 0 }); + }); + + it('needs no calibration pass when it is disabled', () => { + // Without the fast head start the floor cannot outrun a noisy room, and the + // detector latches open on noise a calibrated one rejects. + const vad = createVoiceActivityDetector({ calibrationFrames: 0 }); + expect(events(feed(vad, tone(0.04), 60)).map((e) => e.type)).toEqual(['speech-start']); + }); + + it('reports a zero floor when adaptation is disabled', () => { + const vad = fixed(); + const results = feed(vad, tone(0.5), 5); + expect(results.every((r) => r.noiseFloor === 0)).toBe(true); + }); +}); + +describe('VoiceActivityDetector - lifecycle', () => { + it('reset clears the open utterance and the frame clock', () => { + const vad = fixed({ enterFrames: 2 }); + feed(vad, tone(0.3), 5); + expect(vad.state).toBe('speech'); + + vad.reset(); + expect(vad.state).toBe('silence'); + expect(vad.elapsedMs).toBe(0); + expect(vad.noiseFloor).toBe(0); + + // No stale speech-end for audio from the previous run. + expect(events(feed(vad, silence(), 60))).toEqual([]); + }); + + it('accepts an AudioFrame straight off the wire', () => { + const vad = fixed({ enterFrames: 2 }); + const pcm = tone(0.3); + const buffer = new ArrayBuffer(pcm.byteLength); + new Int16Array(buffer).set(pcm); + const frame: AudioFrame = { + seq: 1, + capturedAt: 1_700_000_000_000, + // Deliberately wrong: the detector measures the samples, not this field. + rms: 0, + pcm: buffer, + }; + + expect(vad.processFrame(frame).rms).toBeCloseTo(measure(pcm).rms, 5); + expect(vad.processFrame(frame).event).toEqual({ type: 'speech-start', atMs: 0 }); + }); + + it('advances its clock in frames, independent of wall time', () => { + const vad = fixed({ frameMs: 20 }); + feed(vad, silence(), 50); + expect(vad.elapsedMs).toBe(1000); + }); +}); + +describe('resolveVadConfig', () => { + it('fills in the defaults', () => { + expect(resolveVadConfig()).toEqual(DEFAULT_VAD_CONFIG); + }); + + it('pins the exit threshold below the enter threshold', () => { + // An exit at or above the enter threshold would open and close on one frame. + const config = resolveVadConfig({ enterRms: 0.03, exitRms: 0.5 }); + expect(config.exitRms).toBe(0.03); + }); + + it('clamps rather than throws on nonsense, because these come from user settings', () => { + const config = resolveVadConfig({ + frameMs: -5, + enterFrames: 0, + hangoverFrames: -3, + endpointSilenceMs: 1, + enterRms: 5, + maxZeroCrossingRate: 9, + noiseFloorEnterMargin: 0.1, + }); + + expect(config.frameMs).toBe(1); + expect(config.enterFrames).toBe(1); + expect(config.hangoverFrames).toBe(0); + expect(config.endpointSilenceMs).toBeGreaterThanOrEqual(config.frameMs); + expect(config.enterRms).toBe(1); + expect(config.maxZeroCrossingRate).toBe(1); + expect(config.noiseFloorEnterMargin).toBe(1); + }); + + it('falls back to the default for a non-finite value', () => { + const config = resolveVadConfig({ endpointSilenceMs: Number.NaN, enterRms: Number.NaN }); + expect(config.endpointSilenceMs).toBe(DEFAULT_VAD_CONFIG.endpointSilenceMs); + expect(config.enterRms).toBe(DEFAULT_VAD_CONFIG.enterRms); + }); + + it('never lets the endpoint fall below a single frame', () => { + const config = resolveVadConfig({ frameMs: 20, endpointSilenceMs: 0 }); + expect(config.endpointSilenceMs).toBe(20); + }); +}); + +describe('VoiceActivityDetector construction', () => { + it('exposes the resolved config', () => { + const vad = new VoiceActivityDetector({ endpointSilenceMs: 450 }); + expect(vad.config.endpointSilenceMs).toBe(450); + expect(vad.config.enterRms).toBe(DEFAULT_VAD_CONFIG.enterRms); + }); +}); diff --git a/src/__tests__/main/acappella/echo-stt.test.ts b/src/__tests__/main/acappella/echo-stt.test.ts new file mode 100644 index 0000000000..0685dc7e56 --- /dev/null +++ b/src/__tests__/main/acappella/echo-stt.test.ts @@ -0,0 +1,311 @@ +/** + * @file echo-stt.test.ts + * + * The development echo recogniser: PCM in, speech segments out. + * + * Everything here is generated PCM against recording callbacks. No audio device, + * no pipeline, no audio host - the provider owns its own segmentation precisely + * so it can be driven from an `Int16Array` and nothing else. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +import { EchoSttProvider, createEchoSttProvider } from '../../../main/acappella/providers/echo-stt'; +import { + ACAPPELLA_AUDIO_FRAME_SAMPLES, + ACAPPELLA_AUDIO_SAMPLE_RATE, +} from '../../../shared/acappella/audio-host'; +import type { SttCallbacks } from '../../../shared/acappella/providers'; + +// --------------------------------------------------------------------------- +// Signal generators +// --------------------------------------------------------------------------- + +function build(fill: (index: number) => number): Int16Array { + const samples = new Int16Array(ACAPPELLA_AUDIO_FRAME_SAMPLES); + for (let i = 0; i < ACAPPELLA_AUDIO_FRAME_SAMPLES; i++) { + const value = Math.max(-1, Math.min(1, fill(i))); + samples[i] = value < 0 ? value * 0x8000 : value * 0x7fff; + } + return samples; +} + +const silence = (): Int16Array => build(() => 0); + +/** Voiced speech stand-in: 200 Hz sits squarely inside the VAD's zero-crossing band. */ +const tone = (amplitude = 0.4): Int16Array => + build((i) => amplitude * Math.sin((2 * Math.PI * 200 * i) / ACAPPELLA_AUDIO_SAMPLE_RATE)); + +/** Silence long enough for the default 700 ms endpoint to fire. */ +const ENDPOINT_FRAMES = 40; + +// --------------------------------------------------------------------------- +// Harness +// --------------------------------------------------------------------------- + +interface Recorded { + partials: Array<{ text: string; stability: number }>; + finals: Array<{ text: string; confidence: number; durationMs?: number }>; + errors: Error[]; +} + +function recording(): { callbacks: SttCallbacks; recorded: Recorded } { + const recorded: Recorded = { partials: [], finals: [], errors: [] }; + return { + recorded, + callbacks: { + onPartial: (text, stability) => recorded.partials.push({ text, stability }), + onFinal: (text, confidence, durationMs) => + recorded.finals.push({ text, confidence, durationMs }), + onError: (error) => recorded.errors.push(error), + }, + }; +} + +async function started(options: ConstructorParameters[0] = {}) { + const provider = new EchoSttProvider({ finalDelayMs: 0, ...options }); + const { callbacks, recorded } = recording(); + await provider.start(callbacks); + return { provider, recorded }; +} + +function push(provider: EchoSttProvider, samples: Int16Array, count = 1): void { + for (let i = 0; i < count; i++) provider.feed(samples); +} + +// --------------------------------------------------------------------------- + +describe('EchoSttProvider identity', () => { + it('declares itself an audio consumer at the pipeline sample rate', () => { + const provider = createEchoSttProvider(); + + // The flag the audio bridge reads before opening a microphone. + expect(provider.acceptsAudio).toBe(true); + expect(provider.sampleRate).toBe(ACAPPELLA_AUDIO_SAMPLE_RATE); + expect(provider.tier).toBe('mock'); + }); +}); + +describe('EchoSttProvider segmentation', () => { + it('says nothing about a silent room', async () => { + const { provider, recorded } = await started(); + + push(provider, silence(), 100); + + expect(recorded.finals).toEqual([]); + expect(recorded.partials).toEqual([]); + }); + + it('emits a final transcript once a speech segment endpoints', async () => { + const { provider, recorded } = await started(); + + push(provider, tone(), 10); + push(provider, silence(), ENDPOINT_FRAMES); + + expect(recorded.finals).toHaveLength(1); + expect(recorded.finals[0].text).toMatch(/^Echo utterance 1: 0\.2s of speech\.$/); + expect(recorded.finals[0].confidence).toBeGreaterThan(0); + expect(recorded.finals[0].confidence).toBeLessThan(1); + }); + + it('reports the speech duration without the endpoint silence in it', async () => { + const { provider, recorded } = await started(); + + // 25 frames of tone is 500 ms. The 800 ms of silence that ends the utterance + // is the detector agreeing the user stopped, not something the user said. + push(provider, tone(), 25); + push(provider, silence(), ENDPOINT_FRAMES); + + expect(recorded.finals[0].durationMs).toBe(500); + }); + + it('numbers consecutive segments so one transcript is told from the next', async () => { + const { provider, recorded } = await started(); + + for (let segment = 0; segment < 3; segment++) { + push(provider, tone(), 10); + push(provider, silence(), ENDPOINT_FRAMES); + } + + expect(recorded.finals.map((final) => final.text)).toEqual([ + 'Echo utterance 1: 0.2s of speech.', + 'Echo utterance 2: 0.2s of speech.', + 'Echo utterance 3: 0.2s of speech.', + ]); + }); + + it('streams partials while the segment is open, with rising stability', async () => { + const { provider, recorded } = await started({ partialIntervalMs: 200 }); + + push(provider, tone(), 60); + + expect(recorded.partials.length).toBeGreaterThan(2); + expect(recorded.partials[0].text).toMatch(/^Echo utterance 1: .*\.\.\.$/); + // A hypothesis firms up as more of the utterance arrives. + expect(recorded.partials[1].stability).toBeGreaterThan(recorded.partials[0].stability); + expect(recorded.partials.at(-1)!.stability).toBeLessThanOrEqual(0.9); + // Still open: nothing is final until the floor closes. + expect(recorded.finals).toEqual([]); + }); + + it('emits no partials at all when the interval is disabled', async () => { + const { provider, recorded } = await started({ partialIntervalMs: 0 }); + + push(provider, tone(), 60); + + expect(recorded.partials).toEqual([]); + }); + + it('ignores frames fed before start', async () => { + const provider = new EchoSttProvider({ finalDelayMs: 0 }); + const { callbacks, recorded } = recording(); + + push(provider, tone(), 10); + push(provider, silence(), ENDPOINT_FRAMES); + await provider.start(callbacks); + + expect(recorded.finals).toEqual([]); + }); + + it('starts a fresh run on restart rather than continuing the last one', async () => { + const { provider, recorded } = await started(); + push(provider, tone(), 10); + push(provider, silence(), ENDPOINT_FRAMES); + + await provider.stop(); + const second = recording(); + await provider.start(second.callbacks); + push(provider, tone(), 10); + push(provider, silence(), ENDPOINT_FRAMES); + + // Segment 1 again, and on the new callbacks: a new capture run is a new + // conversation, not the continuation of a session that already ended. + expect(second.recorded.finals.map((f) => f.text)).toEqual([ + 'Echo utterance 1: 0.2s of speech.', + ]); + expect(recorded.finals).toHaveLength(1); + }); +}); + +describe('EchoSttProvider endpointing', () => { + it('finalises the open segment immediately on flush', async () => { + const { provider, recorded } = await started(); + + push(provider, tone(), 10); + expect(recorded.finals).toEqual([]); + + // Push-to-talk release, or the pipeline forwarding its own VAD endpoint. The + // user already said they were finished; waiting out 700 ms of silence to + // agree with them would be latency bought with nothing. + await provider.flush(); + + expect(recorded.finals).toHaveLength(1); + expect(recorded.finals[0].durationMs).toBe(200); + }); + + it('produces nothing when flushed with no speech in hand', async () => { + const { provider, recorded } = await started(); + + push(provider, silence(), 10); + await provider.flush(); + + // Silence is not an empty transcript, it is no transcript. + expect(recorded.finals).toEqual([]); + }); + + it('hears the next sentence right after a flush', async () => { + const { provider, recorded } = await started(); + + push(provider, tone(), 10); + await provider.flush(); + push(provider, tone(), 10); + push(provider, silence(), ENDPOINT_FRAMES); + + // A manual endpoint ends an utterance, not the capture run: someone who + // keeps talking must not have to pause before being heard again. + expect(recorded.finals.map((f) => f.text)).toEqual([ + 'Echo utterance 1: 0.2s of speech.', + 'Echo utterance 2: 0.2s of speech.', + ]); + }); + + it('does not double-report a segment that flush already closed', async () => { + const { provider, recorded } = await started(); + + push(provider, tone(), 10); + await provider.flush(); + push(provider, silence(), ENDPOINT_FRAMES); + + expect(recorded.finals).toHaveLength(1); + }); +}); + +describe('EchoSttProvider text-in seam', () => { + it('routes typed text in as a synthetic final transcript', async () => { + const { provider, recorded } = await started(); + + provider.injectUtterance(' open the auth tab '); + + // No partials: the text was already settled when it arrived, so there is no + // hypothesis to revise. + expect(recorded.partials).toEqual([]); + expect(recorded.finals).toEqual([ + { text: 'open the auth tab', confidence: 1, durationMs: expect.any(Number) }, + ]); + expect(recorded.finals[0].durationMs).toBeGreaterThan(0); + }); + + it('passes an empty utterance straight through with no estimated duration', async () => { + const { provider, recorded } = await started(); + + provider.injectUtterance(' '); + + // The session service has its own empty-utterance path; inventing a duration + // for it would put a lie on the transcript timeline. + expect(recorded.finals).toEqual([{ text: '', confidence: 1, durationMs: 0 }]); + }); + + it('supersedes audio that was being spoken over it', async () => { + const { provider, recorded } = await started(); + + push(provider, tone(), 10); + provider.injectUtterance('typed instead'); + push(provider, silence(), ENDPOINT_FRAMES); + + // The open segment is abandoned, not endpointed behind the typed text. + expect(recorded.finals.map((f) => f.text)).toEqual(['typed instead']); + }); +}); + +describe('EchoSttProvider decoder latency', () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it('delivers the final after the simulated decode, not on the endpoint frame', async () => { + const provider = new EchoSttProvider({ finalDelayMs: 250 }); + const { callbacks, recorded } = recording(); + await provider.start(callbacks); + + push(provider, tone(), 10); + push(provider, silence(), ENDPOINT_FRAMES); + expect(recorded.finals).toEqual([]); + + vi.advanceTimersByTime(250); + expect(recorded.finals).toHaveLength(1); + }); + + it('drops a final that was still decoding when the session ended', async () => { + const provider = new EchoSttProvider({ finalDelayMs: 250 }); + const { callbacks, recorded } = recording(); + await provider.start(callbacks); + + push(provider, tone(), 10); + push(provider, silence(), ENDPOINT_FRAMES); + await provider.stop(); + vi.advanceTimersByTime(1_000); + + // A transcript for a session that is gone would arrive with no floor to take + // it and no envelope to travel in. + expect(recorded.finals).toEqual([]); + }); +}); diff --git a/src/__tests__/main/acappella/hotkeys/press-hold.test.ts b/src/__tests__/main/acappella/hotkeys/press-hold.test.ts new file mode 100644 index 0000000000..4bb3ac0fcf --- /dev/null +++ b/src/__tests__/main/acappella/hotkeys/press-hold.test.ts @@ -0,0 +1,303 @@ +/** + * @file press-hold.test.ts + * + * Tap versus hold on a global hotkey, including the boundary itself and the + * tap-only fallback that every platform gets today. + * + * Time and key state are both injected, so nothing here depends on a real + * keyboard or on how fast the machine running the suite happens to be. The + * boundary cases are the point: a threshold that classified a 299 ms press and a + * 301 ms press the same way would make push-to-talk feel random. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + DEFAULT_HOLD_THRESHOLD_MS, + MAX_HOLD_MS, + MAX_HOLD_THRESHOLD_MS, + MIN_HOLD_THRESHOLD_MS, + PressHoldDetector, + createPressHoldDetector, + describePressHoldCapability, + resolveHoldThresholdMs, + resolvePlatformKeyStateProbe, + resolvePressHoldCapability, + setKeyStateProbe, + type KeyStateProbe, +} from '../../../../main/acappella/hotkeys/press-hold'; + +const ACCELERATOR = 'Command+Alt+V'; +const POLL_MS = 10; + +interface Harness { + detector: PressHoldDetector; + taps: number[]; + holdStarts: number[]; + holdEnds: number[]; + /** Let the key back up. */ + release: () => void; + /** Advance the fake clock and run the polls that fall inside the step. */ + advance: (ms: number) => void; +} + +function makeHarness(options: { threshold?: number; probe?: KeyStateProbe | null } = {}): Harness { + let now = 0; + let down = true; + const taps: number[] = []; + const holdStarts: number[] = []; + const holdEnds: number[] = []; + + const probe = + options.probe === undefined ? (((): boolean => down) as KeyStateProbe) : options.probe; + + const detector = createPressHoldDetector({ + accelerator: ACCELERATOR, + holdThresholdMs: options.threshold, + pollIntervalMs: POLL_MS, + probe, + now: () => now, + onTap: () => taps.push(now), + onHoldStart: () => holdStarts.push(now), + onHoldEnd: () => holdEnds.push(now), + }); + + return { + detector, + taps, + holdStarts, + holdEnds, + release: () => { + down = false; + }, + // Stepped at the poll interval rather than jumped, so the detector sees the + // same sequence of elapsed times it would see on a real clock. Jumping + // would hand the first poll the whole span and skip the threshold entirely. + advance: (ms) => { + let remaining = ms; + while (remaining > 0) { + const step = Math.min(POLL_MS, remaining); + now += step; + vi.advanceTimersByTime(step); + remaining -= step; + } + }, + }; +} + +describe('resolveHoldThresholdMs', () => { + it('clamps into the usable band and falls back on nonsense', () => { + expect(resolveHoldThresholdMs(400)).toBe(400); + expect(resolveHoldThresholdMs(1)).toBe(MIN_HOLD_THRESHOLD_MS); + expect(resolveHoldThresholdMs(999_999)).toBe(MAX_HOLD_THRESHOLD_MS); + expect(resolveHoldThresholdMs('later')).toBe(DEFAULT_HOLD_THRESHOLD_MS); + expect(resolveHoldThresholdMs(Number.NaN)).toBe(DEFAULT_HOLD_THRESHOLD_MS); + }); +}); + +describe('capability reporting', () => { + afterEach(() => setKeyStateProbe(null)); + + it('is tap-only with no probe and hold-and-tap with one', () => { + expect(resolvePressHoldCapability(null)).toBe('tap-only'); + expect(resolvePressHoldCapability(() => true)).toBe('hold-and-tap'); + }); + + it('never silently degrades: the tap-only sentence says holding will not work', () => { + const note = describePressHoldCapability('tap-only'); + expect(note.toLowerCase()).toContain('hold'); + expect(note).not.toBe(describePressHoldCapability('hold-and-tap')); + }); + + it('picks up a probe installed at runtime', () => { + expect(resolvePlatformKeyStateProbe()).toBeNull(); + const probe: KeyStateProbe = () => true; + setKeyStateProbe(probe); + expect(resolvePlatformKeyStateProbe()).toBe(probe); + }); +}); + +describe('PressHoldDetector', () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + describe('tap-only fallback', () => { + it('taps immediately on every trigger, with no timer at all', () => { + const harness = makeHarness({ probe: null }); + expect(harness.detector.capability).toBe('tap-only'); + + harness.detector.trigger(); + harness.detector.trigger(); + + expect(harness.taps).toHaveLength(2); + expect(harness.holdStarts).toHaveLength(0); + expect(vi.getTimerCount()).toBe(0); + }); + }); + + describe('threshold boundary', () => { + it('classifies a press released just under the threshold as a tap', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(290); + harness.release(); + harness.advance(POLL_MS); + + expect(harness.taps).toHaveLength(1); + expect(harness.holdStarts).toHaveLength(0); + expect(harness.holdEnds).toHaveLength(0); + }); + + it('opens the floor once the key is still down at the threshold', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(300); + + expect(harness.holdStarts).toEqual([300]); + expect(harness.taps).toHaveLength(0); + expect(harness.detector.isHolding).toBe(true); + }); + + it('ends the utterance on release after a hold, and never taps', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(500); + harness.release(); + harness.advance(POLL_MS); + + expect(harness.holdStarts).toHaveLength(1); + expect(harness.holdEnds).toHaveLength(1); + expect(harness.taps).toHaveLength(0); + expect(harness.detector.isHolding).toBe(false); + }); + + it('fires hold-start exactly once across a long hold', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(2000); + expect(harness.holdStarts).toHaveLength(1); + }); + }); + + it('ignores auto-repeat while a press is being classified', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(50); + harness.detector.trigger(); + harness.detector.trigger(); + harness.advance(300); + harness.release(); + harness.advance(POLL_MS); + + expect(harness.holdStarts).toHaveLength(1); + expect(harness.holdEnds).toHaveLength(1); + }); + + it('starts a fresh gesture after the previous one resolved', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(50); + harness.release(); + harness.advance(POLL_MS); + expect(harness.taps).toHaveLength(1); + + harness.detector.trigger(); + harness.advance(POLL_MS); + expect(harness.taps).toHaveLength(2); + }); + + it('stops polling once a gesture resolves', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.release(); + harness.advance(POLL_MS); + expect(vi.getTimerCount()).toBe(0); + }); + + it('releases the floor when a probe lies about the key staying down', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(MAX_HOLD_MS); + + expect(harness.holdStarts).toHaveLength(1); + expect(harness.holdEnds).toHaveLength(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it('treats a probe that throws as a release rather than holding forever', () => { + let now = 0; + const taps: number[] = []; + const detector = createPressHoldDetector({ + accelerator: ACCELERATOR, + pollIntervalMs: POLL_MS, + probe: () => { + throw new Error('probe exploded'); + }, + now: () => now, + onTap: () => taps.push(now), + onHoldStart: vi.fn(), + onHoldEnd: vi.fn(), + }); + + detector.trigger(); + now += POLL_MS; + vi.advanceTimersByTime(POLL_MS); + + expect(taps).toHaveLength(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it('dispose ends a hold in flight rather than abandoning an open floor', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(400); + expect(harness.holdStarts).toHaveLength(1); + + harness.detector.dispose(); + expect(harness.holdEnds).toHaveLength(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it('dispose before the threshold neither taps nor holds', () => { + const harness = makeHarness({ threshold: 300 }); + harness.detector.trigger(); + harness.advance(100); + harness.detector.dispose(); + + expect(harness.taps).toHaveLength(0); + expect(harness.holdEnds).toHaveLength(0); + }); + + it('ignores triggers after dispose', () => { + const harness = makeHarness({ probe: null }); + harness.detector.dispose(); + harness.detector.trigger(); + expect(harness.taps).toHaveLength(0); + }); + + it('does not let a throwing callback leave the poll timer running', () => { + let now = 0; + let down = true; + const detector = new PressHoldDetector({ + accelerator: ACCELERATOR, + pollIntervalMs: POLL_MS, + probe: () => down, + now: () => now, + onTap: () => { + throw new Error('floor exploded'); + }, + onHoldStart: vi.fn(), + onHoldEnd: vi.fn(), + }); + + detector.trigger(); + down = false; + now += POLL_MS; + expect(() => vi.advanceTimersByTime(POLL_MS)).not.toThrow(); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/src/__tests__/main/acappella/hotkeys/voice-hotkeys.test.ts b/src/__tests__/main/acappella/hotkeys/voice-hotkeys.test.ts new file mode 100644 index 0000000000..4f852cccb0 --- /dev/null +++ b/src/__tests__/main/acappella/hotkeys/voice-hotkeys.test.ts @@ -0,0 +1,269 @@ +/** + * @file voice-hotkeys.test.ts + * + * The two A Cappella hotkeys: what each one does to window focus, how a scope is + * resolved, and every way a press is refused rather than silently doing nothing. + * + * The focus rule is the interesting one and it is asymmetric on purpose. Talking + * to the Conductor must NOT steal focus, because the whole point is speaking to + * Maestro from inside another application; talking to the current agent must, + * because "current" is a thing you have to be looking at to mean. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock('electron', () => ({ + app: { show: vi.fn() }, + BrowserWindow: class {}, + globalShortcut: { register: vi.fn(() => true), unregister: vi.fn() }, +})); + +vi.mock('../../../../shared/platformDetection', () => ({ + isMacOS: () => true, + isWindows: () => false, + isLinux: () => false, +})); + +import type { VoiceScope, WakeSource } from '../../../../shared/acappella/protocol'; +import { + VOICE_AGENT_HOTKEY_ID, + VOICE_CONDUCTOR_HOTKEY_ID, +} from '../../../../shared/global-hotkeys'; +import { + GlobalHotkeyRegistry, + type GlobalShortcutBackend, +} from '../../../../main/global-hotkey-manager'; +import type { FloorControlConfig } from '../../../../main/acappella/audio/floor-control'; +import { + VoiceHotkeyController, + createVoiceHotkeyController, + defaultVoiceHotkeyKeys, + type VoiceFloorSurface, + type VoiceHotkeyRefusalInfo, +} from '../../../../main/acappella/hotkeys/voice-hotkeys'; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +class FakeBackend implements GlobalShortcutBackend { + readonly bound = new Map void>(); + register(accelerator: string, callback: () => void): boolean { + this.bound.set(accelerator, callback); + return true; + } + unregister(accelerator: string): void { + this.bound.delete(accelerator); + } + fire(accelerator: string): void { + this.bound.get(accelerator)?.(); + } +} + +class FakeFloor implements VoiceFloorSurface { + mode: FloorControlConfig['mode'] = 'tap-to-toggle'; + readonly presses: WakeSource[] = []; + readonly releases: WakeSource[] = []; + readonly modes: string[] = []; + + configure(overrides: Partial): void { + if (overrides.mode) { + this.mode = overrides.mode; + this.modes.push(overrides.mode); + } + } + async press(source: WakeSource = 'client-button'): Promise { + this.presses.push(source); + } + async release(source: WakeSource = 'client-button'): Promise { + this.releases.push(source); + } +} + +describe('VoiceHotkeyController', () => { + let backend: FakeBackend; + let registry: GlobalHotkeyRegistry; + let floor: FakeFloor; + let scopes: VoiceScope[]; + let summons: number; + let refusals: VoiceHotkeyRefusalInfo[]; + let available: { ok: true } | { ok: false; reason: 'feature-disabled'; message: string }; + let focusedAgent: string | null; + + function build(overrides: Record = {}): VoiceHotkeyController { + return createVoiceHotkeyController({ + registry, + checkAvailability: () => available, + acquireFloor: (scope) => { + scopes.push(scope); + return floor; + }, + resolveFocusedAgent: () => (focusedAgent ? { kind: 'agent', sessionId: focusedAgent } : null), + summon: () => { + summons += 1; + }, + onRefused: (info) => refusals.push(info), + // Forces tap-only, which is what every platform gets today. + probe: null, + ...overrides, + }); + } + + beforeEach(() => { + backend = new FakeBackend(); + registry = new GlobalHotkeyRegistry(backend); + floor = new FakeFloor(); + scopes = []; + summons = 0; + refusals = []; + available = { ok: true }; + focusedAgent = 'agent-7'; + }); + + it('binds both hotkeys to their shipped defaults', () => { + const controller = build(); + const statuses = controller.sync(); + + expect(statuses[VOICE_CONDUCTOR_HOTKEY_ID].registered).toBe(true); + expect(statuses[VOICE_AGENT_HOTKEY_ID].registered).toBe(true); + expect(statuses[VOICE_CONDUCTOR_HOTKEY_ID].keys).toEqual( + defaultVoiceHotkeyKeys()[VOICE_CONDUCTOR_HOTKEY_ID] + ); + }); + + it('ships defaults that do not collide with each other', () => { + const controller = build(); + const statuses = controller.sync(); + expect(statuses[VOICE_AGENT_HOTKEY_ID].reason).toBeUndefined(); + expect(statuses[VOICE_CONDUCTOR_HOTKEY_ID].accelerator).not.toBe( + statuses[VOICE_AGENT_HOTKEY_ID].accelerator + ); + }); + + it('honours an explicitly cleared binding rather than restoring the default', () => { + const controller = build(); + const statuses = controller.sync({ [VOICE_CONDUCTOR_HOTKEY_ID]: [] }); + expect(statuses[VOICE_CONDUCTOR_HOTKEY_ID].accelerator).toBeNull(); + expect(statuses[VOICE_AGENT_HOTKEY_ID].registered).toBe(true); + }); + + it('opens a Conductor session without touching window focus', () => { + const controller = build(); + const statuses = controller.sync(); + backend.fire(statuses[VOICE_CONDUCTOR_HOTKEY_ID].accelerator!); + + expect(summons).toBe(0); + expect(scopes).toEqual([{ kind: 'conductor' }]); + expect(floor.presses).toEqual(['hotkey']); + expect(floor.mode).toBe('tap-to-toggle'); + }); + + it('summons Maestro and binds the focused agent', () => { + const controller = build(); + const statuses = controller.sync(); + backend.fire(statuses[VOICE_AGENT_HOTKEY_ID].accelerator!); + + expect(summons).toBe(1); + expect(scopes).toEqual([{ kind: 'agent', sessionId: 'agent-7' }]); + }); + + it('refuses the agent hotkey rather than guessing when nothing is focused', () => { + focusedAgent = null; + const controller = build(); + const statuses = controller.sync(); + backend.fire(statuses[VOICE_AGENT_HOTKEY_ID].accelerator!); + + expect(refusals.map((r) => r.reason)).toEqual(['no-focused-agent']); + // Notably NOT a fall back to the Conductor: a spoken instruction must not + // land somewhere the user did not aim it. + expect(scopes).toHaveLength(0); + expect(floor.presses).toHaveLength(0); + }); + + it('refuses with a reason when the Encore Feature is off', () => { + available = { ok: false, reason: 'feature-disabled', message: 'A Cappella is switched off.' }; + const controller = build(); + const statuses = controller.sync(); + backend.fire(statuses[VOICE_CONDUCTOR_HOTKEY_ID].accelerator!); + + expect(refusals[0]).toMatchObject({ + id: VOICE_CONDUCTOR_HOTKEY_ID, + reason: 'feature-disabled', + }); + expect(floor.presses).toHaveLength(0); + }); + + it('refuses when nothing can give it a floor', () => { + const controller = build({ acquireFloor: () => null }); + const statuses = controller.sync(); + backend.fire(statuses[VOICE_CONDUCTOR_HOTKEY_ID].accelerator!); + + expect(refusals.map((r) => r.reason)).toEqual(['no-floor']); + }); + + it('reports tap-only on a platform with no key-release signal', () => { + expect(build().capability).toBe('tap-only'); + }); + + it('holds the floor open while the key is down when a probe exists', () => { + vi.useFakeTimers(); + try { + let now = 0; + let down = true; + const controller = new VoiceHotkeyController({ + registry, + checkAvailability: () => available, + acquireFloor: () => floor, + resolveFocusedAgent: () => null, + summon: vi.fn(), + probe: () => down, + getHoldThresholdMs: () => 300, + }); + const statuses = controller.sync(); + const accelerator = statuses[VOICE_CONDUCTOR_HOTKEY_ID].accelerator!; + + vi.setSystemTime(now); + backend.fire(accelerator); + // Past the threshold with the key still down: the floor opens in hold mode. + now += 400; + vi.setSystemTime(now); + vi.advanceTimersByTime(400); + expect(floor.mode).toBe('hold-to-talk'); + expect(floor.presses).toEqual(['hotkey']); + + down = false; + now += 50; + vi.setSystemTime(now); + vi.advanceTimersByTime(50); + expect(floor.releases).toEqual(['hotkey']); + // The user's own mode is restored: a push-to-talk gesture must not convert + // the session to push-to-talk forever. + expect(floor.mode).toBe('tap-to-toggle'); + } finally { + vi.useRealTimers(); + } + }); + + it('dispose releases both combos', () => { + const controller = build(); + controller.sync(); + controller.dispose(); + + expect(backend.bound.size).toBe(0); + expect(registry.status(VOICE_CONDUCTOR_HOTKEY_ID)).toBeNull(); + }); + + it('ignores presses after dispose', () => { + const controller = build(); + const statuses = controller.sync(); + const accelerator = statuses[VOICE_CONDUCTOR_HOTKEY_ID].accelerator!; + controller.dispose(); + backend.fire(accelerator); + + expect(floor.presses).toHaveLength(0); + }); +}); diff --git a/src/__tests__/main/acappella/ice-config.test.ts b/src/__tests__/main/acappella/ice-config.test.ts new file mode 100644 index 0000000000..4844e7ba08 --- /dev/null +++ b/src/__tests__/main/acappella/ice-config.test.ts @@ -0,0 +1,144 @@ +/** + * ICE configuration: what a connection can reach, and what it is called. + * + * The two things worth pinning down are honesty properties rather than + * behaviours. A TURN server switched on with no URL must be off, because the + * alternative is an ICE configuration that throws where nobody sees it; and a + * candidate pair with a relay on either end must be called relayed, because + * saying "direct" would describe a path the audio is not taking. + */ + +import { describe, expect, it } from 'vitest'; + +import { + CANDIDATE_TYPE_LABELS, + DEFAULT_ICE_SETTINGS, + DEFAULT_STUN_URLS, + TUNNEL_MEDIA_NOTE, + buildIceServers, + classifyCandidatePair, + classifyCandidateType, + describeIceReach, + iceTransportPolicy, + isOverlayAddress, + readIceSettings, +} from '../../../main/acappella/transport/ice-config'; + +describe('readIceSettings', () => { + it('fills in the defaults for an empty store', () => { + expect(readIceSettings(undefined)).toEqual(DEFAULT_ICE_SETTINGS); + expect(readIceSettings({})).toEqual(DEFAULT_ICE_SETTINGS); + }); + + it('keeps an explicitly empty STUN list, because that is a real choice', () => { + // LAN and overlay only, with nothing reflecting an address off anyone. + expect(readIceSettings({ stunUrls: [] }).stunUrls).toEqual([]); + }); + + it('treats a TURN server with no URL as off, whatever the flag says', () => { + const settings = readIceSettings({ turn: { enabled: true, url: ' ' } }); + expect(settings.turn.enabled).toBe(false); + }); + + it('does not throw on junk from a settings pane', () => { + const settings = readIceSettings({ stunUrls: 'nope', turn: 7, hostCandidates: 'yes' }); + expect(settings.stunUrls).toEqual([...DEFAULT_STUN_URLS]); + expect(settings.turn.enabled).toBe(false); + expect(settings.hostCandidates).toBe(true); + }); +}); + +describe('buildIceServers', () => { + it('puts TURN last so ICE tries the free paths first', () => { + const servers = buildIceServers({ + ...DEFAULT_ICE_SETTINGS, + turn: { enabled: true, url: 'turns:relay.example.com:5349', username: 'u', credential: 'c' }, + }); + expect(servers).toHaveLength(2); + expect(servers[1]).toMatchObject({ urls: 'turns:relay.example.com:5349', username: 'u' }); + }); + + it('produces nothing at all for a LAN-only configuration', () => { + expect(buildIceServers({ ...DEFAULT_ICE_SETTINGS, stunUrls: [] })).toEqual([]); + }); + + it('forces a relay only when asked', () => { + expect(iceTransportPolicy(DEFAULT_ICE_SETTINGS)).toBe('all'); + expect(iceTransportPolicy({ ...DEFAULT_ICE_SETTINGS, forceRelay: true })).toBe('relay'); + }); +}); + +describe('describeIceReach', () => { + it('says cellular will not work without TURN', () => { + expect(describeIceReach(DEFAULT_ICE_SETTINGS)).toMatch(/cellular will not connect/i); + }); + + it('says a LAN-only configuration reaches nothing outside it', () => { + expect(describeIceReach({ ...DEFAULT_ICE_SETTINGS, stunUrls: [] })).toMatch( + /this network and overlay networks only/i + ); + }); + + it('names cellular as covered once a relay is configured', () => { + const reach = describeIceReach({ + ...DEFAULT_ICE_SETTINGS, + turn: { enabled: true, url: 'turn:r', username: 'u', credential: 'c' }, + }); + expect(reach).toMatch(/cellular/i); + }); + + it('says so when relay-only is on with no relay configured', () => { + expect(describeIceReach({ ...DEFAULT_ICE_SETTINGS, forceRelay: true })).toMatch( + /no device can connect/i + ); + }); +}); + +describe('candidate classification', () => { + it('collapses the ICE vocabulary onto three words a person can act on', () => { + expect(classifyCandidateType('host')).toBe('lan'); + expect(classifyCandidateType('srflx')).toBe('stun'); + expect(classifyCandidateType('prflx')).toBe('stun'); + expect(classifyCandidateType('relay')).toBe('relay'); + expect(classifyCandidateType(undefined)).toBe('unknown'); + }); + + it('takes the worse end of a pair', () => { + expect(classifyCandidatePair('host', 'host')).toBe('lan'); + expect(classifyCandidatePair('host', 'srflx')).toBe('stun'); + expect(classifyCandidatePair('host', 'relay')).toBe('relay'); + expect(classifyCandidatePair('relay', 'host')).toBe('relay'); + }); + + it('reports unknown until both ends are known', () => { + expect(classifyCandidatePair('host', undefined)).toBe('unknown'); + }); + + it('has a label for every type', () => { + for (const type of ['lan', 'stun', 'relay', 'unknown'] as const) { + expect(CANDIDATE_TYPE_LABELS[type]).toEqual(expect.any(String)); + } + }); +}); + +describe('overlay addresses', () => { + it('recognises the CGNAT block Tailscale allocates out of', () => { + expect(isOverlayAddress('100.64.0.1')).toBe(true); + expect(isOverlayAddress('100.127.255.254')).toBe(true); + }); + + it('does not mistake an ordinary address for an overlay', () => { + expect(isOverlayAddress('192.168.1.10')).toBe(false); + expect(isOverlayAddress('100.128.0.1')).toBe(false); + expect(isOverlayAddress('not an address')).toBe(false); + }); +}); + +describe('the tunnel note', () => { + it('says plainly that the quick tunnel cannot carry the media', () => { + // The user who does not know this blames the wrong thing every single time. + expect(TUNNEL_MEDIA_NOTE).toMatch(/cloudflare/i); + expect(TUNNEL_MEDIA_NOTE).toMatch(/cannot carry/i); + expect(TUNNEL_MEDIA_NOTE).toMatch(/turn relay/i); + }); +}); diff --git a/src/__tests__/main/acappella/mock-providers.test.ts b/src/__tests__/main/acappella/mock-providers.test.ts new file mode 100644 index 0000000000..a6fa4e0813 --- /dev/null +++ b/src/__tests__/main/acappella/mock-providers.test.ts @@ -0,0 +1,487 @@ +/** + * @file mock-providers.test.ts + * + * Unit tests for the mock provider tier: partial then final ordering out of the + * mock STT, deterministic keyword routing out of the mock Brain, and cancellable + * sentence streaming out of the mock TTS. + * + * Resolution rules live in `provider-registry.test.ts`, because they are about + * what gets CHOSEN rather than about what the mocks do. + * + * Every provider is constructed with zero-delay timing so the suite runs + * synchronously: the timers are a UX affordance, not behaviour under test. + */ + +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { MockBrainProvider } from '../../../main/acappella/providers/mock/mock-brain'; +import { MockSttProvider } from '../../../main/acappella/providers/mock/mock-stt'; +import { MockTtsProvider } from '../../../main/acappella/providers/mock/mock-tts'; +import { createMockProviderTrio } from '../../../main/acappella/providers/mock'; +import type { RosterAgent } from '../../../shared/acappella/protocol'; +import type { SttCallbacks, TtsChunk } from '../../../shared/acappella/providers'; +import { splitIntoSpokenSentences } from '../../../shared/acappella/sentences'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeRoster(): RosterAgent[] { + return [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + tabs: [ + { id: 'tab-auth', name: 'Auth Refactor', lastActiveAt: 1_000 }, + { id: 'tab-migrations', name: 'DB Migrations', lastActiveAt: 5_000 }, + ], + }, + { + sessionId: 'agent-frontend', + name: 'Frontend', + agentType: 'codex', + cwd: '/repo/web', + tabs: [{ id: 'tab-ui', name: 'Sidebar', lastActiveAt: 2_000 }], + }, + ]; +} + +/** Records every callback the STT fires, in order. */ +function recordingCallbacks(): { + callbacks: SttCallbacks; + events: Array<{ kind: 'partial' | 'final' | 'error'; text: string; value: number }>; +} { + const events: Array<{ kind: 'partial' | 'final' | 'error'; text: string; value: number }> = []; + return { + events, + callbacks: { + onPartial: (text, stability) => events.push({ kind: 'partial', text, value: stability }), + onFinal: (text, confidence) => events.push({ kind: 'final', text, value: confidence }), + onError: (error) => events.push({ kind: 'error', text: error.message, value: 0 }), + }, + }; +} + +async function collect(iterable: AsyncIterable): Promise { + const chunks: TtsChunk[] = []; + for await (const chunk of iterable) chunks.push(chunk); + return chunks; +} + +// --------------------------------------------------------------------------- +// Mock STT +// --------------------------------------------------------------------------- + +describe('MockSttProvider', () => { + it('emits two partials before the final, in that order', async () => { + const stt = new MockSttProvider({ partialDelayMs: 0 }); + const { callbacks, events } = recordingCallbacks(); + await stt.start(callbacks); + + stt.injectUtterance('open a new tab on the backend agent'); + + expect(events.map((event) => event.kind)).toEqual(['partial', 'partial', 'final']); + expect(events[2].text).toBe('open a new tab on the backend agent'); + }); + + it('grows the partial hypothesis and its stability', async () => { + const stt = new MockSttProvider({ partialDelayMs: 0 }); + const { callbacks, events } = recordingCallbacks(); + await stt.start(callbacks); + + stt.injectUtterance('one two three four five six'); + + const [first, second] = events; + expect(second.text.startsWith(first.text)).toBe(true); + expect(second.text.length).toBeGreaterThan(first.text.length); + expect(second.value).toBeGreaterThan(first.value); + }); + + it('still emits two partials for a single word', async () => { + const stt = new MockSttProvider({ partialDelayMs: 0 }); + const { callbacks, events } = recordingCallbacks(); + await stt.start(callbacks); + + stt.injectUtterance('stop'); + + expect(events.filter((event) => event.kind === 'partial')).toHaveLength(2); + }); + + it('reports an empty utterance as a final with no partials', async () => { + const stt = new MockSttProvider({ partialDelayMs: 0 }); + const { callbacks, events } = recordingCallbacks(); + await stt.start(callbacks); + + stt.injectUtterance(' '); + + expect(events).toEqual([{ kind: 'final', text: '', value: 1 }]); + }); + + it('drops pending emissions after stop()', async () => { + vi.useFakeTimers(); + try { + const stt = new MockSttProvider({ partialDelayMs: 10 }); + const { callbacks, events } = recordingCallbacks(); + await stt.start(callbacks); + + stt.injectUtterance('this will be abandoned'); + await stt.stop(); + vi.advanceTimersByTime(100); + + expect(events).toHaveLength(0); + } finally { + vi.useRealTimers(); + } + }); + + it('supersedes a pending utterance rather than interleaving the two', async () => { + vi.useFakeTimers(); + try { + const stt = new MockSttProvider({ partialDelayMs: 10 }); + const { callbacks, events } = recordingCallbacks(); + await stt.start(callbacks); + + stt.injectUtterance('first utterance'); + vi.advanceTimersByTime(10); + stt.injectUtterance('second utterance'); + vi.advanceTimersByTime(100); + + const finals = events.filter((event) => event.kind === 'final'); + expect(finals).toHaveLength(1); + expect(finals[0].text).toBe('second utterance'); + } finally { + vi.useRealTimers(); + } + }); + + it('ignores fed audio instead of inventing a transcript', async () => { + const stt = new MockSttProvider({ partialDelayMs: 0 }); + const { callbacks, events } = recordingCallbacks(); + await stt.start(callbacks); + + stt.feed(new Int16Array(1024)); + await stt.flush(); + + expect(events).toHaveLength(0); + }); +}); + +// --------------------------------------------------------------------------- +// Mock Brain +// --------------------------------------------------------------------------- + +describe('MockBrainProvider routing', () => { + const brain = new MockBrainProvider(); + const roster = makeRoster(); + + it('targets the agent whose name is in the utterance', async () => { + const decision = await brain.route('ask backend to run the migrations', { + roster, + scope: { kind: 'conductor' }, + }); + + expect(decision.target).toEqual({ sessionId: 'agent-backend' }); + }); + + it('targets the conductor when no name is mentioned and nothing is bound', async () => { + const decision = await brain.route('what is running right now', { + roster, + scope: { kind: 'conductor' }, + }); + + expect(decision.target).toBe('conductor'); + }); + + it('falls back to the bound agent when no name is mentioned', async () => { + const decision = await brain.route('run the tests', { + roster, + scope: { kind: 'agent', sessionId: 'agent-frontend' }, + }); + + expect(decision.target).toEqual({ sessionId: 'agent-frontend' }); + }); + + it('ignores a leading "hey maestro" when an agent is called Maestro', async () => { + // Found by driving the running app: an agent named "Maestro" is what you + // call the agent working on Maestro itself, so it is a very common name, + // and "hey maestro" is how you address the conductor. Matching the address + // as a name sent every routed sentence to that one agent. + const withConductorName: RosterAgent[] = [ + { + sessionId: 'agent-maestro', + name: 'Maestro', + agentType: 'claude-code', + cwd: '/repo/maestro', + tabs: [], + }, + ...roster, + ]; + + const decision = await brain.route('hey maestro ask backend to run the migrations', { + roster: withConductorName, + scope: { kind: 'conductor' }, + }); + + expect(decision.target).toEqual({ sessionId: 'agent-backend' }); + }); + + it('still targets an agent named Maestro when it is named mid-utterance', async () => { + const withConductorName: RosterAgent[] = [ + { + sessionId: 'agent-maestro', + name: 'Maestro', + agentType: 'claude-code', + cwd: '/repo/maestro', + tabs: [], + }, + ...roster, + ]; + + const decision = await brain.route('hey maestro what is maestro working on', { + roster: withConductorName, + scope: { kind: 'conductor' }, + }); + + expect(decision.target).toEqual({ sessionId: 'agent-maestro' }); + }); + + it('lets a named agent beat the bound one', async () => { + const decision = await brain.route('backend, run the tests', { + roster, + scope: { kind: 'agent', sessionId: 'agent-frontend' }, + }); + + expect(decision.target).toEqual({ sessionId: 'agent-backend' }); + }); + + it('picks new from a new-tab cue and names the tab', async () => { + const decision = await brain.route( + 'start a new tab on the backend agent about the auth refactor', + { roster, scope: { kind: 'conductor' } } + ); + + expect(decision.target).toEqual({ sessionId: 'agent-backend' }); + expect(decision.tabAction).toBe('new'); + expect(decision.prompt).toBe('the auth refactor'); + expect(decision.tabName).toBe('Auth Refactor'); + }); + + it('picks recall from a back-to cue and resolves the tab by name', async () => { + const decision = await brain.route('go back to the auth refactor on backend', { + roster, + scope: { kind: 'conductor' }, + }); + + expect(decision.tabAction).toBe('recall'); + expect(decision.tabId).toBe('tab-auth'); + }); + + it('recalls the most recent tab when the utterance names none', async () => { + const decision = await brain.route('back to backend', { + roster, + scope: { kind: 'conductor' }, + }); + + expect(decision.tabAction).toBe('recall'); + expect(decision.tabId).toBe('tab-migrations'); + }); + + it('downgrades recall to current when the target has no tabs', async () => { + const decision = await brain.route('back to the auth one', { + roster: [{ ...roster[0], tabs: [] }], + scope: { kind: 'conductor' }, + }); + + expect(decision.tabAction).toBe('current'); + expect(decision.tabId).toBeUndefined(); + }); + + it('defaults to the current tab with no cue', async () => { + const decision = await brain.route('frontend, tighten the sidebar spacing', { + roster, + scope: { kind: 'conductor' }, + }); + + expect(decision.tabAction).toBe('current'); + expect(decision.tabName).toBeUndefined(); + }); + + it('is deterministic: the same utterance routes the same way twice', async () => { + const context = { roster, scope: { kind: 'conductor' as const } }; + const first = await brain.route('new tab on frontend about dark mode', context); + const second = await brain.route('new tab on frontend about dark mode', context); + + expect(first).toEqual(second); + }); + + it('scores a named agent with a cue above a bare guess', async () => { + const context = { roster, scope: { kind: 'conductor' as const } }; + const strong = await brain.route('new tab on backend about caching', context); + const weak = await brain.route('what changed', context); + + expect(strong.confidence).toBeGreaterThan(weak.confidence); + expect(strong.confidence).toBeLessThanOrEqual(1); + expect(weak.confidence).toBeGreaterThanOrEqual(0); + }); + + it('never returns an empty prompt', async () => { + const decision = await brain.route('backend', { roster, scope: { kind: 'conductor' } }); + + expect(decision.prompt.length).toBeGreaterThan(0); + }); +}); + +describe('MockBrainProvider converse', () => { + const brain = new MockBrainProvider(); + const context = { agentSessionId: 'agent-backend', tabId: 'tab-auth' }; + + it('reshapes markdown into at most two spoken sentences', async () => { + const spoken = await brain.converse( + '## Done\n\nI updated `auth.ts` and **two** tests. The suite is green. One more thing to check later.', + context + ); + + expect(splitIntoSpokenSentences(spoken)).toHaveLength(2); + expect(spoken).not.toContain('#'); + expect(spoken).not.toContain('`'); + expect(spoken).not.toContain('**'); + }); + + it('honours an explicit sentence budget', async () => { + const spoken = await brain.converse('One. Two. Three. Four.', { ...context, maxSentences: 3 }); + + expect(splitIntoSpokenSentences(spoken)).toHaveLength(3); + }); + + it('keeps the sentence count the session announces', async () => { + const spoken = await brain.converse(`${'word '.repeat(80)}. Second sentence here.`, context); + + // The service emits `speak-start` with this count and the TTS splits the + // same text, so a truncated sentence must not become two. + expect(splitIntoSpokenSentences(spoken)).toHaveLength(2); + }); + + it('returns nothing to speak for empty agent output', async () => { + expect(await brain.converse(' ', context)).toBe(''); + }); +}); + +// --------------------------------------------------------------------------- +// Mock TTS +// --------------------------------------------------------------------------- + +describe('MockTtsProvider', () => { + it('emits one chunk per sentence, in order', async () => { + const tts = new MockTtsProvider({ msPerCharacter: 0 }); + const chunks = await collect( + tts.speak('First sentence. Second sentence. Third one.', { utteranceId: 'u1' }) + ); + + expect(chunks.map((chunk) => chunk.text)).toEqual([ + 'First sentence.', + 'Second sentence.', + 'Third one.', + ]); + expect(chunks.map((chunk) => chunk.index)).toEqual([0, 1, 2]); + expect(chunks.every((chunk) => chunk.utteranceId === 'u1')).toBe(true); + }); + + it('splits exactly like the shared splitter the service counted with', async () => { + const text = 'Dr. Reed shipped it. The tests pass!'; + const tts = new MockTtsProvider({ msPerCharacter: 0 }); + const chunks = await collect(tts.speak(text, { utteranceId: 'u1' })); + + expect(chunks.map((chunk) => chunk.text)).toEqual(splitIntoSpokenSentences(text)); + }); + + it('carries no audio, so the mock tier is silent by construction', async () => { + const tts = new MockTtsProvider({ msPerCharacter: 0 }); + const [chunk] = await collect(tts.speak('Only one.', { utteranceId: 'u1' })); + + expect(chunk.format).toBe('none'); + expect(chunk.audio).toBeNull(); + }); + + it('stops emitting sentences after cancel()', async () => { + const tts = new MockTtsProvider({ msPerCharacter: 0 }); + const seen: string[] = []; + + for await (const chunk of tts.speak('One. Two. Three. Four.', { utteranceId: 'u1' })) { + seen.push(chunk.text); + if (seen.length === 2) tts.cancel(); + } + + expect(seen).toEqual(['One.', 'Two.']); + }); + + it('cuts the in-flight sentence delay short instead of waiting it out', async () => { + const tts = new MockTtsProvider({ msPerCharacter: 50, minSentenceMs: 5_000 }); + const seen: string[] = []; + + const run = (async () => { + for await (const chunk of tts.speak('One. Two. Three.', { utteranceId: 'u1' })) { + seen.push(chunk.text); + tts.cancel(); + } + })(); + + // Real timers: if cancel() did not wake the sleep this would hang for 5s. + await run; + expect(seen).toEqual(['One.']); + }); + + it('supersedes a previous run so its stragglers are dropped', async () => { + const tts = new MockTtsProvider({ msPerCharacter: 0 }); + const first = tts.speak('One. Two. Three.', { utteranceId: 'u1' })[Symbol.asyncIterator](); + + await first.next(); + const second = await collect(tts.speak('Fresh run.', { utteranceId: 'u2' })); + + expect((await first.next()).done).toBe(true); + expect(second.map((chunk) => chunk.text)).toEqual(['Fresh run.']); + }); + + it('yields nothing for text with no sentences in it', async () => { + const tts = new MockTtsProvider({ msPerCharacter: 0 }); + + expect(await collect(tts.speak(' ', { utteranceId: 'u1' }))).toHaveLength(0); + }); +}); + +// --------------------------------------------------------------------------- +// Registry +// --------------------------------------------------------------------------- +// Trio wiring +// --------------------------------------------------------------------------- + +describe('createMockProviderTrio', () => { + it('drives an utterance from typed text through to spoken sentences', async () => { + const trio = createMockProviderTrio({ stt: { partialDelayMs: 0 }, tts: { msPerCharacter: 0 } }); + const { callbacks, events } = recordingCallbacks(); + await trio.stt.start(callbacks); + + trio.stt.injectUtterance?.('new tab on backend about the auth refactor'); + const final = events.at(-1); + expect(final?.kind).toBe('final'); + + const decision = await trio.brain.route(final!.text, { + roster: makeRoster(), + scope: { kind: 'conductor' }, + }); + expect(decision.tabAction).toBe('new'); + + const spoken = await trio.brain.converse('Opened it. The refactor branch is checked out.', { + agentSessionId: 'agent-backend', + tabId: 'tab-auth', + }); + const chunks = await collect(trio.tts.speak(spoken, { utteranceId: 'u1' })); + + expect(chunks).toHaveLength(splitIntoSpokenSentences(spoken).length); + }); +}); diff --git a/src/__tests__/main/acappella/models/capability-gate.test.ts b/src/__tests__/main/acappella/models/capability-gate.test.ts new file mode 100644 index 0000000000..1db8b5d669 --- /dev/null +++ b/src/__tests__/main/acappella/models/capability-gate.test.ts @@ -0,0 +1,661 @@ +/** + * @file capability-gate.test.ts + * + * The gate has one job with two halves: say exactly why a slot is not ready, and + * never, under any configuration, hand back a different provider than the one + * that was asked for. + * + * The second half is the load-bearing one. Routing audio to a cloud API the user + * did not pick is an unasked-for charge and a privacy break, so the last test in + * this file walks every combination of provider selection and disk state and + * asserts that no verdict ever names a provider other than the configured one. + */ + +import { describe, it, expect, vi } from 'vitest'; + +/** + * Which runtimes this "build" ships, flipped per test. + * + * The gate now asks the loader whether a runtime will load at all, and the real + * registry has every one at `declared: false` until the providers land. Against + * that, every test below would block on the runtime and never reach the provider + * and model logic they exist to cover. Declared by default, so a test that cares + * about the runtime dimension is the one that says so. + */ +// `vi.hoisted` because the mock factory runs during the import phase, before a +// plain `const` at this scope has been initialised. +const { declared } = vi.hoisted(() => ({ + declared: { llama: true, whisper: true, onnx: true } as Record, +})); + +vi.mock('../../../../shared/acappella/native-runtimes', () => { + const descriptor = (id: string, moduleId: string, label: string) => ({ + id, + moduleId, + versionPin: '1.0.0', + label, + slots: [], + get declared() { + return declared[id]; + }, + requiresElectronRebuild: false, + prebuilds: { + 'darwin-arm64': 'prebuilt', + 'darwin-x64': 'prebuilt', + 'win32-x64': 'prebuilt', + 'linux-x64': 'prebuilt', + }, + asarUnpack: [], + packagedBinaries: { + 'darwin-arm64': [], + 'darwin-x64': [], + 'win32-x64': [], + 'linux-x64': [], + }, + rationale: '', + notes: '', + }); + + const runtimes = [ + descriptor('llama', 'fake-llama', 'Fake llama'), + descriptor('whisper', 'fake-whisper', 'Fake whisper'), + descriptor('onnx', 'fake-onnx', 'Fake onnx'), + ]; + + return { + NATIVE_RUNTIMES: runtimes, + getNativeRuntime: (id: string) => runtimes.find((runtime) => runtime.id === id), + nativePlatformKey: (platform: string, arch: string) => { + const key = `${platform}-${arch}`; + return ['darwin-arm64', 'darwin-x64', 'win32-x64', 'linux-x64'].includes(key) ? key : null; + }, + }; +}); + +vi.mock('electron', () => ({ + app: { getPath: () => '/tmp/acappella-capability-gate-test' }, + shell: { openExternal: vi.fn() }, + // The gate reads the microphone permission, which is a pure query and never + // prompts. Granted here so these tests stay about providers and models; the + // permission's own behaviour is covered in mic-permission.test.ts. + systemPreferences: { + getMediaAccessStatus: () => 'granted', + askForMediaAccess: vi.fn(), + }, +})); + +import { + LOCAL_PROVIDER_IDS, + WAKE_WORD_PROVIDER_ID, + resolveVoiceReadiness, +} from '../../../../main/acappella/models/capability-gate'; +import type { ModelStatus } from '../../../../main/acappella/models/model-store'; +import { + VOICE_SLOT_UNSATISFIED_REASONS, + readinessErrorMessage, + type VoiceReadiness, + type VoiceSlot, + type VoiceSlotReadiness, + type VoiceSlotUnsatisfiedReason, +} from '../../../../shared/acappella/readiness'; +import type { NativeRuntimeUnavailable } from '../../../../main/acappella/runtime/native-loader'; +import { + KOKORO_82M_ID, + OPENWAKEWORD_BASE_ID, + QWEN3_1_7B_ID, + WHISPER_BASE_EN_ID, +} from '../../../../shared/acappella/model-catalog'; + +type StatusKind = ModelStatus['status']; + +/** A fake store: every model reports whatever the map says, installed by default. */ +function statusReader(overrides: Record = {}) { + return async (modelId: string): Promise => { + const status = overrides[modelId] ?? 'installed'; + return { + id: modelId, + status, + manifest: null, + detail: status === 'corrupt' ? 'hash mismatch' : undefined, + bytesOnDisk: status === 'not-installed' ? 0 : 1024, + }; + }; +} + +const ALL_LOCAL = { + stt: LOCAL_PROVIDER_IDS.stt, + tts: LOCAL_PROVIDER_IDS.tts, + brain: LOCAL_PROVIDER_IDS.brain, +}; + +describe('capability-gate', () => { + describe('satisfied slots', () => { + it('is ready when every local model is installed', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + }); + + expect(readiness.canStartSession).toBe(true); + expect(readiness.canRunHandsFree).toBe(true); + expect(readiness.blocking).toHaveLength(0); + // The microphone leads: it is the one requirement that holds regardless of + // which providers are configured, and a user who reads "microphone access + // denied" first does not need to read the rest. + expect(readiness.slots.map((slot) => slot.slot)).toEqual([ + 'microphone', + 'stt', + 'tts', + 'brain', + 'wake-word', + ]); + }); + + it('treats the mock tier as satisfied, since it needs nothing', async () => { + const readiness = await resolveVoiceReadiness({ + settings: { stt: 'mock-stt', tts: 'mock-tts', brain: 'mock-brain' }, + readModelStatus: statusReader({ + [WHISPER_BASE_EN_ID]: 'not-installed', + [KOKORO_82M_ID]: 'not-installed', + [QWEN3_1_7B_ID]: 'not-installed', + }), + }); + + expect(readiness.canStartSession).toBe(true); + }); + }); + + describe('every unsatisfied reason', () => { + it('reports model-not-installed with a download action', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader({ [WHISPER_BASE_EN_ID]: 'not-installed' }), + }); + + const stt = readiness.slots.find((slot) => slot.slot === 'stt')!; + expect(stt.satisfied).toBe(false); + expect(stt.reason).toBe('model-not-installed'); + expect(stt.requiredModelId).toBe(WHISPER_BASE_EN_ID); + expect(stt.detail).toContain('is not installed'); + expect(stt.suggestedAction).toContain('Download'); + expect(readiness.canStartSession).toBe(false); + }); + + it('reports model-corrupt with a re-verify action', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader({ [KOKORO_82M_ID]: 'corrupt' }), + }); + + const tts = readiness.slots.find((slot) => slot.slot === 'tts')!; + expect(tts.reason).toBe('model-corrupt'); + expect(tts.detail).toContain('failed verification'); + expect(tts.detail).toContain('hash mismatch'); + expect(tts.suggestedAction).toContain('Re-verify'); + }); + + it('reports api-key-missing for a cloud provider with no key', async () => { + const readiness = await resolveVoiceReadiness({ + settings: { ...ALL_LOCAL, tts: 'elevenlabs-tts' }, + readModelStatus: statusReader(), + hasApiKey: () => false, + }); + + const tts = readiness.slots.find((slot) => slot.slot === 'tts')!; + expect(tts.reason).toBe('api-key-missing'); + expect(tts.detail).toContain('ElevenLabs'); + // The honest alternative is named, because it is often the real recovery. + expect(tts.suggestedAction).toContain('local model'); + }); + + it('treats an absent stored key as missing', async () => { + const readiness = await resolveVoiceReadiness({ + settings: { ...ALL_LOCAL, brain: 'openai-realtime' }, + readModelStatus: statusReader(), + hasApiKey: () => false, + }); + + expect(readiness.slots.find((slot) => slot.slot === 'brain')?.reason).toBe('api-key-missing'); + }); + + it('reports provider-unreachable when the probe says so', async () => { + const readiness = await resolveVoiceReadiness({ + settings: { ...ALL_LOCAL, brain: 'openai-realtime' }, + readModelStatus: statusReader(), + hasApiKey: () => true, + probeProvider: () => false, + }); + + const brain = readiness.slots.find((slot) => slot.slot === 'brain')!; + expect(brain.reason).toBe('provider-unreachable'); + expect(brain.detail).toContain('could not be reached'); + }); + + it('reports runtime-unavailable, and reports it INSTEAD of a download', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + // Model missing AND runtime broken: the runtime wins, because + // downloading 1.1 GB does not fix a binary that will not load. + readModelStatus: statusReader({ [QWEN3_1_7B_ID]: 'not-installed' }), + readRuntimeFailure: (runtimeId) => + runtimeId === 'llama' + ? { + kind: 'runtime-unavailable', + runtimeId: 'llama', + moduleId: 'node-llama-cpp', + platform: 'linux', + arch: 'x64', + failure: 'load-failed', + message: 'llama.cpp failed to load on linux-x64.', + suggestedAction: 'Run the voice self-test and include the result.', + } + : null, + }); + + const brain = readiness.slots.find((slot) => slot.slot === 'brain')!; + expect(brain.reason).toBe('runtime-unavailable'); + expect(brain.detail).toContain('failed to load'); + expect(brain.suggestedAction).toContain('self-test'); + expect(readiness.canStartSession).toBe(false); + }); + + it('leaves a slot alone when its runtime has never failed', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + readRuntimeFailure: () => null, + }); + + expect(readiness.canStartSession).toBe(true); + }); + + it('blocks a runtime that is not in this build, before anything tries to load it', async () => { + // The default reader, deliberately: a gate that only knows about loads + // that have already been attempted says "ready" on a fresh boot for a + // runtime the build does not contain, and the user finds out when the + // session dies. Every model is on disk here; the runtime still decides. + declared.whisper = false; + try { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + }); + + const stt = readiness.slots.find((slot) => slot.slot === 'stt')!; + expect(stt.satisfied).toBe(false); + expect(stt.reason).toBe('runtime-unavailable'); + expect(stt.detail).toContain('not part of this build'); + expect(readiness.canStartSession).toBe(false); + } finally { + declared.whisper = true; + } + }); + + it('assumes reachable when no probe is wired', async () => { + const readiness = await resolveVoiceReadiness({ + settings: { ...ALL_LOCAL, brain: 'openai-realtime' }, + readModelStatus: statusReader(), + hasApiKey: () => true, + }); + + expect(readiness.canStartSession).toBe(true); + }); + }); + + describe('the wake word', () => { + it('blocks hands-free but not a click-to-talk session', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader({ [OPENWAKEWORD_BASE_ID]: 'not-installed' }), + }); + + expect(readiness.canStartSession).toBe(true); + expect(readiness.canRunHandsFree).toBe(false); + expect(readiness.blocking).toHaveLength(0); + const wake = readiness.slots.find((slot) => slot.slot === 'wake-word')!; + expect(wake.satisfied).toBe(false); + expect(wake.providerId).toBe(WAKE_WORD_PROVIDER_ID); + }); + + it('is always local: no setting can point it at a cloud provider', async () => { + const readiness = await resolveVoiceReadiness({ + // Deliberately hostile settings: nothing here may reach the wake word. + settings: { stt: 'openai-realtime', tts: 'elevenlabs-tts', brain: 'openai-realtime' }, + readModelStatus: statusReader(), + hasApiKey: () => true, + }); + + expect(readiness.slots.find((slot) => slot.slot === 'wake-word')?.providerId).toBe( + WAKE_WORD_PROVIDER_ID + ); + }); + }); + + describe('no implicit provider substitution', () => { + const providerChoices = [ + LOCAL_PROVIDER_IDS.stt, + 'openai-realtime', + 'mock-stt', + 'not-a-registered-provider', + ]; + const diskStates: StatusKind[] = ['installed', 'not-installed', 'corrupt']; + + it('never names a provider other than the one configured', async () => { + for (const stt of providerChoices) { + for (const disk of diskStates) { + for (const key of [false, true]) { + for (const reachable of [true, false]) { + const readiness = await resolveVoiceReadiness({ + settings: { ...ALL_LOCAL, stt }, + readModelStatus: statusReader({ + [WHISPER_BASE_EN_ID]: disk, + [KOKORO_82M_ID]: disk, + [QWEN3_1_7B_ID]: disk, + [OPENWAKEWORD_BASE_ID]: disk, + }), + hasApiKey: () => key, + probeProvider: () => reachable, + }); + + const sttSlot = readiness.slots.find((slot) => slot.slot === 'stt')!; + // The verdict reports what was ASKED for. If the gate ever + // "helpfully" resolved to a working provider, this is where it + // would show up. + expect(sttSlot.providerId).toBe(stt); + + // And an unsatisfied slot is never quietly satisfied by another. + if (!sttSlot.satisfied) { + expect(readiness.canStartSession).toBe(false); + expect(readiness.blocking.map((slot) => slot.slot)).toContain('stt'); + } + } + } + } + } + }); + + it('leaves an unknown provider satisfied only because it demands nothing', async () => { + // An id the gate has never heard of has no requirement to check, so it + // cannot be reported as blocked here. It is the registry, not the gate, + // that refuses to run it - and the registry's only fallback is the mock. + const readiness = await resolveVoiceReadiness({ + settings: { ...ALL_LOCAL, brain: 'not-a-registered-provider' }, + readModelStatus: statusReader(), + }); + + const brain = readiness.slots.find((slot) => slot.slot === 'brain')!; + expect(brain.providerId).toBe('not-a-registered-provider'); + expect(brain.satisfied).toBe(true); + }); + }); + + describe('the microphone slot', () => { + it('reports mic-permission-denied as a permission, with the privacy pane as the fix', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + readMicPermission: () => 'denied', + }); + + const mic = readiness.slots.find((slot) => slot.slot === 'microphone')!; + expect(mic.satisfied).toBe(false); + expect(mic.reason).toBe('mic-permission-denied'); + // Named as a permission, never as "voice unavailable": a user with every + // model on disk and a denied microphone has a one-checkbox problem. + expect(mic.detail).toMatch(/microphone access/i); + expect(mic.suggestedAction).toMatch(/privacy settings/i); + expect(readiness.canStartSession).toBe(false); + }); + + it('reports mic-permission-restricted without sending the user to a checkbox they cannot tick', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + readMicPermission: () => 'restricted', + }); + + const mic = readiness.slots.find((slot) => slot.slot === 'microphone')!; + expect(mic.reason).toBe('mic-permission-restricted'); + expect(mic.suggestedAction).toMatch(/manages this machine/i); + expect(mic.suggestedAction).not.toMatch(/privacy settings/i); + }); + + it.each(['granted', 'not-determined', 'unknown'] as const)( + 'does not block on %s, which is a machine that has not been asked yet', + async (permission) => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + readMicPermission: () => permission, + }); + + const mic = readiness.slots.find((slot) => slot.slot === 'microphone')!; + expect(mic.satisfied).toBe(true); + // Reported even when satisfied, so Voice Setup can say "you will be asked + // when you start" instead of describing it as a problem. + expect(mic.micPermission).toBe(permission); + } + ); + }); + + describe('no unsatisfied slot is a dead end', () => { + /** A runtime that has already failed to load in this process. */ + const whisperRuntimeFailure: NativeRuntimeUnavailable = { + kind: 'runtime-unavailable', + runtimeId: 'whisper', + moduleId: 'whisper-node', + platform: 'darwin', + arch: 'arm64', + failure: 'load-failed', + message: 'whisper.cpp could not be loaded on this machine.', + suggestedAction: 'Reinstall Maestro, or switch Speech-to-Text to a hosted provider.', + detail: 'dlopen failed', + }; + + /** + * Every reason in the union, each produced by a real configuration. + * + * "Voice mode unavailable" with no reason is indistinguishable from a bug, + * so the gate's contract is that an unsatisfied slot ALWAYS carries a + * sentence naming the missing piece and a sentence saying what to do. A new + * reason added without either would otherwise reach a user as a disabled + * button with nothing next to it. + */ + const CASES: Array<[VoiceSlotUnsatisfiedReason, () => Promise]> = [ + [ + 'model-not-installed', + () => + resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader({ [WHISPER_BASE_EN_ID]: 'not-installed' }), + }), + ], + [ + 'model-corrupt', + () => + resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader({ [KOKORO_82M_ID]: 'corrupt' }), + }), + ], + [ + 'api-key-missing', + () => + resolveVoiceReadiness({ + settings: { ...ALL_LOCAL, stt: 'openai-stt' }, + readModelStatus: statusReader(), + hasApiKey: () => false, + }), + ], + [ + 'provider-unreachable', + () => + resolveVoiceReadiness({ + settings: { ...ALL_LOCAL, stt: 'openai-stt' }, + readModelStatus: statusReader(), + hasApiKey: () => true, + probeProvider: () => false, + }), + ], + [ + 'runtime-unavailable', + () => + resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + readRuntimeFailure: (runtimeId) => + runtimeId === 'whisper' ? whisperRuntimeFailure : null, + }), + ], + [ + 'mic-permission-denied', + () => + resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + readMicPermission: () => 'denied', + }), + ], + [ + 'mic-permission-restricted', + () => + resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + readMicPermission: () => 'restricted', + }), + ], + ]; + + it('covers every reason in the union, so a new one cannot be added silently', () => { + expect(CASES.map(([reason]) => reason).sort()).toEqual( + [...VOICE_SLOT_UNSATISFIED_REASONS].sort() + ); + }); + + it.each(CASES)('%s carries a detail and a recovery', async (reason, resolve) => { + const readiness = await resolve(); + const slot = readiness.slots.find((entry: VoiceSlotReadiness) => entry.reason === reason); + + expect(slot, `no slot reported ${reason}`).toBeDefined(); + expect(slot!.satisfied).toBe(false); + expect(slot!.detail).toEqual(expect.any(String)); + expect(slot!.detail!.length).toBeGreaterThan(0); + expect(slot!.suggestedAction).toEqual(expect.any(String)); + expect(slot!.suggestedAction!.length).toBeGreaterThan(0); + // The recovery has to be a different sentence from the diagnosis, or it is + // not a recovery. + expect(slot!.suggestedAction).not.toBe(slot!.detail); + }); + }); + + describe('readinessErrorMessage', () => { + it('names every blocking slot and its recovery', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader({ + [WHISPER_BASE_EN_ID]: 'not-installed', + [QWEN3_1_7B_ID]: 'corrupt', + }), + }); + + const message = readinessErrorMessage(readiness); + expect(message).toContain('Speech-to-Text'); + expect(message).toContain('Conductor Brain'); + expect(message).toContain('Download it in Settings'); + expect(message).toContain('Re-verify'); + }); + + it('is empty when nothing is blocking', async () => { + const readiness = await resolveVoiceReadiness({ + settings: ALL_LOCAL, + readModelStatus: statusReader(), + }); + expect(readinessErrorMessage(readiness)).toBe(''); + }); + + it('states one shared recovery once, not once per slot', () => { + // The real shape of a build without the native runtimes: three slots fail + // together with the same fix. Repeating it three times was two thirds of + // the message and read as a wall rather than as an instruction. + const action = 'Use a hosted provider or the mock tier until the local runtime ships.'; + const message = readinessErrorMessage( + blockedOn( + ( + [ + { slot: 'stt', detail: 'Speech-to-Text: whisper.cpp is not part of this build yet.' }, + { + slot: 'tts', + detail: 'Text-to-Speech: ONNX Runtime is not part of this build yet.', + }, + { + slot: 'brain', + detail: 'Conductor Brain: llama.cpp is not part of this build yet.', + }, + ] as const + ).map((slot) => ({ ...slot, suggestedAction: action })) + ) + ); + + expect(message.split(action)).toHaveLength(2); // stated exactly once + expect(message).toContain('whisper.cpp'); + expect(message).toContain('ONNX Runtime'); + expect(message).toContain('llama.cpp'); + expect(message.endsWith(action)).toBe(true); + }); + + it('keeps recoveries per slot when they differ', () => { + // A denied microphone and a missing model are two problems with two + // different next steps; collapsing them would drop one of the fixes. + const message = readinessErrorMessage( + blockedOn([ + { + slot: 'microphone', + detail: 'Microphone: Maestro does not have microphone access.', + suggestedAction: 'Grant microphone access in your system privacy settings.', + }, + { + slot: 'stt', + detail: 'Speech-to-Text: Whisper Base is not installed.', + suggestedAction: 'Download it in Settings.', + }, + ]) + ); + + expect(message).toContain('Grant microphone access'); + expect(message).toContain('Download it in Settings'); + }); + + it('does not lend one slot recovery to a slot that has none', () => { + const action = 'Download it in Settings.'; + const message = readinessErrorMessage( + blockedOn([ + { + slot: 'stt', + detail: 'Speech-to-Text: Whisper Base is not installed.', + suggestedAction: action, + }, + { slot: 'tts', detail: 'Text-to-Speech: something went wrong.' }, + ]) + ); + + // Hoisting here would read as though the second slot were fixed by the + // first one's action. It stays attached to the slot that stated it. + expect(message).toBe( + `Speech-to-Text: Whisper Base is not installed. ${action} Text-to-Speech: something went wrong.` + ); + }); + }); +}); + +/** A readiness verdict blocked on exactly these slots. */ +function blockedOn( + blocking: Array & { slot: VoiceSlot }> +): VoiceReadiness { + const slots = blocking.map((slot) => ({ + providerId: 'test-provider', + satisfied: false as const, + ...slot, + })) as VoiceSlotReadiness[]; + return { canStartSession: false, canRunHandsFree: false, slots, blocking: slots }; +} diff --git a/src/__tests__/main/acappella/models/model-downloader.test.ts b/src/__tests__/main/acappella/models/model-downloader.test.ts new file mode 100644 index 0000000000..a795be3bab --- /dev/null +++ b/src/__tests__/main/acappella/models/model-downloader.test.ts @@ -0,0 +1,346 @@ +/** + * @file model-downloader.test.ts + * + * The downloader's contract in four properties: + * + * - a resumed download CONTINUES from the `.part` rather than restarting, + * - a hash mismatch is rejected, the `.part` is deleted, and both hashes are + * reported, + * - cancel leaves nothing behind, + * - the final file only ever appears AFTER verification. + * + * The transport is injected, so nothing here touches the network. The catalog is + * replaced with one tiny entry whose real SHA-256 is computed in the test, which + * is the only way to exercise the success path without a 141 MB download. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; + +const tempRoot = { dir: '' }; + +vi.mock('electron', () => ({ + app: { getPath: () => tempRoot.dir }, +})); + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +const fixture = vi.hoisted(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { createHash } = require('crypto') as typeof import('crypto'); + const contents = Buffer.from('a cappella downloader fixture payload, long enough to slice'); + return { + contents, + id: 'test-tiny-model', + url: 'https://example.invalid/tiny.bin', + sha256: createHash('sha256').update(contents).digest('hex'), + bytes: contents.length, + }; +}); + +vi.mock('../../../../shared/acappella/model-catalog', async (importOriginal) => { + const actual = + await importOriginal(); + const testEntry = Object.freeze({ + id: fixture.id, + displayName: 'Test Tiny Model', + role: 'stt' as const, + repo: 'maestro/test', + revision: '0000000000000000000000000000000000000000', + license: 'MIT', + licenseUrl: 'https://example.invalid/license', + requiredFor: 'local-speech-to-text' as const, + description: 'Fixture entry, present only under test.', + files: Object.freeze([ + { + path: 'tiny.bin', + sourceUrl: fixture.url, + sha256: fixture.sha256, + bytes: fixture.bytes, + }, + ]), + bytes: fixture.bytes, + }); + const catalog = Object.freeze([...actual.VOICE_MODEL_CATALOG, testEntry]); + const byId = new Map(catalog.map((entry) => [entry.id, entry])); + return { + ...actual, + VOICE_MODEL_CATALOG: catalog, + getVoiceModel: (id: string) => byId.get(id), + isVoiceModelId: (id: string) => byId.has(id), + }; +}); + +import { + ModelDownloader, + type DownloadProgress, + type FetchLike, +} from '../../../../main/acappella/models/model-downloader'; +import { + modelDir, + modelFilePath, + readManifest, +} from '../../../../main/acappella/models/model-store'; + +const FILE_PATH = 'tiny.bin'; + +function finalPath(): string { + return modelFilePath(fixture.id, FILE_PATH); +} + +function partPath(): string { + return `${finalPath()}.part`; +} + +async function exists(target: string): Promise { + try { + await fs.stat(target); + return true; + } catch { + return false; + } +} + +/** A fetch that serves `body` and honours `Range: bytes=N-`. */ +function rangeAwareFetch( + body: Buffer, + record: { ranges: string[] } = { ranges: [] } +): { fetchImpl: FetchLike; record: { ranges: string[] } } { + const fetchImpl: FetchLike = async (_url, init) => { + const range = (init?.headers as Record | undefined)?.Range; + record.ranges.push(range ?? ''); + if (!range) return new Response(new Uint8Array(body), { status: 200 }); + const start = Number(/bytes=(\d+)-/.exec(range)?.[1] ?? 0); + return new Response(new Uint8Array(body.subarray(start)), { status: 206 }); + }; + return { fetchImpl, record }; +} + +function makeDownloader(fetchImpl: FetchLike): ModelDownloader { + return new ModelDownloader({ fetchImpl, retryDelayMs: () => 0, progressIntervalMs: 0 }); +} + +describe('model-downloader', () => { + let previousUserData: string | undefined; + + beforeEach(async () => { + tempRoot.dir = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-downloader-')); + // MAESTRO_USER_DATA wins over the electron mock in `dataDir()`, and a + // Maestro-run agent has it pointed at the live install. + previousUserData = process.env.MAESTRO_USER_DATA; + process.env.MAESTRO_USER_DATA = tempRoot.dir; + }); + + afterEach(async () => { + if (previousUserData === undefined) delete process.env.MAESTRO_USER_DATA; + else process.env.MAESTRO_USER_DATA = previousUserData; + await fs.rm(tempRoot.dir, { recursive: true, force: true }); + }); + + it('downloads, verifies, and only then renames into place', async () => { + const seenDuringTransfer: boolean[] = []; + const fetchImpl: FetchLike = async () => { + // The final path must not exist while bytes are still arriving. + seenDuringTransfer.push(await exists(finalPath())); + return new Response(new Uint8Array(fixture.contents), { status: 200 }); + }; + + const downloader = makeDownloader(fetchImpl); + const result = await downloader.download(fixture.id); + + expect(result.status).toBe('complete'); + expect(seenDuringTransfer).toEqual([false]); + expect(await exists(finalPath())).toBe(true); + expect(await exists(partPath())).toBe(false); + + const manifest = await readManifest(fixture.id); + expect(manifest?.id).toBe(fixture.id); + expect(manifest?.bytes).toBe(fixture.bytes); + }); + + it('resumes from a partial .part instead of restarting', async () => { + const { fetchImpl, record } = rangeAwareFetch(fixture.contents); + + // Simulate a killed app: the first 20 bytes are already on disk. + await fs.mkdir(modelDir(fixture.id), { recursive: true }); + await fs.writeFile(partPath(), fixture.contents.subarray(0, 20)); + + const downloader = makeDownloader(fetchImpl); + const result = await downloader.download(fixture.id); + + expect(result.status).toBe('complete'); + expect(record.ranges).toEqual(['bytes=20-']); + // The digest has to cover the resumed bytes too, or the completed file + // would fail verification despite being byte-perfect. + expect(await fs.readFile(finalPath())).toEqual(fixture.contents); + }); + + it('restarts when the server ignores the range header', async () => { + const requests: string[] = []; + const fetchImpl: FetchLike = async (_url, init) => { + requests.push((init?.headers as Record | undefined)?.Range ?? ''); + // 200, not 206: the whole file, despite the range request. + return new Response(new Uint8Array(fixture.contents), { status: 200 }); + }; + + await fs.mkdir(modelDir(fixture.id), { recursive: true }); + await fs.writeFile(partPath(), fixture.contents.subarray(0, 20)); + + const result = await makeDownloader(fetchImpl).download(fixture.id); + + expect(requests).toEqual(['bytes=20-']); + expect(result.status).toBe('complete'); + // Appending onto the stale partial would have produced 20 extra bytes. + expect(await fs.readFile(finalPath())).toEqual(fixture.contents); + }); + + it('rejects a hash mismatch, deletes the .part, and reports both hashes', async () => { + const corrupted = Buffer.from(fixture.contents); + corrupted[0] = corrupted[0] ^ 0xff; + const fetchImpl: FetchLike = async () => + new Response(new Uint8Array(corrupted), { status: 200 }); + + const downloader = makeDownloader(fetchImpl); + const progress: DownloadProgress[] = []; + downloader.onProgress((event) => progress.push(event)); + + const result = await downloader.download(fixture.id); + + expect(result.status).toBe('error'); + expect(result.mismatch?.expected).toBe(fixture.sha256); + expect(result.mismatch?.actual).not.toBe(fixture.sha256); + // Nothing verified, so nothing may be at the final path, and the bad bytes + // must not survive to be "resumed" forever. + expect(await exists(finalPath())).toBe(false); + expect(await exists(partPath())).toBe(false); + expect(await readManifest(fixture.id)).toBeNull(); + expect(progress.at(-1)?.phase).toBe('error'); + }); + + it('cancel leaves no stray files', async () => { + let release: () => void = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + + const fetchImpl: FetchLike = async () => { + const stream = new ReadableStream({ + async start(controller) { + controller.enqueue(new Uint8Array(fixture.contents.subarray(0, 10))); + await gate; + controller.close(); + }, + }); + return new Response(stream, { status: 200 }); + }; + + const downloader = makeDownloader(fetchImpl); + const running = downloader.download(fixture.id); + // Let the first chunk land so there is something on disk to clean up. + await new Promise((resolve) => setTimeout(resolve, 20)); + + const cancelPromise = downloader.cancel(fixture.id); + release(); + await cancelPromise; + const result = await running; + + expect(result.status).toBe('cancelled'); + expect(await exists(partPath())).toBe(false); + expect(await exists(finalPath())).toBe(false); + expect(await exists(modelDir(fixture.id))).toBe(false); + }); + + it('pause keeps the partial file so the next start resumes', async () => { + let release: () => void = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + let served = 0; + + const fetchImpl: FetchLike = async (_url, init) => { + served++; + const range = (init?.headers as Record | undefined)?.Range; + if (served === 1) { + const stream = new ReadableStream({ + async start(controller) { + controller.enqueue(new Uint8Array(fixture.contents.subarray(0, 12))); + await gate; + controller.close(); + }, + }); + return new Response(stream, { status: 200 }); + } + const start = Number(/bytes=(\d+)-/.exec(range ?? '')?.[1] ?? 0); + return new Response(new Uint8Array(fixture.contents.subarray(start)), { status: 206 }); + }; + + const downloader = makeDownloader(fetchImpl); + const running = downloader.download(fixture.id); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(downloader.pause(fixture.id)).toBe(true); + release(); + const paused = await running; + + expect(paused.status).toBe('paused'); + expect(await exists(partPath())).toBe(true); + expect(await exists(finalPath())).toBe(false); + + const resumed = await downloader.resume(fixture.id); + expect(resumed.status).toBe('complete'); + expect(await fs.readFile(finalPath())).toEqual(fixture.contents); + }); + + it('retries a transient server error and then succeeds', async () => { + let attempts = 0; + const fetchImpl: FetchLike = async () => { + attempts++; + if (attempts < 3) return new Response('boom', { status: 503 }); + return new Response(new Uint8Array(fixture.contents), { status: 200 }); + }; + + const result = await makeDownloader(fetchImpl).download(fixture.id); + expect(result.status).toBe('complete'); + expect(attempts).toBe(3); + }); + + it('does not retry a 404', async () => { + let attempts = 0; + const fetchImpl: FetchLike = async () => { + attempts++; + return new Response('nope', { status: 404 }); + }; + + const result = await makeDownloader(fetchImpl).download(fixture.id); + expect(result.status).toBe('error'); + expect(result.error).toContain('404'); + expect(attempts).toBe(1); + }); + + it('rejects an unknown model without touching the transport', async () => { + const fetchImpl = vi.fn(); + const result = await makeDownloader(fetchImpl).download('not-a-model'); + expect(result.status).toBe('error'); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('reports bytes, total, and a rate on progress', async () => { + const fetchImpl: FetchLike = async () => + new Response(new Uint8Array(fixture.contents), { status: 200 }); + + const downloader = makeDownloader(fetchImpl); + const events: DownloadProgress[] = []; + downloader.onProgress((event) => events.push(event)); + + await downloader.download(fixture.id); + + expect(events.length).toBeGreaterThan(0); + expect(events.every((event) => event.bytesTotal === fixture.bytes)).toBe(true); + expect(events.at(-1)?.phase).toBe('complete'); + expect(events.at(-1)?.bytesReceived).toBe(fixture.bytes); + }); +}); diff --git a/src/__tests__/main/acappella/models/model-store.test.ts b/src/__tests__/main/acappella/models/model-store.test.ts new file mode 100644 index 0000000000..7613ad1fef --- /dev/null +++ b/src/__tests__/main/acappella/models/model-store.test.ts @@ -0,0 +1,356 @@ +/** + * @file model-store.test.ts + * + * The store's job is to be pessimistic about disk. These tests pin the four + * properties the rest of the subsystem trusts: + * + * - a manifest round-trips, + * - `isInstalled` REJECTS a truncated file (the whole reason it is not an + * `existsSync`), + * - concurrent manifest writers never produce a half-written file, + * - `remove` reclaims the entire directory, partials included. + * + * Runs against a real temp directory: the failure mode being tested is a + * property of the filesystem, so mocking `fs` would test the mock. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; + +const tempRoot = { dir: '' }; + +vi.mock('electron', () => ({ + app: { getPath: () => tempRoot.dir }, +})); + +/** + * A synthetic catalog entry small enough to actually satisfy. + * + * The real catalog's hashes belong to files of 2 MB to 1 GB, so the SUCCESS path + * of `verify()` cannot be exercised against them without a download. Appending + * one tiny entry (real bytes, real hash, computed here) is what makes "verify + * passes and stamps verifiedAt" testable at all. Every other test still runs + * against the genuine catalog entries. + */ +const fixture = vi.hoisted(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { createHash } = require('crypto') as typeof import('crypto'); + const contents = Buffer.from('a cappella test model payload'); + return { + contents, + id: 'test-tiny-model', + sha256: createHash('sha256').update(contents).digest('hex'), + bytes: contents.length, + }; +}); + +vi.mock('../../../../shared/acappella/model-catalog', async (importOriginal) => { + const actual = + await importOriginal(); + const testEntry = Object.freeze({ + id: fixture.id, + displayName: 'Test Tiny Model', + role: 'stt' as const, + repo: 'maestro/test', + revision: '0000000000000000000000000000000000000000', + license: 'MIT', + licenseUrl: 'https://example.invalid/license', + requiredFor: 'local-speech-to-text' as const, + description: 'Fixture entry, present only under test.', + files: Object.freeze([ + { + path: 'tiny.bin', + sourceUrl: 'https://example.invalid/tiny.bin', + sha256: fixture.sha256, + bytes: fixture.bytes, + }, + ]), + bytes: fixture.bytes, + }); + const catalog = Object.freeze([...actual.VOICE_MODEL_CATALOG, testEntry]); + const byId = new Map(catalog.map((entry) => [entry.id, entry])); + return { + ...actual, + VOICE_MODEL_CATALOG: catalog, + getVoiceModel: (id: string) => byId.get(id), + isVoiceModelId: (id: string) => byId.has(id), + }; +}); + +import { + buildManifest, + getStatus, + isInstalled, + markInstalled, + modelDigest, + modelDir, + modelFilePath, + readManifest, + remove, + removeAll, + totalFootprint, + verify, + writeManifest, +} from '../../../../main/acappella/models/model-store'; +import { + OPENWAKEWORD_BASE_ID, + WHISPER_BASE_EN_ID, + getVoiceModel, +} from '../../../../shared/acappella/model-catalog'; + +const whisper = getVoiceModel(WHISPER_BASE_EN_ID)!; +const wakeWord = getVoiceModel(OPENWAKEWORD_BASE_ID)!; + +/** + * Write files matching the catalog's declared lengths. + * + * The bytes are zeros, so the LENGTH is right and the HASH is not. That is + * deliberate: it is exactly the state a truncated-then-padded file would be in, + * and it lets the length check and the hash check be tested independently. + */ +async function writeFilesOfDeclaredLength(id: string, entry = getVoiceModel(id)!): Promise { + for (const file of entry.files) { + const full = modelFilePath(id, file.path); + await fs.mkdir(path.dirname(full), { recursive: true }); + await fs.writeFile(full, Buffer.alloc(file.bytes)); + } +} + +/** Same, but with a real payload whose hash we control. */ +async function writeFileWithContents( + id: string, + filePath: string, + contents: Buffer +): Promise { + const full = modelFilePath(id, filePath); + await fs.mkdir(path.dirname(full), { recursive: true }); + await fs.writeFile(full, contents); +} + +describe('model-store', () => { + let previousUserData: string | undefined; + + beforeEach(async () => { + tempRoot.dir = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-models-')); + // `dataDir()` checks MAESTRO_USER_DATA BEFORE app.getPath, and a Maestro-run + // agent has that variable set to the real user-data directory. Without this + // override the electron mock is bypassed entirely and these tests write to + // (and delete from) the live install. + previousUserData = process.env.MAESTRO_USER_DATA; + process.env.MAESTRO_USER_DATA = tempRoot.dir; + }); + + afterEach(async () => { + if (previousUserData === undefined) delete process.env.MAESTRO_USER_DATA; + else process.env.MAESTRO_USER_DATA = previousUserData; + await fs.rm(tempRoot.dir, { recursive: true, force: true }); + }); + + describe('manifest round-trip', () => { + it('writes and reads back every recorded field', async () => { + const manifest = buildManifest(wakeWord, 1_700_000_000_000); + await writeManifest(manifest); + + const read = await readManifest(wakeWord.id); + expect(read).toEqual(manifest); + expect(read?.revision).toBe(wakeWord.revision); + expect(read?.license).toBe(wakeWord.license); + expect(read?.installedAt).toBe(1_700_000_000_000); + expect(read?.verifiedAt).toBe(1_700_000_000_000); + expect(read?.files).toHaveLength(wakeWord.files.length); + }); + + it('reads an absent manifest as null rather than throwing', async () => { + expect(await readManifest(whisper.id)).toBeNull(); + }); + + it('reads an unparseable manifest as null, not as an install', async () => { + await fs.mkdir(modelDir(whisper.id), { recursive: true }); + await fs.writeFile(path.join(modelDir(whisper.id), 'manifest.json'), '{ truncated'); + + expect(await readManifest(whisper.id)).toBeNull(); + expect(await isInstalled(whisper.id)).toBe(false); + }); + }); + + describe('isInstalled', () => { + it('rejects a truncated file even though the file exists', async () => { + await markInstalled(wakeWord); + await writeFilesOfDeclaredLength(wakeWord.id); + expect(await isInstalled(wakeWord.id)).toBe(true); + + // Chop one byte off. `existsSync` would still say yes; this must not. + const victim = modelFilePath(wakeWord.id, wakeWord.files[0].path); + await fs.truncate(victim, wakeWord.files[0].bytes - 1); + + expect(await isInstalled(wakeWord.id)).toBe(false); + const status = await getStatus(wakeWord.id); + expect(status.status).toBe('corrupt'); + expect(status.detail).toContain(wakeWord.files[0].path); + }); + + it('rejects a manifest with no files on disk', async () => { + await markInstalled(wakeWord); + expect(await isInstalled(wakeWord.id)).toBe(false); + expect((await getStatus(wakeWord.id)).detail).toContain('Missing file'); + }); + + it('rejects files on disk with no manifest', async () => { + await writeFilesOfDeclaredLength(wakeWord.id); + const status = await getStatus(wakeWord.id); + expect(status.status).toBe('not-installed'); + expect(status.detail).toContain('no manifest'); + }); + + it('reports an install from a superseded catalog revision as not installed', async () => { + await writeFilesOfDeclaredLength(wakeWord.id); + await writeManifest({ + ...buildManifest(wakeWord, Date.now()), + revision: 'deadbeef', + sha256: modelDigest([{ path: 'stale', sha256: 'stale' }]), + }); + + const status = await getStatus(wakeWord.id); + expect(status.status).toBe('not-installed'); + expect(status.detail).toContain('no longer matches the catalog'); + }); + }); + + describe('atomic manifest writes under concurrent callers', () => { + it('never leaves a partially written manifest', async () => { + const writes = Array.from({ length: 25 }, (_, index) => + writeManifest({ ...buildManifest(wakeWord, 1_000 + index), verifiedAt: 2_000 + index }) + ); + await Promise.all(writes); + + // Every interleaving has to produce ONE whole manifest, not a concatenated + // or truncated one. Parsing it is the assertion. + const read = await readManifest(wakeWord.id); + expect(read).not.toBeNull(); + expect(read?.id).toBe(wakeWord.id); + expect(read?.files).toHaveLength(wakeWord.files.length); + + // And no temp file survives the race. + const entries = await fs.readdir(modelDir(wakeWord.id)); + expect(entries.filter((name) => name.endsWith('.tmp'))).toHaveLength(0); + }); + }); + + describe('verify', () => { + it('reports a hash mismatch as corrupt and repairs nothing', async () => { + await markInstalled(wakeWord); + await writeFilesOfDeclaredLength(wakeWord.id); + + const result = await verify(wakeWord.id); + expect(result.ok).toBe(false); + expect(result.status).toBe('corrupt'); + expect(result.mismatch?.expected).toBe(wakeWord.files[0].sha256); + expect(result.mismatch?.actual).not.toBe(wakeWord.files[0].sha256); + + // The files are still exactly where they were: no silent delete, no + // silent re-download. + const size = (await fs.stat(modelFilePath(wakeWord.id, wakeWord.files[0].path))).size; + expect(size).toBe(wakeWord.files[0].bytes); + }); + + it('stamps verifiedAt when every hash matches', async () => { + const tiny = getVoiceModel(fixture.id)!; + await writeFileWithContents(tiny.id, tiny.files[0].path, fixture.contents); + const installed = await markInstalled(tiny); + + const result = await verify(tiny.id); + expect(result.ok).toBe(true); + expect(result.status).toBe('installed'); + expect(result.verifiedAt).toBeGreaterThanOrEqual(installed.installedAt); + + const manifest = await readManifest(tiny.id); + expect(manifest?.verifiedAt).toBe(result.verifiedAt); + expect(manifest?.installedAt).toBe(installed.installedAt); + }); + + it('reports an uninstalled model without hashing anything', async () => { + const result = await verify(whisper.id); + expect(result.ok).toBe(false); + expect(result.status).toBe('not-installed'); + }); + }); + + describe('remove and footprint', () => { + it('reclaims the whole directory including stray partials', async () => { + await markInstalled(wakeWord); + await writeFilesOfDeclaredLength(wakeWord.id); + await fs.writeFile( + `${modelFilePath(wakeWord.id, wakeWord.files[0].path)}.part`, + Buffer.alloc(4096) + ); + + const before = await totalFootprint(); + expect(before.bytes).toBeGreaterThan(wakeWord.bytes); + + const reclaimed = await remove(wakeWord.id); + expect(reclaimed).toBe(before.bytes); + await expect(fs.stat(modelDir(wakeWord.id))).rejects.toThrow(); + expect((await totalFootprint()).bytes).toBe(0); + }); + + it('counts directories that are no longer in the catalog', async () => { + const orphan = path.join(tempRoot.dir, 'models', 'acappella', 'retired-model'); + await fs.mkdir(orphan, { recursive: true }); + await fs.writeFile(path.join(orphan, 'weights.bin'), Buffer.alloc(2048)); + + const footprint = await totalFootprint(); + expect(footprint.bytes).toBe(2048); + expect(footprint.models.map((model) => model.id)).toContain('retired-model'); + }); + + it('removeAll leaves nothing behind: installed, half-downloaded, or orphaned', async () => { + // The reclaim-disk promise in Settings, in one test. A user who switches + // the Encore Feature off and accepts the offer is told a number of bytes; + // anything this misses is disk they were told they got back and did not. + await writeFilesOfDeclaredLength(wakeWord.id); + await writeFilesOfDeclaredLength(whisper.id); + // A download that was paused or interrupted. The bytes are real and they + // are not inside any manifest. + await fs.writeFile( + `${modelFilePath(whisper.id, whisper.files[0].path)}.part`, + Buffer.alloc(4096) + ); + // A directory left by a model the catalog has since dropped. Disk the user + // cannot see is disk they cannot get back. + const orphan = path.join(tempRoot.dir, 'models', 'acappella', 'retired-model'); + await fs.mkdir(orphan, { recursive: true }); + await fs.writeFile(path.join(orphan, 'weights.bin'), Buffer.alloc(2048)); + + const expected = (await totalFootprint()).bytes; + expect(expected).toBe(wakeWord.bytes + whisper.bytes + 4096 + 2048); + + const reclaimed = await removeAll(); + + expect(reclaimed).toBe(expected); + expect(await totalFootprint()).toEqual({ bytes: 0, models: [] }); + }); + + it('removeAll deletes only the A Cappella root', async () => { + await writeFilesOfDeclaredLength(wakeWord.id); + const neighbour = path.join(tempRoot.dir, 'plugins'); + await fs.mkdir(neighbour, { recursive: true }); + await fs.writeFile(path.join(neighbour, 'keep.json'), '{}'); + + const reclaimed = await removeAll(); + expect(reclaimed).toBe(wakeWord.bytes); + await expect(fs.stat(path.join(neighbour, 'keep.json'))).resolves.toBeTruthy(); + }); + }); + + describe('path safety', () => { + it('refuses an id that is not in the catalog', () => { + expect(() => modelDir('../../etc')).toThrow(/UnknownVoiceModel/); + }); + + it('refuses a file path that escapes the model directory', () => { + expect(() => modelFilePath(wakeWord.id, '../../escape.bin')).toThrow(/UnsafeModelFilePath/); + }); + }); +}); diff --git a/src/__tests__/main/acappella/pairing-service.test.ts b/src/__tests__/main/acappella/pairing-service.test.ts new file mode 100644 index 0000000000..95aef8b924 --- /dev/null +++ b/src/__tests__/main/acappella/pairing-service.test.ts @@ -0,0 +1,328 @@ +/** + * Device pairing. + * + * The security model is the thing under test, not the happy path: + * + * - a code expires, and it is consumed by the FIRST claim whatever happens + * next, so a denied request cannot leave a live code behind; + * - a code alone pairs nothing - a human on the desktop has to approve; + * - the long-lived token is never written to disk in plain text; + * - revocation fires immediately, before the write completes, because the one + * situation anybody ever uses it in is a device that is connected right now. + * + * The clock is injected and the file is a temp directory, so nothing here sleeps + * and nothing here touches the real user data path. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'fs/promises'; +import { readFileSync } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { + DEFAULT_PAIRING_TTL_MS, + PairingService, + constantTimeEquals, + generatePairingCode, + hashToken, +} from '../../../main/acappella/pairing/pairing-service'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +let dir: string; +let filePath: string; +let now = 1_000_000; + +function createService(overrides: { hostSecret?: string } = {}): PairingService { + return new PairingService({ + filePath, + hostSecret: overrides.hostSecret ?? 'server-token', + now: () => now, + }); +} + +/** Walk a device all the way to a token: claim, approve, redeem. */ +async function pairDevice( + service: PairingService, + name = 'Test iPhone' +): Promise<{ deviceId: string; token: string }> { + const offer = service.startPairing(); + const claim = service.claim({ code: offer.code, name, platform: 'ios' }); + if (claim.status !== 'pending') throw new Error(`claim failed: ${claim.reason}`); + await service.approve(claim.requestId); + const redeemed = service.redeem(claim.requestId); + if (redeemed.status !== 'approved') throw new Error(`redeem failed: ${redeemed.status}`); + return { deviceId: redeemed.deviceId, token: redeemed.token }; +} + +beforeEach(async () => { + now = 1_000_000; + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-pairing-')); + filePath = path.join(dir, 'devices.json'); +}); + +afterEach(async () => { + await fs.rm(dir, { recursive: true, force: true }); +}); + +describe('pairing codes', () => { + it('draws codes from an alphabet with no lookalike glyphs', () => { + for (let attempt = 0; attempt < 50; attempt += 1) { + expect(generatePairingCode()).toMatch(/^[2-9BCDFGHJKMNPQRSTVWXYZ]{6}$/); + } + }); + + it('expires the window without anyone touching it', () => { + const service = createService(); + service.startPairing(); + expect(service.currentOffer()).not.toBeNull(); + now += DEFAULT_PAIRING_TTL_MS + 1; + expect(service.currentOffer()).toBeNull(); + }); + + it('refuses a claim against an expired code', () => { + const service = createService(); + const offer = service.startPairing(); + now += DEFAULT_PAIRING_TTL_MS + 1; + expect(service.claim({ code: offer.code, name: 'Late', platform: 'ios' })).toEqual({ + status: 'rejected', + reason: 'expired', + }); + }); + + it('refuses a wrong code', () => { + const service = createService(); + service.startPairing(); + expect(service.claim({ code: 'ZZZZZZ', name: 'Guess', platform: 'ios' })).toMatchObject({ + status: 'rejected', + reason: 'unknown-code', + }); + }); + + it('consumes the code on the first claim, even when the human denies it', () => { + const service = createService(); + const offer = service.startPairing(); + const first = service.claim({ code: offer.code, name: 'First', platform: 'ios' }); + expect(first.status).toBe('pending'); + if (first.status !== 'pending') return; + + service.deny(first.requestId); + // The shoulder-surfer's turn. A denied request must not leave a live code. + expect(service.claim({ code: offer.code, name: 'Second', platform: 'ios' })).toMatchObject({ + status: 'rejected', + }); + }); + + it('shows the same fingerprint for the same host secret and a different one otherwise', () => { + expect(createService({ hostSecret: 'a' }).fingerprint()).toBe( + createService({ hostSecret: 'a' }).fingerprint() + ); + expect(createService({ hostSecret: 'a' }).fingerprint()).not.toBe( + createService({ hostSecret: 'b' }).fingerprint() + ); + }); +}); + +describe('desktop approval', () => { + it('hands out nothing until a human approves', async () => { + const service = createService(); + const offer = service.startPairing(); + const claim = service.claim({ code: offer.code, name: 'iPhone', platform: 'ios' }); + expect(claim.status).toBe('pending'); + if (claim.status !== 'pending') return; + + // The whole point: the code is known and the device still cannot connect. + expect(service.redeem(claim.requestId)).toEqual({ status: 'pending' }); + expect(await service.list()).toHaveLength(0); + + await service.approve(claim.requestId); + const redeemed = service.redeem(claim.requestId); + expect(redeemed.status).toBe('approved'); + }); + + it('reports the waiting request so the desktop can render it', () => { + const service = createService(); + const seen: Array = []; + service.onPairingRequest((request) => seen.push(request?.name ?? null)); + const offer = service.startPairing(); + service.claim({ code: offer.code, name: 'Pedram iPhone', platform: 'ios' }); + expect(service.pendingRequest()?.name).toBe('Pedram iPhone'); + expect(seen).toContain('Pedram iPhone'); + }); + + it('returns nothing at all on a denial', () => { + const service = createService(); + const offer = service.startPairing(); + const claim = service.claim({ code: offer.code, name: 'iPhone', platform: 'ios' }); + if (claim.status !== 'pending') throw new Error('claim failed'); + service.deny(claim.requestId); + expect(service.redeem(claim.requestId)).toEqual({ status: 'denied' }); + }); + + it('redeems exactly once', async () => { + const service = createService(); + const offer = service.startPairing(); + const claim = service.claim({ code: offer.code, name: 'iPhone', platform: 'ios' }); + if (claim.status !== 'pending') throw new Error('claim failed'); + await service.approve(claim.requestId); + + expect(service.redeem(claim.requestId).status).toBe('approved'); + // A replayed redemption gets nothing: the token is deleted as it is handed + // over, so it exists in memory for one call and no longer. + expect(service.redeem(claim.requestId).status).toBe('expired'); + }); + + it('lets the desktop rename the device at approval time', async () => { + const service = createService(); + const offer = service.startPairing(); + const claim = service.claim({ code: offer.code, name: 'iPhone', platform: 'ios' }); + if (claim.status !== 'pending') throw new Error('claim failed'); + const device = await service.approve(claim.requestId, 'Walking phone'); + expect(device?.name).toBe('Walking phone'); + }); +}); + +describe('token storage', () => { + it('never writes the token in plain text', async () => { + const service = createService(); + const { token } = await pairDevice(service); + + const raw = await fs.readFile(filePath, 'utf-8'); + expect(raw).not.toContain(token); + const parsed = JSON.parse(raw) as { devices: Array> }; + expect(parsed.devices[0].tokenHash).toEqual(expect.any(String)); + expect(parsed.devices[0].tokenHash).not.toBe(token); + }); + + it('salts per device, so two devices with the same token would not collide', async () => { + const service = createService(); + await pairDevice(service, 'One'); + await pairDevice(service, 'Two'); + const parsed = JSON.parse(await fs.readFile(filePath, 'utf-8')) as { + devices: Array<{ tokenSalt: string }>; + }; + expect(parsed.devices[0].tokenSalt).not.toBe(parsed.devices[1].tokenSalt); + expect(hashToken('same', 'a')).not.toBe(hashToken('same', 'b')); + }); + + it('authenticates a returning device from the stored hash alone', async () => { + const service = createService(); + const { deviceId, token } = await pairDevice(service); + + // A fresh service, as after a restart: nothing in memory, only the file. + const restarted = createService(); + expect(await restarted.authenticate(deviceId, token)).toMatchObject({ id: deviceId }); + }); + + it('refuses a wrong token and an unknown device the same way', async () => { + const service = createService(); + const { deviceId } = await pairDevice(service); + expect(await service.authenticate(deviceId, 'wrong')).toBeNull(); + expect(await service.authenticate('no-such-device', 'wrong')).toBeNull(); + }); + + it('compares in constant time without throwing on a length mismatch', () => { + expect(constantTimeEquals('abc', 'abc')).toBe(true); + expect(constantTimeEquals('abc', 'abcdefghij')).toBe(false); + }); + + it('survives a corrupt device file rather than refusing to run', async () => { + await fs.writeFile(filePath, 'not json at all', 'utf-8'); + const service = createService(); + expect(await service.list()).toEqual([]); + }); +}); + +describe('revocation', () => { + it('fires before the write lands, so a live peer is not held up by a disk flush', async () => { + const service = createService(); + const { deviceId, token } = await pairDevice(service); + + const torn: string[] = []; + /** What was on disk at the moment the teardown listener ran. */ + let diskAtTeardown = ''; + service.onRevoke((id) => { + torn.push(id); + diskAtTeardown = readFileSync(filePath, 'utf-8'); + }); + + await service.revoke(deviceId); + + expect(torn).toEqual([deviceId]); + // The file still said the device was live when the peer was torn down: the + // urgent half does not wait on a disk flush. A revocation that did would + // leave a revoked phone holding the microphone while a slow disk caught up. + expect(JSON.parse(diskAtTeardown).devices[0].revokedAt).toBeNull(); + expect(await service.authenticate(deviceId, token)).toBeNull(); + }); + + it('refuses the revoked device on its next attempt', async () => { + const service = createService(); + const { deviceId, token } = await pairDevice(service); + expect(await service.authenticate(deviceId, token)).not.toBeNull(); + + await service.revoke(deviceId); + expect(await service.authenticate(deviceId, token)).toBeNull(); + + // And after a restart, because the revocation is persisted rather than held + // in the memory of the process that performed it. + expect(await createService().authenticate(deviceId, token)).toBeNull(); + }); + + it('is idempotent', async () => { + const service = createService(); + const { deviceId } = await pairDevice(service); + expect(await service.revoke(deviceId)).toBe(true); + expect(await service.revoke(deviceId)).toBe(false); + }); + + it('revokes everything at once and reports how many', async () => { + const service = createService(); + await pairDevice(service, 'One'); + await pairDevice(service, 'Two'); + const torn: string[] = []; + service.onRevoke((id) => torn.push(id)); + + expect(await service.revokeAll()).toBe(2); + expect(torn).toHaveLength(2); + expect(await service.revokeAll()).toBe(0); + }); + + it('keeps a revoked device visible until it is explicitly forgotten', async () => { + const service = createService(); + const { deviceId } = await pairDevice(service); + await service.revoke(deviceId); + expect(await service.list()).toHaveLength(1); + expect(await service.forget(deviceId)).toBe(true); + expect(await service.list()).toHaveLength(0); + }); +}); + +describe('device list', () => { + it('records how a device last connected', async () => { + const service = createService(); + const { deviceId } = await pairDevice(service); + await service.noteConnected(deviceId, 'relay'); + const [device] = await service.list(); + expect(device.lastCandidateType).toBe('relay'); + expect(device.lastConnectedAt).toBe(now); + }); + + it('renames', async () => { + const service = createService(); + const { deviceId } = await pairDevice(service); + expect(await service.rename(deviceId, 'Bedroom phone')).toBe(true); + expect((await service.list())[0].name).toBe('Bedroom phone'); + }); + + it('never exposes credential material to a caller', async () => { + const service = createService(); + await pairDevice(service); + const [device] = await service.list(); + expect(device).not.toHaveProperty('tokenHash'); + expect(device).not.toHaveProperty('tokenSalt'); + }); +}); diff --git a/src/__tests__/main/acappella/permissions/mic-permission.test.ts b/src/__tests__/main/acappella/permissions/mic-permission.test.ts new file mode 100644 index 0000000000..63f0a166a8 --- /dev/null +++ b/src/__tests__/main/acappella/permissions/mic-permission.test.ts @@ -0,0 +1,309 @@ +/** + * @file mic-permission.test.ts + * + * Two properties, and the second one is the reason this module exists at all: + * + * 1. Every OS state is distinguished. `not-determined` is not `denied`, and + * `restricted` is not `denied` either, because the recovery differs: wait + * to be asked, tick a checkbox, or talk to whoever manages the machine. + * 2. A microphone problem and a model problem produce DIFFERENT gate reasons. + * Conflating them is what turns a one-checkbox fix into a support ticket, + * so the gate is tested here alongside the permission itself. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + +const getMediaAccessStatus = vi.fn(); +const askForMediaAccess = vi.fn(); +const openExternal = vi.fn(); + +vi.mock('electron', () => ({ + app: { getPath: () => '/tmp/acappella-mic-permission-test' }, + shell: { openExternal: (url: string) => openExternal(url) }, + systemPreferences: { + getMediaAccessStatus: (type: string) => getMediaAccessStatus(type), + askForMediaAccess: (type: string) => askForMediaAccess(type), + }, +})); + +import { + getMicPermission, + noteCaptureFailure, + noteCaptureStarted, + openMicSystemSettings, + requestMicPermission, + resetMicPermissionObservation, +} from '../../../../main/acappella/permissions/mic-permission'; +import { resolveVoiceReadiness } from '../../../../main/acappella/models/capability-gate'; +import type { ModelStatus } from '../../../../main/acappella/models/model-store'; +import { WHISPER_BASE_EN_ID } from '../../../../shared/acappella/model-catalog'; + +const REAL_PLATFORM = process.platform; + +function setPlatform(platform: string): void { + Object.defineProperty(process, 'platform', { value: platform, configurable: true }); +} + +function statusReader(overrides: Record = {}) { + return async (modelId: string): Promise => ({ + id: modelId, + status: overrides[modelId] ?? 'installed', + manifest: null, + bytesOnDisk: 1024, + }); +} + +describe('mic-permission', () => { + beforeEach(() => { + resetMicPermissionObservation(); + getMediaAccessStatus.mockReset(); + askForMediaAccess.mockReset(); + openExternal.mockReset(); + setPlatform('darwin'); + }); + + afterEach(() => { + setPlatform(REAL_PLATFORM); + resetMicPermissionObservation(); + }); + + describe('every state stays its own state', () => { + it.each([ + ['not-determined', 'not-determined'], + ['granted', 'granted'], + ['denied', 'denied'], + ['restricted', 'restricted'], + ])('reports %s as %s', (osStatus, expected) => { + getMediaAccessStatus.mockReturnValue(osStatus); + expect(getMicPermission().state).toBe(expected); + }); + + it('offers a prompt only when the OS has one to show', () => { + getMediaAccessStatus.mockReturnValue('not-determined'); + expect(getMicPermission().canPrompt).toBe(true); + + getMediaAccessStatus.mockReturnValue('denied'); + // Once the user has said no, the app cannot ask again. A "Grant access" + // button here would do nothing, which is worse than no button. + expect(getMicPermission().canPrompt).toBe(false); + }); + + it('never prompts while querying', () => { + getMediaAccessStatus.mockReturnValue('not-determined'); + getMicPermission(); + // The whole point: readiness is resolved on every Settings render, so a + // query that could raise a TCC dialog would ask for the microphone behind + // a user who has not asked for voice. + expect(askForMediaAccess).not.toHaveBeenCalled(); + }); + + it('falls back to unknown when the OS query throws', () => { + getMediaAccessStatus.mockImplementation(() => { + throw new Error('not implemented on this platform'); + }); + expect(getMicPermission().state).toBe('unknown'); + }); + }); + + describe('requesting', () => { + it('asks once on macOS when the state is undetermined', async () => { + getMediaAccessStatus.mockReturnValue('not-determined'); + askForMediaAccess.mockResolvedValue(true); + + const info = await requestMicPermission(); + + expect(askForMediaAccess).toHaveBeenCalledWith('microphone'); + expect(info.state).toBe('granted'); + }); + + it('does not re-ask once the user has answered', async () => { + getMediaAccessStatus.mockReturnValue('denied'); + + const info = await requestMicPermission(); + + expect(askForMediaAccess).not.toHaveBeenCalled(); + expect(info.state).toBe('denied'); + }); + + it('reports the refusal when the user says no', async () => { + getMediaAccessStatus.mockReturnValue('not-determined'); + askForMediaAccess.mockResolvedValue(false); + + expect((await requestMicPermission()).state).toBe('denied'); + }); + + it('does not claim a denial when the API itself throws', async () => { + getMediaAccessStatus.mockReturnValue('not-determined'); + askForMediaAccess.mockRejectedValue(new Error('unavailable')); + + // Reporting "denied" here would send the user to fix a checkbox that is + // already correct. + expect((await requestMicPermission()).state).toBe('not-determined'); + }); + + it('never pretends to prompt on Windows', async () => { + setPlatform('win32'); + getMediaAccessStatus.mockReturnValue('denied'); + + const info = await requestMicPermission(); + + expect(askForMediaAccess).not.toHaveBeenCalled(); + expect(info.state).toBe('denied'); + expect(info.canPrompt).toBe(false); + }); + }); + + describe('the getUserMedia failure path', () => { + it('is the only permission signal on Linux', () => { + setPlatform('linux'); + expect(getMicPermission().state).toBe('unknown'); + + noteCaptureFailure('permission-denied'); + + expect(getMicPermission().state).toBe('denied'); + // Linux has no privacy-pane deep link that works across desktops, so the + // UI has to offer words rather than a button. + expect(getMicPermission().settingsUrl).toBeNull(); + }); + + it('does not mistake a missing device for a denial', () => { + setPlatform('linux'); + noteCaptureFailure('no-device'); + noteCaptureFailure('device-lost'); + expect(getMicPermission().state).toBe('unknown'); + }); + + it('does not outrank an OS that has an answer', () => { + setPlatform('win32'); + getMediaAccessStatus.mockReturnValue('granted'); + noteCaptureFailure('permission-denied'); + + // The OS wins because it is what will actually decide the next capture, + // and it updates the instant the user changes the setting. A remembered + // denial that outranked it would keep blocking sessions after the user + // had already fixed the problem. + expect(getMicPermission().state).toBe('granted'); + }); + + it('fills the gap when the OS query is unavailable', () => { + setPlatform('win32'); + getMediaAccessStatus.mockImplementation(() => { + throw new Error('not implemented'); + }); + noteCaptureFailure('permission-denied'); + + expect(getMicPermission().state).toBe('denied'); + }); + + it('clears once capture actually starts, which is the only proof of a grant', () => { + setPlatform('linux'); + noteCaptureFailure('permission-denied'); + expect(getMicPermission().state).toBe('denied'); + + noteCaptureStarted(); + expect(getMicPermission().state).toBe('unknown'); + }); + + it('does not survive the next session start, so one denial cannot deadlock voice', async () => { + setPlatform('linux'); + noteCaptureFailure('permission-denied'); + expect(getMicPermission().state).toBe('denied'); + + // The user granted access and pressed the button again. Without this, the + // capability gate would block the session forever on the strength of one + // old failure, and the successful capture that would clear it is exactly + // what the gate is refusing to allow. + await requestMicPermission(); + + expect(getMicPermission().state).toBe('unknown'); + }); + }); + + describe('opening system settings', () => { + it('opens the privacy pane where one exists', async () => { + expect(await openMicSystemSettings()).toBe(true); + expect(openExternal).toHaveBeenCalledWith( + 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone' + ); + }); + + it('reports false rather than opening nothing on Linux', async () => { + setPlatform('linux'); + expect(await openMicSystemSettings()).toBe(false); + expect(openExternal).not.toHaveBeenCalled(); + }); + }); + + describe('the capability gate keeps permission and models apart', () => { + it('blocks on a denied microphone with its own reason and recovery', async () => { + const readiness = await resolveVoiceReadiness({ + readModelStatus: statusReader(), + readMicPermission: () => 'denied', + }); + + const mic = readiness.slots.find((slot) => slot.slot === 'microphone')!; + expect(mic.satisfied).toBe(false); + expect(mic.reason).toBe('mic-permission-denied'); + expect(mic.detail).toContain('microphone access'); + expect(mic.suggestedAction).toContain('privacy settings'); + expect(readiness.canStartSession).toBe(false); + }); + + it('distinguishes a restricted microphone, which the user cannot fix', async () => { + const readiness = await resolveVoiceReadiness({ + readModelStatus: statusReader(), + readMicPermission: () => 'restricted', + }); + + const mic = readiness.slots.find((slot) => slot.slot === 'microphone')!; + expect(mic.reason).toBe('mic-permission-restricted'); + // No privacy-pane instruction: sending someone to a checkbox they are not + // allowed to tick is a dead end. + expect(mic.suggestedAction).not.toContain('privacy settings'); + }); + + it.each(['not-determined', 'unknown', 'granted'] as const)( + 'does not block on %s, because nothing has been refused', + async (permission) => { + const readiness = await resolveVoiceReadiness({ + readModelStatus: statusReader(), + readMicPermission: () => permission, + }); + + expect(readiness.slots.find((slot) => slot.slot === 'microphone')?.satisfied).toBe(true); + expect(readiness.canStartSession).toBe(true); + } + ); + + it('gives a missing model and a denied microphone two different reasons at once', async () => { + const readiness = await resolveVoiceReadiness({ + settings: { stt: 'whisper-local' }, + readModelStatus: statusReader({ [WHISPER_BASE_EN_ID]: 'not-installed' }), + readMicPermission: () => 'denied', + // This test is about permission and model being separate answers, so + // the third dimension is held out of it: against the real registry the + // local runtime is not in the build yet, and the gate reports that + // ahead of a download it would not fix. + readRuntimeFailure: () => null, + }); + + // The failure this whole module exists to prevent: one generic "voice + // unavailable" covering two unrelated problems with two unrelated fixes. + const reasons = readiness.blocking.map((slot) => slot.reason); + expect(reasons).toContain('mic-permission-denied'); + expect(reasons).toContain('model-not-installed'); + expect(new Set(reasons).size).toBe(2); + }); + + it('reports the microphone permission on the slot even when satisfied', async () => { + const readiness = await resolveVoiceReadiness({ + readModelStatus: statusReader(), + readMicPermission: () => 'not-determined', + }); + + expect(readiness.slots.find((slot) => slot.slot === 'microphone')?.micPermission).toBe( + 'not-determined' + ); + }); + }); +}); diff --git a/src/__tests__/main/acappella/providers/brain-prompt.test.ts b/src/__tests__/main/acappella/providers/brain-prompt.test.ts new file mode 100644 index 0000000000..82ec0063c4 --- /dev/null +++ b/src/__tests__/main/acappella/providers/brain-prompt.test.ts @@ -0,0 +1,199 @@ +/** + * @file brain-prompt.test.ts + * + * The parser every Brain shares. It exists because a model asked for JSON will + * eventually return a fenced block, a preamble, an agent id that closed while it + * was thinking, or a confidence of 7 - and none of those may become a dispatch. + * Sending a spoken instruction to the wrong agent is the worst thing this feature + * can do, and it is worse than doing nothing. + */ + +import { describe, it, expect } from 'vitest'; + +import { + buildRouteUserPrompt, + extractJsonObject, + limitSpokenReply, + parseRouteDecision, +} from '../../../../main/acappella/providers/brain-prompt'; +import type { RosterAgent } from '../../../../shared/acappella/protocol'; +import type { VoiceRouteContext } from '../../../../shared/acappella/providers'; + +const ROSTER: RosterAgent[] = [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + tabs: [ + { id: 'tab-auth', name: 'Auth', lastActiveAt: 1 }, + { id: 'tab-db', name: 'Migrations', lastActiveAt: 2 }, + ], + }, +]; + +const CONTEXT: VoiceRouteContext = { roster: ROSTER, scope: { kind: 'conductor' } }; + +describe('extractJsonObject', () => { + it('finds an object inside a fenced block with a preamble', () => { + const raw = 'Sure! Here you go:\n```json\n{"target":"conductor"}\n```'; + expect(extractJsonObject(raw)).toEqual({ target: 'conductor' }); + }); + + it('keeps braces that belong to a string value', () => { + // The non-greedy regex that "worked" would truncate here. + const raw = '{"prompt":"replace {old} with {new}","confidence":0.5}'; + expect(extractJsonObject(raw)).toMatchObject({ prompt: 'replace {old} with {new}' }); + }); + + it('returns null for text with no object in it', () => { + expect(extractJsonObject('I think you want the backend agent.')).toBeNull(); + }); + + it('returns null rather than throwing on malformed JSON', () => { + expect(extractJsonObject('{"target": }')).toBeNull(); + }); +}); + +describe('parseRouteDecision', () => { + it('accepts a well-formed decision for a running agent', () => { + const decision = parseRouteDecision( + JSON.stringify({ + target: { sessionId: 'agent-backend' }, + tabAction: 'recall', + tabId: 'tab-auth', + prompt: 'what happened to auth', + confidence: 0.82, + }), + CONTEXT, + 'fallback' + ); + + expect(decision).toEqual({ + target: { sessionId: 'agent-backend' }, + tabAction: 'recall', + tabId: 'tab-auth', + tabName: undefined, + prompt: 'what happened to auth', + confidence: 0.82, + }); + }); + + it('sends an unknown agent id to the conductor', () => { + const decision = parseRouteDecision( + JSON.stringify({ target: { sessionId: 'ghost' }, tabAction: 'current', prompt: 'x' }), + CONTEXT, + 'fallback' + ); + + expect(decision.target).toBe('conductor'); + }); + + it('downgrades a recall of a tab that does not exist', () => { + const decision = parseRouteDecision( + JSON.stringify({ + target: { sessionId: 'agent-backend' }, + tabAction: 'recall', + tabId: 'tab-that-closed', + prompt: 'x', + }), + CONTEXT, + 'fallback' + ); + + // A recall the executor cannot perform fails the turn; the tab the user is + // already looking at is the honest downgrade. + expect(decision.tabAction).toBe('current'); + expect(decision.tabId).toBeUndefined(); + }); + + it('drops a tabId on an action that does not take one', () => { + const decision = parseRouteDecision( + JSON.stringify({ target: 'conductor', tabAction: 'new', tabId: 'tab-auth', prompt: 'x' }), + CONTEXT, + 'fallback' + ); + + expect(decision.tabId).toBeUndefined(); + }); + + it('falls back to current for an action it has never heard of', () => { + const decision = parseRouteDecision( + JSON.stringify({ target: 'conductor', tabAction: 'teleport', prompt: 'x' }), + CONTEXT, + 'fallback' + ); + + expect(decision.tabAction).toBe('current'); + }); + + it('uses the user own words when the model gave no prompt', () => { + const decision = parseRouteDecision( + JSON.stringify({ target: 'conductor', tabAction: 'current' }), + CONTEXT, + ' open the auth tab ' + ); + + // A verbatim utterance is a worse prompt than a cleaned one and an + // infinitely better outcome than a turn that silently did nothing. + expect(decision.prompt).toBe('open the auth tab'); + }); + + it('clamps a confidence outside 0 to 1', () => { + const high = parseRouteDecision( + JSON.stringify({ target: 'conductor', tabAction: 'current', prompt: 'x', confidence: 7 }), + CONTEXT, + 'x' + ); + const missing = parseRouteDecision( + JSON.stringify({ target: 'conductor', tabAction: 'current', prompt: 'x' }), + CONTEXT, + 'x' + ); + + expect(high.confidence).toBe(1); + expect(missing.confidence).toBe(0.5); + }); + + it('survives a response with no JSON at all', () => { + const decision = parseRouteDecision('I could not decide.', CONTEXT, 'open the auth tab'); + + expect(decision.target).toBe('conductor'); + expect(decision.prompt).toBe('open the auth tab'); + }); +}); + +describe('buildRouteUserPrompt', () => { + it('names every running agent and its tabs', () => { + const prompt = buildRouteUserPrompt('open auth', CONTEXT); + + expect(prompt).toContain('agent-backend'); + expect(prompt).toContain('tab-auth'); + expect(prompt).toContain('Utterance: open auth'); + }); + + it('says which agent the session is bound to', () => { + const prompt = buildRouteUserPrompt('do it', { + roster: ROSTER, + scope: { kind: 'agent', sessionId: 'agent-backend' }, + }); + + expect(prompt).toContain('bound to agent agent-backend'); + }); + + it('says so plainly when nothing is running', () => { + expect(buildRouteUserPrompt('hello', { roster: [], scope: { kind: 'conductor' } })).toContain( + '(none)' + ); + }); +}); + +describe('limitSpokenReply', () => { + it('strips markdown and trims to the budget', () => { + expect(limitSpokenReply('**One.** Two. Three.', 2)).toBe('One. Two.'); + }); + + it('returns nothing for a reply with nothing in it', () => { + expect(limitSpokenReply(' ', 2)).toBe(''); + }); +}); diff --git a/src/__tests__/main/acappella/providers/credentials.test.ts b/src/__tests__/main/acappella/providers/credentials.test.ts new file mode 100644 index 0000000000..36267620ac --- /dev/null +++ b/src/__tests__/main/acappella/providers/credentials.test.ts @@ -0,0 +1,256 @@ +/** + * @file credentials.test.ts + * + * The credential layer's two promises: + * + * 1. A key round-trips through the OS keychain and NOWHERE else. The test that + * matters here is the negative one: no code path writes a key into the + * settings store or into a log line. That is checked by scanning what the + * module actually did with a fake store and a fake logger, rather than by + * reading the source and trusting it. + * 2. Validation tells the three outcomes apart. A rate limit is not a bad key, + * and telling a throttled user to paste a new one would have them fix a + * problem that fixes itself. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +const settingsWrites: Array<[string, unknown]> = []; +const logLines: string[] = []; + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { + info: (message: string) => logLines.push(message), + warn: (message: string) => logLines.push(message), + error: (message: string) => logLines.push(message), + debug: (message: string) => logLines.push(message), + }, +})); + +import { + __setCredentialEntryFactory, + clearCredential, + getCredential, + hasCredential, + listCredentialStates, + redactSecrets, + redactSecretsDeep, + setCredential, + validateCredential, + REDACTED, +} from '../../../../main/acappella/providers/credentials'; +import type { KeyringEntry } from '../../../../main/utils/keyring'; + +const SECRET = 'sk-test-abcdefghijklmnopqrstuvwxyz012345'; + +/** An in-memory keyring entry, standing in for the OS credential store. */ +function fakeEntry(): KeyringEntry & { value: string | null } { + return { + value: null, + getPassword() { + return this.value; + }, + setPassword(password: string) { + this.value = password; + }, + deletePassword() { + const had = this.value !== null; + this.value = null; + return had; + }, + }; +} + +let entries: Record>; + +beforeEach(() => { + settingsWrites.length = 0; + logLines.length = 0; + entries = {}; + __setCredentialEntryFactory((service) => { + entries[service] ??= fakeEntry(); + return entries[service]; + }); +}); + +afterEach(() => { + __setCredentialEntryFactory(null); +}); + +describe('credential storage', () => { + it('round-trips a key through the keychain', () => { + expect(setCredential('openai', SECRET)).toEqual({ ok: true }); + expect(getCredential('openai')).toBe(SECRET); + expect(hasCredential('openai')).toBe(true); + expect(entries.openai.value).toBe(SECRET); + }); + + it('keeps services apart', () => { + setCredential('openai', SECRET); + expect(hasCredential('elevenlabs')).toBe(false); + expect(hasCredential('anthropic')).toBe(false); + }); + + it('trims, and treats a whitespace-only key as a clear', () => { + setCredential('anthropic', ` ${SECRET} `); + expect(getCredential('anthropic')).toBe(SECRET); + + setCredential('anthropic', ' '); + expect(getCredential('anthropic')).toBeNull(); + }); + + it('clears a key', () => { + setCredential('elevenlabs', 'abc123'); + expect(clearCredential('elevenlabs')).toEqual({ ok: true }); + expect(hasCredential('elevenlabs')).toBe(false); + }); + + it('reports, rather than crashes, when the machine has no keyring', () => { + __setCredentialEntryFactory(() => null); + + const result = setCredential('openai', SECRET); + expect(result.ok).toBe(false); + expect(result.error).toContain('credential store'); + // The important half: no fallback to a file. A machine without a keychain + // simply cannot use the hosted tier. + expect(result.error).toContain('never written to disk'); + expect(hasCredential('openai')).toBe(false); + }); + + it('survives a locked keychain that throws on read', () => { + __setCredentialEntryFactory(() => ({ + getPassword: () => { + throw new Error('keychain is locked'); + }, + setPassword: () => {}, + deletePassword: () => false, + })); + + expect(getCredential('openai')).toBeNull(); + expect(logLines.join('\n')).toContain('keychain is locked'); + }); + + it('lists configured state without ever returning a key', () => { + setCredential('openai', SECRET); + const states = listCredentialStates(); + + expect(states.find((state) => state.service === 'openai')).toMatchObject({ + configured: true, + keyringAvailable: true, + }); + expect(JSON.stringify(states)).not.toContain(SECRET); + }); + + it('never writes a key into the settings store', () => { + setCredential('openai', SECRET); + setCredential('elevenlabs', 'el-secret-value'); + void listCredentialStates(); + void getCredential('openai'); + + // Nothing in this module touches a settings store at all; this asserts the + // property rather than the implementation, so a future change that reaches + // for one fails here. + expect(settingsWrites).toEqual([]); + }); + + it('never logs a key', () => { + setCredential('openai', SECRET); + void getCredential('openai'); + void hasCredential('openai'); + + expect(logLines.join('\n')).not.toContain(SECRET); + }); +}); + +describe('credential validation', () => { + it('accepts a key the service accepts', async () => { + const fetchImpl = vi.fn(async () => new Response('{}', { status: 200 })); + const result = await validateCredential('openai', SECRET, fetchImpl); + + expect(result.status).toBe('valid'); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('tells a rate limit apart from a bad key', async () => { + const limited = await validateCredential( + 'openai', + SECRET, + async () => new Response('', { status: 429 }) + ); + expect(limited.status).toBe('rate-limited'); + expect(limited.message).toContain('looks fine'); + + const rejected = await validateCredential( + 'openai', + SECRET, + async () => new Response('', { status: 401 }) + ); + expect(rejected.status).toBe('invalid'); + expect(rejected.message).toContain('revoked'); + }); + + it('reports a server error as a network problem, not a bad key', async () => { + const result = await validateCredential( + 'elevenlabs', + 'el-key-value', + async () => new Response('', { status: 503 }) + ); + expect(result.status).toBe('network-error'); + }); + + it('reports an unreachable service without quoting the request', async () => { + const result = await validateCredential('anthropic', 'sk-ant-abcdefghijklmnop', async () => { + throw new Error('getaddrinfo ENOTFOUND api.anthropic.com'); + }); + + expect(result.status).toBe('network-error'); + expect(result.message).not.toContain('sk-ant-'); + }); + + it('rejects an obviously wrong key shape without a round trip', async () => { + const fetchImpl = vi.fn(); + const result = await validateCredential('openai', 'not-a-key', fetchImpl as never); + + expect(result.status).toBe('invalid'); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('says so when nothing is stored', async () => { + const result = await validateCredential('openai', undefined, async () => new Response('{}')); + expect(result.status).toBe('missing'); + }); + + it('validates the STORED key when none is passed', async () => { + setCredential('openai', SECRET); + const fetchImpl = vi.fn(async () => new Response('{}', { status: 200 })); + + expect((await validateCredential('openai', undefined, fetchImpl)).status).toBe('valid'); + const [, init] = fetchImpl.mock.calls[0] as unknown as [string, RequestInit]; + expect((init.headers as Record).Authorization).toContain(SECRET); + }); +}); + +describe('redaction', () => { + it('scrubs key shapes out of free text', () => { + expect(redactSecrets(`Authorization: Bearer ${SECRET}`)).not.toContain(SECRET); + expect(redactSecrets(`{"api_key": "abcdef1234567890"}`)).toContain(REDACTED); + expect(redactSecrets('xi-api-key: 0123456789abcdef0123')).toContain(REDACTED); + }); + + it('leaves ordinary text alone', () => { + expect(redactSecrets('the session started in 4ms')).toBe('the session started in 4ms'); + }); + + it('scrubs a nested payload by value and by key name', () => { + const scrubbed = redactSecretsDeep({ + provider: 'openai', + headers: { authorization: `Bearer ${SECRET}` }, + nested: [{ apiKey: 'short' }], + }) as { headers: { authorization: string }; nested: Array<{ apiKey: string }> }; + + expect(scrubbed.headers.authorization).toBe(REDACTED); + // A short value that no pattern would match is still redacted, because the + // KEY said what it was. + expect(scrubbed.nested[0].apiKey).toBe(REDACTED); + }); +}); diff --git a/src/__tests__/main/acappella/providers/hosted-providers.test.ts b/src/__tests__/main/acappella/providers/hosted-providers.test.ts new file mode 100644 index 0000000000..3b153dfba0 --- /dev/null +++ b/src/__tests__/main/acappella/providers/hosted-providers.test.ts @@ -0,0 +1,534 @@ +/** + * @file hosted-providers.test.ts + * + * The hosted tier, against a mocked transport. Three properties per backend, + * because these are the three that fail silently in production: + * + * - **Streaming.** Partial transcripts arrive as the deltas do, rather than in + * one lump at the end. + * - **Cancellation aborts the REQUEST.** A barge-in that only stops iterating + * leaves the socket open and the account paying for audio nobody will hear, + * so the test asserts the abort signal actually fired. + * - **Classification.** Quota, auth, and network come back as distinct kinds + * with distinct protocol codes. Collapsing them is what tells a user with an + * expired key to go and download a model. + */ + +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { OpenAiSttProvider } from '../../../../main/acappella/providers/hosted/openai-stt'; +import { ElevenLabsTtsProvider } from '../../../../main/acappella/providers/hosted/elevenlabs-tts'; +import { OpenAiBrainProvider } from '../../../../main/acappella/providers/hosted/openai-brain'; +import { AnthropicBrainProvider } from '../../../../main/acappella/providers/hosted/anthropic-brain'; +import { hostedRequest, MAX_ATTEMPTS } from '../../../../main/acappella/providers/hosted/http'; +import { isVoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import type { SttCallbacks, TtsChunk } from '../../../../shared/acappella/providers'; +import type { RosterAgent } from '../../../../shared/acappella/protocol'; + +const KEY = () => 'sk-test-abcdefghijklmnopqrstuvwxyz'; + +const ROSTER: RosterAgent[] = [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + tabs: [{ id: 'tab-auth', name: 'Auth', lastActiveAt: 1 }], + }, +]; + +function recorder(): { + callbacks: SttCallbacks; + partials: string[]; + finals: string[]; + errors: Error[]; +} { + const partials: string[] = []; + const finals: string[] = []; + const errors: Error[] = []; + return { + partials, + finals, + errors, + callbacks: { + onPartial: (text) => partials.push(text), + onFinal: (text) => finals.push(text), + onError: (error) => errors.push(error), + }, + }; +} + +/** A `text/event-stream` body built from event objects. */ +function sseResponse(events: unknown[]): Response { + const body = new ReadableStream({ + start(controller) { + const encoder = new TextEncoder(); + for (const event of events) { + controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`)); + } + controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.close(); + }, + }); + return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }); +} + +/** One second of silence, enough to clear the minimum-utterance floor. */ +function utterancePcm(): Int16Array { + return new Int16Array(16_000); +} + +// --------------------------------------------------------------------------- +// Transport +// --------------------------------------------------------------------------- + +describe('hosted transport', () => { + const base = { + providerId: 'openai-stt', + service: 'openai' as const, + url: 'https://example.test/v1/thing', + timeoutMs: 1_000, + delayMs: async () => {}, + }; + + it('retries a 429 and succeeds', async () => { + let calls = 0; + const response = await hostedRequest({ + ...base, + fetchImpl: async () => { + calls += 1; + return calls === 1 + ? new Response('', { status: 429 }) + : new Response('{}', { status: 200 }); + }, + }); + + expect(calls).toBe(2); + expect(response.status).toBe(200); + }); + + it('gives up after a bounded number of attempts', async () => { + let calls = 0; + await expect( + hostedRequest({ + ...base, + fetchImpl: async () => { + calls += 1; + return new Response('', { status: 503 }); + }, + }) + ).rejects.toMatchObject({ kind: 'server' }); + + expect(calls).toBe(MAX_ATTEMPTS); + }); + + it('never retries an auth failure', async () => { + let calls = 0; + const error = await hostedRequest({ + ...base, + fetchImpl: async () => { + calls += 1; + return new Response('', { status: 401 }); + }, + }).catch((err: unknown) => err); + + expect(calls).toBe(1); + expect(isVoiceProviderError(error) && error.kind).toBe('auth'); + expect(isVoiceProviderError(error) && error.sessionErrorCode).toBe('provider-auth-failed'); + }); + + it('classifies quota, network, and request failures distinctly', async () => { + const quota = await hostedRequest({ + ...base, + retry: false, + fetchImpl: async () => new Response('', { status: 402 }), + }).catch((err: unknown) => err); + expect(isVoiceProviderError(quota) && quota.sessionErrorCode).toBe('provider-quota-exceeded'); + + const network = await hostedRequest({ + ...base, + retry: false, + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }).catch((err: unknown) => err); + expect(isVoiceProviderError(network) && network.sessionErrorCode).toBe( + 'provider-network-error' + ); + + const bad = await hostedRequest({ + ...base, + retry: false, + fetchImpl: async () => new Response('', { status: 400 }), + }).catch((err: unknown) => err); + expect(isVoiceProviderError(bad) && bad.kind).toBe('request'); + }); + + it('never quotes the response body, which can echo the key', async () => { + const error = await hostedRequest({ + ...base, + retry: false, + fetchImpl: async () => + new Response(JSON.stringify({ error: { message: `bad key ${KEY()}` } }), { status: 401 }), + }).catch((err: unknown) => err); + + expect((error as Error).message).not.toContain(KEY()); + }); + + it('aborts when the caller cancels', async () => { + const controller = new AbortController(); + controller.abort(); + + await expect( + hostedRequest({ ...base, signal: controller.signal, fetchImpl: async () => new Response() }) + ).rejects.toMatchObject({ kind: 'network' }); + }); +}); + +// --------------------------------------------------------------------------- +// OpenAI STT +// --------------------------------------------------------------------------- + +describe('OpenAiSttProvider', () => { + it('refuses to start without a key, before any audio is buffered', async () => { + const provider = new OpenAiSttProvider({ readCredential: () => null }); + const { callbacks } = recorder(); + + await expect(provider.start(callbacks)).rejects.toThrow(/API key/i); + }); + + it('sends nothing until the floor has been opened', async () => { + const fetchImpl = vi.fn(async () => sseResponse([])); + const provider = new OpenAiSttProvider({ readCredential: KEY, fetchImpl }); + + // `feed` before `start`: audio that arrives with no session behind it must + // never reach a hosted service. + provider.feed(utterancePcm()); + await provider.flush(); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('streams partials as deltas arrive and a final at the end', async () => { + const provider = new OpenAiSttProvider({ + readCredential: KEY, + fetchImpl: async () => + sseResponse([ + { type: 'transcript.text.delta', delta: 'open the ' }, + { type: 'transcript.text.delta', delta: 'auth tab' }, + { type: 'transcript.text.done', text: 'open the auth tab' }, + ]), + }); + const { callbacks, partials, finals } = recorder(); + + await provider.start(callbacks); + provider.feed(utterancePcm()); + await provider.flush(); + + expect(partials).toEqual(['open the ', 'open the auth tab']); + expect(finals).toEqual(['open the auth tab']); + }); + + it('reports a classified failure through onError rather than throwing', async () => { + const provider = new OpenAiSttProvider({ + readCredential: KEY, + fetchImpl: async () => new Response('', { status: 429 }), + }); + const { callbacks, errors } = recorder(); + + await provider.start(callbacks); + provider.feed(utterancePcm()); + await provider.flush(); + + expect(errors).toHaveLength(1); + expect(isVoiceProviderError(errors[0]) && errors[0].kind).toBe('quota'); + }); + + it('aborts the in-flight upload on stop', async () => { + let signal: AbortSignal | undefined; + const provider = new OpenAiSttProvider({ + readCredential: KEY, + fetchImpl: async (_url, init) => { + signal = init?.signal ?? undefined; + return sseResponse([{ type: 'transcript.text.done', text: 'hello' }]); + }, + }); + const { callbacks } = recorder(); + + await provider.start(callbacks); + provider.feed(utterancePcm()); + const inflight = provider.flush(); + await provider.stop(); + await inflight; + + expect(signal?.aborted).toBe(true); + }); + + it('does not upload a cough', async () => { + const fetchImpl = vi.fn(async () => sseResponse([])); + const provider = new OpenAiSttProvider({ readCredential: KEY, fetchImpl }); + const { callbacks } = recorder(); + + await provider.start(callbacks); + provider.feed(new Int16Array(100)); + await provider.flush(); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('sends nothing for a client that did its own transcription', async () => { + const fetchImpl = vi.fn(async () => sseResponse([])); + const provider = new OpenAiSttProvider({ readCredential: KEY, fetchImpl }); + const { callbacks, finals } = recorder(); + + await provider.start(callbacks); + provider.injectUtterance('typed instead of spoken'); + + expect(finals).toEqual(['typed instead of spoken']); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// ElevenLabs TTS +// --------------------------------------------------------------------------- + +describe('ElevenLabsTtsProvider', () => { + function pcmResponse(): Response { + return new Response(new Uint8Array(320).buffer, { status: 200 }); + } + + it('synthesises one chunk per sentence', async () => { + const fetchImpl = vi.fn(async () => pcmResponse()); + const provider = new ElevenLabsTtsProvider({ readCredential: KEY, fetchImpl }); + + const chunks = []; + for await (const chunk of provider.speak('First one. Second one.', { utteranceId: 'u1' })) { + chunks.push(chunk); + } + + expect(chunks.map((chunk) => chunk.text)).toEqual(['First one.', 'Second one.']); + expect(chunks[0].format).toBe('pcm16'); + expect(chunks[0].sampleRate).toBe(16_000); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it('cancels an in-flight synthesis by aborting the request', async () => { + let signal: AbortSignal | undefined; + let started: () => void; + const requestStarted = new Promise((resolve) => { + started = resolve; + }); + + const provider = new ElevenLabsTtsProvider({ + readCredential: KEY, + // A request that never settles on its own, so the only way this run can + // end is the abort. That is the property under test: a `cancel()` that + // merely stopped iterating would leave this socket open and the account + // paying for audio nobody will hear. + fetchImpl: (_url, init) => + new Promise((_resolve, reject) => { + signal = init?.signal ?? undefined; + signal?.addEventListener('abort', () => reject(new Error('aborted'))); + started(); + }), + }); + + const chunks: TtsChunk[] = []; + const run = (async () => { + for await (const chunk of provider.speak('First. Second.', { utteranceId: 'u1' })) { + chunks.push(chunk); + } + })(); + + await requestStarted; + provider.cancel(); + await run; + + expect(signal?.aborted).toBe(true); + // The run ends quietly: a barge-in is not a failure to report. + expect(chunks).toHaveLength(0); + }); + + it('drops the rest of a run that was cancelled between sentences', async () => { + const provider = new ElevenLabsTtsProvider({ + readCredential: KEY, + fetchImpl: async () => pcmResponse(), + }); + + const chunks = []; + for await (const chunk of provider.speak('First. Second. Third.', { utteranceId: 'u1' })) { + chunks.push(chunk); + provider.cancel(); + } + + expect(chunks).toHaveLength(1); + }); + + it('clamps the speed to the range the service accepts', async () => { + let body: Record = {}; + const provider = new ElevenLabsTtsProvider({ + readCredential: KEY, + fetchImpl: async (_url, init) => { + body = JSON.parse(String(init?.body)) as Record; + return pcmResponse(); + }, + }); + + for await (const _chunk of provider.speak('Hello.', { utteranceId: 'u1', rate: 9 })) { + // drain + } + + expect((body.voice_settings as Record).speed).toBe(1.2); + }); + + it('classifies an auth failure rather than reporting a generic one', async () => { + const provider = new ElevenLabsTtsProvider({ + readCredential: KEY, + fetchImpl: async () => new Response('', { status: 401 }), + }); + + const iterate = async () => { + for await (const _chunk of provider.speak('Hello.', { utteranceId: 'u1' })) { + // drain + } + }; + + await expect(iterate()).rejects.toMatchObject({ kind: 'auth' }); + }); + + it('lists the voices on the account', async () => { + const provider = new ElevenLabsTtsProvider({ + readCredential: KEY, + fetchImpl: async () => + new Response( + JSON.stringify({ + voices: [{ voice_id: 'v1', name: 'Rachel', labels: { accent: 'us' } }], + }), + { status: 200 } + ), + }); + + expect(await provider.listVoices()).toEqual([ + { id: 'v1', name: 'Rachel', description: 'us', previewUrl: undefined }, + ]); + }); +}); + +// --------------------------------------------------------------------------- +// Hosted Brains +// --------------------------------------------------------------------------- + +describe('hosted Brain providers', () => { + const decision = { + target: { sessionId: 'agent-backend' }, + tabAction: 'new', + tabName: 'Auth', + prompt: 'refactor the auth module', + confidence: 0.9, + }; + + it('OpenAI routes through the shared parser', async () => { + const provider = new OpenAiBrainProvider({ + readCredential: KEY, + fetchImpl: async () => + new Response( + JSON.stringify({ choices: [{ message: { content: JSON.stringify(decision) } }] }), + { status: 200 } + ), + }); + + const result = await provider.route('ask backend to refactor auth', { + roster: ROSTER, + scope: { kind: 'conductor' }, + }); + + expect(result).toMatchObject({ target: { sessionId: 'agent-backend' }, tabAction: 'new' }); + }); + + it('Anthropic restores its JSON prefill before parsing', async () => { + const provider = new AnthropicBrainProvider({ + readCredential: () => 'sk-ant-abcdefghijklmnop', + fetchImpl: async () => + new Response( + // The prefill `{` is not echoed, so the body starts mid-object. + JSON.stringify({ + content: [{ type: 'text', text: JSON.stringify(decision).slice(1) }], + }), + { status: 200 } + ), + }); + + const result = await provider.route('ask backend to refactor auth', { + roster: ROSTER, + scope: { kind: 'conductor' }, + }); + + expect(result.target).toEqual({ sessionId: 'agent-backend' }); + }); + + it('sends the conductor an utterance naming an agent that is not running', async () => { + const provider = new OpenAiBrainProvider({ + readCredential: KEY, + fetchImpl: async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { + content: JSON.stringify({ ...decision, target: { sessionId: 'ghost' } }), + }, + }, + ], + }), + { status: 200 } + ), + }); + + const result = await provider.route('ask ghost something', { + roster: ROSTER, + scope: { kind: 'conductor' }, + }); + + // A hallucinated id must never be dispatched to. The conductor takes it. + expect(result.target).toBe('conductor'); + }); + + it('trims a spoken rewrite to the sentence budget', async () => { + const provider = new OpenAiBrainProvider({ + readCredential: KEY, + fetchImpl: async () => + new Response( + JSON.stringify({ + choices: [{ message: { content: 'One. Two. Three. Four.' } }], + }), + { status: 200 } + ), + }); + + const spoken = await provider.converse('...', { + agentSessionId: 'agent-backend', + tabId: 'tab-auth', + maxSentences: 2, + }); + + expect(spoken).toBe('One. Two.'); + }); + + it('refuses without a key before any request is made', async () => { + const fetchImpl = vi.fn(); + const provider = new OpenAiBrainProvider({ + readCredential: () => null, + fetchImpl: fetchImpl as never, + }); + + await expect( + provider.route('anything', { roster: ROSTER, scope: { kind: 'conductor' } }) + ).rejects.toMatchObject({ kind: 'unavailable' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/main/acappella/providers/local-providers.test.ts b/src/__tests__/main/acappella/providers/local-providers.test.ts new file mode 100644 index 0000000000..1c8d7b1274 --- /dev/null +++ b/src/__tests__/main/acappella/providers/local-providers.test.ts @@ -0,0 +1,498 @@ +/** + * @file local-providers.test.ts + * + * The local tier against mocked native runtimes. Nothing here loads a model or a + * `.node` binary: the runtime loader is the seam, which is the whole reason it + * exists. + * + * What is worth testing about a local provider is not the inference - that + * belongs to whisper.cpp and llama.cpp - but everything around it, which is where + * this codebase's bugs would be: chunked partials that stop when the session + * does, a cancel that actually cuts a run, a model that unloads when idle and + * loads again on the next turn, and a load failure that reports itself as its own + * provider rather than reaching for another one. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('electron', () => ({ app: { getPath: () => '/tmp/maestro-test' } })); +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { WhisperSttProvider } from '../../../../main/acappella/providers/local/whisper-stt'; +import { + KokoroTtsProvider, + styleVectorFor, +} from '../../../../main/acappella/providers/local/kokoro-tts'; +import { LlamaBrainProvider } from '../../../../main/acappella/providers/local/llama-brain'; +import { VoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import type { SttCallbacks, TtsChunk } from '../../../../shared/acappella/providers'; +import type { RosterAgent } from '../../../../shared/acappella/protocol'; + +const ROSTER: RosterAgent[] = [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + tabs: [{ id: 'tab-auth', name: 'Auth', lastActiveAt: 1 }], + }, +]; + +/** A runtime loader that always fails, as an uninstalled runtime would. */ +const failingRuntime = async (_id: string, providerId: string) => { + throw new VoiceProviderError('llama.cpp is not part of this build yet.', { + kind: 'unavailable', + providerId, + }); +}; + +function recorder(): { + callbacks: SttCallbacks; + partials: string[]; + finals: string[]; + errors: Error[]; +} { + const partials: string[] = []; + const finals: string[] = []; + const errors: Error[] = []; + return { + partials, + finals, + errors, + callbacks: { + onPartial: (text) => partials.push(text), + onFinal: (text) => finals.push(text), + onError: (error) => errors.push(error), + }, + }; +} + +/** 20 ms of 16 kHz audio. */ +function frame(): Int16Array { + return new Int16Array(320); +} + +// --------------------------------------------------------------------------- +// Whisper +// --------------------------------------------------------------------------- + +describe('WhisperSttProvider', () => { + function whisperRuntime(segments: string[] = ['open the auth tab']) { + const free = vi.fn(); + const transcribe = vi.fn(async () => ({ + result: Promise.resolve(segments.map((text) => ({ text }))), + })); + const module = { + Whisper: class { + transcribe = transcribe; + free = free; + }, + }; + return { module, transcribe, free, loadRuntime: async () => module as never }; + } + + it('reports a runtime that will not load as its own failure, never another provider', async () => { + const provider = new WhisperSttProvider({ loadRuntime: failingRuntime as never }); + + await expect(provider.start(recorder().callbacks)).rejects.toMatchObject({ + kind: 'unavailable', + providerId: 'whisper-local', + }); + }); + + it('emits a partial once enough audio has accumulated', async () => { + const runtime = whisperRuntime(['open the']); + const provider = new WhisperSttProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/model.bin', + partialIntervalMs: 40, + }); + const { callbacks, partials } = recorder(); + + await provider.start(callbacks); + // Two frames is 40 ms, which is the configured interval. + provider.feed(frame()); + provider.feed(frame()); + await vi.waitFor(() => expect(partials).toEqual(['open the'])); + }); + + it('publishes the final on endpointing and clears the buffer', async () => { + const runtime = whisperRuntime(['open the auth tab']); + const provider = new WhisperSttProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/model.bin', + // No partials, so exactly one decode happens and it is the final. + partialIntervalMs: 0, + }); + const { callbacks, finals } = recorder(); + + await provider.start(callbacks); + provider.feed(frame()); + await provider.flush(); + + expect(finals).toEqual(['open the auth tab']); + expect(runtime.transcribe).toHaveBeenCalledTimes(1); + + // A second flush with nothing buffered must not decode silence. + await provider.flush(); + expect(runtime.transcribe).toHaveBeenCalledTimes(1); + }); + + it('drops a decode that lands after the session ended', async () => { + const runtime = whisperRuntime(['too late']); + const provider = new WhisperSttProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/model.bin', + partialIntervalMs: 0, + }); + const { callbacks, finals } = recorder(); + + await provider.start(callbacks); + provider.feed(frame()); + const pending = provider.flush(); + await provider.stop(); + await pending; + + // The transcript belongs to a session that is over; publishing it would put + // an old utterance on the next turn. + expect(finals).toEqual([]); + }); + + it('frees the model on stop', async () => { + const runtime = whisperRuntime(); + const provider = new WhisperSttProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/model.bin', + }); + + await provider.start(recorder().callbacks); + await provider.stop(); + + expect(runtime.free).toHaveBeenCalled(); + }); + + it('reports a decode failure through onError rather than throwing at the frame path', async () => { + const module = { + Whisper: class { + transcribe = async () => { + throw new Error('ggml assert'); + }; + free = vi.fn(); + }, + }; + const provider = new WhisperSttProvider({ + loadRuntime: async () => module as never, + modelPath: '/tmp/model.bin', + partialIntervalMs: 0, + }); + const { callbacks, errors } = recorder(); + + await provider.start(callbacks); + provider.feed(frame()); + await provider.flush(); + + expect(errors[0].message).toContain('ggml assert'); + }); + + it('takes typed text without decoding anything', async () => { + const runtime = whisperRuntime(); + const provider = new WhisperSttProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/model.bin', + }); + const { callbacks, finals } = recorder(); + + await provider.start(callbacks); + provider.injectUtterance('typed instead of spoken'); + + expect(finals).toEqual(['typed instead of spoken']); + expect(runtime.transcribe).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// Kokoro +// --------------------------------------------------------------------------- + +describe('KokoroTtsProvider', () => { + function onnxRuntime() { + const run = vi.fn(async () => ({ waveform: { data: new Float32Array(240), dims: [1, 240] } })); + const release = vi.fn(); + const module = { + InferenceSession: { create: async () => ({ run, release }) }, + Tensor: class { + constructor( + readonly type: string, + readonly data: unknown, + readonly dims: readonly number[] + ) {} + }, + }; + return { module, run, release, loadRuntime: async () => module as never }; + } + + const phonemize = (text: string) => text.split('').map((_char, index) => index + 1); + const readVoicePack = async () => new Float32Array(256 * 64); + + async function collect(iterable: AsyncIterable): Promise { + const chunks: TtsChunk[] = []; + for await (const chunk of iterable) chunks.push(chunk); + return chunks; + } + + it('refuses, by name, when there is no phoneme front end', async () => { + const runtime = onnxRuntime(); + const provider = new KokoroTtsProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/kokoro.onnx', + readVoicePack, + }); + + // It does NOT approximate. A character-level fallback would synthesise + // confident nonsense, which is worse than silence with an explanation. + await expect(collect(provider.speak('Hello.', { utteranceId: 'u1' }))).rejects.toMatchObject({ + kind: 'unavailable', + }); + }); + + it('synthesises one chunk per sentence at the model rate', async () => { + const runtime = onnxRuntime(); + const provider = new KokoroTtsProvider({ + phonemize, + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/kokoro.onnx', + voicePackPath: '/tmp/voice.bin', + readVoicePack, + }); + + const chunks = await collect(provider.speak('First one. Second one.', { utteranceId: 'u1' })); + + expect(chunks.map((chunk) => chunk.text)).toEqual(['First one.', 'Second one.']); + expect(chunks[0].sampleRate).toBe(24_000); + expect(runtime.run).toHaveBeenCalledTimes(2); + }); + + it('cancels a run without delivering the sentence it interrupted', async () => { + const runtime = onnxRuntime(); + const provider = new KokoroTtsProvider({ + phonemize, + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/kokoro.onnx', + voicePackPath: '/tmp/voice.bin', + readVoicePack, + }); + + const chunks: TtsChunk[] = []; + for await (const chunk of provider.speak('First. Second. Third.', { utteranceId: 'u1' })) { + chunks.push(chunk); + provider.cancel(); + } + + expect(chunks).toHaveLength(1); + }); + + it('clamps the speed to the range the model can render', async () => { + const runtime = onnxRuntime(); + const provider = new KokoroTtsProvider({ + phonemize, + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/kokoro.onnx', + voicePackPath: '/tmp/voice.bin', + readVoicePack, + }); + + await collect(provider.speak('Hello.', { utteranceId: 'u1', rate: 9 })); + + const feeds = ( + runtime.run.mock.calls[0] as unknown as [Record] + )[0]; + expect(feeds.speed.data[0]).toBe(2); + }); + + it('releases the session when disposed', async () => { + const runtime = onnxRuntime(); + const provider = new KokoroTtsProvider({ + phonemize, + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/kokoro.onnx', + voicePackPath: '/tmp/voice.bin', + readVoicePack, + }); + + await collect(provider.speak('Hello.', { utteranceId: 'u1' })); + await provider.dispose(); + + expect(runtime.release).toHaveBeenCalled(); + }); + + it('clamps a style row rather than reading past the end of the pack', () => { + const pack = new Float32Array(256 * 4).fill(1); + // A sentence longer than the pack has rows for: reading past the end would + // hand the model whatever followed it in memory, which comes out as a burst + // of noise at full volume. + expect(styleVectorFor(pack, 10_000)).toHaveLength(256); + expect(styleVectorFor(pack, 0)).toHaveLength(256); + }); +}); + +// --------------------------------------------------------------------------- +// Qwen3 via llama.cpp +// --------------------------------------------------------------------------- + +describe('LlamaBrainProvider', () => { + function llamaRuntime(reply: string) { + const prompt = vi.fn(async () => reply); + const contextDispose = vi.fn(); + const modelDispose = vi.fn(); + const module = { + getLlama: async () => ({ + loadModel: async () => ({ + createContext: async () => ({ + getSequence: () => ({}), + dispose: contextDispose, + }), + dispose: modelDispose, + }), + createGrammarForJsonSchema: async () => ({}), + }), + LlamaChatSession: class { + prompt = prompt; + }, + }; + return { + module, + prompt, + contextDispose, + modelDispose, + loadRuntime: async () => module as never, + }; + } + + const decision = JSON.stringify({ + target: { sessionId: 'agent-backend' }, + tabAction: 'new', + tabName: 'Auth', + prompt: 'refactor auth', + confidence: 0.9, + }); + + it('reports a runtime that will not load as its own failure', async () => { + const provider = new LlamaBrainProvider({ loadRuntime: failingRuntime as never }); + + await expect( + provider.route('anything', { roster: ROSTER, scope: { kind: 'conductor' } }) + ).rejects.toMatchObject({ kind: 'unavailable', providerId: 'qwen3-local' }); + }); + + it('routes through the shared parser', async () => { + const runtime = llamaRuntime(decision); + const provider = new LlamaBrainProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/qwen.gguf', + idleUnloadMs: 0, + }); + + const result = await provider.route('ask backend to refactor auth', { + roster: ROSTER, + scope: { kind: 'conductor' }, + }); + + expect(result).toMatchObject({ target: { sessionId: 'agent-backend' }, tabAction: 'new' }); + }); + + it('keeps the context loaded across turns', async () => { + const runtime = llamaRuntime(decision); + const provider = new LlamaBrainProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/qwen.gguf', + // Never unload, so this test is about reuse rather than about the timer. + idleUnloadMs: 0, + }); + const context = { roster: ROSTER, scope: { kind: 'conductor' as const } }; + + await provider.route('one', context); + await provider.route('two', context); + + expect(provider.isLoaded).toBe(true); + expect(runtime.prompt).toHaveBeenCalledTimes(2); + // One load, two turns: per-turn latency is inference only. + expect(runtime.modelDispose).not.toHaveBeenCalled(); + }); + + it('unloads after the idle window and reloads on the next turn', async () => { + vi.useFakeTimers(); + try { + const runtime = llamaRuntime(decision); + const provider = new LlamaBrainProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/qwen.gguf', + idleUnloadMs: 1_000, + }); + const context = { roster: ROSTER, scope: { kind: 'conductor' as const } }; + + await provider.route('one', context); + expect(provider.isLoaded).toBe(true); + + await vi.advanceTimersByTimeAsync(1_100); + // A gigabyte of resident memory for a conversation that ended is not + // acceptable, however warm it would have been. + expect(provider.isLoaded).toBe(false); + expect(runtime.contextDispose).toHaveBeenCalled(); + expect(runtime.modelDispose).toHaveBeenCalled(); + + await provider.route('two', context); + expect(provider.isLoaded).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + it('trims a spoken rewrite to the sentence budget', async () => { + const runtime = llamaRuntime('One. Two. Three.'); + const provider = new LlamaBrainProvider({ + loadRuntime: runtime.loadRuntime, + modelPath: '/tmp/qwen.gguf', + idleUnloadMs: 0, + }); + + const spoken = await provider.converse('...', { + agentSessionId: 'agent-backend', + tabId: 'tab-auth', + maxSentences: 2, + }); + + expect(spoken).toBe('One. Two.'); + }); + + it('reports an inference failure as a classified provider error', async () => { + const module = { + getLlama: async () => ({ + loadModel: async () => ({ + createContext: async () => ({ getSequence: () => ({}), dispose: vi.fn() }), + dispose: vi.fn(), + }), + createGrammarForJsonSchema: async () => ({}), + }), + LlamaChatSession: class { + prompt = async () => { + throw new Error('context is full'); + }; + }, + }; + const provider = new LlamaBrainProvider({ + loadRuntime: async () => module as never, + modelPath: '/tmp/qwen.gguf', + idleUnloadMs: 0, + }); + + await expect( + provider.route('anything', { roster: ROSTER, scope: { kind: 'conductor' } }) + ).rejects.toMatchObject({ kind: 'unavailable' }); + }); +}); + +beforeEach(() => { + vi.clearAllMocks(); +}); diff --git a/src/__tests__/main/acappella/providers/provider-registry.test.ts b/src/__tests__/main/acappella/providers/provider-registry.test.ts new file mode 100644 index 0000000000..7c7bb1abd8 --- /dev/null +++ b/src/__tests__/main/acappella/providers/provider-registry.test.ts @@ -0,0 +1,540 @@ +/** + * @file provider-registry.test.ts + * + * The rules the registry exists to enforce, one test each: + * + * - Every slot combination resolves to what was ASKED for, independently. + * - A missing local provider NEVER resolves to a hosted one. This is the test + * that would fail if someone added a "helpful" fallback, and it is checked + * exhaustively across every combination rather than on one example, because a + * fallback added for one role would be trivially easy to miss on another. + * - The mock tier is selected explicitly and is never substituted in. + * - A hot swap tears the old pipeline down, and is refused mid-utterance. + * + * The concrete providers are stubbed at the module boundary. Constructing a real + * one would try to open a keychain and a native runtime, and none of that is what + * these tests are about. + */ + +import { describe, it, expect, vi, beforeAll, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +// Every real backend is replaced with an inert stand-in: these tests are about +// which one is chosen, and a real one would reach for a keychain entry, a model +// file, or a native addon at construction. +vi.mock('../../../../main/acappella/providers/local/whisper-stt', () => ({ + WhisperSttProvider: class { + readonly id = 'whisper-local'; + readonly label = 'Whisper (local)'; + readonly tier = 'local'; + readonly sampleRate = 16_000; + readonly acceptsAudio = true; + async start() {} + feed() {} + async flush() {} + async stop() {} + }, +})); +vi.mock('../../../../main/acappella/providers/local/kokoro-tts', () => ({ + KokoroTtsProvider: class { + readonly id = 'kokoro-local'; + readonly label = 'Kokoro (local)'; + readonly tier = 'local'; + speak() { + return (async function* () {})(); + } + cancel() {} + }, +})); +vi.mock('../../../../main/acappella/providers/local/llama-brain', () => ({ + LlamaBrainProvider: class { + readonly id = 'qwen3-local'; + readonly label = 'Qwen3 1.7B (local)'; + readonly tier = 'local'; + async route() { + return { target: 'conductor', tabAction: 'current', prompt: '', confidence: 1 }; + } + async converse() { + return ''; + } + }, +})); +vi.mock('../../../../main/acappella/providers/hosted/openai-stt', () => ({ + OpenAiSttProvider: class { + readonly id = 'openai-stt'; + readonly label = 'OpenAI (hosted)'; + readonly tier = 'cloud'; + readonly sampleRate = 16_000; + readonly acceptsAudio = true; + async start() {} + feed() {} + async flush() {} + async stop() {} + }, +})); +vi.mock('../../../../main/acappella/providers/hosted/elevenlabs-tts', () => ({ + ElevenLabsTtsProvider: class { + readonly id = 'elevenlabs-tts'; + readonly label = 'ElevenLabs (hosted)'; + readonly tier = 'cloud'; + speak() { + return (async function* () {})(); + } + cancel() {} + }, +})); +vi.mock('../../../../main/acappella/providers/hosted/openai-brain', () => ({ + OpenAiBrainProvider: class { + readonly id = 'openai-brain'; + readonly label = 'OpenAI (hosted)'; + readonly tier = 'cloud'; + async route() { + return { target: 'conductor', tabAction: 'current', prompt: '', confidence: 1 }; + } + async converse() { + return ''; + } + }, +})); +vi.mock('../../../../main/acappella/providers/hosted/anthropic-brain', () => ({ + AnthropicBrainProvider: class { + readonly id = 'anthropic-brain'; + readonly label = 'Anthropic (hosted)'; + readonly tier = 'cloud'; + async route() { + return { target: 'conductor', tabAction: 'current', prompt: '', confidence: 1 }; + } + async converse() { + return ''; + } + }, +})); + +const realtimeDispose = vi.fn(async () => {}); +vi.mock('../../../../main/acappella/providers/realtime/realtime-session', () => { + const adapter = { + id: 'openai-realtime', + label: 'OpenAI Realtime', + tier: 'cloud', + sampleRate: 16_000, + acceptsAudio: true, + }; + return { + createRealtimePipeline: () => ({ + shape: 'realtime', + providers: { stt: adapter, tts: adapter, brain: adapter }, + dispose: realtimeDispose, + }), + }; +}); + +import { logger } from '../../../../main/utils/logger'; +import { + MockBrainProvider, + MockSttProvider, + MockTtsProvider, +} from '../../../../main/acappella/providers/mock'; +import { ECHO_STT_PROVIDER_ID } from '../../../../main/acappella/providers/echo-stt'; +import { + DEFAULT_PROVIDER_IDS, + MOCK_PROVIDER_IDS, + buildProviderState, + listVoiceProviders, + pipelineKey, + readVoiceProviderSettings, + registerVoiceProvider, + resolveVoicePipeline, + resolveVoiceProviders, + swapVoicePipeline, +} from '../../../../main/acappella/providers/provider-registry'; +import { unresolvedProviderId } from '../../../../main/acappella/providers/unresolved'; +import type { VoiceProviderRole } from '../../../../shared/acappella/providers'; +import { + DEFAULT_SEND_HOLD_MS, + DEFAULT_TURN_SETTLE_MS, +} from '../../../../shared/acappella/voice-controls'; + +describe('provider registry', () => { + beforeAll(() => { + registerVoiceProvider({ + role: 'tts', + id: 'test-local-tts', + label: 'Test Local TTS', + tier: 'local', + create: () => new MockTtsProvider({ msPerCharacter: 0 }), + }); + registerVoiceProvider({ + role: 'stt', + id: 'test-cloud-stt', + label: 'Test Cloud STT', + tier: 'cloud', + create: () => new MockSttProvider({ partialDelayMs: 0 }), + }); + registerVoiceProvider({ + role: 'brain', + id: 'test-unavailable-brain', + label: 'Test Unavailable Brain', + tier: 'local', + isAvailable: () => false, + create: () => new MockBrainProvider(), + }); + }); + + beforeEach(() => { + vi.mocked(logger.warn).mockClear(); + realtimeDispose.mockClear(); + }); + + // -- Resolution ---------------------------------------------------------- + + it('defaults to a trio that can hear, and says nothing about it', () => { + const { providers, substitutions, resolvedIds } = resolveVoiceProviders(); + + // STT is the exception to the mock default, and the important one: an + // unconfigured install has to be able to establish that its microphone + // reaches the app at all. + expect(resolvedIds.stt).toBe(ECHO_STT_PROVIDER_ID); + expect(providers.stt.acceptsAudio).toBe(true); + expect(resolvedIds.tts).toBe(MOCK_PROVIDER_IDS.tts); + expect(resolvedIds.brain).toBe(MOCK_PROVIDER_IDS.brain); + // The default path is documented behaviour, not something to warn about. + expect(substitutions).toEqual([]); + }); + + it('hands out a fresh trio each time', () => { + expect(resolveVoiceProviders().providers.stt).not.toBe(resolveVoiceProviders().providers.stt); + }); + + it('resolves every slot combination to exactly what was asked for', () => { + const choices: Record = { + stt: ['mock-stt', 'whisper-local', 'openai-stt'], + tts: ['mock-tts', 'kokoro-local', 'elevenlabs-tts'], + brain: ['mock-brain', 'qwen3-local', 'openai-brain', 'anthropic-brain'], + }; + + for (const stt of choices.stt) { + for (const tts of choices.tts) { + for (const brain of choices.brain) { + const { resolvedIds, substitutions } = resolveVoicePipeline({ + settings: { stt, tts, brain }, + }); + expect(resolvedIds).toEqual({ stt, tts, brain }); + expect(substitutions).toEqual([]); + } + } + } + }); + + it('never resolves a missing local provider to a hosted one', () => { + // Hosted providers for every role are registered and perfectly usable. None + // of them may be chosen for a local id that does not exist. + const missing: Record = { + stt: 'whisper-that-is-not-registered', + tts: 'kokoro-that-is-not-registered', + brain: 'qwen-that-is-not-registered', + }; + + for (const role of ['stt', 'tts', 'brain'] as VoiceProviderRole[]) { + const { providers, resolvedIds, substitutions } = resolveVoicePipeline({ + settings: { [role]: missing[role] }, + }); + + expect(resolvedIds[role]).toBe(unresolvedProviderId(role)); + expect(providers[role].tier).not.toBe('cloud'); + expect(substitutions).toEqual([ + expect.objectContaining({ role, requestedId: missing[role], reason: 'unknown-provider' }), + ]); + expect(logger.warn).toHaveBeenCalled(); + } + }); + + it('refuses by name rather than falling back to the mock', async () => { + const { providers } = resolveVoicePipeline({ settings: { stt: 'not-a-provider' } }); + + // The distinction that matters: a mock STT would "work" and transcribe + // nothing, which is indistinguishable from a broken feature. + expect(providers.stt.tier).not.toBe('mock'); + await expect( + providers.stt.start({ onPartial: () => {}, onFinal: () => {}, onError: () => {} }) + ).rejects.toThrow(/not-a-provider/); + }); + + it('marks a registered but unrunnable provider unavailable, not unknown', () => { + const { substitutions } = resolveVoicePipeline({ + settings: { brain: 'test-unavailable-brain' }, + }); + + expect(substitutions[0].reason).toBe('unavailable'); + }); + + it('substitutes only the broken role and leaves the others alone', () => { + const { resolvedIds, substitutions } = resolveVoicePipeline({ + settings: { stt: 'test-missing', tts: 'test-local-tts' }, + }); + + expect(resolvedIds.stt).toBe(unresolvedProviderId('stt')); + expect(resolvedIds.tts).toBe('test-local-tts'); + expect(resolvedIds.brain).toBe(MOCK_PROVIDER_IDS.brain); + expect(substitutions).toHaveLength(1); + }); + + it.each(['development', 'production'])( + 'defaults STT to a provider that consumes audio in a %s build', + (env) => { + // The microphone check used to be development-only, which left a packaged + // app with NO provider that opens a capture device: the session reported + // "Listening" and the microphone was never touched. The build must not + // decide whether the user can hear themselves. + const previous = process.env.NODE_ENV; + process.env.NODE_ENV = env; + try { + const { resolvedIds, providers, substitutions } = resolveVoicePipeline(); + + expect(resolvedIds.stt).toBe(ECHO_STT_PROVIDER_ID); + expect(providers.stt.acceptsAudio).toBe(true); + // Nobody asked for it, so it is a default rather than a substitution. + expect(substitutions).toEqual([]); + expect(DEFAULT_PROVIDER_IDS.stt).toBe(ECHO_STT_PROVIDER_ID); + } finally { + process.env.NODE_ENV = previous; + } + } + ); + + it('reports a DEFAULT that fell back to the mock instead of falling back silently', () => { + // The silence was the bug. A slot that lands on a text-in recogniser nobody + // chose is the one fact that explains a session which cannot hear, and it + // used to exist only inside the resolver. + const previous = process.env.NODE_ENV; + process.env.NODE_ENV = 'production'; + registerVoiceProvider({ + role: 'tts', + id: 'test-unavailable-default-tts', + label: 'Unavailable', + tier: 'local', + isAvailable: () => false, + create: () => ({}) as never, + }); + const restoreDefault = DEFAULT_PROVIDER_IDS.tts; + DEFAULT_PROVIDER_IDS.tts = 'test-unavailable-default-tts'; + try { + const { resolvedIds, substitutions } = resolveVoicePipeline(); + + expect(resolvedIds.tts).toBe(MOCK_PROVIDER_IDS.tts); + expect(substitutions).toEqual([ + expect.objectContaining({ + role: 'tts', + requestedId: 'test-unavailable-default-tts', + resolvedId: MOCK_PROVIDER_IDS.tts, + reason: 'unavailable', + }), + ]); + } finally { + DEFAULT_PROVIDER_IDS.tts = restoreDefault; + process.env.NODE_ENV = previous; + } + }); + + it('lists the mock tier as selectable and available', () => { + expect(listVoiceProviders('stt')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: MOCK_PROVIDER_IDS.stt, tier: 'mock', available: true }), + ]) + ); + }); + + // -- Pipeline shape ------------------------------------------------------ + + it('builds a cascade pipeline by default', () => { + const { shape, pipeline } = resolveVoicePipeline(); + expect(shape).toBe('cascade'); + expect(pipeline.shape).toBe('cascade'); + }); + + it('builds a realtime pipeline with one adapter in all three slots', () => { + const { shape, providers } = resolveVoicePipeline({ settings: { pipeline: 'realtime' } }); + + expect(shape).toBe('realtime'); + expect(providers.stt).toBe(providers.tts); + expect(providers.tts).toBe(providers.brain); + }); + + it('refuses an unknown realtime provider rather than using the one that exists', () => { + const { shape, resolvedIds, substitutions } = resolveVoicePipeline({ + settings: { pipeline: 'realtime', realtime: 'some-other-realtime' }, + }); + + expect(shape).toBe('cascade'); + expect(resolvedIds.stt).toBe(unresolvedProviderId('stt')); + expect(substitutions).toHaveLength(3); + }); + + // -- Provider state ------------------------------------------------------ + + it('describes the live engines and where audio goes', () => { + const resolution = resolveVoicePipeline({ + settings: { stt: 'whisper-local', tts: 'kokoro-local', brain: 'qwen3-local' }, + }); + + const state = buildProviderState(resolution); + expect(state.pipeline).toBe('cascade'); + expect(state.audioLeavesMachine).toBe(false); + expect(state.egressStatement).toBe('Audio stays on this machine.'); + expect(state.slots.map((slot) => slot.providerId)).toEqual([ + 'whisper-local', + 'kokoro-local', + 'qwen3-local', + ]); + }); + + it('says where audio goes when the recogniser is hosted', () => { + const state = buildProviderState( + resolveVoicePipeline({ settings: { stt: 'openai-stt', tts: 'kokoro-local' } }) + ); + + expect(state.audioLeavesMachine).toBe(true); + expect(state.egressStatement).toBe('Audio is sent to OpenAI.'); + }); + + it('reports a substituted slot rather than presenting it as configured', () => { + const state = buildProviderState(resolveVoicePipeline({ settings: { tts: 'gone' } })); + const tts = state.slots.find((slot) => slot.role === 'tts')!; + + expect(tts.substitutedFor).toBe('gone'); + // A slot that resolved to nothing sends nothing anywhere, whatever it was + // configured with. + expect(state.audioLeavesMachine).toBe(false); + }); + + // -- Hot swap ------------------------------------------------------------ + + it('does nothing when the selection has not changed', async () => { + const settings = { stt: 'whisper-local' }; + const current = resolveVoicePipeline({ settings }); + const dispose = vi.spyOn(current.pipeline, 'dispose'); + + const result = await swapVoicePipeline({ + settings, + current: { pipeline: current.pipeline, key: pipelineKey(settings) }, + isBusy: false, + }); + + expect(result.status).toBe('unchanged'); + expect(dispose).not.toHaveBeenCalled(); + }); + + it('tears the old pipeline down before building the new one', async () => { + const current = resolveVoicePipeline({ settings: { stt: 'whisper-local' } }); + const dispose = vi.spyOn(current.pipeline, 'dispose'); + + const result = await swapVoicePipeline({ + settings: { stt: 'openai-stt' }, + current: { pipeline: current.pipeline, key: pipelineKey({ stt: 'whisper-local' }) }, + isBusy: false, + }); + + expect(dispose).toHaveBeenCalledTimes(1); + expect(result.status).toBe('swapped'); + expect(result.resolution?.resolvedIds.stt).toBe('openai-stt'); + }); + + it('refuses a swap mid-utterance and leaves the live pipeline alone', async () => { + const current = resolveVoicePipeline({ settings: { stt: 'whisper-local' } }); + const dispose = vi.spyOn(current.pipeline, 'dispose'); + + const result = await swapVoicePipeline({ + settings: { stt: 'openai-stt' }, + current: { pipeline: current.pipeline, key: pipelineKey({ stt: 'whisper-local' }) }, + isBusy: true, + }); + + expect(result.status).toBe('refused'); + expect(result.reason).toMatch(/middle of a turn/i); + // The important half: nothing was torn down, so the turn in flight still has + // the engines it started with. + expect(dispose).not.toHaveBeenCalled(); + expect(result.resolution).toBeUndefined(); + }); + + it('disposes a realtime pipeline on swap, closing its socket', async () => { + const current = resolveVoicePipeline({ settings: { pipeline: 'realtime' } }); + + await swapVoicePipeline({ + settings: { pipeline: 'cascade' }, + current: { pipeline: current.pipeline, key: pipelineKey({ pipeline: 'realtime' }) }, + isBusy: false, + }); + + expect(realtimeDispose).toHaveBeenCalledTimes(1); + }); + + // -- Settings ------------------------------------------------------------ + + it('reads provider ids out of settings and ignores malformed values', () => { + const store = { + get: () => ({ + providers: { stt: ' whisper-local ', tts: 42, brain: ' ' }, + pipeline: 'realtime', + voice: { voiceId: 'af_heart', rate: 1.1 }, + }), + }; + + expect(readVoiceProviderSettings(store)).toEqual({ + stt: 'whisper-local', + tts: undefined, + brain: undefined, + pipeline: 'realtime', + realtime: undefined, + voiceId: 'af_heart', + rate: 1.1, + // Clamped rather than passed through: this becomes a gain on a live + // output node, so an absent or nonsensical value reads as full volume. + volume: 1, + // Same reasoning, different consequence: this one becomes a timer in + // front of every dispatch, so an absent value reads as the default wait + // rather than as zero. + turnSettleMs: DEFAULT_TURN_SETTLE_MS, + // Off unless explicitly stored: conversational mode changes what a + // spoken sentence means, so it is never on by inference. + conversationalMode: false, + holdUntilSend: false, + sendHoldMs: DEFAULT_SEND_HOLD_MS, + // Absent takes the built-in phrases; a stored empty string is a + // deliberate "no spoken send", so the two stay distinguishable. + sendPhrases: undefined, + }); + }); + + it('treats a missing settings key as unset and the shape as cascade', () => { + const store = { get: (_key: string, defaultValue: unknown) => defaultValue }; + + expect(readVoiceProviderSettings(store)).toEqual({ + stt: undefined, + tts: undefined, + brain: undefined, + pipeline: 'cascade', + realtime: undefined, + voiceId: undefined, + rate: undefined, + volume: 1, + turnSettleMs: DEFAULT_TURN_SETTLE_MS, + conversationalMode: false, + holdUntilSend: false, + sendHoldMs: DEFAULT_SEND_HOLD_MS, + // Absent takes the built-in phrases; a stored empty string is a + // deliberate "no spoken send", so the two stay distinguishable. + sendPhrases: undefined, + }); + }); + + it('keys a selection so an unchanged one is recognised', () => { + expect(pipelineKey({ stt: 'a', tts: 'b', brain: 'c' })).toBe( + pipelineKey({ stt: 'a', tts: 'b', brain: 'c' }) + ); + expect(pipelineKey({ stt: 'a' })).not.toBe(pipelineKey({ stt: 'b' })); + expect(pipelineKey({ voiceId: 'x' })).not.toBe(pipelineKey({ voiceId: 'y' })); + }); +}); diff --git a/src/__tests__/main/acappella/providers/realtime-session.test.ts b/src/__tests__/main/acappella/providers/realtime-session.test.ts new file mode 100644 index 0000000000..e80f44bf18 --- /dev/null +++ b/src/__tests__/main/acappella/providers/realtime-session.test.ts @@ -0,0 +1,315 @@ +/** + * @file realtime-session.test.ts + * + * The two properties that make the realtime tier safe to swap in: + * + * 1. **Routing stays Maestro's.** The model asks for an agent and a tab through + * a tool call; Maestro validates it against the live roster and performs it. + * A model that names an agent nobody is running does NOT get to dispatch + * there, exactly as in the cascade. + * 2. **Interruption propagates.** Barge-in has to reach the SERVER, cancelling + * generation and dropping audio already queued there. A cancel that only + * stopped the local iterator would leave the assistant talking over the + * person who interrupted it. + * + * The socket is injected, so the whole protocol is exercised with no network and + * no API key. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { + RealtimePipeline, + RealtimeVoiceAdapter, + ROUTE_TOOL_NAME, + type RealtimeSocket, +} from '../../../../main/acappella/providers/realtime/realtime-session'; +import type { RosterAgent } from '../../../../shared/acappella/protocol'; +import type { SttCallbacks, TtsChunk } from '../../../../shared/acappella/providers'; + +const ROSTER: RosterAgent[] = [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + tabs: [{ id: 'tab-auth', name: 'Auth', lastActiveAt: 1 }], + }, +]; + +/** An in-memory socket that records what was sent and can push events back. */ +class FakeSocket implements RealtimeSocket { + readonly sent: Array> = []; + closed = false; + private handlers: Record void>> = {}; + + send(data: string): void { + this.sent.push(JSON.parse(data) as Record); + } + + close(): void { + this.closed = true; + this.emit('close'); + } + + on(event: string, handler: (...args: never[]) => void): void { + (this.handlers[event] ??= []).push(handler); + } + + emit(event: string, ...args: unknown[]): void { + for (const handler of this.handlers[event] ?? []) { + (handler as (...inner: unknown[]) => void)(...args); + } + } + + /** Push one server event, as the API would. */ + server(payload: Record): void { + this.emit('message', JSON.stringify(payload)); + } + + /** Everything sent of a given type. */ + sentOfType(type: string): Array> { + return this.sent.filter((message) => message.type === type); + } +} + +function noopCallbacks(): SttCallbacks { + return { onPartial: () => {}, onFinal: () => {}, onError: () => {} }; +} + +let socket: FakeSocket; + +/** Build an adapter over a socket that opens on the next tick. */ +async function startAdapter(callbacks: SttCallbacks = noopCallbacks()) { + socket = new FakeSocket(); + const adapter = new RealtimeVoiceAdapter({ + readCredential: () => 'sk-test-abcdefghijklmnop', + socketFactory: () => socket, + routeTimeoutMs: 50, + responseTimeoutMs: 50, + }); + + const started = adapter.start(callbacks); + socket.emit('open'); + await started; + return adapter; +} + +beforeEach(() => { + socket = new FakeSocket(); +}); + +describe('RealtimeVoiceAdapter', () => { + it('declares the routing tool on the session', async () => { + await startAdapter(); + + const update = socket.sentOfType('session.update')[0]; + const session = update.session as { tools: Array<{ name: string }> }; + expect(session.tools.map((tool) => tool.name)).toEqual([ROUTE_TOOL_NAME]); + }); + + it('refuses to connect without a key', async () => { + const adapter = new RealtimeVoiceAdapter({ + readCredential: () => null, + socketFactory: () => new FakeSocket(), + }); + + await expect(adapter.start(noopCallbacks())).rejects.toThrow(/API key/i); + }); + + it('turns the transcription events into partials and a final', async () => { + const partials: string[] = []; + const finals: string[] = []; + await startAdapter({ + onPartial: (text) => partials.push(text), + onFinal: (text) => finals.push(text), + onError: () => {}, + }); + + socket.server({ type: 'conversation.item.input_audio_transcription.delta', delta: 'open ' }); + socket.server({ + type: 'conversation.item.input_audio_transcription.completed', + transcript: 'open the auth tab', + }); + + expect(partials).toEqual(['open ']); + expect(finals).toEqual(['open the auth tab']); + }); + + it('executes a tool-call route decision through the shared parser', async () => { + const adapter = await startAdapter(); + + const routing = adapter.route('ask backend about auth', { + roster: ROSTER, + scope: { kind: 'conductor' }, + }); + + socket.server({ + type: 'response.function_call_arguments.done', + arguments: JSON.stringify({ + target: { sessionId: 'agent-backend' }, + tabAction: 'recall', + tabId: 'tab-auth', + prompt: 'what happened to auth', + confidence: 0.8, + }), + }); + + // The decision Maestro will EXECUTE, not something the model performed. + expect(await routing).toEqual({ + target: { sessionId: 'agent-backend' }, + tabAction: 'recall', + tabId: 'tab-auth', + tabName: undefined, + prompt: 'what happened to auth', + confidence: 0.8, + }); + }); + + it('sends a tool call naming an agent that is not running to the conductor', async () => { + const adapter = await startAdapter(); + + const routing = adapter.route('ask ghost something', { + roster: ROSTER, + scope: { kind: 'conductor' }, + }); + + socket.server({ + type: 'response.function_call_arguments.done', + arguments: JSON.stringify({ + target: { sessionId: 'ghost' }, + tabAction: 'current', + prompt: 'something', + confidence: 0.9, + }), + }); + + expect((await routing).target).toBe('conductor'); + }); + + it('hands the turn to the conductor when no tool call arrives', async () => { + const adapter = await startAdapter(); + + // No tool call at all: the turn must still resolve, or the session hangs + // with the user waiting on silence. + const decision = await adapter.route('do the thing', { + roster: ROSTER, + scope: { kind: 'conductor' }, + }); + + expect(decision.target).toBe('conductor'); + expect(decision.prompt).toBe('do the thing'); + }); + + it('cuts the spoken reply into sentences with their own audio', async () => { + const adapter = await startAdapter(); + + const conversing = adapter.converse('The migration finished and the tests pass.', { + agentSessionId: 'agent-backend', + tabId: 'tab-auth', + }); + + socket.server({ type: 'response.audio_transcript.delta', delta: 'Migration done. ' }); + socket.server({ type: 'response.audio.delta', delta: Buffer.from([1, 2]).toString('base64') }); + socket.server({ type: 'response.audio_transcript.delta', delta: 'Tests pass.' }); + socket.server({ type: 'response.audio.delta', delta: Buffer.from([3, 4]).toString('base64') }); + socket.server({ type: 'response.done' }); + + const spoken = await conversing; + expect(spoken).toBe('Migration done. Tests pass.'); + + const chunks: TtsChunk[] = []; + for await (const chunk of adapter.speak(spoken, { utteranceId: 'u1' })) chunks.push(chunk); + + expect(chunks.map((chunk) => chunk.text)).toEqual(['Migration done.', 'Tests pass.']); + // Audio, not a re-synthesis: the model already made it. + expect(chunks[0].format).toBe('pcm16'); + expect(chunks[0].audio).toBeInstanceOf(Uint8Array); + }); + + it('propagates an interruption to the server and drops queued audio', async () => { + const adapter = await startAdapter(); + + adapter.cancel(); + + // Both matter. `response.cancel` stops generation; without the buffer clear + // the server keeps streaming what it had already made and the assistant + // talks over the interruption. + expect(socket.sentOfType('response.cancel')).toHaveLength(1); + expect(socket.sentOfType('output_audio_buffer.clear')).toHaveLength(1); + }); + + it('stops delivering sentences once cancelled', async () => { + const adapter = await startAdapter(); + + const conversing = adapter.converse('Something happened.', { + agentSessionId: 'agent-backend', + tabId: 'tab-auth', + }); + socket.server({ type: 'response.audio_transcript.delta', delta: 'One. Two. Three.' }); + socket.server({ type: 'response.done' }); + const spoken = await conversing; + + const chunks: TtsChunk[] = []; + for await (const chunk of adapter.speak(spoken, { utteranceId: 'u1' })) { + chunks.push(chunk); + adapter.cancel(); + } + + expect(chunks).toHaveLength(1); + }); + + it('reports a server error as a classified network failure', async () => { + const errors: Error[] = []; + await startAdapter({ + onPartial: () => {}, + onFinal: () => {}, + onError: (error) => errors.push(error), + }); + + socket.server({ type: 'error', error: { message: 'session expired' } }); + + expect(errors[0].message).toContain('session expired'); + }); + + it('closes the socket on stop', async () => { + const adapter = await startAdapter(); + await adapter.stop(); + expect(socket.closed).toBe(true); + }); + + it('upsamples capture audio to the rate the API speaks', async () => { + const adapter = await startAdapter(); + + // 320 samples of 16 kHz is 20 ms; at 24 kHz that is 480 samples, so 960 + // bytes. Sending 16 kHz samples labelled as 24 kHz would make every voice + // sound like a chipmunk and every transcript wrong. + adapter.feed(new Int16Array(320)); + + const appended = socket.sentOfType('input_audio_buffer.append')[0]; + expect(Buffer.from(String(appended.audio), 'base64').byteLength).toBe(960); + }); +}); + +describe('RealtimePipeline', () => { + it('puts one adapter in all three slots', async () => { + const adapter = await startAdapter(); + const pipeline = new RealtimePipeline(adapter); + + expect(pipeline.shape).toBe('realtime'); + expect(pipeline.providers.stt).toBe(adapter); + expect(pipeline.providers.tts).toBe(adapter); + expect(pipeline.providers.brain).toBe(adapter); + }); + + it('closes the session when disposed', async () => { + const adapter = await startAdapter(); + await new RealtimePipeline(adapter).dispose(); + + expect(socket.closed).toBe(true); + }); +}); diff --git a/src/__tests__/main/acappella/remote-session.test.ts b/src/__tests__/main/acappella/remote-session.test.ts new file mode 100644 index 0000000000..94b4972669 --- /dev/null +++ b/src/__tests__/main/acappella/remote-session.test.ts @@ -0,0 +1,309 @@ +/** + * Remote session semantics: what it means for a phone to hold the microphone. + * + * The claims under test are the ones a user would notice being wrong: + * + * - a remote utterance takes the IDENTICAL path a local one takes, which here + * means it presses the same floor controller and starts an ordinary session + * whose only difference is a `remote` origin; + * - exactly one device holds the floor, last press wins, and a stale release + * from the displaced device cannot shut the new holder's microphone; + * - a lost connection ends the session cleanly and stops speech, while an ICE + * `disconnected` (a WiFi to LTE handover) does not. + * + * No peer connection and no audio: the floor is a fake with the same interface + * the real `FloorController` presents. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { RemoteSessionCoordinator } from '../../../main/acappella/transport/remote-session'; +import type { + RemoteFloor, + RemoteMessageSink, + RemoteVoiceSession, +} from '../../../main/acappella/transport/remote-session'; +import type { DeviceMessage } from '../../../shared/acappella/device-protocol'; +import type { VoiceEvent, VoiceOrigin, VoiceScope } from '../../../shared/acappella/protocol'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +/** Every press the coordinator made, with what it was credited to. */ +let presses: Array<{ scope: VoiceScope; origin: VoiceOrigin }>; +let releases: number; +let floor: RemoteFloor; +let session: RemoteVoiceSession & { + interrupt: ReturnType; + stopSession: ReturnType; + hardStop: ReturnType; +}; +let listeners: Array<(event: VoiceEvent) => void>; +let sink: RemoteMessageSink & { sent: Array<{ deviceId: string; message: DeviceMessage }> }; +let broadcasts: DeviceMessage[]; +let coordinator: RemoteSessionCoordinator; + +function emit(event: Partial & { type: VoiceEvent['type'] }): void { + const full = { sessionId: 's1', seq: 1, ts: 0, ...event } as VoiceEvent; + for (const listener of listeners) listener(full); +} + +/** Sent messages of one type, for one device. */ +function sentTo(deviceId: string, type: DeviceMessage['type']): DeviceMessage[] { + return sink.sent + .filter((entry) => entry.deviceId === deviceId && entry.message.type === type) + .map((entry) => entry.message); +} + +beforeEach(() => { + presses = []; + releases = 0; + listeners = []; + broadcasts = []; + floor = { + press: vi.fn(async () => {}), + release: vi.fn(async () => { + releases += 1; + }), + close: vi.fn(async () => {}), + isFloorOpen: false, + }; + session = { + subscribe: (listener: (event: VoiceEvent) => void) => { + listeners.push(listener); + return () => { + listeners = listeners.filter((entry) => entry !== listener); + }; + }, + interrupt: vi.fn(() => true), + stopSession: vi.fn(async () => {}), + hardStop: vi.fn(async () => {}), + getState: () => 'listening', + } as unknown as typeof session; + sink = { + sent: [], + send: (deviceId: string, message: DeviceMessage) => sink.sent.push({ deviceId, message }), + broadcast: (message: DeviceMessage) => broadcasts.push(message), + } as unknown as typeof sink; + + coordinator = new RemoteSessionCoordinator({ + session, + sink, + acquireFloor: (scope, origin) => { + presses.push({ scope, origin }); + return floor; + }, + getDeviceName: (deviceId) => `Device ${deviceId}`, + }); +}); + +describe('a remote utterance takes the local path', () => { + it('presses the same floor controller a hotkey presses', async () => { + coordinator.handleConnected('phone'); + await coordinator.requestFloor('phone'); + + expect(floor.press).toHaveBeenCalledWith('remote-device'); + expect(presses).toHaveLength(1); + }); + + it('credits the session to the device without changing anything else', async () => { + await coordinator.requestFloor('phone', { kind: 'agent', sessionId: 'agent-7' }); + + // The scope, and therefore the routing, dispatch, and TTS behind it, is + // exactly what a desktop press with the same scope would produce. + expect(presses[0].scope).toEqual({ kind: 'agent', sessionId: 'agent-7' }); + expect(presses[0].origin).toEqual({ + kind: 'remote', + deviceId: 'phone', + deviceName: 'Device phone', + }); + }); + + it('defaults to conductor scope', async () => { + await coordinator.requestFloor('phone'); + expect(presses[0].scope).toEqual({ kind: 'conductor' }); + }); +}); + +describe('floor takeover', () => { + it('gives the floor to the device that pressed last', async () => { + coordinator.handleConnected('phone'); + coordinator.handleConnected('laptop'); + + await coordinator.requestFloor('phone'); + expect(coordinator.floorHolder).toBe('phone'); + + await coordinator.requestFloor('laptop'); + expect(coordinator.floorHolder).toBe('laptop'); + }); + + it('tells the displaced device it lost the floor, and who to', async () => { + coordinator.handleConnected('phone'); + coordinator.handleConnected('laptop'); + await coordinator.requestFloor('phone'); + sink.sent.length = 0; + + await coordinator.requestFloor('laptop'); + + const takeover = sentTo('phone', 'floor-state').find( + (message) => message.type === 'floor-state' && message.takenOverBy + ); + expect(takeover).toMatchObject({ holder: 'laptop', isSelf: false }); + }); + + it('ignores a repeated press from the holder', async () => { + await coordinator.requestFloor('phone'); + await coordinator.requestFloor('phone'); + expect(floor.press).toHaveBeenCalledTimes(1); + }); + + it('ignores a stale release from a device that already lost the floor', async () => { + await coordinator.requestFloor('phone'); + await coordinator.requestFloor('laptop'); + + // The phone's release lands a few milliseconds after the takeover. Acting on + // it would shut the microphone of the device that just took the floor. + await coordinator.releaseFloor('phone'); + expect(releases).toBe(0); + expect(coordinator.floorHolder).toBe('laptop'); + + await coordinator.releaseFloor('laptop'); + expect(releases).toBe(1); + }); + + it('hands the floor back to the desktop when a local session starts', async () => { + await coordinator.requestFloor('phone'); + emit({ type: 'listen-start', origin: { kind: 'local' } } as Partial & { + type: 'listen-start'; + }); + expect(coordinator.floorHolder).toBe('local'); + }); + + it('clears the holder when the session ends', async () => { + coordinator.handleConnected('phone'); + await coordinator.requestFloor('phone'); + emit({ type: 'listen-stop', reason: 'stopped' } as Partial & { + type: 'listen-stop'; + }); + expect(coordinator.floorHolder).toBeNull(); + }); +}); + +describe('interrupts from a device', () => { + it('barges in without ending the session', async () => { + await coordinator.requestFloor('phone'); + coordinator.handleDeviceMessage('phone', { type: 'interrupt', kind: 'barge-in' }); + expect(session.interrupt).toHaveBeenCalledWith('client-button'); + expect(session.hardStop).not.toHaveBeenCalled(); + }); + + it('ends the session on a stop word', async () => { + await coordinator.requestFloor('phone'); + coordinator.handleDeviceMessage('phone', { type: 'interrupt', kind: 'stop-word' }); + expect(session.hardStop).toHaveBeenCalledWith('client-button'); + }); + + it('refuses an interrupt from a device that is not holding the floor', async () => { + await coordinator.requestFloor('phone'); + coordinator.handleDeviceMessage('laptop', { type: 'interrupt', kind: 'barge-in' }); + expect(session.interrupt).not.toHaveBeenCalled(); + }); + + it('routes a floor message through the same path as a direct request', async () => { + coordinator.handleDeviceMessage('phone', { type: 'floor', action: 'press' }); + await coordinator.whenSettled(); + expect(coordinator.floorHolder).toBe('phone'); + + coordinator.handleDeviceMessage('phone', { type: 'floor', action: 'release' }); + await coordinator.whenSettled(); + expect(releases).toBe(1); + }); +}); + +describe('connection loss', () => { + it('ends the session cleanly when the holder disappears', async () => { + coordinator.handleConnected('phone'); + await coordinator.requestFloor('phone'); + + coordinator.handleDisconnected('phone', 'socket closed'); + await coordinator.whenSettled(); + + // Speech is cancelled before the session is closed: the chunks are already + // queued in the audio host and the interrupt is what discards them. + expect(session.interrupt).toHaveBeenCalled(); + expect(session.stopSession).toHaveBeenCalledWith('user'); + expect(coordinator.floorHolder).toBeNull(); + }); + + it('leaves the session alone when a device that was not holding it drops', async () => { + coordinator.handleConnected('phone'); + coordinator.handleConnected('laptop'); + await coordinator.requestFloor('phone'); + + coordinator.handleDisconnected('laptop', 'socket closed'); + await coordinator.whenSettled(); + + expect(session.stopSession).not.toHaveBeenCalled(); + expect(coordinator.floorHolder).toBe('phone'); + }); + + it('survives an ICE disconnect, because that is a WiFi to LTE handover', async () => { + coordinator.handleConnected('phone'); + await coordinator.requestFloor('phone'); + + coordinator.handlePeerState('phone', 'disconnected'); + await coordinator.whenSettled(); + + expect(session.stopSession).not.toHaveBeenCalled(); + expect(coordinator.floorHolder).toBe('phone'); + }); + + it.each(['failed', 'closed'] as const)('ends the session on a %s peer', async (state) => { + coordinator.handleConnected('phone'); + await coordinator.requestFloor('phone'); + + coordinator.handlePeerState('phone', state); + await coordinator.whenSettled(); + + expect(session.stopSession).toHaveBeenCalledWith('user'); + expect(coordinator.floorHolder).toBeNull(); + }); +}); + +describe('event fan-out', () => { + it('forwards the session stream to connected devices', () => { + coordinator.handleConnected('phone'); + emit({ type: 'partial-transcript', text: 'hello', stability: 0.4 } as Partial & { + type: 'partial-transcript'; + }); + expect(broadcasts).toContainEqual( + expect.objectContaining({ + type: 'voice-event', + event: expect.objectContaining({ type: 'partial-transcript' }), + }) + ); + }); + + it('sends nothing when no device is connected', () => { + emit({ type: 'listen-stop', reason: 'stopped' } as Partial & { + type: 'listen-stop'; + }); + expect(broadcasts).toHaveLength(0); + }); + + it('keeps every connected device up to date on who holds the floor', async () => { + coordinator.handleConnected('phone'); + coordinator.handleConnected('laptop'); + sink.sent.length = 0; + + await coordinator.requestFloor('phone'); + + expect(sentTo('phone', 'floor-state')).toContainEqual( + expect.objectContaining({ holder: 'phone', isSelf: true }) + ); + expect(sentTo('laptop', 'floor-state')).toContainEqual( + expect.objectContaining({ holder: 'phone', isSelf: false }) + ); + }); +}); diff --git a/src/__tests__/main/acappella/route-executor.test.ts b/src/__tests__/main/acappella/route-executor.test.ts new file mode 100644 index 0000000000..e6c93e3b4c --- /dev/null +++ b/src/__tests__/main/acappella/route-executor.test.ts @@ -0,0 +1,772 @@ +/** + * @file route-executor.test.ts + * + * Unit tests for the dispatch executor: building the agent roster out of the + * persisted sessions, and turning each of the three tab actions into the + * `remote:*` operations the renderer already implements. + * + * The renderer round trip is behind `VoiceRendererBridge`, so a fake bridge + * drives the whole suite: no Electron window, no store, no timers. The one + * exception is the bridge's own section, which asserts the exact channel names + * and argument ORDER - a shifted argument there is invisible at the type level + * and would silently deliver a response channel into a `force` flag. + */ + +import { describe, it, expect, vi, beforeEach, type MockedFunction } from 'vitest'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); +vi.mock('../../../main/utils/safe-send', () => ({ + isWebContentsAvailable: (win: unknown) => !!win, +})); +vi.mock('../../../main/web-server/callbacks/remoteRequest', () => ({ + requestFromRenderer: vi.fn(), +})); +vi.mock('../../../main/stores/getters', () => ({ + getSessionsStore: vi.fn(), +})); + +import { requestFromRenderer } from '../../../main/web-server/callbacks/remoteRequest'; +import { + buildAgentRoster, + createRendererVoiceBridge, + createVoiceRouteExecutor, + type CommandReceipt, + type FocusTabResult, + type NewTabWithPromptResult, + type VoiceRendererBridge, +} from '../../../main/acappella/dispatch/route-executor'; +import { VoiceDispatchError } from '../../../main/acappella/voice-session-service'; +import type { VoiceScope } from '../../../shared/acappella/protocol'; +import type { RouteDecision } from '../../../shared/acappella/route-decision'; +import type { StoredSession } from '../../../main/stores/types'; +import { createMockSession } from '../../helpers/mockSession'; +import { createMockAITab } from '../../helpers/mockTab'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeSession(overrides: Partial = {}): StoredSession { + return createMockSession(overrides as never) as unknown as StoredSession; +} + +/** Two agents: Backend with two tabs, Frontend with one. */ +function makeSessions(): StoredSession[] { + return [ + makeSession({ + id: 'agent-backend', + name: 'Backend', + toolType: 'claude-code', + cwd: '/repo/api', + activeTabId: 'tab-auth', + aiTabs: [ + createMockAITab({ + id: 'tab-auth', + name: 'Auth Refactor', + createdAt: 1_000, + logs: [{ id: 'l1', timestamp: 4_000, source: 'stdout', text: 'hi' }] as never, + }), + createMockAITab({ id: 'tab-migrations', name: 'DB Migrations', createdAt: 9_000 }), + ], + }), + makeSession({ + id: 'agent-frontend', + name: 'Frontend', + toolType: 'codex', + cwd: '/repo/web', + activeTabId: 'tab-ui', + aiTabs: [createMockAITab({ id: 'tab-ui', name: 'Sidebar', createdAt: 2_000 })], + }), + ]; +} + +type FakeBridge = { + [K in keyof VoiceRendererBridge]: MockedFunction; +}; + +function makeBridge(overrides: Partial = {}): FakeBridge { + return { + selectSession: vi.fn(), + renameTab: vi.fn(), + newTab: vi.fn(async () => 'tab-created'), + newTabWithPrompt: vi.fn( + async (): Promise => ({ success: true, tabId: 'tab-created' }) + ), + executeCommand: vi.fn( + async (): Promise => ({ accepted: true }) + ), + focusTab: vi.fn( + async (_agentSessionId, tabId): Promise => ({ + ok: true, + tabId, + action: 'focused', + }) + ), + ...overrides, + }; +} + +function makeDecision(overrides: Partial = {}): RouteDecision { + return { + target: { sessionId: 'agent-backend' }, + tabAction: 'current', + prompt: 'refactor the auth middleware', + confidence: 0.8, + ...overrides, + }; +} + +const CONDUCTOR_SCOPE: VoiceScope = { kind: 'conductor' }; + +/** Bind an executor over a fixed session list. */ +function makeExecutor(options: { + bridge: VoiceRendererBridge; + sessions?: StoredSession[]; + activeSessionId?: string | null; +}) { + return createVoiceRouteExecutor({ + bridge: options.bridge, + getSessions: () => options.sessions ?? makeSessions(), + getActiveSessionId: () => options.activeSessionId ?? null, + }); +} + +// --------------------------------------------------------------------------- +// Roster +// --------------------------------------------------------------------------- + +describe('buildAgentRoster', () => { + it('compacts sessions into agents with their AI tabs', () => { + const roster = buildAgentRoster(makeSessions()); + + expect(roster).toHaveLength(2); + expect(roster[0]).toMatchObject({ + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + }); + expect(roster[0].tabs.map((tab) => tab.id)).toEqual(['tab-auth', 'tab-migrations']); + expect(roster[1].tabs[0]).toMatchObject({ + id: 'tab-ui', + name: 'Sidebar', + lastActiveAt: 2_000, + state: 'open', + }); + }); + + it('dates a tab by its last log, not just its creation', () => { + const [backend] = buildAgentRoster(makeSessions()); + + // tab-auth was created at 1000 but last spoke at 4000. + expect(backend.tabs[0].lastActiveAt).toBe(4_000); + expect(backend.tabs[1].lastActiveAt).toBe(9_000); + }); + + it('omits hidden consult tabs and reports an unnamed tab as null', () => { + const roster = buildAgentRoster([ + makeSession({ + id: 'agent-1', + aiTabs: [ + createMockAITab({ id: 'tab-visible', name: null, createdAt: 1 }), + createMockAITab({ id: 'tab-consult', name: 'Consult', hidden: true }), + ], + }), + ]); + + expect(roster[0].tabs).toHaveLength(1); + expect(roster[0].tabs[0]).toMatchObject({ id: 'tab-visible', name: null, lastActiveAt: 1 }); + }); + + it('survives a session with no tabs at all', () => { + const roster = buildAgentRoster([makeSession({ id: 'agent-1', aiTabs: undefined })]); + + expect(roster[0].tabs).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Tab actions +// --------------------------------------------------------------------------- + +describe('executeRouteDecision - current', () => { + let bridge: FakeBridge; + + beforeEach(() => { + bridge = makeBridge(); + }); + + it('focuses the active tab and delivers the prompt', async () => { + const execute = makeExecutor({ bridge }); + + const result = await execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE }); + + expect(bridge.selectSession).toHaveBeenCalledWith('agent-backend', 'tab-auth'); + expect(bridge.executeCommand).toHaveBeenCalledWith( + 'agent-backend', + 'tab-auth', + 'refactor the auth middleware' + ); + expect(result).toEqual({ + agentSessionId: 'agent-backend', + agentName: 'Backend', + tabId: 'tab-auth', + tabName: 'Auth Refactor', + action: 'focused', + promptSent: true, + }); + }); + + it('falls back to the most recently active tab when activeTabId is stale', async () => { + const sessions = makeSessions(); + sessions[0].activeTabId = 'tab-closed-yesterday'; + const execute = makeExecutor({ bridge, sessions }); + + const result = await execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE }); + + expect(result.tabId).toBe('tab-migrations'); + expect(result.action).toBe('focused'); + }); + + it('creates a tab when the agent has none, and says so', async () => { + const sessions = [makeSession({ id: 'agent-backend', name: 'Backend', aiTabs: [] })]; + const execute = makeExecutor({ bridge, sessions }); + + const result = await execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE }); + + expect(bridge.newTabWithPrompt).toHaveBeenCalledWith( + 'agent-backend', + 'refactor the auth middleware' + ); + expect(result).toMatchObject({ tabId: 'tab-created', action: 'created', promptSent: true }); + }); + + it('treats a rejected delivery receipt as a dispatch failure', async () => { + bridge.executeCommand.mockResolvedValue({ accepted: false, reason: 'session-busy' }); + const execute = makeExecutor({ bridge }); + + await expect(execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE })).rejects.toThrow( + VoiceDispatchError + ); + }); +}); + +describe('executeRouteDecision - new', () => { + it('opens a tab with the prompt in one operation and names it', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge }); + + const result = await execute(makeDecision({ tabAction: 'new', tabName: 'Auth Refactor' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }); + + expect(bridge.newTabWithPrompt).toHaveBeenCalledWith( + 'agent-backend', + 'refactor the auth middleware' + ); + expect(bridge.renameTab).toHaveBeenCalledWith('agent-backend', 'tab-created', 'Auth Refactor'); + expect(bridge.executeCommand).not.toHaveBeenCalled(); + expect(result).toEqual({ + agentSessionId: 'agent-backend', + agentName: 'Backend', + tabId: 'tab-created', + tabName: 'Auth Refactor', + action: 'created', + promptSent: true, + }); + }); + + it('opens an empty tab without dispatching when there is no prompt', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge }); + + const result = await execute(makeDecision({ tabAction: 'new', prompt: ' ' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }); + + expect(bridge.newTab).toHaveBeenCalledWith('agent-backend'); + expect(bridge.newTabWithPrompt).not.toHaveBeenCalled(); + expect(result).toMatchObject({ action: 'created', promptSent: false }); + }); + + it('fails the dispatch when the renderer does not create the tab', async () => { + const bridge = makeBridge({ + newTabWithPrompt: vi.fn(async () => ({ + success: false, + })), + }); + const execute = makeExecutor({ bridge }); + + await expect( + execute(makeDecision({ tabAction: 'new' }), { roster: [], scope: CONDUCTOR_SCOPE }) + ).rejects.toThrow(VoiceDispatchError); + }); +}); + +describe('executeRouteDecision - recall', () => { + it('returns to the named tab', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge }); + + const result = await execute(makeDecision({ tabAction: 'recall', tabId: 'tab-migrations' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }); + + // A REQUEST, not a fire-and-forget send: the renderer is the only side that + // can tell a focus from a wake from a reopen. + expect(bridge.focusTab).toHaveBeenCalledWith('agent-backend', 'tab-migrations'); + expect(result).toMatchObject({ + tabId: 'tab-migrations', + tabName: 'DB Migrations', + action: 'recalled', + promptSent: true, + }); + }); + + it('fails rather than guessing when the recalled tab is gone', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge }); + + await expect( + execute(makeDecision({ tabAction: 'recall', tabId: 'tab-closed' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }) + ).rejects.toThrow(/no longer open/); + expect(bridge.focusTab).not.toHaveBeenCalled(); + expect(bridge.executeCommand).not.toHaveBeenCalled(); + }); + + it('fails when a recall arrives with no tab id', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge }); + + await expect( + execute(makeDecision({ tabAction: 'recall' }), { roster: [], scope: CONDUCTOR_SCOPE }) + ).rejects.toThrow(VoiceDispatchError); + }); + + it('lands on the tab the renderer actually landed on', async () => { + // Waking a snooze whose conversation is already open focuses the copy that + // exists rather than restoring a duplicate. + const bridge = makeBridge({ + focusTab: vi.fn(async () => ({ + ok: true, + tabId: 'tab-auth', + action: 'woke', + })), + }); + const execute = makeExecutor({ bridge }); + + const result = await execute(makeDecision({ tabAction: 'recall', tabId: 'tab-migrations' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }); + + expect(result.tabId).toBe('tab-auth'); + expect(bridge.executeCommand).toHaveBeenCalledWith( + 'agent-backend', + 'tab-auth', + 'refactor the auth middleware' + ); + }); + + it('fails rather than announcing a recall the renderer did not perform', async () => { + const bridge = makeBridge({ + focusTab: vi.fn(async () => ({ + ok: false, + reason: 'renderer-timeout', + })), + }); + const execute = makeExecutor({ bridge }); + + await expect( + execute(makeDecision({ tabAction: 'recall', tabId: 'tab-migrations' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }) + ).rejects.toThrow(/renderer-timeout/); + expect(bridge.executeCommand).not.toHaveBeenCalled(); + }); + + it('wakes a snoozed tab rather than treating it as gone', async () => { + const sessions = makeSessions(); + sessions[0].snoozedTabs = [ + { + id: 'snooze-1', + tab: createMockAITab({ id: 'tab-spike', name: 'Rate Limit Spike', createdAt: 5 }), + unifiedIndex: 0, + snoozedAt: 1, + wakeAt: 999_999, + }, + ]; + const bridge = makeBridge(); + const execute = makeExecutor({ bridge, sessions }); + + const result = await execute(makeDecision({ tabAction: 'recall', tabId: 'tab-spike' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }); + + expect(bridge.focusTab).toHaveBeenCalledWith('agent-backend', 'tab-spike'); + expect(result.action).toBe('recalled'); + }); +}); + +// --------------------------------------------------------------------------- +// Tab state +// --------------------------------------------------------------------------- + +describe('executeRouteDecision - current', () => { + it('never treats a snoozed or closed tab as the current one', async () => { + // The roster lists them so recall can name them. "Carry on where we were" + // landing on a tab the user put away last week would be the worst possible + // reading of "current". + const sessions = [ + makeSession({ + id: 'agent-backend', + name: 'Backend', + activeTabId: null, + aiTabs: [createMockAITab({ id: 'tab-open', name: 'Open', createdAt: 1 })], + snoozedTabs: [ + { + id: 'snooze-1', + tab: createMockAITab({ id: 'tab-snoozed', name: 'Snoozed', createdAt: 9_000 }), + unifiedIndex: 0, + snoozedAt: 1, + wakeAt: 999_999, + }, + ], + }), + ]; + const bridge = makeBridge(); + const execute = makeExecutor({ bridge, sessions }); + + const result = await execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE }); + + expect(result.tabId).toBe('tab-open'); + }); +}); + +// --------------------------------------------------------------------------- +// Idempotency +// --------------------------------------------------------------------------- + +describe('executeRouteDecision - idempotency', () => { + it('replays an identical decision instead of opening a second tab', async () => { + const bridge = makeBridge(); + const execute = createVoiceRouteExecutor({ + bridge, + getSessions: () => makeSessions(), + getActiveSessionId: () => null, + }); + const decision = makeDecision({ tabAction: 'new', tabName: 'Auth Refactor' }); + + const first = await execute(decision, { roster: [], scope: CONDUCTOR_SCOPE }); + const second = await execute(decision, { roster: [], scope: CONDUCTOR_SCOPE }); + + expect(bridge.newTabWithPrompt).toHaveBeenCalledTimes(1); + expect(second).toEqual(first); + }); + + it('treats a different prompt to the same tab as a new dispatch', async () => { + const bridge = makeBridge(); + const execute = createVoiceRouteExecutor({ + bridge, + getSessions: () => makeSessions(), + getActiveSessionId: () => null, + }); + + await execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE }); + await execute(makeDecision({ prompt: 'and run the linter' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }); + + expect(bridge.executeCommand).toHaveBeenCalledTimes(2); + }); + + it('lets the replay window expire, because a repeat later is intent', async () => { + const bridge = makeBridge(); + const execute = createVoiceRouteExecutor({ + bridge, + getSessions: () => makeSessions(), + getActiveSessionId: () => null, + replayWindowMs: 0, + }); + + await execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE }); + await execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE }); + + expect(bridge.executeCommand).toHaveBeenCalledTimes(2); + }); +}); + +// --------------------------------------------------------------------------- +// Clarifications +// --------------------------------------------------------------------------- + +describe('executeRouteDecision - clarifications', () => { + it('refuses to dispatch a question', async () => { + // Reaching the executor with one means a caller skipped the guard, and + // dispatching it would send the user their own half-finished request. + const bridge = makeBridge(); + const execute = makeExecutor({ bridge }); + + await expect( + execute(makeDecision({ clarify: 'Backend or Frontend?' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }) + ).rejects.toThrow(/question, not a dispatch/); + expect(bridge.executeCommand).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// Target resolution +// --------------------------------------------------------------------------- + +describe('executeRouteDecision - target resolution', () => { + it('fails when the targeted agent closed while the decision was in flight', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge, sessions: [] }); + + await expect(execute(makeDecision(), { roster: [], scope: CONDUCTOR_SCOPE })).rejects.toThrow( + /no longer running/ + ); + }); + + it('sends a conductor decision to the session scope agent', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge, activeSessionId: 'agent-backend' }); + + const result = await execute(makeDecision({ target: 'conductor' }), { + roster: [], + scope: { kind: 'agent', sessionId: 'agent-frontend' }, + }); + + // The bound scope outranks whichever agent the desktop happens to show. + expect(result.agentSessionId).toBe('agent-frontend'); + }); + + it('falls back to the active desktop agent for an unscoped conductor decision', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge, activeSessionId: 'agent-frontend' }); + + const result = await execute(makeDecision({ target: 'conductor' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }); + + expect(result.agentSessionId).toBe('agent-frontend'); + }); + + it('uses the only agent there is', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge, sessions: [makeSessions()[1]] }); + + const result = await execute(makeDecision({ target: 'conductor' }), { + roster: [], + scope: CONDUCTOR_SCOPE, + }); + + expect(result.agentSessionId).toBe('agent-frontend'); + }); + + it('refuses to guess between several agents with nothing active', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge }); + + await expect( + execute(makeDecision({ target: 'conductor' }), { roster: [], scope: CONDUCTOR_SCOPE }) + ).rejects.toThrow(VoiceDispatchError); + expect(bridge.executeCommand).not.toHaveBeenCalled(); + }); + + it('reports having no agents at all distinctly', async () => { + const bridge = makeBridge(); + const execute = makeExecutor({ bridge, sessions: [] }); + + await expect( + execute(makeDecision({ target: 'conductor' }), { roster: [], scope: CONDUCTOR_SCOPE }) + ).rejects.toThrow(/No agents are open/); + }); +}); + +// --------------------------------------------------------------------------- +// Renderer bridge +// --------------------------------------------------------------------------- + +describe('createRendererVoiceBridge', () => { + const send = vi.fn(); + const show = vi.fn(); + const win = { + webContents: { send }, + isMinimized: () => false, + restore: vi.fn(), + show, + } as never; + + beforeEach(() => { + send.mockClear(); + vi.mocked(requestFromRenderer).mockReset(); + }); + + it('focuses and renames over the existing remote channels', () => { + const bridge = createRendererVoiceBridge(() => win); + + bridge.selectSession('agent-1', 'tab-1'); + bridge.renameTab('agent-1', 'tab-1', 'Auth Refactor'); + + expect(send).toHaveBeenNthCalledWith(1, 'remote:selectSession', 'agent-1', 'tab-1'); + expect(send).toHaveBeenNthCalledWith( + 2, + 'remote:renameTab', + 'agent-1', + 'tab-1', + 'Auth Refactor' + ); + }); + + it('creates a tab with a prompt atomically, leaving room for the response channel', async () => { + vi.mocked(requestFromRenderer).mockResolvedValue({ success: true, tabId: 'tab-9' }); + const bridge = createRendererVoiceBridge(() => win); + + const result = await bridge.newTabWithPrompt('agent-1', 'do the thing'); + + expect(result).toEqual({ success: true, tabId: 'tab-9' }); + const [, channel, options] = vi.mocked(requestFromRenderer).mock.calls[0]; + expect(channel).toBe('remote:newAITabWithPrompt'); + // `background` is deliberately omitted so the response channel lands in + // its slot AND the new tab takes focus. + expect(options.args).toEqual(['agent-1', 'do the thing']); + }); + + it('sends a prompt with the full positional argument list', async () => { + vi.mocked(requestFromRenderer).mockResolvedValue({ accepted: true }); + const bridge = createRendererVoiceBridge(() => win); + + await bridge.executeCommand('agent-1', 'tab-1', 'do the thing'); + + const [, channel, options] = vi.mocked(requestFromRenderer).mock.calls[0]; + expect(channel).toBe('remote:executeCommand'); + expect(options.args).toEqual([ + 'agent-1', + 'do the thing', + 'ai', + 'tab-1', + false, + undefined, + false, + ]); + }); + + it('reads the tab id out of a new-tab reply', async () => { + vi.mocked(requestFromRenderer).mockResolvedValue({ tabId: 'tab-3' }); + const bridge = createRendererVoiceBridge(() => win); + + await expect(bridge.newTab('agent-1')).resolves.toBe('tab-3'); + }); + + it('classifies a missing renderer as a dispatch failure', () => { + const bridge = createRendererVoiceBridge(() => null); + + expect(() => bridge.selectSession('agent-1')).toThrow(VoiceDispatchError); + }); + + it('asks the renderer to focus a tab and reads back what that took', async () => { + vi.mocked(requestFromRenderer).mockResolvedValue({ + ok: true, + tabId: 'tab-1', + action: 'woke', + }); + const bridge = createRendererVoiceBridge(() => win); + + const result = await bridge.focusTab('agent-1', 'tab-1'); + + const [, channel, options] = vi.mocked(requestFromRenderer).mock.calls[0]; + expect(channel).toBe('remote:focusAiTab'); + expect(options.args).toEqual(['agent-1', 'tab-1']); + expect(result).toEqual({ ok: true, tabId: 'tab-1', action: 'woke' }); + }); + + it('treats a malformed focus reply as a failure rather than a success', async () => { + vi.mocked(requestFromRenderer).mockResolvedValue({ ok: true }); + const bridge = createRendererVoiceBridge(() => win); + + await bridge.focusTab('agent-1', 'tab-1'); + + // The parser runs inside `requestFromRenderer`, which is mocked here, so it + // is exercised directly: a truthy-looking reply must not become an `ok`. + const [, , options] = vi.mocked(requestFromRenderer).mock.calls[0]; + expect(options.parse!({ ok: 'yes' })).toMatchObject({ ok: false }); + expect(options.parse!('sure')).toMatchObject({ ok: false, reason: 'malformed-result' }); + expect(options.fallback).toMatchObject({ ok: false, reason: 'renderer-timeout' }); + }); + + // -- Multi-window -------------------------------------------------------- + + describe('multi-window', () => { + function makeWindow(label: string) { + return { + label, + webContents: { send: vi.fn() }, + isMinimized: () => false, + restore: vi.fn(), + show: vi.fn(), + }; + } + + it('dispatches into the window that owns the agent, not whichever is main', () => { + // Agent ownership is per window while `activeSessionId` is global, so + // dispatching to "main" would activate an agent that window does not own. + const main = makeWindow('main'); + const secondary = makeWindow('secondary'); + const bridge = createRendererVoiceBridge( + () => main as never, + (sessionId) => (sessionId === 'agent-2' ? (secondary as never) : null) + ); + + bridge.selectSession('agent-2', 'tab-1'); + + expect(secondary.webContents.send).toHaveBeenCalledWith( + 'remote:selectSession', + 'agent-2', + 'tab-1' + ); + expect(main.webContents.send).not.toHaveBeenCalled(); + }); + + it('raises the owning window: a dispatch behind another window did nothing', () => { + const secondary = makeWindow('secondary'); + secondary.isMinimized = () => true; + const bridge = createRendererVoiceBridge( + () => null, + () => secondary as never + ); + + bridge.selectSession('agent-2', 'tab-1'); + + expect(secondary.restore).toHaveBeenCalled(); + expect(secondary.show).toHaveBeenCalled(); + }); + + it('falls back to the main window when no window claims the agent', () => { + const main = makeWindow('main'); + const bridge = createRendererVoiceBridge( + () => main as never, + () => null + ); + + bridge.selectSession('agent-2', 'tab-1'); + + expect(main.webContents.send).toHaveBeenCalled(); + }); + }); +}); diff --git a/src/__tests__/main/acappella/router/conductor-agent.test.ts b/src/__tests__/main/acappella/router/conductor-agent.test.ts new file mode 100644 index 0000000000..6bf02027be --- /dev/null +++ b/src/__tests__/main/acappella/router/conductor-agent.test.ts @@ -0,0 +1,267 @@ +/** + * @file conductor-agent.test.ts + * + * The Conductor run as a real agent: the busy refusal, the deadline, and the + * SSH rule that a configured remote which cannot be resolved is a loud failure + * rather than a quiet local run. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); +vi.mock('../../../../main/utils/ssh-spawn-wrapper', () => ({ wrapSpawnWithSsh: vi.fn() })); +vi.mock('../../../../main/prompt-manager', () => ({ + getPrompt: () => { + throw new Error('prompts not initialised'); + }, +})); + +import { wrapSpawnWithSsh } from '../../../../main/utils/ssh-spawn-wrapper'; +import { + createConductorAgentBrain, + type ConductorProcessManager, +} from '../../../../main/acappella/router/conductor-agent'; +import { VoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import type { VoiceRouteContext } from '../../../../shared/acappella/providers'; + +const CONTEXT: VoiceRouteContext = { + roster: [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + tabs: [{ id: 'tab-auth', name: 'Auth', lastActiveAt: 1 }], + }, + ], + scope: { kind: 'conductor' }, +}; + +const DECISION_JSON = JSON.stringify({ + target: { sessionId: 'agent-backend' }, + tabAction: 'current', + prompt: 'run the tests', + confidence: 0.9, +}); + +/** A process manager that replays one scripted response per spawn. */ +function fakeProcessManager(script: { output?: string; exits?: boolean; spawns?: boolean } = {}) { + const handlers = new Map void>>(); + const spawned: Array> = []; + const killed: string[] = []; + + const emit = (event: string, ...args: unknown[]): void => { + for (const handler of handlers.get(event) ?? []) handler(...args); + }; + + const manager: ConductorProcessManager = { + spawn(config) { + spawned.push(config); + if (script.spawns === false) return null; + // Asynchronous, like the real thing: the collector must have attached + // its listeners before anything arrives. + setTimeout(() => { + if (script.output) emit('data', config.sessionId, script.output); + if (script.exits !== false) emit('exit', config.sessionId, 0); + }, 0); + return { pid: 1234 }; + }, + on(event, handler) { + handlers.set(event, [...(handlers.get(event) ?? []), handler]); + }, + off(event, handler) { + handlers.set( + event, + (handlers.get(event) ?? []).filter((entry) => entry !== handler) + ); + }, + kill(sessionId) { + killed.push(sessionId); + }, + }; + + return { manager, spawned, killed, listenerCount: () => (handlers.get('data') ?? []).length }; +} + +function agentDetector(available = true) { + return { + getAgent: vi.fn(async () => ({ + id: 'claude-code', + command: 'claude', + binaryName: 'claude', + args: ['--print'], + available, + promptArgs: undefined, + noPromptSeparator: false, + })), + } as never; +} + +function makeBrain( + script: Parameters[0] = { output: DECISION_JSON }, + overrides: Record = {} +) { + const process = fakeProcessManager(script); + const brain = createConductorAgentBrain({ + processManager: process.manager, + agentDetector: agentDetector(), + agentType: 'claude-code', + cwd: '/repo/api', + timeoutMs: 50, + ...overrides, + }); + return { brain, ...process }; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('ConductorAgentBrain - routing', () => { + it('spawns read-only and parses the decision through the shared schema', async () => { + const { brain, spawned } = makeBrain(); + + const decision = await brain.route('run the tests', CONTEXT); + + expect(decision).toMatchObject({ + target: { sessionId: 'agent-backend' }, + tabAction: 'current', + confidence: 0.9, + }); + // A router reads; it does not edit, and read-only also means no workspace + // lock, so the Conductor can think while its agents work. + expect(spawned[0]).toMatchObject({ readOnlyMode: true, toolType: 'claude-code' }); + }); + + it('validates a hallucinated id away rather than returning it', async () => { + const { brain } = makeBrain({ + output: JSON.stringify({ + target: { sessionId: 'agent-ghost' }, + tabAction: 'current', + prompt: 'x', + confidence: 0.9, + }), + }); + + const decision = await brain.route('run the tests', CONTEXT); + + expect(decision.target).toBe('conductor'); + }); + + it('drops its listeners once the run is over', async () => { + const { brain, listenerCount } = makeBrain(); + + await brain.route('run the tests', CONTEXT); + + expect(listenerCount()).toBe(0); + }); +}); + +describe('ConductorAgentBrain - never blocks the floor', () => { + it('refuses a second turn while one is in flight, out loud', async () => { + const { brain } = makeBrain(); + + const first = brain.route('run the tests', CONTEXT); + const second = brain.route('and the linter', CONTEXT); + + await expect(second).rejects.toThrow(/Conductor is busy/); + await first; + // The refusal is recoverable: wait, then say it again. + await expect(brain.route('and the linter', CONTEXT)).resolves.toBeTruthy(); + }); + + it('kills the process when the deadline passes', async () => { + const { brain, killed } = makeBrain({ exits: false }); + + await expect(brain.route('run the tests', CONTEXT)).rejects.toThrow(/did not answer in time/); + // An abandoned agent left running still holds a model and a token budget. + expect(killed).toHaveLength(1); + }); + + it('reports a spawn that never started', async () => { + const { brain } = makeBrain({ spawns: false }); + + await expect(brain.route('run the tests', CONTEXT)).rejects.toThrow(/could not be started/); + }); + + it('reports an unavailable agent as a provider failure', async () => { + const { manager } = fakeProcessManager(); + const brain = createConductorAgentBrain({ + processManager: manager, + agentDetector: agentDetector(false), + agentType: 'claude-code', + cwd: '/repo/api', + }); + + await expect(brain.route('run the tests', CONTEXT)).rejects.toThrow(VoiceProviderError); + }); +}); + +describe('ConductorAgentBrain - SSH', () => { + it('wraps the spawn when a remote is configured', async () => { + vi.mocked(wrapSpawnWithSsh).mockResolvedValue({ + command: 'ssh', + args: ['host', 'claude'], + cwd: '/remote/api', + prompt: 'hello', + customEnvVars: {}, + sshRemoteUsed: { id: 'remote-1', name: 'box' } as never, + } as never); + + const { brain, spawned } = makeBrain( + { output: DECISION_JSON }, + { + sshRemoteConfig: { enabled: true, remoteId: 'remote-1' }, + sshStore: {} as never, + } + ); + + await brain.route('run the tests', CONTEXT); + + expect(wrapSpawnWithSsh).toHaveBeenCalled(); + expect(spawned[0]).toMatchObject({ + command: 'ssh', + cwd: '/remote/api', + sshRemoteId: 'remote-1', + }); + }); + + it('fails loudly when the remote cannot be resolved, rather than running locally', async () => { + vi.mocked(wrapSpawnWithSsh).mockResolvedValue({ + command: 'claude', + args: [], + cwd: '/repo/api', + sshRemoteUsed: null, + } as never); + + const { brain, spawned } = makeBrain( + { output: DECISION_JSON }, + { sshRemoteConfig: { enabled: true, remoteId: 'gone' }, sshStore: {} as never } + ); + + await expect(brain.route('run the tests', CONTEXT)).rejects.toThrow(/could not be resolved/); + // The user opted into a remote. A routing prompt carries the names and paths + // of everything they have open, so it must not run here instead. + expect(spawned).toHaveLength(0); + }); + + it('fails loudly when SSH is enabled with no store to resolve it', async () => { + const { brain, spawned } = makeBrain( + { output: DECISION_JSON }, + { sshRemoteConfig: { enabled: true, remoteId: 'remote-1' } } + ); + + await expect(brain.route('run the tests', CONTEXT)).rejects.toThrow(/could not be resolved/); + expect(spawned).toHaveLength(0); + }); + + it('does not touch the SSH wrapper when no remote is configured', async () => { + const { brain } = makeBrain(); + + await brain.route('run the tests', CONTEXT); + + expect(wrapSpawnWithSsh).not.toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/main/acappella/router/conductor-router.test.ts b/src/__tests__/main/acappella/router/conductor-router.test.ts new file mode 100644 index 0000000000..6767602fa8 --- /dev/null +++ b/src/__tests__/main/acappella/router/conductor-router.test.ts @@ -0,0 +1,421 @@ +/** + * @file conductor-router.test.ts + * + * The decision layer, driven by a fake Brain. + * + * Every case here is a fixture pair - an utterance and a roster - because that + * is the only honest way to test routing: the rules are about what a decision + * means against a specific set of agents, and a suite that asserted "the Brain + * was called" would pass for a router that dispatched everything to the first + * agent in the list. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + createConductorRouter, + isCorrectionUtterance, + planCorrection, +} from '../../../../main/acappella/router/conductor-router'; +import type { RoutingContext } from '../../../../main/acappella/router/routing-context'; +import type { RosterAgent } from '../../../../shared/acappella/protocol'; +import type { BrainProvider, VoiceRouteContext } from '../../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const NOW = 1_000_000_000; + +function roster(): RosterAgent[] { + return [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + status: 'idle', + tabs: [ + { + id: 'tab-auth', + name: 'Auth Refactor', + lastActiveAt: NOW - 1000, + state: 'open', + topic: 'rewrite the auth middleware', + }, + { id: 'tab-db', name: 'DB Migrations', lastActiveAt: NOW, state: 'open', topic: null }, + ], + }, + { + sessionId: 'agent-api', + name: 'API', + agentType: 'codex', + cwd: '/repo/gateway', + status: 'idle', + tabs: [{ id: 'tab-gw', name: 'Gateway', lastActiveAt: NOW, state: 'open', topic: null }], + }, + ]; +} + +function context(overrides: Partial = {}): VoiceRouteContext { + return { + roster: roster(), + scope: { kind: 'conductor' }, + activeAgentSessionId: null, + recentUtterances: [], + ...overrides, + }; +} + +function routingContext(agents: RosterAgent[] = roster()): RoutingContext { + return { + agents, + activeAgentSessionId: null, + recentUtterances: [], + droppedTabs: 0, + serializedChars: 400, + }; +} + +function decision(overrides: Partial = {}): RouteDecision { + return { + target: { sessionId: 'agent-backend' }, + tabAction: 'current', + prompt: 'run the tests', + confidence: 0.9, + ...overrides, + }; +} + +/** A Brain that returns a scripted decision per call. */ +function fakeBrain(...decisions: RouteDecision[]): BrainProvider & { calls: VoiceRouteContext[] } { + const calls: VoiceRouteContext[] = []; + let index = 0; + return { + id: 'fake-brain', + label: 'Fake', + tier: 'mock', + calls, + async route(_input, ctx) { + calls.push(ctx); + return decisions[Math.min(index++, decisions.length - 1)]; + }, + async converse(text) { + return text; + }, + }; +} + +function makeRouter( + brain: BrainProvider, + options: { agents?: RosterAgent[]; threshold?: number } = {} +) { + const recorded: Array> = []; + const router = createConductorRouter({ + brain, + confidenceThreshold: options.threshold, + loadContext: async () => routingContext(options.agents ?? roster()), + record: (entry) => { + recorded.push(entry as unknown as Record); + return entry.id; + }, + now: () => NOW, + }); + return { router, recorded }; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +// --------------------------------------------------------------------------- +// The four shapes of a decision +// --------------------------------------------------------------------------- + +describe('createConductorRouter - decisions', () => { + it('passes a same-topic continuation through as current', async () => { + const { router } = makeRouter(fakeBrain(decision())); + + const result = await router.route('run the tests', context()); + + expect(result).toMatchObject({ + target: { sessionId: 'agent-backend' }, + tabAction: 'current', + prompt: 'run the tests', + }); + expect(result.clarify).toBeUndefined(); + }); + + it('passes a topic switch through as a new named tab', async () => { + const { router } = makeRouter( + fakeBrain(decision({ tabAction: 'new', tabName: 'Rate Limiting', confidence: 0.8 })) + ); + + const result = await router.route('add a rate limiter', context()); + + expect(result).toMatchObject({ tabAction: 'new', tabName: 'Rate Limiting' }); + }); + + it('passes a recall of an open tab through untouched', async () => { + const { router } = makeRouter( + fakeBrain(decision({ tabAction: 'recall', tabId: 'tab-auth', confidence: 0.8 })) + ); + + const result = await router.route('back to the auth thing', context()); + + expect(result).toMatchObject({ tabAction: 'recall', tabId: 'tab-auth' }); + expect(result.clarify).toBeUndefined(); + }); + + it('passes a conductor-targeted question through', async () => { + const { router } = makeRouter( + fakeBrain( + decision({ target: 'conductor', prompt: 'how many agents are running', confidence: 0.9 }) + ) + ); + + const result = await router.route('how many agents are running', context()); + + expect(result.target).toBe('conductor'); + }); + + it('reshapes the roster it hands the Brain into the shortlist', async () => { + const brain = fakeBrain(decision()); + const { router } = makeRouter(brain); + + await router.route('back to the auth thing', context()); + + // Every agent survives - dropping one makes it unroutable - and the enriched + // roster from the assembler is what the Brain sees, not the caller's. + expect(brain.calls[0].roster.map((agent) => agent.sessionId)).toEqual([ + 'agent-backend', + 'agent-api', + ]); + expect(brain.calls[0].roster[0].tabs[0].topic).toBe('rewrite the auth middleware'); + }); +}); + +// --------------------------------------------------------------------------- +// Low confidence +// --------------------------------------------------------------------------- + +describe('createConductorRouter - disambiguation', () => { + it('asks rather than dispatching below the threshold', async () => { + const { router } = makeRouter(fakeBrain(decision({ confidence: 0.3 }))); + + const result = await router.route('run it', context()); + + expect(result.clarify).toBe('Backend or API?'); + }); + + it('names the two best tabs when the low-confidence decision is a recall', async () => { + const { router } = makeRouter( + fakeBrain(decision({ tabAction: 'recall', tabId: 'tab-auth', confidence: 0.3 })) + ); + + const result = await router.route('back to that auth gateway thing', context()); + + expect(result.clarify).toMatch(/Auth Refactor|Gateway/); + }); + + it('dispatches a low-confidence decision when there is only one agent', async () => { + // Asking "Backend?" of someone who has one agent is worse than acting. + const only = [roster()[0]]; + const { router } = makeRouter(fakeBrain(decision({ confidence: 0.2 })), { agents: only }); + + const result = await router.route('run it', context({ roster: only })); + + expect(result.clarify).toBeUndefined(); + }); + + it('honours a threshold override', async () => { + const { router } = makeRouter(fakeBrain(decision({ confidence: 0.6 })), { threshold: 0.9 }); + + const result = await router.route('run it', context()); + + expect(result.clarify).toBe('Backend or API?'); + }); + + it('leaves a question the Brain asked for itself alone', async () => { + const { router } = makeRouter( + fakeBrain(decision({ confidence: 0.2, clarify: 'the payments one or the gateway?' })) + ); + + const result = await router.route('do the thing', context()); + + expect(result.clarify).toBe('the payments one or the gateway?'); + }); +}); + +// --------------------------------------------------------------------------- +// Recovery +// --------------------------------------------------------------------------- + +describe('createConductorRouter - validation failures', () => { + it('retries once, telling the model what was wrong', async () => { + const brain = fakeBrain(decision({ target: { sessionId: 'agent-ghost' } }), decision()); + const { router, recorded } = makeRouter(brain); + + const result = await router.route('run the tests', context()); + + expect(brain.calls).toHaveLength(2); + expect(brain.calls[1].retryNotes?.join(' ')).toContain('decision.target'); + expect(result.target).toEqual({ sessionId: 'agent-backend' }); + expect(recorded[0].retries).toBe(1); + }); + + it('asks out loud rather than guessing after a second rejection', async () => { + const bad = decision({ target: { sessionId: 'agent-ghost' } }); + const { router } = makeRouter(fakeBrain(bad, bad)); + + const result = await router.route('deploy the thing', context()); + + expect(result.target).toBe('conductor'); + expect(result.clarify).toContain('Backend or API?'); + // The user's own words survive, so answering the question routes the + // request they actually made. + expect(result.prompt).toBe('deploy the thing'); + }); + + it('never dispatches a decision that failed validation twice', async () => { + const bad = decision({ tabAction: 'recall', tabId: 'tab-ghost' }); + const { router } = makeRouter(fakeBrain(bad, bad)); + + const result = await router.route('back to that thing', context()); + + expect(result.clarify).toBeTruthy(); + }); + + it('routes on the caller roster when the context assembler is unavailable', async () => { + const brain = fakeBrain(decision()); + const router = createConductorRouter({ + brain, + loadContext: async () => { + throw new Error('no store'); + }, + record: (entry) => entry.id, + }); + + const result = await router.route('run the tests', context()); + + expect(result.target).toEqual({ sessionId: 'agent-backend' }); + expect(brain.calls[0].roster).toHaveLength(2); + }); +}); + +// --------------------------------------------------------------------------- +// Closed tabs +// --------------------------------------------------------------------------- + +describe('createConductorRouter - closed tab recall', () => { + function withClosedAuthTab(): RosterAgent[] { + const agents = roster(); + agents[0].tabs[0] = { ...agents[0].tabs[0], state: 'closed' }; + return agents; + } + + it('offers to reopen rather than silently creating a duplicate', async () => { + const agents = withClosedAuthTab(); + const { router } = makeRouter( + fakeBrain(decision({ tabAction: 'recall', tabId: 'tab-auth', confidence: 0.9 })), + { agents } + ); + + const result = await router.route('back to the auth thing', context({ roster: agents })); + + expect(result.clarify).toContain('reopen'); + expect(result.tabAction).toBe('recall'); + }); + + it('acts on the second pass, once the offer has been answered', async () => { + const agents = withClosedAuthTab(); + const { router } = makeRouter( + fakeBrain(decision({ tabAction: 'recall', tabId: 'tab-auth', confidence: 0.9 })), + { agents } + ); + + const result = await router.route( + 'yes', + context({ + roster: agents, + clarification: { question: 'Reopen it?', utterance: 'back to the auth thing' }, + }) + ); + + expect(result.clarify).toBeUndefined(); + expect(result.tabId).toBe('tab-auth'); + }); +}); + +// --------------------------------------------------------------------------- +// Logging +// --------------------------------------------------------------------------- + +describe('createConductorRouter - routing log', () => { + it('records the turn with the context size and the latency', async () => { + const { router, recorded } = makeRouter(fakeBrain(decision())); + + await router.route('run the tests', context()); + + expect(recorded[0]).toMatchObject({ + utterance: 'run the tests', + brainProviderId: 'fake-brain', + contextChars: 400, + droppedTabs: 0, + latencyMs: 0, + }); + expect(router.lastTurnId()).toBe(recorded[0].id); + }); +}); + +// --------------------------------------------------------------------------- +// Correction +// --------------------------------------------------------------------------- + +describe('isCorrectionUtterance', () => { + it('recognises the short interjections', () => { + for (const phrase of ['no, the other one', 'Wrong tab.', 'not that one', 'the other one']) { + expect(isCorrectionUtterance(phrase)).toBe(true); + } + }); + + it('does not treat a sentence containing one as a correction', () => { + // A false positive silently moves a prompt the user never asked to move. + expect(isCorrectionUtterance('no, not that one, use the other endpoint')).toBe(false); + expect(isCorrectionUtterance('the other one is failing its tests')).toBe(false); + }); +}); + +describe('planCorrection', () => { + it('moves to the only alternative without asking', () => { + expect(planCorrection(roster(), 'agent-backend')).toEqual({ + kind: 'move', + agentSessionId: 'agent-api', + }); + }); + + it('asks when there are several alternatives', () => { + const three = [ + ...roster(), + { sessionId: 'agent-web', name: 'Web', agentType: 'codex', cwd: '/repo/web', tabs: [] }, + ]; + + const plan = planCorrection(three, 'agent-backend'); + + expect(plan.kind).toBe('ask'); + if (plan.kind !== 'ask') throw new Error('expected a question'); + expect(plan.question).toBe('API, or Web?'); + }); + + it('says so when there is nowhere else to send it', () => { + expect(planCorrection([roster()[0]], 'agent-backend')).toEqual({ + kind: 'ask', + question: 'There is nowhere else to send that.', + }); + }); +}); diff --git a/src/__tests__/main/acappella/router/conversation-buffer.test.ts b/src/__tests__/main/acappella/router/conversation-buffer.test.ts new file mode 100644 index 0000000000..26fed7619b --- /dev/null +++ b/src/__tests__/main/acappella/router/conversation-buffer.test.ts @@ -0,0 +1,98 @@ +/** + * @file conversation-buffer.test.ts + * + * The memory that lets the Conductor hold a conversation. Two of these tests are + * about forgetting rather than remembering, which is the half that goes wrong: + * a buffer that survives a dispatch makes the next request arrive wearing the + * last one's context. + */ + +import { describe, it, expect } from 'vitest'; +import { ConversationBuffer } from '../../../../main/acappella/router/conversation-buffer'; + +describe('ConversationBuffer', () => { + it('keeps both halves of the exchange in order', () => { + const buffer = new ConversationBuffer(); + + buffer.add('user', 'the refresh keeps failing'); + buffer.add('conductor', 'On the second load, or every time?'); + buffer.add('user', 'second load'); + + expect(buffer.history).toEqual([ + { role: 'user', text: 'the refresh keeps failing' }, + { role: 'conductor', text: 'On the second load, or every time?' }, + { role: 'user', text: 'second load' }, + ]); + }); + + it('starts empty and reports it', () => { + const buffer = new ConversationBuffer(); + + expect(buffer.active).toBe(false); + expect(buffer.history).toEqual([]); + }); + + it('forgets everything on clear, which is what a dispatch does', () => { + // The discussion that produced a request is finished the moment it is sent. + const buffer = new ConversationBuffer(); + buffer.add('user', 'fix the auth bug'); + + buffer.clear(); + + expect(buffer.active).toBe(false); + expect(buffer.history).toEqual([]); + }); + + it('ignores an empty line rather than recording a blank turn', () => { + const buffer = new ConversationBuffer(); + + buffer.add('user', ' '); + + expect(buffer.active).toBe(false); + }); + + it('trims the text it records', () => { + const buffer = new ConversationBuffer(); + + buffer.add('user', ' spaced out '); + + expect(buffer.history[0].text).toBe('spaced out'); + }); + + it('drops the oldest turns once it is full', () => { + const buffer = new ConversationBuffer({ maxTurns: 3 }); + + for (const text of ['one', 'two', 'three', 'four']) buffer.add('user', text); + + expect(buffer.history.map((turn) => turn.text)).toEqual(['two', 'three', 'four']); + }); + + it('drops by size too, since ten paragraphs is a prompt that keeps growing', () => { + const buffer = new ConversationBuffer({ maxTurns: 50, maxChars: 30 }); + + buffer.add('user', 'x'.repeat(25)); + buffer.add('user', 'y'.repeat(25)); + + expect(buffer.history).toHaveLength(1); + expect(buffer.history[0].text.startsWith('y')).toBe(true); + }); + + it('never drops below one turn, however long that turn is', () => { + // The thing just said is the least droppable part of the context, even when + // it blows the budget on its own. + const buffer = new ConversationBuffer({ maxTurns: 50, maxChars: 10 }); + + buffer.add('user', 'z'.repeat(500)); + + expect(buffer.history).toHaveLength(1); + }); + + it('hands out a copy, so a caller cannot edit the memory in place', () => { + const buffer = new ConversationBuffer(); + buffer.add('user', 'original'); + + buffer.history.push({ role: 'user', text: 'injected' }); + + expect(buffer.history).toHaveLength(1); + }); +}); diff --git a/src/__tests__/main/acappella/router/grammar.test.ts b/src/__tests__/main/acappella/router/grammar.test.ts new file mode 100644 index 0000000000..f8aad0db0c --- /dev/null +++ b/src/__tests__/main/acappella/router/grammar.test.ts @@ -0,0 +1,225 @@ +/** + * @file grammar.test.ts + * + * The compiled `RouteDecision` grammar, from both ends. + * + * The GBNF text is asserted structurally (it is what llama.cpp masks its sampler + * against, and a missing id alternative there is invisible until a model invents + * an agent), and the acceptance rules are asserted through `validate`, which is + * rendered from the SAME compiled node tree. That is the point of the two + * renderers: the suite can prove "the grammar rejects an unknown session id" + * without embedding a GBNF engine, and a drift between the grammar and the + * validator would have to be a drift within one tree. + */ + +import { describe, it, expect } from 'vitest'; + +import { + compileRouteDecisionGrammar, + rosterScope, + routeDecisionSchema, + validateRouteDecision, +} from '../../../../main/acappella/router/grammar'; +import type { RosterAgent } from '../../../../shared/acappella/protocol'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; + +const ROSTER: RosterAgent[] = [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + tabs: [ + { id: 'tab-auth', name: 'Auth', lastActiveAt: 10 }, + { id: 'tab-db', name: 'DB', lastActiveAt: 20 }, + ], + }, + { + sessionId: 'agent-frontend', + name: 'Frontend', + agentType: 'codex', + cwd: '/repo/web', + tabs: [{ id: 'tab-ui', name: 'Sidebar', lastActiveAt: 30 }], + }, +]; + +function decision(overrides: Partial = {}): RouteDecision { + return { + target: { sessionId: 'agent-backend' }, + tabAction: 'current', + prompt: 'run the tests', + confidence: 0.9, + ...overrides, + }; +} + +describe('compileRouteDecisionGrammar - GBNF', () => { + it('emits a root rule and the JSON lexical prelude', () => { + const { gbnf } = compileRouteDecisionGrammar(); + + expect(gbnf).toMatch(/^root ::= /); + for (const rule of ['ws ::=', 'string ::=', 'number ::=', 'char ::=', 'hex ::=']) { + expect(gbnf).toContain(rule); + } + }); + + it('constrains the ids to exactly the ones in the roster', () => { + const { gbnf } = compileRouteDecisionGrammar(rosterScope(ROSTER)); + + expect(gbnf).toContain('"\\"agent-backend\\""'); + expect(gbnf).toContain('"\\"agent-frontend\\""'); + expect(gbnf).toContain('"\\"tab-auth\\""'); + // Nothing that is not in the roster may appear as an alternative. + expect(gbnf).not.toContain('agent-ghost'); + }); + + it('closes the tab action to the three known values', () => { + const { gbnf } = compileRouteDecisionGrammar(); + + expect(gbnf).toContain('"\\"current\\"" | "\\"new\\"" | "\\"recall\\""'); + }); + + it('marks the optional fields optional and the required ones not', () => { + const { gbnf } = compileRouteDecisionGrammar(); + + // `tabId` may be omitted; `prompt` may not. + expect(gbnf).toMatch(/\("," ws "\\"tabId\\"" ws ":" ws string\)\?/); + expect(gbnf).toMatch(/"," ws "\\"prompt\\"" ws ":" ws string/); + expect(gbnf).not.toMatch(/\("," ws "\\"prompt\\""[^)]*\)\?/); + }); +}); + +describe('compileRouteDecisionGrammar - acceptance', () => { + const grammar = compileRouteDecisionGrammar(rosterScope(ROSTER)); + + it('accepts a well-formed decision', () => { + expect(grammar.validate(decision()).ok).toBe(true); + }); + + it('accepts the conductor as a target', () => { + expect(grammar.validate(decision({ target: 'conductor' })).ok).toBe(true); + }); + + it('rejects a value that is not an object at all', () => { + expect(grammar.validate('sure, here you go: {}').ok).toBe(false); + expect(grammar.validate(null).ok).toBe(false); + expect(grammar.validate([decision()]).ok).toBe(false); + }); + + it('rejects an out-of-set tab action', () => { + const result = grammar.validate(decision({ tabAction: 'switch' as never })); + + expect(result.ok).toBe(false); + expect(result.errors.join(' ')).toContain('decision.tabAction'); + }); + + it('rejects an invented session id', () => { + const result = grammar.validate(decision({ target: { sessionId: 'agent-ghost' } })); + + expect(result.ok).toBe(false); + expect(result.errors.join(' ')).toContain('decision.target'); + }); + + it('rejects an invented tab id', () => { + const result = grammar.validate(decision({ tabAction: 'recall', tabId: 'tab-ghost' })); + + expect(result.ok).toBe(false); + expect(result.errors.join(' ')).toContain('decision.tabId'); + }); + + it('rejects a missing required field', () => { + const { prompt, ...withoutPrompt } = decision(); + void prompt; + + const result = grammar.validate(withoutPrompt); + + expect(result.ok).toBe(false); + expect(result.errors.join(' ')).toContain('decision.prompt is required'); + }); + + it('rejects a confidence outside 0 to 1', () => { + expect(grammar.validate(decision({ confidence: 7 })).ok).toBe(false); + expect(grammar.validate(decision({ confidence: -1 })).ok).toBe(false); + }); + + it('rejects a field nobody declared', () => { + const result = grammar.validate({ ...decision(), reasoning: 'because' }); + + expect(result.ok).toBe(false); + expect(result.errors.join(' ')).toContain('reasoning'); + }); + + it('leaves the ids free when the roster is empty', () => { + // An empty alternation would be a grammar that matches nothing, so an + // empty roster falls back to a plain string and validation carries the + // refusal instead. + const empty = compileRouteDecisionGrammar(rosterScope([])); + + expect(empty.gbnf).not.toContain('()'); + expect(empty.validate(decision()).ok).toBe(true); + }); +}); + +describe('routeDecisionSchema', () => { + it('narrows the hosted schema to the roster, leaving the shape intact', () => { + const schema = routeDecisionSchema(rosterScope(ROSTER)) as any; + + expect(schema.properties.tabId.enum).toEqual(['tab-auth', 'tab-db', 'tab-ui']); + const agentShape = schema.properties.target.oneOf.find((o: any) => o.type === 'object'); + expect(agentShape.properties.sessionId.enum).toEqual(['agent-backend', 'agent-frontend']); + expect(schema.required).toEqual(['target', 'tabAction', 'prompt', 'confidence']); + }); + + it('does not mutate the shared schema', () => { + const first = routeDecisionSchema(rosterScope(ROSTER)) as any; + const second = routeDecisionSchema() as any; + + expect(first.properties.tabId.enum).toBeDefined(); + expect(second.properties.tabId.enum).toBeUndefined(); + }); +}); + +describe('validateRouteDecision', () => { + it('runs for every provider, hosted ones included', () => { + // The hosted tier asks for a structured output and gets one; the roster is + // still the only thing that knows whether the id is real. + const fromHostedProvider = decision({ target: { sessionId: 'agent-ghost' } }); + + expect(validateRouteDecision(fromHostedProvider, ROSTER).ok).toBe(false); + }); + + it('rejects a recall whose tab belongs to a different agent', () => { + const result = validateRouteDecision( + decision({ target: { sessionId: 'agent-backend' }, tabAction: 'recall', tabId: 'tab-ui' }), + ROSTER + ); + + expect(result.ok).toBe(false); + expect(result.errors.join(' ')).toContain('is not a tab on "Backend"'); + }); + + it('rejects a recall with no tab id', () => { + const result = validateRouteDecision(decision({ tabAction: 'recall' }), ROSTER); + + expect(result.ok).toBe(false); + expect(result.errors.join(' ')).toContain('tabId is required'); + }); + + it('accepts a decision carrying a clarification', () => { + const result = validateRouteDecision( + decision({ target: 'conductor', confidence: 0.2, clarify: 'Backend or Frontend?' }), + ROSTER + ); + + expect(result.ok).toBe(true); + }); + + it('ignores undefined optional fields rather than calling them unknown', () => { + const result = validateRouteDecision( + { ...decision(), tabId: undefined, tabName: undefined, clarify: undefined }, + ROSTER + ); + + expect(result.ok).toBe(true); + }); +}); diff --git a/src/__tests__/main/acappella/router/routing-context.test.ts b/src/__tests__/main/acappella/router/routing-context.test.ts new file mode 100644 index 0000000000..4645336652 --- /dev/null +++ b/src/__tests__/main/acappella/router/routing-context.test.ts @@ -0,0 +1,212 @@ +/** + * @file routing-context.test.ts + * + * The bounding rules, the tab states, and the promise that no second summarizer + * exists: every topic here comes out of data the app already had. + */ + +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('../../../../main/stores/getters', () => ({ getSessionsStore: vi.fn() })); + +import { + buildRoutingContext, + buildRoutingRoster, + deriveTabTopic, + serializeRoutingContext, +} from '../../../../main/acappella/router/routing-context'; +import type { StoredSession } from '../../../../main/stores/types'; +import { createMockSession } from '../../../helpers/mockSession'; +import { createMockAITab } from '../../../helpers/mockTab'; + +function makeSession(overrides: Partial = {}): StoredSession { + return createMockSession(overrides as never) as unknown as StoredSession; +} + +function sessions(): StoredSession[] { + return [ + makeSession({ + id: 'agent-backend', + name: 'Backend', + toolType: 'claude-code', + cwd: '/repo/api', + state: 'idle', + aiTabs: [ + createMockAITab({ + id: 'tab-auth', + name: 'Auth Refactor', + createdAt: 1_000, + logs: [ + { id: 'l1', timestamp: 2_000, source: 'user', text: 'rewrite the auth middleware' }, + { id: 'l2', timestamp: 4_000, source: 'ai', text: 'done' }, + ] as never, + }), + createMockAITab({ id: 'tab-db', name: 'DB Migrations', createdAt: 9_000 }), + ], + }), + ]; +} + +describe('buildRoutingRoster', () => { + it('lists open tabs with a topic derived from the conversation', () => { + const [agent] = buildRoutingRoster(sessions()); + + expect(agent.tabs[0]).toMatchObject({ + id: 'tab-auth', + name: 'Auth Refactor', + state: 'open', + topic: 'rewrite the auth middleware', + lastActiveAt: 4_000, + }); + }); + + it('lists snoozed tabs so recall can reach them', () => { + const list = sessions(); + list[0].snoozedTabs = [ + { + id: 'snooze-1', + tab: createMockAITab({ id: 'tab-spike', name: 'Rate Limit Spike', createdAt: 500 }), + unifiedIndex: 0, + snoozedAt: 100, + wakeAt: 999_999, + }, + ]; + + const [agent] = buildRoutingRoster(list); + + expect(agent.tabs.find((tab) => tab.id === 'tab-spike')).toMatchObject({ state: 'snoozed' }); + }); + + it('lists closed AI tabs and ignores closed file and terminal tabs', () => { + const list = sessions(); + list[0].unifiedClosedTabHistory = [ + { type: 'ai', tab: createMockAITab({ id: 'tab-old', name: 'Old Spike' }), index: 0 }, + { type: 'terminal', tab: { id: 'term-1' }, index: 1 }, + ]; + + const [agent] = buildRoutingRoster(list); + + expect(agent.tabs.find((tab) => tab.id === 'tab-old')).toMatchObject({ state: 'closed' }); + expect(agent.tabs.some((tab) => tab.id === 'term-1')).toBe(false); + }); + + it('omits hidden consult tabs', () => { + const list = sessions(); + list[0].aiTabs.push(createMockAITab({ id: 'tab-consult', name: 'Consult', hidden: true })); + + const [agent] = buildRoutingRoster(list); + + expect(agent.tabs.some((tab) => tab.id === 'tab-consult')).toBe(false); + }); + + it('keeps the open copy when a tab appears open and closed at once', () => { + const list = sessions(); + list[0].unifiedClosedTabHistory = [ + { type: 'ai', tab: createMockAITab({ id: 'tab-auth', name: 'Auth Refactor' }), index: 0 }, + ]; + + const [agent] = buildRoutingRoster(list); + + expect(agent.tabs.filter((tab) => tab.id === 'tab-auth')).toHaveLength(1); + expect(agent.tabs.find((tab) => tab.id === 'tab-auth')?.state).toBe('open'); + }); + + it('survives a session with no tabs at all', () => { + expect(buildRoutingRoster([makeSession({ id: 'a1', aiTabs: undefined })])[0].tabs).toEqual([]); + }); +}); + +describe('deriveTabTopic', () => { + it('prefers the opening message, which the name already compresses', () => { + const tab = { + logs: [{ source: 'user', text: 'why is the migration locking the users table' }], + }; + + expect(deriveTabTopic(tab, 'DB Migrations')).toBe( + 'why is the migration locking the users table' + ); + }); + + it('falls back to the name when the transcript has no user message', () => { + expect(deriveTabTopic({ logs: [] }, 'DB Migrations')).toBe('DB Migrations'); + }); + + it('is null when there is nothing to say', () => { + expect(deriveTabTopic({ logs: [] }, null)).toBeNull(); + }); + + it('truncates and collapses so a pasted stack trace is still one line', () => { + const topic = deriveTabTopic({ logs: [{ source: 'user', text: 'x\n\ty '.repeat(200) }] }, null); + + expect(topic).toMatch(/…$/); + expect(topic!.length).toBeLessThanOrEqual(90); + expect(topic).not.toContain('\n'); + }); +}); + +describe('buildRoutingContext', () => { + it('carries the agent status and the history synopsis', () => { + const context = buildRoutingContext({ + sessions: sessions(), + synopses: new Map([['agent-backend', 'Landed the auth refactor']]), + }); + + expect(context.agents[0].status).toBe('idle'); + expect(context.agents[0].recentWork).toBe('Landed the auth refactor'); + expect(serializeRoutingContext(context)).toContain('recently: Landed the auth refactor'); + }); + + it('reports its own serialized size', () => { + const context = buildRoutingContext({ sessions: sessions() }); + + expect(context.serializedChars).toBe(serializeRoutingContext(context).length); + }); + + it('drops the least recently used tabs to stay under the cap, and says how many', () => { + const many = makeSession({ + id: 'agent-busy', + name: 'Busy', + aiTabs: Array.from({ length: 40 }, (_, index) => + createMockAITab({ + id: `tab-${index}`, + name: `Conversation number ${index}`, + createdAt: index, + }) + ), + }); + + const context = buildRoutingContext({ sessions: [many], maxChars: 400 }); + + expect(context.droppedTabs).toBeGreaterThan(0); + expect(context.serializedChars).toBeLessThanOrEqual(400); + // What survives is what the user was most recently doing. + expect(context.agents[0].tabs.map((tab) => tab.id)).toContain('tab-39'); + expect(context.agents[0].tabs.map((tab) => tab.id)).not.toContain('tab-0'); + }); + + it('never drops an agent, even under an impossible cap', () => { + const context = buildRoutingContext({ sessions: sessions(), maxChars: 1 }); + + // An agent missing from the roster cannot be routed to at all; a missing + // tab only costs a recall the user can repeat with more words. + expect(context.agents).toHaveLength(1); + expect(context.agents[0].tabs.length).toBeGreaterThan(0); + }); + + it('serializes an empty roster without pretending anything is running', () => { + const context = buildRoutingContext({ sessions: [] }); + + expect(serializeRoutingContext(context)).toContain('(none)'); + }); + + it('includes the voice conversation, not the agent transcripts', () => { + const context = buildRoutingContext({ + sessions: sessions(), + recentUtterances: ['run the tests', 'what broke'], + }); + + const text = serializeRoutingContext(context); + expect(text).toContain('Earlier in this conversation:'); + expect(text).toContain('what broke'); + }); +}); diff --git a/src/__tests__/main/acappella/router/routing-log.test.ts b/src/__tests__/main/acappella/router/routing-log.test.ts new file mode 100644 index 0000000000..743159def3 --- /dev/null +++ b/src/__tests__/main/acappella/router/routing-log.test.ts @@ -0,0 +1,209 @@ +/** + * @file routing-log.test.ts + * + * The log, and the one number it exists to produce: a hit rate that counts a + * dispatch the user immediately corrected as a miss. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; + +vi.mock('electron', () => ({ app: { getPath: () => '/nonexistent' } })); +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +import { + flushRoutingLog, + lastRoutingTurn, + loadRoutingLog, + MAX_ENTRIES, + noteRoutingOutcome, + readRoutingLog, + recordRoutingTurn, + resetRoutingLog, + routingQuality, + setRoutingLogPath, +} from '../../../../main/acappella/router/routing-log'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; + +function decision(overrides: Partial = {}): RouteDecision { + return { + target: { sessionId: 'agent-backend' }, + tabAction: 'current', + prompt: 'run the tests', + confidence: 0.9, + ...overrides, + }; +} + +function record(id: string, overrides: Partial[0]> = {}) { + return recordRoutingTurn({ + id, + utterance: 'run the tests', + decision: decision(), + brainProviderId: 'qwen3-local', + latencyMs: 120, + contextChars: 900, + ...overrides, + }); +} + +let tempDir: string; + +beforeEach(async () => { + resetRoutingLog(); + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-routing-log-')); + setRoutingLogPath(path.join(tempDir, 'routing-log.json')); +}); + +afterEach(async () => { + resetRoutingLog(); + setRoutingLogPath(null); + await fs.rm(tempDir, { recursive: true, force: true }); +}); + +describe('recordRoutingTurn', () => { + it('flattens the decision into one readable entry', () => { + record('turn-1'); + + expect(readRoutingLog()[0]).toMatchObject({ + id: 'turn-1', + utterance: 'run the tests', + targetSessionId: 'agent-backend', + tabAction: 'current', + confidence: 0.9, + latencyMs: 120, + contextChars: 900, + outcome: 'dispatched', + }); + }); + + it('records a question as clarified rather than dispatched', () => { + record('turn-1', { decision: decision({ clarify: 'Backend or API?', confidence: 0.3 }) }); + + expect(readRoutingLog()[0].outcome).toBe('clarified'); + }); + + it('truncates the utterance: this is a log, not a transcript', () => { + record('turn-1', { utterance: 'a'.repeat(1000) }); + + expect(readRoutingLog()[0].utterance.length).toBeLessThanOrEqual(200); + }); + + it('keeps the newest entries when it overflows', () => { + for (let index = 0; index < MAX_ENTRIES + 10; index++) record(`turn-${index}`); + + const entries = readRoutingLog(); + expect(entries).toHaveLength(MAX_ENTRIES); + expect(entries[entries.length - 1].id).toBe(`turn-${MAX_ENTRIES + 9}`); + }); + + it('hands back a copy, so a reader cannot rewrite history', () => { + record('turn-1'); + + readRoutingLog()[0].outcome = 'failed'; + + expect(readRoutingLog()[0].outcome).toBe('dispatched'); + }); +}); + +describe('noteRoutingOutcome', () => { + it('attaches what actually happened', () => { + record('turn-1'); + + noteRoutingOutcome('turn-1', 'corrected', 'moved to API'); + + expect(readRoutingLog()[0]).toMatchObject({ outcome: 'corrected', detail: 'moved to API' }); + }); + + it('ignores an id that has aged out', () => { + record('turn-1'); + + expect(() => noteRoutingOutcome('turn-gone', 'failed')).not.toThrow(); + expect(readRoutingLog()).toHaveLength(1); + }); +}); + +describe('routingQuality', () => { + it('counts a correction as a miss even though nothing errored', () => { + record('turn-1'); + record('turn-2'); + record('turn-3'); + noteRoutingOutcome('turn-2', 'corrected'); + + const quality = routingQuality(); + + expect(quality).toMatchObject({ turns: 3, dispatched: 2, corrected: 1 }); + expect(quality.hitRate).toBeCloseTo(2 / 3); + }); + + it('excludes clarifications from the hit rate entirely', () => { + record('turn-1'); + record('turn-2', { decision: decision({ clarify: 'Backend or API?' }) }); + + const quality = routingQuality(); + + expect(quality.clarified).toBe(1); + // Asking is the correct behaviour below the threshold. Counting it either + // way would make the threshold impossible to tune. + expect(quality.hitRate).toBe(1); + }); + + it('reports no hit rate before anything has been decided', () => { + expect(routingQuality().hitRate).toBeNull(); + }); + + it('averages the routing latency', () => { + record('turn-1', { latencyMs: 100 }); + record('turn-2', { latencyMs: 300 }); + + expect(routingQuality().meanLatencyMs).toBe(200); + }); +}); + +describe('persistence', () => { + it('writes atomically and reads back', async () => { + record('turn-1'); + await flushRoutingLog(); + + resetRoutingLog(); + await loadRoutingLog(); + + expect(readRoutingLog()[0].id).toBe('turn-1'); + }); + + it('starts fresh rather than failing when the file is unreadable', async () => { + await fs.writeFile(path.join(tempDir, 'routing-log.json'), 'not json'); + + await loadRoutingLog(); + + expect(readRoutingLog()).toEqual([]); + }); + + it('survives a directory that does not exist yet', async () => { + setRoutingLogPath(path.join(tempDir, 'nested', 'deeper', 'routing-log.json')); + record('turn-1'); + + await flushRoutingLog(); + + const raw = await fs.readFile( + path.join(tempDir, 'nested', 'deeper', 'routing-log.json'), + 'utf-8' + ); + expect(JSON.parse(raw)[0].id).toBe('turn-1'); + }); +}); + +describe('lastRoutingTurn', () => { + it('is null before the first decision, then the newest one', () => { + expect(lastRoutingTurn()).toBeNull(); + + record('turn-1'); + record('turn-2'); + + expect(lastRoutingTurn()?.id).toBe('turn-2'); + }); +}); diff --git a/src/__tests__/main/acappella/router/tab-recall.test.ts b/src/__tests__/main/acappella/router/tab-recall.test.ts new file mode 100644 index 0000000000..f1aa1cc04d --- /dev/null +++ b/src/__tests__/main/acappella/router/tab-recall.test.ts @@ -0,0 +1,217 @@ +/** + * @file tab-recall.test.ts + * + * Ranking, and the two tab states that have a wrong answer which looks like + * success: a snoozed tab focused without being woken, and a closed tab quietly + * replaced by an empty new one. + * + * `now` is injected throughout, because recency is half the score and a suite + * whose expectations drift with the wall clock is worse than no suite. + */ + +import { describe, it, expect } from 'vitest'; + +import { + narrowRosterForRecall, + rankRecallCandidates, + resolveRecall, +} from '../../../../main/acappella/router/tab-recall'; +import type { RosterAgent, RosterTab } from '../../../../shared/acappella/protocol'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; + +const NOW = 1_000_000_000; +const MINUTE = 60_000; +const DAY = 24 * 60 * MINUTE; + +function tab(overrides: Partial & { id: string }): RosterTab { + return { name: null, lastActiveAt: NOW - MINUTE, state: 'open', topic: null, ...overrides }; +} + +function roster(): RosterAgent[] { + return [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/payments-api', + tabs: [ + tab({ id: 'tab-auth', name: 'Auth Refactor', lastActiveAt: NOW - 3 * DAY }), + tab({ id: 'tab-db', name: 'DB Migrations', lastActiveAt: NOW - MINUTE }), + ], + }, + { + sessionId: 'agent-frontend', + name: 'Frontend', + agentType: 'codex', + cwd: '/repo/web', + tabs: [ + tab({ + id: 'tab-ui', + name: 'Sidebar', + topic: 'make the sidebar collapse on narrow screens', + lastActiveAt: NOW - 2 * MINUTE, + }), + ], + }, + ]; +} + +function decision(overrides: Partial = {}): RouteDecision { + return { + target: { sessionId: 'agent-backend' }, + tabAction: 'recall', + prompt: 'did we land that fix?', + confidence: 0.7, + ...overrides, + }; +} + +// --------------------------------------------------------------------------- +// Ranking +// --------------------------------------------------------------------------- + +describe('rankRecallCandidates', () => { + it('puts a name match ahead of a much more recent tab', () => { + const ranked = rankRecallCandidates('back to the auth thing', roster(), { now: NOW }); + + // tab-db was touched a minute ago and tab-auth three days ago. Recall that + // meant "the most recent tab" would be a feature nobody needs to speak. + expect(ranked[0].tab.id).toBe('tab-auth'); + }); + + it('matches on the topic when the tab was never named after it', () => { + const ranked = rankRecallCandidates('the sidebar collapse conversation', roster(), { + now: NOW, + }); + + expect(ranked[0].tab.id).toBe('tab-ui'); + expect(ranked[0].reasons.join(' ')).toContain('Sidebar'); + }); + + it('scores a mentioned project path', () => { + const ranked = rankRecallCandidates('what did payments say', roster(), { now: NOW }); + + expect(ranked.map((candidate) => candidate.agentSessionId)).toContain('agent-backend'); + expect(ranked[0].reasons.join(' ')).toContain('project path mentioned'); + }); + + it('biases toward the agent already in play when scores are otherwise level', () => { + const withBias = rankRecallCandidates('migrations', roster(), { + now: NOW, + activeAgentSessionId: 'agent-backend', + }); + + expect(withBias[0].agentSessionId).toBe('agent-backend'); + expect(withBias[0].reasons).toContain('same agent as the current turn'); + }); + + it('returns nothing rather than padding the list with noise', () => { + expect(rankRecallCandidates('what is the weather', roster(), { now: NOW })).toEqual([]); + }); + + it('honours the limit', () => { + const ranked = rankRecallCandidates('auth db sidebar', roster(), { now: NOW, limit: 1 }); + + expect(ranked).toHaveLength(1); + }); + + it('ignores stop words, so a filler-only utterance matches nothing', () => { + expect(rankRecallCandidates('go back to the one thing', roster(), { now: NOW })).toEqual([]); + }); +}); + +describe('narrowRosterForRecall', () => { + it('keeps every agent, even one with no shortlisted tab', () => { + const agents = roster(); + const candidates = rankRecallCandidates('auth', agents, { now: NOW }); + + const narrowed = narrowRosterForRecall(agents, candidates); + + expect(narrowed.map((agent) => agent.sessionId)).toEqual(['agent-backend', 'agent-frontend']); + }); + + it('keeps open tabs and drops the put-away ones that did not shortlist', () => { + const agents = roster(); + agents[0].tabs.push(tab({ id: 'tab-old', name: 'Old Spike', state: 'closed' })); + const candidates = rankRecallCandidates('auth', agents, { now: NOW }); + + const narrowed = narrowRosterForRecall(agents, candidates); + + const ids = narrowed.flatMap((agent) => agent.tabs.map((entry) => entry.id)); + expect(ids).toContain('tab-db'); + expect(ids).not.toContain('tab-old'); + }); + + it('keeps a shortlisted closed tab', () => { + const agents = roster(); + agents[0].tabs.push(tab({ id: 'tab-old', name: 'Auth Spike', state: 'closed' })); + const candidates = rankRecallCandidates('auth spike', agents, { now: NOW }); + + const narrowed = narrowRosterForRecall(agents, candidates); + + expect(narrowed[0].tabs.map((entry) => entry.id)).toContain('tab-old'); + }); +}); + +// --------------------------------------------------------------------------- +// Resolution +// --------------------------------------------------------------------------- + +describe('resolveRecall', () => { + it('focuses an open tab', () => { + const result = resolveRecall(decision({ tabId: 'tab-auth' }), roster()); + + expect(result).toMatchObject({ kind: 'focus', agentSessionId: 'agent-backend' }); + }); + + it('wakes a snoozed tab rather than focusing a tab that is not on screen', () => { + const agents = roster(); + agents[0].tabs[0] = { ...agents[0].tabs[0], state: 'snoozed' }; + + const result = resolveRecall(decision({ tabId: 'tab-auth' }), agents); + + expect(result.kind).toBe('wake'); + }); + + it('offers to reopen a closed tab instead of duplicating it', () => { + const agents = roster(); + agents[0].tabs[0] = { ...agents[0].tabs[0], state: 'closed' }; + + const result = resolveRecall(decision({ tabId: 'tab-auth' }), agents); + + expect(result.kind).toBe('offer'); + if (result.kind !== 'offer') throw new Error('expected an offer'); + expect(result.question).toContain('Auth Refactor'); + expect(result.question).toContain('reopen'); + }); + + it('reopens a closed tab once the offer has been answered', () => { + const agents = roster(); + agents[0].tabs[0] = { ...agents[0].tabs[0], state: 'closed' }; + + const result = resolveRecall(decision({ tabId: 'tab-auth' }), agents, { confirmed: true }); + + expect(result.kind).toBe('reopen'); + }); + + it('finds a tab the Brain attributed to the wrong agent', () => { + // The conversation was identified correctly; only the owner was wrong. + const result = resolveRecall( + decision({ target: { sessionId: 'agent-backend' }, tabId: 'tab-ui' }), + roster() + ); + + expect(result).toMatchObject({ kind: 'focus', agentSessionId: 'agent-frontend' }); + }); + + it('reports a tab that no longer exists as missing', () => { + expect(resolveRecall(decision({ tabId: 'tab-ghost' }), roster())).toEqual({ + kind: 'missing', + tabId: 'tab-ghost', + }); + }); + + it('reports a recall with no tab id as missing', () => { + expect(resolveRecall(decision(), roster())).toEqual({ kind: 'missing', tabId: undefined }); + }); +}); diff --git a/src/__tests__/main/acappella/runtime/native-loader.test.ts b/src/__tests__/main/acappella/runtime/native-loader.test.ts new file mode 100644 index 0000000000..7cf6a6c628 --- /dev/null +++ b/src/__tests__/main/acappella/runtime/native-loader.test.ts @@ -0,0 +1,442 @@ +/** + * @file native-loader.test.ts + * + * The loader has three promises, and every one of them is invisible in + * development and expensive in production: + * + * 1. Nothing native is imported until someone asks for it. A user with the + * Encore Feature off must not pay a single dlopen, and the way that breaks + * is a static import added anywhere else in the codebase, so the first test + * here scans the source rather than the runtime. + * 2. A failure comes back structured. A dlopen error string in front of a user + * is a bug report with no information in it. + * 3. A failed load never takes the app down. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import fs from 'fs'; +import path from 'path'; + +/** + * A stand-in registry. + * + * The real one has every runtime at `declared: false` (the packages land with + * the providers in Phase 05), so testing the load paths against it would only + * ever exercise the not-a-dependency branch. This mock keeps the same shape and + * the same helper behaviour with one declared and one undeclared runtime. + */ +vi.mock('../../../../shared/acappella/native-runtimes', () => { + const runtimes = [ + { + id: 'llama', + moduleId: 'fake-llama', + versionPin: '1.0.0', + label: 'Fake llama', + slots: ['brain'], + declared: true, + requiresElectronRebuild: false, + prebuilds: { + 'darwin-arm64': 'prebuilt', + 'darwin-x64': 'prebuilt', + 'win32-x64': 'prebuilt', + 'linux-x64': 'prebuilt', + }, + asarUnpack: [], + packagedBinaries: { + 'darwin-arm64': [], + 'darwin-x64': [], + 'win32-x64': [], + 'linux-x64': [], + }, + rationale: '', + notes: '', + }, + { + id: 'whisper', + moduleId: 'fake-whisper', + versionPin: '1.0.0', + label: 'Fake whisper', + slots: ['stt'], + declared: false, + requiresElectronRebuild: false, + prebuilds: { + 'darwin-arm64': 'source-build', + 'darwin-x64': 'source-build', + 'win32-x64': 'source-build', + 'linux-x64': 'source-build', + }, + asarUnpack: [], + packagedBinaries: { + 'darwin-arm64': [], + 'darwin-x64': [], + 'win32-x64': [], + 'linux-x64': [], + }, + rationale: '', + notes: '', + }, + { + id: 'onnx', + moduleId: 'fake-onnx', + versionPin: '1.0.0', + label: 'Fake onnx', + slots: ['tts', 'wake-word'], + declared: true, + requiresElectronRebuild: false, + // Deliberately shipped nowhere, to exercise the unsupported-platform path. + prebuilds: { + 'darwin-arm64': 'unavailable', + 'darwin-x64': 'unavailable', + 'win32-x64': 'unavailable', + 'linux-x64': 'unavailable', + }, + asarUnpack: [], + packagedBinaries: { + 'darwin-arm64': [], + 'darwin-x64': [], + 'win32-x64': [], + 'linux-x64': [], + }, + rationale: '', + notes: '', + }, + ]; + + return { + NATIVE_RUNTIMES: runtimes, + getNativeRuntime: (id: string) => runtimes.find((runtime) => runtime.id === id), + nativePlatformKey: (platform: string, arch: string) => { + const key = `${platform}-${arch}`; + return ['darwin-arm64', 'darwin-x64', 'win32-x64', 'linux-x64'].includes(key) ? key : null; + }, + }; +}); + +import { + NativeRuntimeUnavailableError, + __setNativeImporter, + allNativeRuntimeFailures, + describeRuntimeUnavailable, + isNativeRuntimeLoaded, + knownNativeRuntimeUnavailability, + lastNativeRuntimeFailure, + loadNativeRuntime, + resetNativeRuntimes, + tryLoadNativeRuntime, + unloadNativeRuntime, +} from '../../../../main/acappella/runtime/native-loader'; + +const REAL_PLATFORM = process.platform; +const REAL_ARCH = process.arch; + +function setPlatform(platform: string, arch: string): void { + Object.defineProperty(process, 'platform', { value: platform, configurable: true }); + Object.defineProperty(process, 'arch', { value: arch, configurable: true }); +} + +function moduleNotFound(): NodeJS.ErrnoException { + const error: NodeJS.ErrnoException = new Error("Cannot find module 'fake-llama'"); + error.code = 'MODULE_NOT_FOUND'; + return error; +} + +describe('native-loader', () => { + beforeEach(() => { + resetNativeRuntimes(); + setPlatform('darwin', 'arm64'); + }); + + afterEach(() => { + __setNativeImporter(null); + resetNativeRuntimes(); + setPlatform(REAL_PLATFORM, REAL_ARCH); + }); + + describe('lazy loading', () => { + it('imports nothing until a runtime is asked for', async () => { + const importer = vi.fn().mockResolvedValue({}); + __setNativeImporter(importer); + + // Importing this module, and reading the registry through it, must not + // have reached the module system. + expect(importer).not.toHaveBeenCalled(); + expect(isNativeRuntimeLoaded('llama')).toBe(false); + + await tryLoadNativeRuntime('llama'); + expect(importer).toHaveBeenCalledWith('fake-llama'); + }); + + it('loads a runtime once, however many callers ask', async () => { + const importer = vi.fn().mockResolvedValue({ marker: 1 }); + __setNativeImporter(importer); + + const [first, second] = await Promise.all([ + loadNativeRuntime<{ marker: number }>('llama'), + loadNativeRuntime<{ marker: number }>('llama'), + ]); + + expect(first).toBe(second); + expect(importer).toHaveBeenCalledTimes(1); + }); + + it('is the only module in the codebase that imports a native runtime', () => { + // The runtime invariant cannot be observed at runtime: a stray top-level + // `import 'node-llama-cpp'` in some provider would break the "Encore off + // costs nothing" property silently and permanently. So the source itself + // is the assertion. + const roots = ['src/main', 'src/renderer', 'src/shared', 'src/cli']; + const nativeModules = ['node-llama-cpp', 'smart-whisper', 'onnxruntime-node']; + const offenders: string[] = []; + + for (const root of roots) { + for (const file of walkTypeScript(path.resolve(process.cwd(), root))) { + const source = fs.readFileSync(file, 'utf8'); + for (const moduleId of nativeModules) { + // A static import or require of the package by name. The loader + // reaches these through a variable specifier, so it never matches. + const staticImport = new RegExp( + `(from\\s+['"]${moduleId}['"])|(require\\(['"]${moduleId}['"]\\))|(import\\(['"]${moduleId}['"]\\))` + ); + if (staticImport.test(source)) offenders.push(`${file} -> ${moduleId}`); + } + } + } + + expect(offenders).toEqual([]); + }); + }); + + describe('structured failures', () => { + it('reports a missing module with the runtime, module, platform, and cause', async () => { + __setNativeImporter(vi.fn().mockRejectedValue(moduleNotFound())); + + const result = await tryLoadNativeRuntime('llama'); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.kind).toBe('runtime-unavailable'); + expect(result.error.failure).toBe('module-not-found'); + expect(result.error.runtimeId).toBe('llama'); + expect(result.error.moduleId).toBe('fake-llama'); + expect(result.error.platform).toBe('darwin'); + expect(result.error.arch).toBe('arm64'); + expect(result.error.detail).toContain('Cannot find module'); + // Never a bare dlopen string: there is always a sentence and a next step. + expect(result.error.message).not.toBe(''); + expect(result.error.suggestedAction).not.toBe(''); + }); + + it('classifies an arbitrary load error as load-failed, keeping the cause', async () => { + __setNativeImporter( + vi.fn().mockRejectedValue(new Error('dlopen(libggml.dylib): symbol not found')) + ); + + const result = await tryLoadNativeRuntime('llama'); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.failure).toBe('load-failed'); + expect(result.error.detail).toContain('symbol not found'); + }); + + it('names the Visual C++ redistributable when Windows cannot find a dependent DLL', async () => { + setPlatform('win32', 'x64'); + __setNativeImporter( + vi + .fn() + .mockRejectedValue( + new Error('\\\\?\\C:\\app\\llama-addon.node: The specified module could not be found.') + ) + ); + + const result = await tryLoadNativeRuntime('llama'); + + expect(result.ok).toBe(false); + if (result.ok) return; + // Windows names the addon rather than the DLL it actually wanted, so the + // raw message reads like a corrupt install and sends the user to reinstall + // the app, which changes nothing. + expect(result.error.suggestedAction).toContain('Visual C++ Redistributable'); + }); + + it('says "not a dependency" without touching the module system', async () => { + const importer = vi.fn(); + __setNativeImporter(importer); + + const result = await tryLoadNativeRuntime('whisper'); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.failure).toBe('not-a-dependency'); + // The distinction that matters: this is not a broken install, so nothing + // was attempted and nothing should be reinstalled. + expect(importer).not.toHaveBeenCalled(); + }); + + it('says "unsupported platform" for a runtime with no build here', async () => { + const importer = vi.fn(); + __setNativeImporter(importer); + + const result = await tryLoadNativeRuntime('onnx'); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.failure).toBe('unsupported-platform'); + expect(importer).not.toHaveBeenCalled(); + }); + + it('reports an unknown platform/arch pair as unsupported rather than crashing', async () => { + setPlatform('sunos', 'sparc'); + __setNativeImporter(vi.fn()); + + const result = await tryLoadNativeRuntime('llama'); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.failure).toBe('unsupported-platform'); + expect(result.error.message).toContain('sunos-sparc'); + }); + + it('formats a failure as one line for a log or support report', () => { + const line = describeRuntimeUnavailable({ + kind: 'runtime-unavailable', + runtimeId: 'llama', + moduleId: 'fake-llama', + platform: 'darwin', + arch: 'arm64', + failure: 'load-failed', + message: 'Fake llama failed to load.', + suggestedAction: 'Run the self-test.', + detail: 'symbol not found', + }); + + expect(line).toContain('fake-llama'); + expect(line).toContain('load-failed'); + expect(line).toContain('symbol not found'); + }); + }); + + describe('a failed load does not crash the app', () => { + it('never rejects from tryLoadNativeRuntime', async () => { + __setNativeImporter(vi.fn().mockRejectedValue(new Error('boom'))); + await expect(tryLoadNativeRuntime('llama')).resolves.toMatchObject({ ok: false }); + }); + + it('throws a typed error from loadNativeRuntime, carrying the same structure', async () => { + __setNativeImporter(vi.fn().mockRejectedValue(moduleNotFound())); + + await expect(loadNativeRuntime('llama')).rejects.toBeInstanceOf( + NativeRuntimeUnavailableError + ); + await expect(loadNativeRuntime('llama')).rejects.toMatchObject({ + info: { failure: 'module-not-found', moduleId: 'fake-llama' }, + }); + }); + + it('reports an unknown runtime id instead of throwing', async () => { + const result = await tryLoadNativeRuntime('nope' as 'llama'); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.message).toContain('Unknown native runtime'); + }); + }); + + describe('remembered failures', () => { + it('remembers the last failure so the capability gate can explain it without loading', async () => { + __setNativeImporter(vi.fn().mockRejectedValue(new Error('boom'))); + await tryLoadNativeRuntime('llama'); + + expect(lastNativeRuntimeFailure('llama')?.failure).toBe('load-failed'); + expect(allNativeRuntimeFailures()).toHaveLength(1); + }); + + it('retries after a failure rather than replaying it forever', async () => { + const importer = vi + .fn() + .mockRejectedValueOnce(new Error('boom')) + .mockResolvedValueOnce({ ok: true }); + __setNativeImporter(importer); + + expect((await tryLoadNativeRuntime('llama')).ok).toBe(false); + // The user installed the missing piece; the next attempt must be real. + expect((await tryLoadNativeRuntime('llama')).ok).toBe(true); + expect(importer).toHaveBeenCalledTimes(2); + expect(lastNativeRuntimeFailure('llama')).toBeNull(); + }); + + it('forgets a runtime on unload so a later load is a fresh attempt', async () => { + const importer = vi.fn().mockResolvedValue({}); + __setNativeImporter(importer); + + await tryLoadNativeRuntime('llama'); + expect(isNativeRuntimeLoaded('llama')).toBe(true); + + unloadNativeRuntime('llama'); + expect(isNativeRuntimeLoaded('llama')).toBe(false); + + await tryLoadNativeRuntime('llama'); + expect(importer).toHaveBeenCalledTimes(2); + }); + }); + + /** + * The question a capability gate actually has is "will this work here", not + * "has this already gone wrong here". Answering the second one in place of the + * first is how readiness came back "everything satisfied" on a fresh boot for + * runtimes that are not in the build at all, and the opposite answer once + * anything had attempted a load. + */ + describe('known unavailability, without loading', () => { + it('reports a runtime that is not a dependency before anything tries it', () => { + const importer = vi.fn(); + __setNativeImporter(importer); + + const verdict = knownNativeRuntimeUnavailability('whisper'); + + expect(verdict?.failure).toBe('not-a-dependency'); + expect(importer).not.toHaveBeenCalled(); + }); + + it('reports a platform with no build before anything tries it', () => { + setPlatform('darwin', 'arm64'); + + // `onnx` is declared but shipped nowhere in the stand-in registry. + expect(knownNativeRuntimeUnavailability('onnx')?.failure).toBe('unsupported-platform'); + }); + + it('is null for a runtime that should load', () => { + setPlatform('darwin', 'arm64'); + + expect(knownNativeRuntimeUnavailability('llama')).toBeNull(); + }); + + it('prefers what actually happened over what the registry predicts', async () => { + setPlatform('darwin', 'arm64'); + __setNativeImporter(vi.fn().mockRejectedValue(new Error('boom'))); + await tryLoadNativeRuntime('llama'); + + expect(knownNativeRuntimeUnavailability('llama')?.failure).toBe('load-failed'); + }); + + it('does not record a failure nobody hit', () => { + knownNativeRuntimeUnavailability('whisper'); + + // Asking must not put anything in the support report: the debug package + // lists failures that HAPPENED, not answers to hypothetical questions. + expect(allNativeRuntimeFailures()).toHaveLength(0); + expect(lastNativeRuntimeFailure('whisper')).toBeNull(); + }); + }); +}); + +/** Every .ts/.tsx file under a directory, skipping the test tree itself. */ +function walkTypeScript(dir: string, out: string[] = []): string[] { + if (!fs.existsSync(dir)) return out; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.name === 'node_modules' || entry.name === '__tests__') continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walkTypeScript(full, out); + else if (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx')) out.push(full); + } + return out; +} diff --git a/src/__tests__/main/acappella/runtime/runtime-installer.test.ts b/src/__tests__/main/acappella/runtime/runtime-installer.test.ts new file mode 100644 index 0000000000..51cf3f2fcf --- /dev/null +++ b/src/__tests__/main/acappella/runtime/runtime-installer.test.ts @@ -0,0 +1,323 @@ +/** + * @file runtime-installer.test.ts + * + * The install transaction, driven end to end against a real tarball with no + * network. Everything here is offline and deterministic: the archive is built in + * a temp directory, hashed, and handed back through an injected `fetch`, so the + * test exercises the real streaming download, the real SHA-256 comparison, the + * real node-tar extraction, and the real promote-and-commit ordering. + * + * The properties worth protecting, each of which was a design decision rather + * than an accident: + * + * - Nothing appears at the install path until the manifest is written, so a + * failure part-way through can never look like a finished install. + * - A hash mismatch is fatal and leaves no bytes behind, because the payload is + * code that will later be dlopen'd. + * - Only the running platform's subtree is written, which is the entire reason + * a 101 MB download costs 37 MB of disk. + * - A reinstall REPLACES rather than merges, so a stale binary cannot outlive + * the version that shipped it. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { createHash } from 'crypto'; +import * as fs from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; +import * as tar from 'tar'; + +/** + * A synthetic payload shaped like the real ONNX Runtime tarball: an npm root, a + * little JavaScript, the platform we want, a platform we do not, and a file + * outside every kept prefix. + */ +const ARCHIVE_FILES: Record = { + 'package/package.json': '{"name":"fake-runtime","main":"dist/index.js"}', + 'package/dist/index.js': 'module.exports = {};', + 'package/bin/napi-v6/darwin/arm64/onnxruntime_binding.node': 'ARM64 BINDING', + 'package/bin/napi-v6/darwin/arm64/libonnxruntime.1.dylib': 'ARM64 DYLIB', + 'package/bin/napi-v6/win32/x64/onnxruntime_binding.node': 'WINDOWS BINDING', + 'package/bin/napi-v6/win32/x64/onnxruntime.dll': 'WINDOWS DLL', + 'package/README.md': 'not kept', +}; + +/** + * The artifact the store and installer see, mutated per test. + * + * Hoisted because `vi.mock` factories run before the module body, and a factory + * that closed over an ordinary `const` would read it before initialisation. + */ +const { artifact } = vi.hoisted(() => ({ + artifact: { + runtimeId: 'onnx' as const, + platform: 'darwin-arm64' as const, + url: 'https://registry.npmjs.org/fake-runtime/-/fake-runtime-1.27.0.tgz', + sha256: '', + bytes: 0, + stripComponents: 1, + keep: ['dist', 'package.json', 'bin/napi-v6/darwin/arm64'], + entry: 'dist/index.js', + binary: 'bin/napi-v6/darwin/arm64/onnxruntime_binding.node', + }, +})); + +const ENTRY = artifact.entry; +const BINARY = artifact.binary; +const URL = artifact.url; + +let tempRoot: string; +let tarballBytes: Buffer; +let tarballSha256: string; + +vi.mock('../../../../shared/acappella/runtime-artifacts', () => ({ + nativeRuntimeArtifact: () => artifact, + isNativeRuntimeDownloadable: () => true, + nativeRuntimeDownloadBytes: () => artifact.bytes, + NATIVE_RUNTIME_ARTIFACTS: [artifact], +})); + +import { + installNativeRuntime, + RuntimeBinaryMissingError, + RuntimeHashMismatchError, +} from '../../../../main/acappella/runtime/runtime-installer'; +import { + installedRuntimeEntry, + isRuntimeInstalled, + readRuntimeManifest, + removeRuntime, + runtimeDir, + runtimeStagingDir, +} from '../../../../main/acappella/runtime/runtime-store'; + +/** Build the archive once per test run; it is deterministic. */ +async function buildTarball(): Promise { + const source = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-archive-')); + for (const [name, contents] of Object.entries(ARCHIVE_FILES)) { + const target = path.join(source, name); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.writeFile(target, contents, 'utf8'); + } + + const tarballPath = path.join(source, 'payload.tgz'); + await tar.c( + { file: tarballPath, cwd: source, gzip: true }, + Object.keys(ARCHIVE_FILES).map((name) => name) + ); + tarballBytes = await fs.readFile(tarballPath); + tarballSha256 = createHash('sha256').update(tarballBytes).digest('hex'); + await fs.rm(source, { recursive: true, force: true }); +} + +/** An injected fetch that serves the archive, or whatever bytes it is given. */ +function serve(body: Buffer, status = 200): typeof globalThis.fetch { + return (async () => + ({ + ok: status >= 200 && status < 300, + status, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(body)); + controller.close(); + }, + }), + }) as unknown as Response) as unknown as typeof globalThis.fetch; +} + +async function exists(target: string): Promise { + try { + await fs.access(target); + return true; + } catch { + return false; + } +} + +beforeEach(async () => { + tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-userdata-')); + // The store reads this before it reaches for Electron, which is what lets the + // whole transaction run in a test with no app instance. + process.env.MAESTRO_USER_DATA = tempRoot; + await buildTarball(); + artifact.sha256 = tarballSha256; + artifact.bytes = tarballBytes.length; +}); + +afterEach(async () => { + delete process.env.MAESTRO_USER_DATA; + await fs.rm(tempRoot, { recursive: true, force: true }); +}); + +describe('installNativeRuntime', () => { + it('installs a verified payload and reports it as installed', async () => { + expect(await isRuntimeInstalled('onnx')).toBe(false); + + const manifest = await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + + expect(manifest.runtimeId).toBe('onnx'); + expect(manifest.sha256).toBe(tarballSha256); + expect(manifest.version).toBe('1.27.0'); + expect(await isRuntimeInstalled('onnx')).toBe(true); + }); + + it('writes only the running platform, discarding the rest of the archive', async () => { + await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + const root = runtimeDir('onnx'); + + expect(await exists(path.join(root, BINARY))).toBe(true); + expect(await exists(path.join(root, 'bin/napi-v6/darwin/arm64/libonnxruntime.1.dylib'))).toBe( + true + ); + expect(await exists(path.join(root, ENTRY))).toBe(true); + + // The point of the whole exercise. + expect(await exists(path.join(root, 'bin/napi-v6/win32'))).toBe(false); + expect(await exists(path.join(root, 'README.md'))).toBe(false); + }); + + it('removes the tarball once it has been unpacked', async () => { + await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + expect(await exists(path.join(runtimeDir('onnx'), 'payload.tgz'))).toBe(false); + }); + + it('leaves no staging directory behind on success', async () => { + await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + expect(await exists(runtimeStagingDir('onnx'))).toBe(false); + }); + + it('resolves the entry point to an absolute path that exists', async () => { + await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + const entry = await installedRuntimeEntry('onnx'); + + expect(entry).toBeTruthy(); + expect(path.isAbsolute(entry!)).toBe(true); + expect(await exists(entry!)).toBe(true); + }); + + it('records a footprint that is the extracted size, not the download size', async () => { + const manifest = await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + // Every kept file is tiny here, but the property under test is that the + // number is measured from disk rather than copied from the catalog. + expect(manifest.bytes).toBeGreaterThan(0); + expect(manifest.bytes).not.toBe(artifact.bytes); + }); + + it('refuses a payload whose hash does not match, and keeps nothing', async () => { + const tampered = Buffer.concat([tarballBytes, Buffer.from('extra')]); + + await expect( + installNativeRuntime('onnx', { fetchImpl: serve(tampered) }) + ).rejects.toBeInstanceOf(RuntimeHashMismatchError); + + expect(await isRuntimeInstalled('onnx')).toBe(false); + expect(await exists(runtimeStagingDir('onnx'))).toBe(false); + expect(await exists(runtimeDir('onnx'))).toBe(false); + }); + + it('reports both hashes on a mismatch, so a support report can be acted on', async () => { + const tampered = Buffer.concat([tarballBytes, Buffer.from('extra')]); + const actualHash = createHash('sha256').update(tampered).digest('hex'); + + await expect( + installNativeRuntime('onnx', { fetchImpl: serve(tampered) }) + ).rejects.toMatchObject({ expected: tarballSha256, actual: actualHash }); + }); + + it('fails when the archive verifies but does not contain the promised binary', async () => { + // The archive is genuine and its hash is right; the artifact simply points + // at a file that is not in it. Without this check the install would + // "succeed" and die later inside a dlopen. + artifact.binary = 'bin/napi-v6/darwin/arm64/not-in-the-archive.node'; + + await expect( + installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }) + ).rejects.toBeInstanceOf(RuntimeBinaryMissingError); + + expect(await isRuntimeInstalled('onnx')).toBe(false); + artifact.binary = BINARY; + }); + + it('fails on a non-OK response without leaving a staging directory', async () => { + await expect( + installNativeRuntime('onnx', { fetchImpl: serve(Buffer.from(''), 404) }) + ).rejects.toThrow(/404/); + + expect(await exists(runtimeStagingDir('onnx'))).toBe(false); + }); + + it('replaces a previous install rather than merging into it', async () => { + await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + + // A file from an imaginary older version, sitting where the new payload + // does not put one. A merge would leave it; a replace removes it. + const stale = path.join(runtimeDir('onnx'), 'bin/napi-v6/darwin/arm64/old-engine.dylib'); + await fs.writeFile(stale, 'previous version', 'utf8'); + expect(await exists(stale)).toBe(true); + + await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + expect(await exists(stale)).toBe(false); + expect(await isRuntimeInstalled('onnx')).toBe(true); + }); + + it('clears the wreckage of a killed install before starting a new one', async () => { + const staging = runtimeStagingDir('onnx'); + await fs.mkdir(staging, { recursive: true }); + await fs.writeFile(path.join(staging, 'half-written.tgz'), 'junk', 'utf8'); + + await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + + expect(await isRuntimeInstalled('onnx')).toBe(true); + expect(await exists(path.join(runtimeDir('onnx'), 'half-written.tgz'))).toBe(false); + }); + + it('emits progress that ends in the done phase', async () => { + const phases: string[] = []; + await installNativeRuntime('onnx', { + fetchImpl: serve(tarballBytes), + onProgress: (progress) => phases.push(progress.phase), + }); + + expect(phases).toContain('verifying'); + expect(phases[phases.length - 1]).toBe('done'); + }); +}); + +describe('runtime store, after an install', () => { + beforeEach(async () => { + await installNativeRuntime('onnx', { fetchImpl: serve(tarballBytes) }); + }); + + it('round-trips the manifest', async () => { + const manifest = await readRuntimeManifest('onnx'); + expect(manifest).toMatchObject({ + runtimeId: 'onnx', + platform: 'darwin-arm64', + sourceUrl: URL, + entry: ENTRY, + binary: BINARY, + }); + }); + + it('reports not-installed once the binary is gone, manifest notwithstanding', async () => { + // The manifest is a claim about the past; the binary is what a dlopen + // needs. A store that trusted the manifest alone would send the loader at + // a file that is not there. + await fs.rm(path.join(runtimeDir('onnx'), BINARY)); + + expect(await isRuntimeInstalled('onnx')).toBe(false); + expect(await installedRuntimeEntry('onnx')).toBeNull(); + }); + + it('uninstalls cleanly', async () => { + await removeRuntime('onnx'); + + expect(await isRuntimeInstalled('onnx')).toBe(false); + expect(await exists(runtimeDir('onnx'))).toBe(false); + }); + + it('refuses an unknown runtime id rather than turning it into a path', async () => { + // `runtimeDir` feeds a recursive delete, and ids arrive from IPC. + expect(() => runtimeDir('../../etc' as never)).toThrow(/UnknownVoiceRuntime/); + }); +}); diff --git a/src/__tests__/main/acappella/runtime/runtime-selftest.test.ts b/src/__tests__/main/acappella/runtime/runtime-selftest.test.ts new file mode 100644 index 0000000000..50b2382301 --- /dev/null +++ b/src/__tests__/main/acappella/runtime/runtime-selftest.test.ts @@ -0,0 +1,165 @@ +/** + * @file runtime-selftest.test.ts + * + * The self-test is what a support report carries instead of "voice does not + * work". So the things worth asserting are the ones that would make that report + * misleading: a runtime that is not part of the build reading as broken, a + * failure taking the whole diagnostic down with it, or a hung load turning the + * diagnostic into a second copy of the bug being diagnosed. + */ + +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('electron', () => ({ + app: { getPath: () => '/tmp/acappella-selftest-test' }, + shell: { openExternal: vi.fn() }, + systemPreferences: { + getMediaAccessStatus: () => 'granted', + askForMediaAccess: vi.fn(), + }, +})); + +import { + runSelfTest, + formatSelfTestReport, +} from '../../../../main/acappella/runtime/runtime-selftest'; +import type { NativeRuntimeResult } from '../../../../main/acappella/runtime/native-loader'; +import type { NativeRuntimeId } from '../../../../shared/acappella/native-runtimes'; + +/** Module surfaces the real probes accept. */ +const WORKING_MODULES: Record = { + llama: { getLlama: () => undefined }, + whisper: { Whisper: function Whisper() {} }, + onnx: { InferenceSession: {}, env: { versions: { common: '1.27.0' } } }, +}; + +function loader( + behaviour: Partial> +) { + return async (id: NativeRuntimeId): Promise> => { + const mode = behaviour[id] ?? 'pass'; + if (mode === 'hang') return new Promise(() => undefined); + if (mode === 'bad-surface') return { ok: true, module: {} }; + if (mode === 'pass') return { ok: true, module: WORKING_MODULES[id] }; + return { + ok: false, + error: { + kind: 'runtime-unavailable', + runtimeId: id, + moduleId: `mock-${id}`, + platform: 'darwin', + arch: 'arm64', + failure: mode === 'skip' ? 'not-a-dependency' : 'load-failed', + message: mode === 'skip' ? 'not part of this build yet' : 'dlopen failed', + suggestedAction: 'do the thing', + detail: mode === 'skip' ? undefined : 'symbol not found', + }, + }; + }; +} + +const mic = () => ({ state: 'granted' as const, canPrompt: false }); + +describe('runtime-selftest', () => { + it('reports every registered runtime, in registry order', async () => { + const report = await runSelfTest({ loadRuntime: loader({}), readMicPermission: mic }); + + expect(report.entries.map((entry) => entry.runtimeId)).toEqual(['llama', 'whisper', 'onnx']); + expect(report.entries.every((entry) => entry.status === 'pass')).toBe(true); + expect(report.passed).toBe(true); + }); + + it('reports a per-runtime failure without failing the others', async () => { + const report = await runSelfTest({ + loadRuntime: loader({ whisper: 'fail' }), + readMicPermission: mic, + }); + + const whisper = report.entries.find((entry) => entry.runtimeId === 'whisper')!; + expect(whisper.status).toBe('fail'); + expect(whisper.failure).toBe('load-failed'); + // The underlying cause travels: a support report with "it failed" in it is + // the same as no support report. + expect(whisper.detail).toContain('symbol not found'); + + expect(report.entries.find((entry) => entry.runtimeId === 'llama')?.status).toBe('pass'); + expect(report.passed).toBe(false); + }); + + it('skips a runtime that is not part of the build rather than calling it broken', async () => { + const report = await runSelfTest({ + loadRuntime: loader({ whisper: 'skip' }), + readMicPermission: mic, + }); + + expect(report.entries.find((entry) => entry.runtimeId === 'whisper')?.status).toBe('skipped'); + // A skip is not a failure. Reporting it as one would send someone hunting a + // bug that does not exist. + expect(report.passed).toBe(true); + }); + + it('fails a runtime that loads but has lost the API the provider calls', async () => { + const report = await runSelfTest({ + loadRuntime: loader({ onnx: 'bad-surface' }), + readMicPermission: mic, + }); + + const onnx = report.entries.find((entry) => entry.runtimeId === 'onnx')!; + expect(onnx.status).toBe('fail'); + expect(onnx.failure).toBe('probe-failed'); + expect(onnx.detail).toContain('InferenceSession'); + }); + + it('reports the probe detail on success, so a version reaches the report', async () => { + const report = await runSelfTest({ loadRuntime: loader({}), readMicPermission: mic }); + expect(report.entries.find((entry) => entry.runtimeId === 'onnx')?.detail).toContain('1.27.0'); + }); + + it('times out instead of hanging, which is the bug it is diagnosing', async () => { + const report = await runSelfTest({ + loadRuntime: loader({ llama: 'hang' }), + readMicPermission: mic, + timeoutMs: 20, + }); + + const llama = report.entries.find((entry) => entry.runtimeId === 'llama')!; + expect(llama.status).toBe('fail'); + expect(llama.failure).toBe('timeout'); + }); + + it('records timings from the injected clock', async () => { + let clock = 1000; + const report = await runSelfTest({ + loadRuntime: loader({}), + readMicPermission: mic, + now: () => (clock += 5), + }); + + expect(report.entries.every((entry) => entry.durationMs > 0)).toBe(true); + expect(report.ranAt).toBeGreaterThan(1000); + }); + + it('carries the microphone permission, because it is the other half of the diagnosis', async () => { + const report = await runSelfTest({ + loadRuntime: loader({}), + readMicPermission: () => ({ state: 'denied', canPrompt: false }), + }); + + expect(report.microphone.permission).toBe('denied'); + // A microphone problem must be visible in the same artifact as a runtime + // problem, or half the reports name the wrong cause. + expect(formatSelfTestReport(report)).toContain('Microphone: denied'); + }); + + it('formats a report that names each runtime and its verdict', async () => { + const report = await runSelfTest({ + loadRuntime: loader({ whisper: 'fail' }), + readMicPermission: mic, + }); + + const text = formatSelfTestReport(report); + expect(text).toContain('FAIL'); + expect(text).toContain('PASS'); + expect(text).toContain('node-llama-cpp'); + }); +}); diff --git a/src/__tests__/main/acappella/signaling.test.ts b/src/__tests__/main/acappella/signaling.test.ts new file mode 100644 index 0000000000..2108e0b03f --- /dev/null +++ b/src/__tests__/main/acappella/signaling.test.ts @@ -0,0 +1,388 @@ +/** + * WebRTC signaling over the authenticated WebSocket. + * + * What is under test is the authorisation boundary, not the SDP: an unpaired + * device, a revoked device, and a device that skipped `auth` must all be unable + * to reach the peer host, and a renegotiation must be able to get through + * because a phone changing network is the normal case rather than an attack. + * + * No network, no Fastify, no Electron: the socket is a function that appends to + * an array, the peer is a spy, and the pairing service writes to a temp file. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; + +import { PairingService } from '../../../main/acappella/pairing/pairing-service'; +import { DEFAULT_ICE_SETTINGS } from '../../../main/acappella/transport/ice-config'; +import { + AUTH_ATTEMPT_LIMIT, + OFFER_RATE_LIMIT, + OFFER_RATE_WINDOW_MS, + SignalingService, + parseClientMessage, + type SignalingPeerHost, + type SignalingServerMessage, +} from '../../../main/acappella/transport/signaling'; +import { DEVICE_PROTOCOL_VERSION } from '../../../shared/acappella/device-protocol'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +let dir: string; +let pairing: PairingService; +let peerHost: { + [K in keyof SignalingPeerHost]: ReturnType; +} & SignalingPeerHost; +let service: SignalingService; +let sent: Record; +let now = 5_000_000; + +const OFFER = { type: 'offer' as const, sdp: 'v=0\r\na=rtpmap:111 opus/48000/2\r\n' }; + +function connect(clientId: string): void { + sent[clientId] = []; + service.register({ + clientId, + send: (message) => sent[clientId].push(message), + remoteAddress: '192.168.1.20', + }); +} + +function last(clientId: string): SignalingServerMessage | undefined { + return sent[clientId][sent[clientId].length - 1]; +} + +async function pairDevice(name = 'Test iPhone'): Promise<{ deviceId: string; token: string }> { + const offer = pairing.startPairing(); + const claim = pairing.claim({ code: offer.code, name, platform: 'ios' }); + if (claim.status !== 'pending') throw new Error('claim failed'); + await pairing.approve(claim.requestId); + const redeemed = pairing.redeem(claim.requestId); + if (redeemed.status !== 'approved') throw new Error('redeem failed'); + return { deviceId: redeemed.deviceId, token: redeemed.token }; +} + +async function authenticate( + clientId: string, + credentials: { deviceId: string; token: string }, + protocolVersion = DEVICE_PROTOCOL_VERSION +): Promise { + await service.handleMessage(clientId, { op: 'auth', ...credentials, protocolVersion }); +} + +beforeEach(async () => { + now = 5_000_000; + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-signaling-')); + pairing = new PairingService({ + filePath: path.join(dir, 'devices.json'), + hostSecret: 'token', + now: () => now, + }); + peerHost = { + acceptOffer: vi.fn(), + addIceCandidate: vi.fn(), + closePeer: vi.fn(), + } as unknown as typeof peerHost; + sent = {}; + service = new SignalingService({ + pairing, + peerHost, + getIceSettings: () => DEFAULT_ICE_SETTINGS, + now: () => now, + }); +}); + +afterEach(async () => { + service.dispose(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +describe('authentication', () => { + it('lets a paired device in and hands it the ICE configuration', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + + expect(last('c1')).toMatchObject({ + op: 'authenticated', + deviceId: credentials.deviceId, + protocolVersion: DEVICE_PROTOCOL_VERSION, + iceTransportPolicy: 'all', + }); + expect(service.isOnline(credentials.deviceId)).toBe(true); + }); + + it('refuses an unpaired device', async () => { + connect('c1'); + await authenticate('c1', { deviceId: 'nope', token: 'nope' }); + expect(last('c1')).toMatchObject({ op: 'auth-failed' }); + expect(service.onlineDeviceIds()).toEqual([]); + }); + + it('refuses a revoked device', async () => { + const credentials = await pairDevice(); + await pairing.revoke(credentials.deviceId); + + connect('c1'); + await authenticate('c1', credentials); + expect(last('c1')).toMatchObject({ op: 'auth-failed' }); + }); + + it('says nothing about WHY, so the failure is not an enumeration oracle', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', { deviceId: credentials.deviceId, token: 'wrong' }); + const wrongToken = last('c1'); + connect('c2'); + await authenticate('c2', { deviceId: 'no-such-device', token: 'wrong' }); + expect(last('c2')).toEqual(wrongToken); + }); + + it('checks the protocol version before the credential', async () => { + connect('c1'); + await authenticate('c1', { deviceId: 'nope', token: 'nope' }, DEVICE_PROTOCOL_VERSION + 1); + expect(last('c1')).toMatchObject({ op: 'error', code: 'protocol-version' }); + }); + + it('cuts off a socket that keeps guessing', async () => { + connect('c1'); + for (let attempt = 0; attempt < AUTH_ATTEMPT_LIMIT; attempt += 1) { + await authenticate('c1', { deviceId: 'nope', token: `guess-${attempt}` }); + } + await authenticate('c1', { deviceId: 'nope', token: 'one more' }); + expect(last('c1')).toMatchObject({ op: 'error', code: 'rate-limited' }); + }); + + it('displaces an older socket when a device connects again', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + connect('c2'); + await authenticate('c2', credentials); + + expect(sent['c1'].some((message) => message.op === 'closed')).toBe(true); + expect(last('c2')).toMatchObject({ op: 'authenticated' }); + // Two sockets claiming one device would both be told to hold the floor. + expect(service.onlineDeviceIds()).toEqual([credentials.deviceId]); + }); +}); + +describe('signaling', () => { + it('refuses an offer from a socket that never authenticated', async () => { + connect('c1'); + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + expect(last('c1')).toMatchObject({ op: 'error', code: 'not-authenticated' }); + expect(peerHost.acceptOffer).not.toHaveBeenCalled(); + }); + + it('refuses a candidate from a socket that never authenticated', async () => { + connect('c1'); + await service.handleMessage('c1', { + op: 'ice-candidate', + candidate: { candidate: 'candidate:1 1 udp 1 10.0.0.1 1 typ host' }, + }); + expect(last('c1')).toMatchObject({ op: 'error', code: 'not-authenticated' }); + expect(peerHost.addIceCandidate).not.toHaveBeenCalled(); + }); + + it('carries an offer to the peer and an answer back', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + + expect(peerHost.acceptOffer).toHaveBeenCalledWith( + expect.objectContaining({ deviceId: credentials.deviceId }) + ); + service.deliverAnswer(credentials.deviceId, { type: 'answer', sdp: 'v=0' }); + expect(last('c1')).toMatchObject({ op: 'answer' }); + }); + + it('trickles candidates in both directions', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + + await service.handleMessage('c1', { + op: 'ice-candidate', + candidate: { candidate: 'candidate:1 1 udp 1 10.0.0.1 1 typ host', sdpMLineIndex: 0 }, + }); + expect(peerHost.addIceCandidate).toHaveBeenCalledWith( + credentials.deviceId, + expect.objectContaining({ sdpMLineIndex: 0 }) + ); + + service.deliverIceCandidate(credentials.deviceId, { candidate: 'candidate:2' }); + expect(last('c1')).toMatchObject({ op: 'ice-candidate' }); + }); + + it('renegotiates on a network change rather than requiring a new session', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + + // WiFi, then the walk out of the door, then LTE. All on the same socket, + // all reaching the same peer: a handover is a hiccup, not a dropped call. + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + + expect(peerHost.acceptOffer).toHaveBeenCalledTimes(2); + expect(peerHost.closePeer).not.toHaveBeenCalled(); + expect(sent['c1'].some((message) => message.op === 'error')).toBe(false); + }); +}); + +describe('offer rate limiting', () => { + it('allows a run of renegotiations and then refuses', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + + for (let attempt = 0; attempt < OFFER_RATE_LIMIT; attempt += 1) { + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + } + expect(peerHost.acceptOffer).toHaveBeenCalledTimes(OFFER_RATE_LIMIT); + + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + expect(last('c1')).toMatchObject({ op: 'error', code: 'rate-limited' }); + expect(peerHost.acceptOffer).toHaveBeenCalledTimes(OFFER_RATE_LIMIT); + }); + + it('slides the window rather than resetting it on a boundary', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + + for (let attempt = 0; attempt < OFFER_RATE_LIMIT; attempt += 1) { + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + } + // Half a window later, a fixed window would have reset and let the whole + // allowance through again. + now += OFFER_RATE_WINDOW_MS / 2; + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + expect(last('c1')).toMatchObject({ op: 'error', code: 'rate-limited' }); + + now += OFFER_RATE_WINDOW_MS; + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + expect(peerHost.acceptOffer).toHaveBeenCalledTimes(OFFER_RATE_LIMIT + 1); + }); +}); + +describe('revocation', () => { + it('tears down a live signaling session the moment the pairing ends', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + expect(service.isOnline(credentials.deviceId)).toBe(true); + + await pairing.revoke(credentials.deviceId); + + expect(last('c1')).toMatchObject({ op: 'closed' }); + expect(peerHost.closePeer).toHaveBeenCalledWith(credentials.deviceId, expect.any(String)); + expect(service.isOnline(credentials.deviceId)).toBe(false); + }); + + it('leaves the socket unable to signal afterwards', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + await pairing.revoke(credentials.deviceId); + + await service.handleMessage('c1', { op: 'offer', sdp: OFFER }); + expect(last('c1')).toMatchObject({ op: 'error', code: 'not-authenticated' }); + }); +}); + +describe('pairing over the socket', () => { + it('walks claim, approval, and redemption', async () => { + const offer = pairing.startPairing(); + connect('c1'); + await service.handleMessage('c1', { + op: 'pair-claim', + code: offer.code, + name: 'New iPhone', + platform: 'ios', + }); + const pending = last('c1'); + expect(pending).toMatchObject({ op: 'pair-pending' }); + if (!pending || pending.op !== 'pair-pending') return; + + // Still nothing usable: the human has not approved. + await service.handleMessage('c1', { op: 'pair-poll', requestId: pending.requestId }); + expect(last('c1')).toMatchObject({ op: 'pair-pending' }); + + await pairing.approve(pending.requestId); + await service.handleMessage('c1', { op: 'pair-poll', requestId: pending.requestId }); + expect(last('c1')).toMatchObject({ op: 'pair-approved' }); + }); + + it('explains a bad code rather than going quiet', async () => { + pairing.startPairing(); + connect('c1'); + await service.handleMessage('c1', { + op: 'pair-claim', + code: 'WRONG1', + name: 'Guess', + platform: 'ios', + }); + expect(last('c1')).toMatchObject({ op: 'pair-rejected', reason: 'unknown-code' }); + }); +}); + +describe('disconnection', () => { + it('closes the peer when the socket goes away', async () => { + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + service.handleDisconnect('c1'); + + expect(peerHost.closePeer).toHaveBeenCalledWith(credentials.deviceId, expect.any(String)); + expect(service.isOnline(credentials.deviceId)).toBe(false); + }); + + it('reports the device offline exactly once', async () => { + const offline: string[] = []; + service = new SignalingService({ + pairing, + peerHost, + getIceSettings: () => DEFAULT_ICE_SETTINGS, + now: () => now, + onDeviceOffline: (deviceId) => offline.push(deviceId), + }); + const credentials = await pairDevice(); + connect('c1'); + await authenticate('c1', credentials); + + service.handleDisconnect('c1'); + service.handleDisconnect('c1'); + expect(offline).toEqual([credentials.deviceId]); + }); +}); + +describe('parseClientMessage', () => { + it.each([ + ['a non-object', 42], + ['an unknown op', { op: 'take-over' }], + ['an auth with no token', { op: 'auth', deviceId: 'd' }], + ['an offer with no sdp', { op: 'offer', sdp: {} }], + ['a candidate with no candidate', { op: 'ice-candidate', candidate: {} }], + ])('returns null for %s', (_label, payload) => { + expect(parseClientMessage(payload)).toBeNull(); + }); + + it('defaults a missing protocol version to one below the floor, so it is refused', () => { + const parsed = parseClientMessage({ op: 'auth', deviceId: 'd', token: 't' }); + expect(parsed).toMatchObject({ op: 'auth' }); + expect((parsed as { protocolVersion: number }).protocolVersion).toBeLessThan(1); + }); + + it('answers a malformed frame rather than throwing inside a socket handler', async () => { + connect('c1'); + await service.handleMessage('c1', { op: 'nonsense' }); + expect(last('c1')).toMatchObject({ op: 'error', code: 'malformed' }); + }); +}); diff --git a/src/__tests__/main/acappella/speech/agent-output-tap.test.ts b/src/__tests__/main/acappella/speech/agent-output-tap.test.ts new file mode 100644 index 0000000000..e6a09c3744 --- /dev/null +++ b/src/__tests__/main/acappella/speech/agent-output-tap.test.ts @@ -0,0 +1,178 @@ +/** + * @file agent-output-tap.test.ts + * + * What reaches a speaker, and what must never. The filters are asserted against + * the shapes agents really emit - fenced code, unified diffs, tool gutters, + * spinner frames, bare paths - because every one of them read aloud is a defect + * a user hears immediately and reports as "it read me a diff". + * + * The source is a bare EventEmitter on purpose: that is what `ProcessManager` is, + * and binding the suite to the real class would test the process manager. + */ + +import { EventEmitter } from 'node:events'; +import { describe, it, expect, vi } from 'vitest'; + +import { + AgentOutputTap, + type AgentOutputChunk, +} from '../../../../main/acappella/speech/agent-output-tap'; + +const AGENT = 'agent-1'; +const TAB = 'tab-7'; +const PROCESS_ID = `${AGENT}-ai-${TAB}`; + +function harness(options: { minChunkChars?: number; hangMs?: number } = {}) { + const source = new EventEmitter(); + const chunks: AgentOutputChunk[] = []; + const timers: { fn: () => void; ms: number }[] = []; + + const tap = new AgentOutputTap({ + source, + onChunk: (chunk) => chunks.push(chunk), + minChunkChars: options.minChunkChars ?? 200, + hangMs: options.hangMs ?? 20_000, + now: () => 1_000, + setTimeoutFn: (fn, ms) => { + timers.push({ fn, ms }); + return timers.length as unknown as ReturnType; + }, + clearTimeoutFn: vi.fn(), + }); + tap.watch({ agentSessionId: AGENT, tabId: TAB }); + + return { + tap, + chunks, + spoken: () => chunks.map((chunk) => chunk.text), + data: (text: string) => source.emit('data', PROCESS_ID, text), + complete: () => source.emit('query-complete', PROCESS_ID, {}), + fail: (message: string) => source.emit('agent-error', PROCESS_ID, { message }), + exit: (code: number) => source.emit('exit', PROCESS_ID, code), + fireHang: () => timers[timers.length - 1]?.fn(), + }; +} + +describe('AgentOutputTap', () => { + it('emits the finished reply as one spoken chunk', () => { + const h = harness(); + h.data('Fixed the auth bug.\nIt was a stale token check.\n'); + h.complete(); + + expect(h.spoken()).toEqual(['Fixed the auth bug. It was a stale token check.']); + expect(h.chunks[0].kind).toBe('final'); + expect(h.chunks[0].agentSessionId).toBe(AGENT); + expect(h.chunks[0].tabId).toBe(TAB); + }); + + it('never speaks a code fence, even when it spans several output events', () => { + const h = harness(); + h.data('Here is the fix:\n\n```ts\nconst token = read'); + h.data('Fresh();\nreturn token;\n```\n\nThat is the whole change.\n'); + h.complete(); + + expect(h.spoken().join(' ')).not.toContain('token'); + expect(h.spoken()).toEqual(['Here is the fix:', 'That is the whole change.']); + }); + + it('drops diffs, tool gutters, bare paths, spinners, and progress readouts', () => { + const h = harness(); + h.data( + [ + '⏺ Bash(npm test)', + '⎿ 18 passed', + 'diff --git a/src/main/auth.ts b/src/main/auth.ts', + '@@ -1,4 +1,4 @@', + '-const token = read();', + '+const token = readFresh();', + 'src/main/auth/middleware.ts', + '⠋ thinking', + '42% done', + '──────────────', + 'The auth fix is in.', + '', + ].join('\n') + ); + h.complete(); + + expect(h.spoken()).toEqual(['The auth fix is in.']); + }); + + it('strips ANSI rather than reading escape codes aloud', () => { + const h = harness(); + h.data('Tests pass.\n'); + h.complete(); + + expect(h.spoken()).toEqual(['Tests pass.']); + }); + + it('speaks the text out of stream-json and never the tool payload around it', () => { + const h = harness(); + h.data( + [ + JSON.stringify({ type: 'text', text: 'Done.' }), + JSON.stringify({ type: 'tool_use', name: 'Bash', input: { command: 'rm -rf /' } }), + '', + ].join('\n') + ); + h.complete(); + + expect(h.spoken().join(' ')).not.toContain('rm -rf'); + expect(h.spoken().join(' ')).toContain('Done.'); + }); + + it('cuts a completed thought loose mid-reply rather than waiting for the whole answer', () => { + const h = harness({ minChunkChars: 20 }); + h.data('The first completed thought is here.\n\nStill writing the rest'); + + // The paragraph before the blank line is spoken while the agent types on. + expect(h.spoken()).toEqual(['The first completed thought is here.']); + expect(h.chunks[0].kind).toBe('text'); + }); + + it('speaks a short honest status when the agent errors instead of going silent', () => { + const h = harness(); + h.data('Working on it.\n'); + h.fail('rate limited by the API'); + + expect(h.spoken()).toEqual(['Working on it.', 'It hit an error: rate limited by the API']); + expect(h.chunks[1].kind).toBe('status'); + }); + + it('says something when the agent goes quiet, once', () => { + const h = harness({ hangMs: 5_000 }); + h.data('Starting.\n'); + h.fireHang(); + h.fireHang(); + + expect(h.spoken().filter((text) => text.includes('still working'))).toEqual([ + 'It is still working on that one.', + ]); + }); + + it('reports an agent that exited without answering', () => { + const h = harness(); + h.exit(1); + + expect(h.spoken()).toEqual(['It stopped without answering, exit code 1.']); + }); + + it('stays silent about output from a tab it is not following', () => { + const h = harness(); + h.tap.unwatch({ agentSessionId: AGENT, tabId: TAB }); + h.data('Something nobody asked to hear.\n'); + h.complete(); + + expect(h.spoken()).toEqual([]); + expect(h.tap.isWatching).toBe(false); + }); + + it('drops every subscription on dispose', () => { + const h = harness(); + h.tap.dispose(); + h.data('Too late.\n'); + h.complete(); + + expect(h.spoken()).toEqual([]); + }); +}); diff --git a/src/__tests__/main/acappella/speech/background-announcer.test.ts b/src/__tests__/main/acappella/speech/background-announcer.test.ts new file mode 100644 index 0000000000..a3d5cf66ab --- /dev/null +++ b/src/__tests__/main/acappella/speech/background-announcer.test.ts @@ -0,0 +1,110 @@ +/** + * @file background-announcer.test.ts + * + * An agent finishing long after its voice turn ended. The failure this module + * exists to prevent is the obvious implementation: a second agent talking over + * the conversation you are having with the first one. + */ + +import { describe, it, expect } from 'vitest'; + +import { + BackgroundAnnouncer, + announcementText, +} from '../../../../main/acappella/speech/background-announcer'; +import { shouldSpeakBackgroundCompletions } from '../../../../shared/acappella/announcements'; +import type { BackgroundAnnouncementSetting } from '../../../../shared/acappella/announcements'; +import type { VoiceScope } from '../../../../shared/acappella/protocol'; + +function announcer( + options: { + scope?: VoiceScope; + setting?: BackgroundAnnouncementSetting; + foreground?: string | null; + queueLimit?: number; + } = {} +) { + return new BackgroundAnnouncer({ + getScope: () => options.scope ?? { kind: 'conductor' }, + getSetting: () => options.setting, + getForegroundAgentSessionId: () => options.foreground ?? null, + queueLimit: options.queueLimit, + now: () => 1_000, + }); +} + +describe('shouldSpeakBackgroundCompletions', () => { + it('defaults to on for the Conductor and off inside a focused agent session', () => { + expect(shouldSpeakBackgroundCompletions(undefined, { kind: 'conductor' })).toBe(true); + expect( + shouldSpeakBackgroundCompletions(undefined, { kind: 'agent', sessionId: 'agent-1' }) + ).toBe(false); + }); + + it('honours an explicit choice in either scope', () => { + expect(shouldSpeakBackgroundCompletions('off', { kind: 'conductor' })).toBe(false); + expect(shouldSpeakBackgroundCompletions('on', { kind: 'agent', sessionId: 'a' })).toBe(true); + }); +}); + +describe('BackgroundAnnouncer', () => { + it('holds an announcement until the conversation reaches a pause', () => { + const queue = announcer(); + queue.queue({ agentSessionId: 'agent-2', agentName: 'Backend', summary: 'the migration' }); + + expect(queue.take(false)).toBeNull(); + expect(queue.take(true)?.text).toBe('the Backend agent finished the migration.'); + expect(queue.take(true)).toBeNull(); + }); + + it('names the source, because the listener has no tab bar to look at', () => { + expect( + announcementText({ + agentSessionId: 'a', + agentName: 'Backend', + summary: 'Fixed the auth bug', + }) + ).toBe('the Backend agent finished fixed the auth bug.'); + + // A name that already says "agent" is not doubled up. + expect(announcementText({ agentSessionId: 'a', agentName: 'Docs Agent' })).toBe( + 'Docs Agent finished.' + ); + + // An acronym keeps its capital. + expect( + announcementText({ agentSessionId: 'a', agentName: 'API', summary: 'API rate limiting' }) + ).toBe('the API agent finished API rate limiting.'); + }); + + it('says nothing inside a focused agent session by default', () => { + const queue = announcer({ scope: { kind: 'agent', sessionId: 'agent-1' } }); + + expect(queue.queue({ agentSessionId: 'agent-2', agentName: 'Backend' })).toBeNull(); + expect(queue.take(true)).toBeNull(); + }); + + it('does not announce the agent the current turn is already about', () => { + const queue = announcer({ foreground: 'agent-1' }); + + expect(queue.queue({ agentSessionId: 'agent-1', agentName: 'Backend' })).toBeNull(); + expect(queue.queue({ agentSessionId: 'agent-2', agentName: 'Frontend' })).not.toBeNull(); + }); + + it('drops the oldest when the backlog outgrows the limit', () => { + const queue = announcer({ queueLimit: 2 }); + queue.queue({ agentSessionId: 'a1', agentName: 'One' }); + queue.queue({ agentSessionId: 'a2', agentName: 'Two' }); + queue.queue({ agentSessionId: 'a3', agentName: 'Three' }); + + expect(queue.queued.map((entry) => entry.agentName)).toEqual(['Two', 'Three']); + }); + + it('drops the backlog with the session it belonged to', () => { + const queue = announcer(); + queue.queue({ agentSessionId: 'a1', agentName: 'One' }); + queue.clear(); + + expect(queue.take(true)).toBeNull(); + }); +}); diff --git a/src/__tests__/main/acappella/speech/barge-in.test.ts b/src/__tests__/main/acappella/speech/barge-in.test.ts new file mode 100644 index 0000000000..de27eb30ae --- /dev/null +++ b/src/__tests__/main/acappella/speech/barge-in.test.ts @@ -0,0 +1,197 @@ +/** + * @file barge-in.test.ts + * + * Talking over the assistant. Four things are pinned, and all four are failures + * you only notice by using it: + * + * - The teardown ORDER. Ducking is the only step the user hears, so it goes + * first; the translator stream is the step everyone forgets, so it is + * asserted explicitly. + * - The pre-roll comes with the floor, or the first word of the interruption is + * lost and the utterance starts mid-word. + * - The self-interrupt guard. Without it the assistant's own first syllable + * leaks past the echo canceller and it interrupts itself, which looks exactly + * like a crash. + * - Heard is not queued. The conversation memory records only what reached the + * speaker. + */ + +import { describe, it, expect, vi } from 'vitest'; + +import { BargeInController } from '../../../../main/acappella/speech/barge-in'; +import type { BargeInOutcome } from '../../../../main/acappella/speech/barge-in'; +import type { SpeechRunResult } from '../../../../main/acappella/speech/speech-scheduler'; + +const NOW = 1_000_000; + +function runResult(overrides: Partial = {}): SpeechRunResult { + return { + utteranceId: 'u1', + reason: 'interrupted', + spoken: ['Done, it was a stale token check.'], + unspoken: ['Two files changed.', 'Want the details?'], + capped: false, + ...overrides, + }; +} + +function harness( + options: { result?: SpeechRunResult | null; guardMs?: number; preRoll?: boolean } = {} +) { + const order: string[] = []; + let clock = NOW; + const outcomes: BargeInOutcome[] = []; + const remembered: string[][] = []; + + const controller = new BargeInController({ + guardMs: options.guardMs ?? 250, + now: () => clock, + duck: (gain, ramp) => order.push(`duck:${gain}:${ramp}`), + flushPlayback: () => order.push('flush'), + cancelSpeech: vi.fn(() => { + order.push('cancel-speech'); + return options.result === undefined ? runResult() : options.result; + }), + cancelTranslation: () => order.push('cancel-translation'), + toListening: (outcome) => { + order.push('listening'); + outcomes.push(outcome); + }, + rememberSpoken: (spoken) => remembered.push(spoken), + }); + + return { + controller, + order, + outcomes, + remembered, + advance: (ms: number) => { + clock += ms; + }, + }; +} + +describe('BargeInController', () => { + it('tears down in order: duck, flush, cancel synthesis, cancel translation, listen', () => { + const h = harness(); + h.controller.noteSpeechStarted(); + h.advance(500); + + const outcome = h.controller.trigger('voice'); + + expect(h.order).toEqual([ + 'duck:0.15:20', + 'flush', + 'cancel-speech', + 'cancel-translation', + 'listening', + ]); + expect(outcome?.steps).toEqual([ + 'duck', + 'flush', + 'cancel-speech', + 'cancel-translation', + 'listening', + ]); + }); + + it('ducks within about 20 ms, which is the only step the user can hear', () => { + const h = harness(); + h.controller.noteSpeechStarted(); + h.advance(500); + h.controller.trigger(); + + const duck = h.order[0]; + expect(duck.startsWith('duck:')).toBe(true); + const rampMs = Number(duck.split(':')[2]); + expect(rampMs).toBeLessThanOrEqual(20); + }); + + it('hands the floor back so the pre-roll can carry the first word of the interruption', () => { + const h = harness(); + h.controller.noteSpeechStarted(); + h.advance(500); + + h.controller.trigger('voice'); + + // `toListening` is the seam the audio pipeline drains its pre-roll into. It + // runs after the teardown, so the buffer it drains is the one that was + // filling while the assistant was still talking. + expect(h.outcomes).toHaveLength(1); + expect(h.order.indexOf('listening')).toBe(h.order.length - 1); + }); + + it('refuses to self-interrupt inside the guard window', () => { + const h = harness({ guardMs: 250 }); + h.controller.noteSpeechStarted(); + + h.advance(100); + expect(h.controller.canInterrupt()).toBe(false); + expect(h.controller.trigger('voice')).toBeNull(); + expect(h.order).toEqual([]); + + h.advance(200); + expect(h.controller.canInterrupt()).toBe(true); + expect(h.controller.trigger('voice')).not.toBeNull(); + }); + + it('is a no-op when nothing is speaking', () => { + const h = harness(); + expect(h.controller.canInterrupt()).toBe(false); + expect(h.controller.trigger('client-button')).toBeNull(); + expect(h.order).toEqual([]); + }); + + it('cannot fire twice for one speech run', () => { + const h = harness(); + h.controller.noteSpeechStarted(); + h.advance(500); + + expect(h.controller.trigger('voice')).not.toBeNull(); + expect(h.controller.trigger('voice')).toBeNull(); + }); + + it('records what was HEARD and keeps what was queued out of the memory', () => { + const h = harness(); + h.controller.noteSpeechStarted(); + h.advance(500); + + const outcome = h.controller.trigger('voice'); + + expect(outcome?.spoken).toEqual(['Done, it was a stale token check.']); + expect(outcome?.unspoken).toEqual(['Two files changed.', 'Want the details?']); + expect(h.remembered).toEqual([['Done, it was a stale token check.']]); + }); + + it('remembers nothing when the user cut in before a single sentence landed', () => { + const h = harness({ result: runResult({ spoken: [], unspoken: ['Done.'] }) }); + h.controller.noteSpeechStarted(); + h.advance(500); + + const outcome = h.controller.trigger('voice'); + + expect(outcome?.spoken).toEqual([]); + expect(h.remembered).toEqual([]); + }); + + it('still tears down when the speech run had already finished on its own', () => { + const h = harness({ result: null }); + h.controller.noteSpeechStarted(); + h.advance(500); + + const outcome = h.controller.trigger('voice'); + + expect(outcome?.utteranceId).toBeNull(); + expect(h.order).toContain('cancel-translation'); + }); + + it('closes the window when speech ends on its own', () => { + const h = harness(); + h.controller.noteSpeechStarted(); + h.advance(500); + h.controller.noteSpeechEnded(); + + expect(h.controller.canInterrupt()).toBe(false); + expect(h.controller.trigger('voice')).toBeNull(); + }); +}); diff --git a/src/__tests__/main/acappella/speech/conversational-translator.test.ts b/src/__tests__/main/acappella/speech/conversational-translator.test.ts new file mode 100644 index 0000000000..a612b1f32a --- /dev/null +++ b/src/__tests__/main/acappella/speech/conversational-translator.test.ts @@ -0,0 +1,243 @@ +/** + * @file conversational-translator.test.ts + * + * The layer that decides what a person actually hears. Four fixtures, chosen + * because each is a different way for a voice reply to be unusable: a four + * hundred line implementation summary (too long), a diff-heavy reply (unspeakable + * shapes), a one-word confirmation (a wasted round trip), and an error trace + * (the case where going silent is worst). + * + * The fake Brain deliberately returns markdown and more sentences than it was + * asked for, because that is what every real backend eventually does. The + * assertions are about what the TRANSLATOR guarantees on top of the model, not + * about the model's prose. + */ + +import { describe, it, expect, vi } from 'vitest'; + +import { ConversationalTranslator } from '../../../../main/acappella/speech/conversational-translator'; +import type { BrainProvider, VoiceConverseContext } from '../../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const LONG_SUMMARY = [ + '## Summary', + '', + 'I refactored the authentication middleware and threaded the refresh token through the session store.', + '', + ...Array.from( + { length: 380 }, + (_, i) => `Step ${i + 1}: touched a call site and updated its test.` + ), +].join('\n'); + +const DIFF_REPLY = [ + 'Here is the change:', + '', + '```diff', + '--- a/src/main/auth.ts', + '+++ b/src/main/auth.ts', + '-const token = read();', + '+const token = readFresh();', + '```', + '', + 'That is the whole fix.', +].join('\n'); + +const ONE_WORD = 'Yes, the tests pass.'; + +const ERROR_TRACE = [ + 'TypeError: cannot read property id of undefined', + ' at resolveSession (src/main/session.ts:42:11)', + ' at dispatch (src/main/dispatch.ts:11:3)', +].join('\n'); + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +interface FakeBrainOptions { + reply?: string; + deltas?: string[]; +} + +function fakeBrain(options: FakeBrainOptions = {}): BrainProvider & { + converse: ReturnType; + seen: VoiceConverseContext[]; +} { + const seen: VoiceConverseContext[] = []; + const reply = options.reply ?? '**Done.** I fixed the auth bug. Want the details?'; + + const converse = vi.fn(async (_text: string, context: VoiceConverseContext) => { + seen.push(context); + return reply; + }); + + const brain: BrainProvider & { converse: typeof converse; seen: VoiceConverseContext[] } = { + id: 'fake-brain', + label: 'Fake', + tier: 'mock', + route: async (): Promise => ({ + target: 'conductor', + tabAction: 'current', + prompt: '', + confidence: 1, + }), + converse, + seen, + }; + + if (options.deltas) { + brain.converseStream = async function* (_text: string, context: VoiceConverseContext) { + seen.push(context); + for (const delta of options.deltas ?? []) { + if (context.signal?.aborted) return; + yield delta; + } + }; + } + + return brain; +} + +async function collect(iterable: AsyncIterable): Promise { + const out: string[] = []; + for await (const value of iterable) out.push(value); + return out; +} + +function request(text: string, overrides: Partial<{ signal: AbortSignal }> = {}) { + return { + agentSessionId: 'agent-1', + tabId: 'tab-1', + text, + kind: 'final' as const, + ...overrides, + }; +} + +// --------------------------------------------------------------------------- + +describe('ConversationalTranslator', () => { + it('turns a four hundred line summary into a short, markdown-free reply that offers detail', async () => { + const brain = fakeBrain({ + reply: '**Done.** I refactored the `auth` middleware, it was a stale token check.', + }); + const translator = new ConversationalTranslator({ brain }); + + const spoken = await collect(translator.translate(request(LONG_SUMMARY))); + + expect(spoken.length).toBeLessThanOrEqual(3); + expect(spoken.join(' ')).not.toMatch(/[*`#|]/); + // The rewrite made no offer, so the translator makes one. The whole point of + // the layer is a headline plus a door back into the detail. + expect(spoken[spoken.length - 1]).toMatch(/details\?$/i); + expect(brain.converse).toHaveBeenCalledOnce(); + }); + + it('never yields more sentences than the budget, whatever the model returns', async () => { + const brain = fakeBrain({ reply: 'One. Two. Three. Four. Five.' }); + const translator = new ConversationalTranslator({ brain, maxSentences: 2 }); + + const spoken = await collect(translator.translate(request(DIFF_REPLY))); + + // Two sentences of content. No offer: the source was short. + expect(spoken).toEqual(['One.', 'Two.']); + }); + + it('passes a short conversational reply through without a translation hop', async () => { + const brain = fakeBrain(); + const translator = new ConversationalTranslator({ brain }); + + const spoken = await collect(translator.translate(request(ONE_WORD))); + + expect(spoken).toEqual([ONE_WORD]); + expect(brain.converse).not.toHaveBeenCalled(); + expect(translator.stats).toEqual({ translations: 0, passthroughs: 1 }); + }); + + it('translates anything path-shaped or code-shaped rather than passing it through', async () => { + const brain = fakeBrain({ reply: 'It failed on a missing session id.' }); + const translator = new ConversationalTranslator({ brain }); + + const spoken = await collect(translator.translate(request(ERROR_TRACE))); + + expect(brain.converse).toHaveBeenCalledOnce(); + expect(spoken).toEqual(['It failed on a missing session id.']); + expect(spoken.join(' ')).not.toContain('src/main/session.ts'); + }); + + it('speaks a status chunk as-is: an error is not worth a round trip', async () => { + const brain = fakeBrain(); + const translator = new ConversationalTranslator({ brain }); + + const spoken = await collect( + translator.translate({ ...request('It hit an error: rate limited.'), kind: 'status' }) + ); + + expect(spoken).toEqual(['It hit an error: rate limited.']); + expect(brain.converse).not.toHaveBeenCalled(); + }); + + it('emits sentences as they are streamed rather than waiting for the whole rewrite', async () => { + const brain = fakeBrain({ + deltas: ['Done, ', 'the auth bug ', 'was a stale token check. ', 'Two files changed.'], + }); + const translator = new ConversationalTranslator({ brain }); + + const spoken = await collect(translator.translate(request(DIFF_REPLY))); + + expect(spoken).toEqual(['Done, the auth bug was a stale token check.', 'Two files changed.']); + // The streaming seam is used INSTEAD of the buffered one, never as well as. + expect(brain.converse).not.toHaveBeenCalled(); + }); + + it('stops mid-stream when the turn is aborted', async () => { + const controller = new AbortController(); + const brain = fakeBrain({ deltas: ['First one. ', 'Second one. ', 'Third one.'] }); + const translator = new ConversationalTranslator({ brain, maxSentences: 5 }); + + const spoken: string[] = []; + for await (const sentence of translator.translate( + request(DIFF_REPLY, { signal: controller.signal }) + )) { + spoken.push(sentence); + controller.abort(); + } + + expect(spoken).toEqual(['First one.']); + }); + + it('carries what was SPOKEN across turns, not what was queued', async () => { + const brain = fakeBrain({ reply: 'All good.' }); + const translator = new ConversationalTranslator({ brain }); + + translator.rememberSpoken(['Done, it was a stale token check.']); + await collect(translator.translate(request(ERROR_TRACE))); + + expect(brain.seen[0].recentSpoken).toEqual(['Done, it was a stale token check.']); + }); + + it('forgets the conversation on reset', async () => { + const brain = fakeBrain({ reply: 'All good.' }); + const translator = new ConversationalTranslator({ brain }); + + translator.rememberSpoken(['Something earlier.']); + translator.reset(); + await collect(translator.translate(request(ERROR_TRACE))); + + expect(brain.seen[0].recentSpoken).toEqual([]); + expect(translator.memory).toEqual([]); + }); + + it('bounds the memory it carries', () => { + const translator = new ConversationalTranslator({ brain: fakeBrain(), memoryLimit: 2 }); + + translator.rememberSpoken(['One.', 'Two.', 'Three.']); + + expect(translator.memory).toEqual(['Two.', 'Three.']); + }); +}); diff --git a/src/__tests__/main/acappella/speech/drill-down.test.ts b/src/__tests__/main/acappella/speech/drill-down.test.ts new file mode 100644 index 0000000000..07959feda4 --- /dev/null +++ b/src/__tests__/main/acappella/speech/drill-down.test.ts @@ -0,0 +1,147 @@ +/** + * @file drill-down.test.ts + * + * "Tell me more", served from the retained output of the last turn. + * + * The property that matters most is the negative one: no follow-up dispatches a + * new agent turn. Re-asking would cost a full round trip AND would answer a + * different question, because the agent has moved on since the sentence the user + * is asking about. + */ + +import { describe, it, expect } from 'vitest'; + +import { + DetailBuffer, + detectDrillDownIntent, + speakPath, +} from '../../../../main/acappella/speech/drill-down'; + +const DETAIL = [ + 'I refactored the authentication middleware in src/main/auth/middleware.ts.', + 'The stale token check was in the refresh path.', + 'Six call sites needed updating.', + 'All eighteen tests pass.', + 'Nothing else changed.', +].join(' '); + +function buffer(): DetailBuffer { + const detail = new DetailBuffer({ sentencesPerSlice: 2 }); + detail.record({ + agentSessionId: 'agent-1', + tabId: 'tab-7', + detail: DETAIL, + spoken: ['Done, the auth bug was a stale token check.'], + }); + return detail; +} + +describe('detectDrillDownIntent', () => { + it('recognises each follow-up', () => { + expect(detectDrillDownIntent('tell me more')).toBe('more'); + expect(detectDrillDownIntent('go on')).toBe('more'); + expect(detectDrillDownIntent('what else?')).toBe('more'); + expect(detectDrillDownIntent('say that again')).toBe('repeat'); + expect(detectDrillDownIntent('read that again')).toBe('repeat'); + expect(detectDrillDownIntent('what was the file?')).toBe('file'); + expect(detectDrillDownIntent('which file was it')).toBe('file'); + expect(detectDrillDownIntent('show me that')).toBe('show'); + expect(detectDrillDownIntent('pull up the diff')).toBe('show'); + }); + + it('leaves a real request alone', () => { + expect(detectDrillDownIntent('run the tests again')).toBeNull(); + expect(detectDrillDownIntent('open a new tab for the migration')).toBeNull(); + expect(detectDrillDownIntent('show me the backlog for next sprint')).toBeNull(); + expect(detectDrillDownIntent('')).toBeNull(); + }); + + it('prefers show over file, because a request to LOOK is answered on screen', () => { + expect(detectDrillDownIntent('show me the file')).toBe('show'); + }); +}); + +describe('DetailBuffer', () => { + it('serves successive slices of detail without a new agent turn', () => { + const detail = buffer(); + + expect(detail.serve('more')).toEqual({ + kind: 'speak', + text: 'I refactored the authentication middleware in src/main/auth/middleware.ts. The stale token check was in the refresh path.', + }); + expect(detail.serve('more')).toEqual({ + kind: 'speak', + text: 'Six call sites needed updating. All eighteen tests pass.', + }); + expect(detail.serve('more')).toEqual({ kind: 'speak', text: 'Nothing else changed.' }); + expect(detail.serve('more')).toEqual({ kind: 'speak', text: "That's everything it said." }); + }); + + it('repeats exactly what was said, not a fresh rewrite of it', () => { + const detail = buffer(); + detail.noteSpoken(['Want the details?']); + + expect(detail.serve('repeat')).toEqual({ + kind: 'speak', + text: 'Done, the auth bug was a stale token check. Want the details?', + }); + }); + + it('names the file the way a person would say it', () => { + expect(buffer().serve('file')).toEqual({ kind: 'speak', text: 'It was middleware dot ts.' }); + }); + + it('says so when no file was named rather than inventing one', () => { + const detail = new DetailBuffer(); + detail.record({ + agentSessionId: 'agent-1', + tabId: 'tab-7', + detail: 'It all worked out fine in the end.', + spoken: [], + }); + + expect(detail.serve('file')).toEqual({ kind: 'speak', text: 'It did not name a file.' }); + }); + + it('answers "show me" on screen and says nothing at all', () => { + expect(buffer().serve('show')).toEqual({ + kind: 'focus', + agentSessionId: 'agent-1', + tabId: 'tab-7', + path: 'src/main/auth/middleware.ts', + }); + }); + + it('has nothing to serve before a turn is recorded, and nothing after it is cleared', () => { + const detail = new DetailBuffer(); + expect(detail.hasTurn).toBe(false); + expect(detail.serve('more')).toEqual({ kind: 'none' }); + + const live = buffer(); + live.clear(); + expect(live.serve('repeat')).toEqual({ kind: 'none' }); + }); + + it('rewinds the read cursor when a new turn replaces the old one', () => { + const detail = buffer(); + detail.serve('more'); + detail.record({ + agentSessionId: 'agent-2', + tabId: 'tab-9', + detail: 'A brand new answer entirely.', + spoken: [], + }); + + expect(detail.serve('more')).toEqual({ kind: 'speak', text: 'A brand new answer entirely.' }); + }); +}); + +describe('speakPath', () => { + it('says the basename and its extension, never the directories', () => { + expect(speakPath('src/main/acappella/speech/speech-scheduler.ts')).toBe( + 'speech-scheduler dot ts' + ); + expect(speakPath('C:\\Users\\dev\\project\\index.ts')).toBe('index dot ts'); + expect(speakPath('Makefile')).toBe('Makefile'); + }); +}); diff --git a/src/__tests__/main/acappella/speech/send-phrase.test.ts b/src/__tests__/main/acappella/speech/send-phrase.test.ts new file mode 100644 index 0000000000..f070630164 --- /dev/null +++ b/src/__tests__/main/acappella/speech/send-phrase.test.ts @@ -0,0 +1,95 @@ +/** + * @file send-phrase.test.ts + * + * The spoken "that's it, go". Two things have to be exactly right: it fires only + * at the END of a turn (mid-sentence agreement is not a send signal), and what + * survives is the request WITHOUT the phrase, because that string becomes the + * prompt an agent receives. + */ + +import { describe, it, expect } from 'vitest'; +import { + DEFAULT_SEND_PHRASES, + matchSendPhrase, + normalisePhrase, +} from '../../../../main/acappella/speech/send-phrase'; + +describe('matchSendPhrase', () => { + it('strips the phrase and keeps the request', () => { + const match = matchSendPhrase('fix the auth bug, good to go'); + + expect(match?.text).toBe('fix the auth bug'); + expect(match?.phrase).toBe('good to go'); + }); + + it('returns empty text when the whole turn was the signal', () => { + // The common case: you pause, then say it on its own. Everything already + // buffered is the request. + const match = matchSendPhrase("That's it."); + + expect(match).not.toBeNull(); + expect(match?.text).toBe(''); + }); + + it('does NOT fire mid-sentence', () => { + // Agreeing and then carrying on is not a send signal. Only position tells + // these apart, which is why the match is anchored to the end. + expect(matchSendPhrase("that's it, the bug is in the auth module")).toBeNull(); + expect(matchSendPhrase('go ahead and look at the tests')).toBeNull(); + }); + + it('ignores casing, punctuation and apostrophes the recogniser chose', () => { + // Whether a transcript contains an apostrophe is a property of the engine, + // not of the speaker. + expect(matchSendPhrase('run the tests. Thats it')?.text).toBe('run the tests'); + expect(matchSendPhrase("run the tests, that's IT!")?.text).toBe('run the tests'); + }); + + it('keeps the original wording of the request', () => { + // Normalising is for MATCHING. The surviving text becomes a prompt, so the + // user's capitals and punctuation have to come through untouched. + const match = matchSendPhrase('Look at OAuth in the API repo, send it'); + + expect(match?.text).toBe('Look at OAuth in the API repo'); + }); + + it('prefers the longest phrase when two could match', () => { + const match = matchSendPhrase('do the thing, go ahead', ['ahead', 'go ahead']); + + expect(match?.phrase).toBe('go ahead'); + expect(match?.text).toBe('do the thing'); + }); + + it('returns null when nothing matches', () => { + expect(matchSendPhrase('fix the auth bug')).toBeNull(); + }); + + it('returns null for an empty utterance', () => { + expect(matchSendPhrase(' ')).toBeNull(); + }); + + it('ignores a blank configured phrase rather than matching everything', () => { + // An empty phrase normalises to '', which every string ends with. + expect(matchSendPhrase('anything at all', ['', ' '])).toBeNull(); + }); + + it('handles a symbol between the request and the phrase', () => { + expect(matchSendPhrase('fix the bug -- good to go')?.text).toBe('fix the bug'); + }); + + it('ships phrases that are natural to say', () => { + for (const phrase of DEFAULT_SEND_PHRASES) { + expect(matchSendPhrase(`do the work, ${phrase}`)?.text).toBe('do the work'); + } + }); +}); + +describe('normalisePhrase', () => { + it('collapses casing, punctuation and spacing', () => { + expect(normalisePhrase(" That's IT! ")).toBe('thats it'); + }); + + it('reduces a string with nothing speakable in it to empty', () => { + expect(normalisePhrase('!!! ...')).toBe(''); + }); +}); diff --git a/src/__tests__/main/acappella/speech/speech-scheduler.test.ts b/src/__tests__/main/acappella/speech/speech-scheduler.test.ts new file mode 100644 index 0000000000..8e240f3024 --- /dev/null +++ b/src/__tests__/main/acappella/speech/speech-scheduler.test.ts @@ -0,0 +1,377 @@ +/** + * @file speech-scheduler.test.ts + * + * The queue between the translator and the speaker. Four things are pinned here + * because each of them is silently wrong in a way a listener notices and a log + * does not: + * + * - Segmentation against the strings agents actually write (`v1.2.3`, + * `src/main/index.ts`, `99.5`, `e.g.`). A splitter that gets one wrong reads + * half a sentence and then stops. + * - No gap between sentences: the next one is synthesized while the current one + * is still being delivered. + * - The length cap wraps up out loud instead of cutting off, because a stop + * with no explanation reads as a crash to someone with no screen. + * - `interrupted` is not `completed`. The conversation memory is built on that + * difference. + */ + +import { describe, it, expect, vi } from 'vitest'; + +import { + SpeechScheduler, + type SpeechRunResult, +} from '../../../../main/acappella/speech/speech-scheduler'; +import { splitIntoSpokenSentences } from '../../../../shared/acappella/sentences'; +import type { TtsChunk, TtsProvider } from '../../../../shared/acappella/providers'; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +interface FakeTts extends TtsProvider { + /** Sentences the provider was asked to synthesize, in order. */ + requested: string[]; + /** Sentences whose synthesis has been started but not resolved. */ + release: (sentence: string) => void; + cancel: ReturnType void>>; +} + +/** + * A provider whose synthesis can be held open, which is the only way to observe + * lookahead: with instant synthesis every schedule looks gapless. + */ +function fakeTts(options: { manual?: boolean } = {}): FakeTts { + const requested: string[] = []; + const gates = new Map void>(); + let cancelled = false; + + const provider: FakeTts = { + id: 'fake-tts', + label: 'Fake', + tier: 'mock', + requested, + cancel: vi.fn(() => { + cancelled = true; + for (const open of gates.values()) open(); + gates.clear(); + }), + release: (sentence: string) => { + gates.get(sentence)?.(); + gates.delete(sentence); + }, + speak: async function* (text: string, speakOptions): AsyncIterable { + requested.push(text); + if (options.manual) { + await new Promise((resolve) => gates.set(text, resolve)); + } + if (cancelled) return; + yield { + utteranceId: speakOptions.utteranceId, + index: requested.length - 1, + text, + format: 'none', + audio: null, + }; + }, + }; + + return provider; +} + +interface Harness { + scheduler: SpeechScheduler; + tts: FakeTts; + starts: { utteranceId: string; sentenceCount: number; streaming: boolean }[]; + sentences: { index: number; text: string }[]; + ends: SpeechRunResult[]; + chunks: TtsChunk[]; +} + +function harness( + overrides: Partial[0]> = {} +): Harness { + const tts = (overrides.tts as FakeTts) ?? fakeTts(); + const starts: Harness['starts'] = []; + const sentences: Harness['sentences'] = []; + const ends: SpeechRunResult[] = []; + const chunks: TtsChunk[] = []; + + const scheduler = new SpeechScheduler({ + tts, + onStart: (event) => starts.push(event), + onSentence: (event) => sentences.push({ index: event.index, text: event.text }), + onEnd: (result) => ends.push(result), + onChunk: (chunk) => chunks.push(chunk), + ...overrides, + }); + + return { scheduler, tts, starts, sentences, ends, chunks }; +} + +/** Let the scheduler's worker run to its next await. */ +const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + +// --------------------------------------------------------------------------- + +describe('sentence segmentation', () => { + it('does not split on abbreviations, decimals, version numbers, or file extensions', () => { + expect(splitIntoSpokenSentences('Coverage is 99.5 percent now.')).toEqual([ + 'Coverage is 99.5 percent now.', + ]); + expect(splitIntoSpokenSentences('Bumped it to v1.2.3 this morning.')).toEqual([ + 'Bumped it to v1.2.3 this morning.', + ]); + expect(splitIntoSpokenSentences('The fix is in src/main/index.ts near the top.')).toEqual([ + 'The fix is in src/main/index.ts near the top.', + ]); + expect(splitIntoSpokenSentences('Check the store, e.g. the session one.')).toEqual([ + 'Check the store, e.g. the session one.', + ]); + expect(splitIntoSpokenSentences('It ships in the U.S. only.')).toEqual([ + 'It ships in the U.S. only.', + ]); + }); + + it('still splits after an acronym, which is what agents write constantly', () => { + expect(splitIntoSpokenSentences('Fixed the API. Then I ran the tests.')).toEqual([ + 'Fixed the API.', + 'Then I ran the tests.', + ]); + }); + + it('splits at the end of a sentence that finishes on a file name', () => { + expect(splitIntoSpokenSentences('It is in src/main/index.ts. Two lines changed.')).toEqual([ + 'It is in src/main/index.ts.', + 'Two lines changed.', + ]); + }); +}); + +describe('SpeechScheduler', () => { + it('announces a streaming run with the count it has, not a count it invented', async () => { + const h = harness(); + h.scheduler.begin('u1'); + await settle(); + + expect(h.starts).toEqual([ + { utteranceId: 'u1', sentenceCount: 0, streaming: true, ttsProviderId: 'fake-tts' }, + ]); + }); + + it('speaks pushed text sentence by sentence and reports each one before its audio', async () => { + const h = harness(); + h.scheduler.begin('u1'); + h.scheduler.push('Done, it was a stale token check. '); + h.scheduler.push('Two files changed.'); + h.scheduler.close(); + + const result = await h.scheduler.drained(); + + expect(h.sentences.map((s) => s.text)).toEqual([ + 'Done, it was a stale token check.', + 'Two files changed.', + ]); + expect(h.chunks.map((c) => c.text)).toEqual(h.sentences.map((s) => s.text)); + expect(result?.reason).toBe('completed'); + expect(result?.spoken).toHaveLength(2); + expect(result?.unspoken).toEqual([]); + }); + + it('holds an unterminated tail rather than speaking half a sentence', async () => { + const h = harness(); + h.scheduler.begin('u1'); + h.scheduler.push('The file is index'); + await settle(); + + expect(h.tts.requested).toEqual([]); + + // The rest of the token arrives and proves it was never a boundary. + h.scheduler.push('.ts and it is fine.'); + h.scheduler.close(); + await h.scheduler.drained(); + + expect(h.tts.requested).toEqual(['The file is index .ts and it is fine.']); + }); + + it('synthesizes the next sentence while the current one is still playing', async () => { + const tts = fakeTts({ manual: true }); + const h = harness({ tts, lookahead: 1 }); + + h.scheduler.begin('u1'); + h.scheduler.push('One. Two. Three. '); + await settle(); + + // Sentence two is already being synthesized while sentence one is still open + // and has not been delivered. That overlap is what removes the gap. + expect(h.tts.requested).toEqual(['One.', 'Two.']); + expect(h.sentences).toEqual([]); + // Not the whole reply, though: a barge-in during sentence one must not throw + // away three sentences of paid-for audio. + expect(h.tts.requested).not.toContain('Three.'); + + tts.release('One.'); + await settle(); + expect(h.sentences.map((s) => s.text)).toEqual(['One.']); + expect(h.tts.requested).toEqual(['One.', 'Two.', 'Three.']); + + tts.release('Two.'); + tts.release('Three.'); + h.scheduler.close(); + await h.scheduler.drained(); + expect(h.sentences.map((s) => s.text)).toEqual(['One.', 'Two.', 'Three.']); + }); + + it('wraps up out loud when the per-turn cap is reached', async () => { + const h = harness({ maxSentencesPerTurn: 2 }); + h.scheduler.begin('u1'); + h.scheduler.push('One. Two. Three. Four. '); + h.scheduler.close(); + + const result = await h.scheduler.drained(); + + expect(h.sentences.map((s) => s.text)).toEqual([ + 'One.', + 'Two.', + "There's more, ask me for the details.", + ]); + expect(result?.capped).toBe(true); + expect(result?.reason).toBe('completed'); + // The sentences it never got to are recorded as unheard, so nothing claims + // the user was told about them. + expect(result?.unspoken).toEqual(['Three.', 'Four.']); + }); + + it('does not promise details that do not exist when the run ends exactly on the cap', async () => { + const h = harness({ maxSentencesPerTurn: 2 }); + h.scheduler.begin('u1'); + h.scheduler.push('One. Two. '); + h.scheduler.close(); + + const result = await h.scheduler.drained(); + + expect(h.sentences.map((s) => s.text)).toEqual(['One.', 'Two.']); + expect(result?.capped).toBe(false); + }); + + it('reports interrupted, not completed, and separates what was heard from what was not', async () => { + const tts = fakeTts({ manual: true }); + const h = harness({ tts }); + + h.scheduler.begin('u1'); + h.scheduler.push('First. Second. Third. '); + await settle(); + tts.release('First.'); + await settle(); + + // Mid-synthesis of the second sentence, with the third still queued. + const result = h.scheduler.cancel('interrupted'); + + expect(result?.reason).toBe('interrupted'); + expect(result?.spoken).toEqual(['First.']); + expect(result?.unspoken).toEqual(['Second.', 'Third.']); + expect(h.tts.cancel).toHaveBeenCalledOnce(); + }); + + it('emits exactly one speak-end however the run ends', async () => { + const h = harness(); + h.scheduler.begin('u1'); + h.scheduler.push('Only one. '); + h.scheduler.close(); + await h.scheduler.drained(); + + expect(h.scheduler.cancel('interrupted')).toBeNull(); + expect(h.ends).toHaveLength(1); + }); + + it('drops sentences that arrive after the run ended', async () => { + const h = harness(); + h.scheduler.begin('u1'); + h.scheduler.push('Done. '); + h.scheduler.close(); + await h.scheduler.drained(); + + h.scheduler.push('Too late.'); + await settle(); + + expect(h.tts.requested).toEqual(['Done.']); + }); + + it('reports an error end reason when the provider fails mid-run', async () => { + const failing: TtsProvider = { + id: 'broken-tts', + label: 'Broken', + tier: 'mock', + cancel: vi.fn(), + // eslint-disable-next-line require-yield + speak: async function* (): AsyncIterable { + throw new Error('synthesis failed'); + }, + }; + const errors: Error[] = []; + const h = harness({ tts: failing as FakeTts, onError: (error) => errors.push(error) }); + + h.scheduler.begin('u1'); + h.scheduler.push('Anything. '); + const result = await h.scheduler.drained(); + + expect(result?.reason).toBe('error'); + expect(errors.map((error) => error.message)).toEqual(['synthesis failed']); + }); +}); + +describe('live voice and rate', () => { + /** A provider that records the options each sentence was synthesized with. */ + function recordingTts(): TtsProvider & { options: Array<{ voiceId?: string; rate?: number }> } { + const options: Array<{ voiceId?: string; rate?: number }> = []; + return { + id: 'recording-tts', + label: 'Recording', + tier: 'mock', + options, + cancel: vi.fn(), + speak: async function* (text: string, speakOptions): AsyncIterable { + options.push({ voiceId: speakOptions.voiceId, rate: speakOptions.rate }); + yield { + utteranceId: speakOptions.utteranceId, + index: options.length - 1, + text, + format: 'none', + audio: null, + }; + }, + }; + } + + it('reads the voice and rate fresh for every sentence', async () => { + // This is what makes the Settings sliders apply to the NEXT SENTENCE + // rather than the next session. Reading them once at construction would + // pin a whole conversation to whatever was configured when it started. + const tts = recordingTts(); + let current = { voiceId: 'alloy', rate: 1 }; + const h = harness({ tts: tts as never, speechOptions: () => current }); + + h.scheduler.begin('u1'); + h.scheduler.push('One sentence. '); + await settle(); + current = { voiceId: 'nova', rate: 1.25 }; + h.scheduler.push('Two sentence. '); + h.scheduler.close(); + await h.scheduler.drained(); + + expect(tts.options[0]).toEqual({ voiceId: 'alloy', rate: 1 }); + expect(tts.options[1]).toEqual({ voiceId: 'nova', rate: 1.25 }); + }); + + it('passes nothing when no getter was supplied, leaving the provider its default', async () => { + const tts = recordingTts(); + const h = harness({ tts: tts as never }); + + h.scheduler.begin('u1'); + h.scheduler.push('One sentence. '); + h.scheduler.close(); + await h.scheduler.drained(); + + expect(tts.options[0]).toEqual({ voiceId: undefined, rate: undefined }); + }); +}); diff --git a/src/__tests__/main/acappella/speech/utterance-composer.test.ts b/src/__tests__/main/acappella/speech/utterance-composer.test.ts new file mode 100644 index 0000000000..1a0ca33dbc --- /dev/null +++ b/src/__tests__/main/acappella/speech/utterance-composer.test.ts @@ -0,0 +1,364 @@ +/** + * @file utterance-composer.test.ts + * + * The thing this component exists to stop: a pause mid-sentence becoming two + * separate requests to an agent. Every test here is about where the boundary of + * one thought is drawn, so they all run on fake timers - the boundary is a + * duration, and asserting it against the wall clock would be a flaky way to + * describe a deterministic rule. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { + UtteranceComposer, + type ComposedUtterance, +} from '../../../../main/acappella/speech/utterance-composer'; + +const SETTLE = 900; + +function makeComposer( + overrides: { settleMs?: number; maxHoldMs?: number; sendPhrases?: readonly string[] } = {} +) { + const settled: ComposedUtterance[] = []; + const composing: string[] = []; + const composer = new UtteranceComposer({ + settleMs: overrides.settleMs ?? SETTLE, + maxHoldMs: overrides.maxHoldMs ?? 30_000, + sendPhrases: overrides.sendPhrases, + onSettled: (utterance) => settled.push(utterance), + onComposing: (text) => composing.push(text), + }); + return { composer, settled, composing }; +} + +beforeEach(() => vi.useFakeTimers()); +afterEach(() => vi.useRealTimers()); + +describe('UtteranceComposer', () => { + it('joins fragments separated by a pause into one request', () => { + // The bug, in one test: a person pausing to think used to produce two + // dispatches, and the agent answered half a sentence. + const { composer, settled } = makeComposer(); + + composer.add('look at the auth module', 0.9); + vi.advanceTimersByTime(SETTLE - 100); + composer.add('and tell me why the refresh is failing', 0.8); + vi.advanceTimersByTime(SETTLE); + + expect(settled).toHaveLength(1); + expect(settled[0].text).toBe('look at the auth module and tell me why the refresh is failing'); + expect(settled[0].fragments).toBe(2); + }); + + it('dispatches nothing while the user is still talking', () => { + const { composer, settled } = makeComposer(); + + composer.add('first', 1); + vi.advanceTimersByTime(SETTLE - 1); + + expect(settled).toEqual([]); + }); + + it('settles a complete thought once the pause is long enough', () => { + const { composer, settled } = makeComposer(); + + composer.add('open the auth tab', 1); + vi.advanceTimersByTime(SETTLE); + + expect(settled).toHaveLength(1); + expect(settled[0].fragments).toBe(1); + }); + + it('reports the worst confidence of the parts, not the average', () => { + // The whole thing is dispatched as one request, so it is only as reliable + // as its most doubtful fragment; averaging lets a clear part vouch for a + // mumbled one. + const { composer, settled } = makeComposer(); + + composer.add('clear part', 0.95); + composer.add('mumbled part', 0.4); + vi.advanceTimersByTime(SETTLE); + + expect(settled[0].confidence).toBe(0.4); + }); + + it('sums the spoken duration across fragments', () => { + const { composer, settled } = makeComposer(); + + composer.add('one', 1, 1_000); + composer.add('two', 1, 500); + vi.advanceTimersByTime(SETTLE); + + expect(settled[0].durationMs).toBe(1_500); + }); + + it('emits the growing text so the transcript does not blank mid-thought', () => { + const { composer, composing } = makeComposer(); + + composer.add('look at', 1); + composer.add('the auth module', 1); + + expect(composing).toEqual(['look at', 'look at the auth module']); + }); + + it('ignores an empty final rather than restarting the clock for silence', () => { + const { composer, settled, composing } = makeComposer(); + + composer.add(' ', 1); + vi.advanceTimersByTime(SETTLE); + + expect(settled).toEqual([]); + expect(composing).toEqual([]); + }); + + it('dispatches on arrival when composing is switched off', () => { + // settleMs 0 is the pre-composer behaviour, kept reachable for anyone who + // wants the old snappiness back. + const { composer, settled } = makeComposer({ settleMs: 0 }); + + composer.add('go', 1); + + expect(settled).toHaveLength(1); + }); + + /** + * The spoken Enter key. A settle timer is a guess at when someone stopped + * talking; a phrase is them saying so, which is why the timer becomes a + * backstop rather than the mechanism once these exist. + */ + describe('a send phrase ends dictation immediately', () => { + it('sends without waiting for the settle timer', () => { + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.add('fix the auth bug, good to go', 1); + + // No timer advanced: the whole point is not waiting. + expect(settled).toHaveLength(1); + expect(settled[0].sentBy).toBe('good to go'); + }); + + it('strips the phrase, so the agent gets the request and not the signal', () => { + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.add('fix the auth bug, good to go', 1); + + expect(settled[0].text).toBe('fix the auth bug'); + }); + + it('sends everything buffered when the phrase is a turn of its own', () => { + // The way it is actually said: you talk, you pause, then you say it. + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.add('look at the auth module', 1); + composer.add('and say why the refresh fails', 1); + composer.add("that's it", 1); + + expect(settled).toHaveLength(1); + expect(settled[0].text).toBe('look at the auth module and say why the refresh fails'); + expect(settled[0].fragments).toBe(2); + }); + + it('ignores the signal when there is no request to send', () => { + // A send phrase with an empty buffer would otherwise dispatch an empty + // prompt, which is worse than doing nothing. + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.add('good to go', 1); + + expect(settled).toEqual([]); + expect(composer.composing).toBe(false); + }); + + it('does not fire on a phrase said mid-sentence', () => { + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.add("that's it exactly, the auth module is the problem", 1); + + expect(settled).toEqual([]); + expect(composer.composing).toBe(true); + }); + + it('leaves the timer as the backstop when nothing is said', () => { + // Forgetting the phrase must not mean the request never goes. + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.add('fix the auth bug', 1); + vi.advanceTimersByTime(30_000); + + expect(settled).toHaveLength(1); + expect(settled[0].sentBy).toBeUndefined(); + }); + + it('can be switched off entirely', () => { + const { composer, settled } = makeComposer({ settleMs: 30_000, sendPhrases: [] }); + + composer.add('fix the auth bug, good to go', 1); + + expect(settled).toEqual([]); + }); + }); + + /** + * Letting go of a push-to-talk key. The recogniser is flushed at the same + * moment, and its final can land either side of the release, which is the + * entire reason this is not just "settle now". + */ + describe('a release gesture', () => { + it('sends what is buffered without waiting for the timer', () => { + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + composer.add('fix the auth bug', 1); + + composer.armImmediateSettle(); + + expect(settled).toHaveLength(1); + expect(settled[0].text).toBe('fix the auth bug'); + expect(settled[0].sentBy).toBe('release'); + }); + + it('waits for a flushed tail that has not arrived yet', () => { + // The failure this exists to prevent: settling only the current buffer + // sends the sentence minus its last few words. + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.armImmediateSettle(); + expect(settled).toEqual([]); + + composer.add('the last few words', 1); + + expect(settled).toHaveLength(1); + expect(settled[0].text).toBe('the last few words'); + }); + + it('includes the tail with what came before it', () => { + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + composer.add('look at the auth module', 1); + + // Buffer settles on the release, then the flushed tail arrives as its own + // thought - the honest reading of words spoken after "I am done". + composer.armImmediateSettle(); + composer.add('and the refresh path', 1); + + expect(settled.map((entry) => entry.text)).toEqual([ + 'look at the auth module', + 'and the refresh path', + ]); + }); + + it('does nothing when nothing was ever said', () => { + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.armImmediateSettle(); + vi.advanceTimersByTime(60_000); + + expect(settled).toEqual([]); + }); + + it('is cleared by a cancel, so a closed floor cannot settle later', () => { + const { composer, settled } = makeComposer({ settleMs: 30_000 }); + + composer.armImmediateSettle(); + composer.cancel(); + composer.add('spoken into a dead session', 1); + + expect(settled).toEqual([]); + }); + }); + + describe('ending a thought by decree', () => { + it('flush settles immediately', () => { + const { composer, settled } = makeComposer(); + + composer.add('half a sentence', 1); + composer.flush(); + + expect(settled).toHaveLength(1); + expect(settled[0].text).toBe('half a sentence'); + }); + + it('flush on an empty buffer dispatches nothing', () => { + const { composer, settled } = makeComposer(); + + composer.flush(); + + expect(settled).toEqual([]); + }); + + it('cancel drops the thought and the timer with it', () => { + // The floor closing must not leave a fragment to be dispatched into a + // session nobody is in. + const { composer, settled } = makeComposer(); + + composer.add('abandoned', 1); + composer.cancel(); + vi.advanceTimersByTime(SETTLE * 5); + + expect(settled).toEqual([]); + expect(composer.composing).toBe(false); + }); + }); + + it('stops holding at the cap, so a noisy room cannot wait forever', () => { + // The backstop, not a normal path: fragments arriving faster than the settle + // window would otherwise never let the thought finish. + const { composer, settled } = makeComposer({ maxHoldMs: 3_000 }); + + for (let elapsed = 0; elapsed < 5_000; elapsed += 300) { + composer.add('noise', 1); + vi.advanceTimersByTime(300); + } + + expect(settled.length).toBeGreaterThan(0); + }); + + it('never lets the cap undercut the wait it is backstopping', () => { + // A 30 s hold under a 30 s cap fires the cap first and splits the thought, + // because the cap starts on the first fragment while the settle restarts on + // every one. + const { composer, settled } = makeComposer({ settleMs: 30_000, maxHoldMs: 30_000 }); + + composer.add('still talking', 1); + vi.advanceTimersByTime(29_000); + composer.add('and still going', 1); + vi.advanceTimersByTime(29_000); + + expect(settled).toEqual([]); + expect(composer.composing).toBe(true); + }); + + it('does not restart the cap on every fragment', () => { + // Restarting it would make the backstop unreachable in exactly the case it + // exists for - continuous fragments. Gaps are shorter than the settle, so + // the settle keeps restarting and only the cap can end this. + const { composer, settled } = makeComposer({ settleMs: 200, maxHoldMs: 800 }); + + for (let i = 0; i < 6; i += 1) { + composer.add('still going', 1); + vi.advanceTimersByTime(150); + } + + expect(settled).toHaveLength(1); + expect(settled[0].fragments).toBeGreaterThan(1); + }); + + it('starts a fresh thought after one settles', () => { + const { composer, settled } = makeComposer(); + + composer.add('first thought', 1); + vi.advanceTimersByTime(SETTLE); + composer.add('second thought', 1); + vi.advanceTimersByTime(SETTLE); + + expect(settled.map((entry) => entry.text)).toEqual(['first thought', 'second thought']); + }); + + it('dispose stops it settling anything afterwards', () => { + const { composer, settled } = makeComposer(); + + composer.add('pending', 1); + composer.dispose(); + composer.add('more', 1); + vi.advanceTimersByTime(SETTLE * 5); + + expect(settled).toEqual([]); + }); +}); diff --git a/src/__tests__/main/acappella/telemetry/turn-metrics.test.ts b/src/__tests__/main/acappella/telemetry/turn-metrics.test.ts new file mode 100644 index 0000000000..a7e4568d95 --- /dev/null +++ b/src/__tests__/main/acappella/telemetry/turn-metrics.test.ts @@ -0,0 +1,161 @@ +/** + * @file turn-metrics.test.ts + * + * The instrumentation that answers "voice feels slow" with a hop instead of a + * shrug. Two things worth pinning down: a milestone is stamped once (a second + * partial is not the first partial), and the breakdown reports time spent IN each + * hop rather than time since the turn began, because the first form is actionable + * and the second is arithmetic homework. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; + +import { + TurnTimer, + describeTurn, + formatTurnBreakdown, + lastTurn, + recordTurn, + resetTurnMetrics, + turnHistory, +} from '../../../../main/acappella/telemetry/turn-metrics'; + +const CONFIG = { + pipeline: 'cascade' as const, + providerIds: { stt: 'whisper-local', tts: 'kokoro-local', brain: 'qwen3-local' }, +}; + +/** A clock the test drives, so no span depends on how fast the machine is. */ +function clock(): { now: () => number; advance: (ms: number) => void } { + let time = 1_000; + return { + now: () => time, + advance: (ms) => { + time += ms; + }, + }; +} + +beforeEach(() => { + resetTurnMetrics(); +}); + +describe('TurnTimer', () => { + it('records every hop from the moment speech ended', () => { + const time = clock(); + const timer = new TurnTimer('turn-1', CONFIG, time.now); + + time.advance(300); + timer.mark('firstPartial'); + time.advance(200); + timer.mark('finalTranscript'); + time.advance(400); + timer.mark('routeDecision'); + time.advance(1_500); + timer.mark('agentFirstToken'); + time.advance(250); + timer.mark('firstSpokenSentence'); + + const metrics = timer.finish(); + expect(metrics.spans).toEqual({ + firstPartial: 300, + finalTranscript: 500, + routeDecision: 900, + agentFirstToken: 2_400, + firstSpokenSentence: 2_650, + total: 2_650, + }); + }); + + it('keeps the FIRST stamp for a span', () => { + const time = clock(); + const timer = new TurnTimer('turn-1', CONFIG, time.now); + + timer.mark('firstPartial'); + time.advance(500); + // A later partial is not the first one; overwriting would quietly turn this + // into a most-recent-event log. + timer.mark('firstPartial'); + + expect(timer.finish().spans.firstPartial).toBe(0); + }); + + it('leaves a milestone that never happened absent', () => { + const timer = new TurnTimer('turn-1', CONFIG, clock().now); + timer.mark('finalTranscript'); + + const metrics = timer.finish(); + // A turn with a transcript and no spoken sentence failed somewhere specific, + // and the gap is the evidence. + expect(metrics.spans.firstSpokenSentence).toBeUndefined(); + }); +}); + +describe('describeTurn', () => { + it('reports time spent in each hop, with the total as the whole turn', () => { + const breakdown = describeTurn({ + turnId: 'turn-1', + startedAt: 0, + configuration: CONFIG, + spans: { firstPartial: 300, finalTranscript: 500, total: 900 }, + }); + + expect(breakdown.deltas.map((delta) => [delta.span, delta.ms])).toEqual([ + ['firstPartial', 300], + ['finalTranscript', 200], + ['total', 900], + ]); + }); + + it('formats spans with the shared duration helper', () => { + const breakdown = describeTurn({ + turnId: 'turn-1', + startedAt: 0, + configuration: CONFIG, + spans: { firstPartial: 1_500, total: 1_500 }, + }); + + expect(breakdown.deltas[0].formatted).toBe('1.50s'); + }); +}); + +describe('the rolling history', () => { + it('remembers the last turn and names the configuration it ran on', () => { + recordTurn({ turnId: 'a', startedAt: 0, configuration: CONFIG, spans: { total: 100 } }); + recordTurn({ turnId: 'b', startedAt: 0, configuration: CONFIG, spans: { total: 200 } }); + + expect(lastTurn()?.turnId).toBe('b'); + expect(turnHistory()).toHaveLength(2); + }); + + it('is empty before anything has been said', () => { + expect(lastTurn()).toBeNull(); + }); + + it('caps what it retains', () => { + for (let index = 0; index < 50; index++) { + recordTurn({ + turnId: `turn-${index}`, + startedAt: 0, + configuration: CONFIG, + spans: { total: index }, + }); + } + + expect(turnHistory().length).toBeLessThanOrEqual(20); + expect(lastTurn()?.turnId).toBe('turn-49'); + }); + + it('formats a breakdown that names the providers it was measured on', () => { + recordTurn({ + turnId: 'a', + startedAt: 0, + configuration: CONFIG, + spans: { firstPartial: 300, total: 300 }, + }); + + const text = formatTurnBreakdown(lastTurn()!); + expect(text).toContain('whisper-local'); + expect(text).toContain('Speech end to first partial'); + }); +}); diff --git a/src/__tests__/main/acappella/transport-stand-down.test.ts b/src/__tests__/main/acappella/transport-stand-down.test.ts new file mode 100644 index 0000000000..df869861d4 --- /dev/null +++ b/src/__tests__/main/acappella/transport-stand-down.test.ts @@ -0,0 +1,154 @@ +/** + * The transport's Encore-flag behaviour. + * + * A Cappella is off by default, and "off" has to mean the same thing to every + * surface that reads the flag. The transport owns the two resources a user most + * expects that switch to control - a Bonjour advert broadcasting this machine's + * name and port, and live connections from paired phones - so it is the one that + * has to stand down. + * + * Contracts defended: + * - `standDown()` takes the advert down, cancels any half-finished pairing, and + * drops live connections. + * - It does NOT revoke anything. Switching the feature off says "stop", not + * "forget my phone"; re-pairing a device because a checkbox was toggled is a + * punishment for reading the settings screen. + * - It is not `dispose()`. The transport is constructed once per process at + * handler registration, so a teardown here would mean switching the feature + * back on did nothing until the next restart. + * - `featureEnabled()` is read from settings on every call, so the signaling + * adapter and the IPC handlers cannot disagree about it. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; + +import { ACappellaTransport } from '../../../main/acappella/transport'; +import type { WebRtcHostCommand } from '../../../shared/acappella/webrtc-host'; + +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +let dir: string; +let settings: Record; +let hostCommands: WebRtcHostCommand[]; +let transport: ACappellaTransport; + +function createTransport(): ACappellaTransport { + return new ACappellaTransport({ + settingsStore: { + get: (key: string, defaultValue?: unknown) => settings[key] ?? defaultValue, + }, + userDataPath: dir, + sendToAudioHost: (command) => hostCommands.push(command), + acquireFloor: () => { + throw new Error('no floor in this test'); + }, + getSession: () => null, + getServerToken: () => 'server-token', + getServerPort: () => 4123, + getAppVersion: () => '0.0.0-test', + getMachineName: () => 'Test Machine', + }); +} + +beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'acappella-transport-')); + settings = { encoreFeatures: { aCappella: true } }; + hostCommands = []; + transport = createTransport(); +}); + +afterEach(async () => { + transport.dispose(); + await fs.rm(dir, { recursive: true, force: true }); +}); + +describe('ACappellaTransport.featureEnabled', () => { + it('mirrors the Encore flag, read fresh each time', () => { + expect(transport.featureEnabled()).toBe(true); + + settings.encoreFeatures = { aCappella: false }; + expect(transport.featureEnabled()).toBe(false); + + settings.encoreFeatures = { aCappella: true }; + expect(transport.featureEnabled()).toBe(true); + }); + + it('is false when no flags have ever been written', () => { + settings = {}; + expect(transport.featureEnabled()).toBe(false); + }); +}); + +describe('ACappellaTransport.standDown', () => { + it('takes the advert down', async () => { + // No mDNS responder is available in a test process, so the advert reports + // itself unavailable rather than advertising. What matters is the transition: + // the service must end up disabled, which is the state that has actually + // released the responder. + await transport.discovery.start(); + const stop = vi.spyOn(transport.discovery, 'stop'); + + transport.standDown(); + await transport.discovery.stop(); + + expect(stop).toHaveBeenCalled(); + expect(transport.discoveryStatus()).toEqual({ state: 'disabled' }); + }); + + it('cancels a pairing window that was open', () => { + expect(transport.startPairing()).not.toBeNull(); + expect(transport.currentPairingPayload()).not.toBeNull(); + + transport.standDown(); + + // A code that outlives the switch is a code somebody can still redeem + // against a feature its owner believes is off. + expect(transport.currentPairingPayload()).toBeNull(); + }); + + it('disconnects live devices without revoking any of them', async () => { + const offer = transport.startPairing(); + const claim = transport.pairing.claim({ + code: offer!.code, + name: 'Test iPhone', + platform: 'ios', + }); + expect(claim.status).toBe('pending'); + if (claim.status !== 'pending') return; + await transport.pairing.approve(claim.requestId); + + const disconnect = vi.spyOn(transport, 'disconnectAll'); + transport.standDown(); + + expect(disconnect).toHaveBeenCalled(); + + const devices = await transport.listDevices(); + expect(devices).toHaveLength(1); + expect(devices[0].revokedAt).toBeNull(); + expect(devices[0].online).toBe(false); + }); + + it('leaves the transport able to serve again when the feature comes back on', async () => { + transport.standDown(); + + // The reason this is standDown() and not dispose(): the transport is built + // once per process, so a user who toggles the feature off and on again must + // not need a restart to pair. + const payload = transport.startPairing(); + expect(payload).not.toBeNull(); + expect(payload?.kind).toBe('maestro-acappella'); + await expect(transport.listDevices()).resolves.toEqual([]); + }); + + it('is safe to call twice, and with nothing running', () => { + expect(() => { + transport.standDown(); + transport.standDown(); + }).not.toThrow(); + }); +}); diff --git a/src/__tests__/main/acappella/voice-session-service.test.ts b/src/__tests__/main/acappella/voice-session-service.test.ts new file mode 100644 index 0000000000..2436bf0fb4 --- /dev/null +++ b/src/__tests__/main/acappella/voice-session-service.test.ts @@ -0,0 +1,1998 @@ +/** + * @file voice-session-service.test.ts + * + * Unit tests for the headless voice session service: the wake -> listen -> + * transcribe -> route -> dispatch -> speak pipeline, monotonic `seq`, the + * barge-in / stop distinction, and the classified failure modes. + * + * No electron, no providers, no timers: a fake trio drives the pipeline + * synchronously so every assertion is about the service's own sequencing. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../main/utils/sentry', () => ({ captureException: vi.fn() })); +vi.mock('../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +/** + * Every transition the service takes, recorded in order. + * + * `transition()` is private and most edges are invisible in the event stream + * (`transcribing` and `routing` emit nothing of their own), so the only honest + * way to prove which edges the pipeline actually walks is to wrap the shared + * assertion the service routes all of them through. The wrapper still delegates, + * so an illegal edge throws exactly as it would in production. + */ +const transitionLog = vi.hoisted(() => ({ edges: [] as string[] })); + +vi.mock('../../../shared/acappella/session-state', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + assertVoiceStateTransition: (from: string, to: string) => { + transitionLog.edges.push(`${from} -> ${to}`); + actual.assertVoiceStateTransition(from as VoiceSessionState, to as VoiceSessionState); + }, + }; +}); + +import { captureException } from '../../../main/utils/sentry'; +import { + VoiceSessionService, + VoiceDispatchError, + type AgentReplyStream, + type VoiceDispatchResult, + type VoiceFocusTarget, + type VoiceRouteExecutor, +} from '../../../main/acappella/voice-session-service'; +import type { AgentOutputChunk } from '../../../main/acappella/speech'; +import type { BackgroundAnnouncementSetting } from '../../../shared/acappella/announcements'; +import type { RosterAgent, VoiceEvent, VoiceEventType } from '../../../shared/acappella/protocol'; +import type { + BrainProvider, + SttCallbacks, + SttProvider, + TtsChunk, + TtsProvider, + VoiceConverseContext, + VoiceProviderTrio, + VoiceRouteContext, +} from '../../../shared/acappella/providers'; +import type { VoiceReadiness, VoiceSlotReadiness } from '../../../shared/acappella/readiness'; +import type { RouteDecision } from '../../../shared/acappella/route-decision'; +import { splitIntoSpokenSentences } from '../../../shared/acappella/sentences'; +import { + InvalidVoiceStateTransitionError, + VOICE_STATE_TRANSITIONS, + type VoiceSessionState, +} from '../../../shared/acappella/session-state'; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +/** Text-in STT: two partials then a final, exactly like the mock tier will. */ +class FakeStt implements SttProvider { + readonly id = 'fake-stt'; + readonly label = 'Fake STT'; + readonly tier = 'mock' as const; + readonly sampleRate = 16_000; + readonly acceptsAudio = false; + + callbacks: SttCallbacks | null = null; + started = false; + stopped = false; + /** When set, `start()` rejects with it. */ + startError: Error | null = null; + + async start(callbacks: SttCallbacks): Promise { + if (this.startError) throw this.startError; + this.callbacks = callbacks; + this.started = true; + } + feed(): void {} + async flush(): Promise {} + async stop(): Promise { + this.stopped = true; + } + injectUtterance(text: string): void { + this.callbacks?.onPartial(text.slice(0, Math.ceil(text.length / 3)), 0.3); + this.callbacks?.onPartial(text.slice(0, Math.ceil((text.length * 2) / 3)), 0.7); + this.callbacks?.onFinal(text, 0.95); + } +} + +class FakeBrain implements BrainProvider { + readonly id = 'fake-brain'; + readonly label = 'Fake Brain'; + readonly tier = 'mock' as const; + + decision: RouteDecision = { + target: 'conductor', + tabAction: 'current', + prompt: 'hello', + confidence: 0.9, + }; + spoken = 'All done. Two files changed.'; + routeError: Error | null = null; + /** When set, `route()` parks here, so a test can act while the brain thinks. */ + routeGate: Promise | null = null; + + /** Every context the brain was routed with, so a test can read the conversation. */ + contexts: VoiceRouteContext[] = []; + async route(_input: string, context: VoiceRouteContext): Promise { + this.contexts.push(context); + if (this.routeGate) await this.routeGate; + if (this.routeError) throw this.routeError; + return this.decision; + } + async converse(_agentText: string, _context: VoiceConverseContext): Promise { + return this.spoken; + } +} + +/** + * Yields one chunk per sentence, checking a cancel flag between them so + * `cancel()` cuts the run off rather than draining it. + */ +class FakeTts implements TtsProvider { + readonly id = 'fake-tts'; + readonly label = 'Fake TTS'; + readonly tier = 'mock' as const; + + cancelled = false; + /** Called after each chunk, so a test can interrupt mid-run. */ + onChunk: (() => void) | null = null; + /** Thrown from inside the iterator, the way a streaming cloud voice fails. */ + speakError: Error | null = null; + + speak(text: string, options: { utteranceId: string }): AsyncIterable { + this.cancelled = false; + const sentences = splitIntoSpokenSentences(text); + const self = this; + return { + async *[Symbol.asyncIterator]() { + for (let index = 0; index < sentences.length; index++) { + if (self.cancelled) return; + if (self.speakError) throw self.speakError; + yield { + utteranceId: options.utteranceId, + index, + text: sentences[index], + format: 'none' as const, + audio: null, + }; + self.onChunk?.(); + } + }, + }; + } + cancel(): void { + this.cancelled = true; + } +} + +/** + * An agent reply the translator will NOT pass through. + * + * Two lines, so it is markdown-shaped by the passthrough test's own rule, which + * is what sends it to the Brain and makes `FakeBrain.spoken` the thing that gets + * said. A short single-line reply is passed through untouched on purpose - that + * is the "no translation hop for `yes, done`" behaviour, covered on its own in + * conversational-translator.test.ts and again below. + */ +const AGENT_REPLY = 'Rewrote the stale token check in the auth middleware.\nTwo files changed.'; + +function makeRoster(): RosterAgent[] { + return [ + { + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo', + tabs: [{ id: 'tab-1', name: 'Auth', lastActiveAt: 1 }], + }, + ]; +} + +interface Harness { + service: VoiceSessionService; + stt: FakeStt; + tts: FakeTts; + brain: FakeBrain; + events: VoiceEvent[]; + types: () => VoiceEventType[]; + executor: ReturnType; +} + +function makeHarness( + overrides: { + executeRoute?: VoiceRouteExecutor; + onSpeechChunk?: (chunk: TtsChunk) => void; + checkReadiness?: () => VoiceReadiness | Promise; + agentReplyStream?: AgentReplyStream; + focusTarget?: VoiceFocusTarget; + getBackgroundAnnouncementSetting?: () => BackgroundAnnouncementSetting | undefined; + bargeInGuardMs?: number; + getUtteranceComposerConfig?: () => { settleMs?: number; maxHoldMs?: number }; + getConversationalMode?: () => boolean; + } = {} +): Harness { + const stt = new FakeStt(); + const tts = new FakeTts(); + const brain = new FakeBrain(); + const providers: VoiceProviderTrio = { stt, tts, brain }; + + const dispatchResult: VoiceDispatchResult = { + agentSessionId: 'agent-backend', + agentName: 'Backend', + tabId: 'tab-1', + action: 'focused', + promptSent: true, + }; + const executor = vi.fn(async () => dispatchResult); + + const service = new VoiceSessionService({ + providers, + getRoster: () => makeRoster(), + executeRoute: overrides.executeRoute ?? (executor as unknown as VoiceRouteExecutor), + onSpeechChunk: overrides.onSpeechChunk, + checkReadiness: overrides.checkReadiness, + agentReplyStream: overrides.agentReplyStream, + focusTarget: overrides.focusTarget, + getBackgroundAnnouncementSetting: overrides.getBackgroundAnnouncementSetting, + // Off unless a test asks for it. The guard is real-time dead time after + // speech starts, so leaving it on would make every barge-in assertion here a + // race against the wall clock; it has its own tests below. + bargeInGuardMs: overrides.bargeInGuardMs ?? 0, + // Off unless a test asks: `FakeStt` is text-in, so the composer is bypassed + // for every existing test here, which is the production rule too. + getUtteranceComposerConfig: overrides.getUtteranceComposerConfig, + // Off unless a test asks, which is the shipped default: every utterance is + // a command until someone turns conversation on. + getConversationalMode: overrides.getConversationalMode, + }); + + const events: VoiceEvent[] = []; + service.subscribe((event) => events.push(event)); + + return { service, stt, tts, brain, events, types: () => events.map((e) => e.type), executor }; +} + +/** Start a session and drain the events emitted by startup. */ +async function start(h: Harness): Promise { + await h.service.startSession({ scope: { kind: 'conductor' }, source: 'hotkey' }); + h.events.length = 0; + takeEdges(); +} + +/** Read and clear the recorded transitions. */ +function takeEdges(): string[] { + return transitionLog.edges.splice(0, transitionLog.edges.length); +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('VoiceSessionService lifecycle', () => { + let h: Harness; + + beforeEach(() => { + h = makeHarness(); + }); + + it('starts idle with no session', () => { + const snapshot = h.service.getSnapshot(); + expect(snapshot.state).toBe('idle'); + expect(snapshot.sessionId).toBeNull(); + expect(snapshot.providerIds).toEqual({ stt: 'fake-stt', tts: 'fake-tts', brain: 'fake-brain' }); + }); + + it('wakes into listening and announces the provider', async () => { + const snapshot = await h.service.startSession({ + scope: { kind: 'agent', sessionId: 'agent-backend' }, + source: 'wake-word', + }); + + expect(snapshot.state).toBe('listening'); + expect(snapshot.sessionId).toBeTruthy(); + expect(h.stt.started).toBe(true); + expect(h.types()).toEqual(['wake', 'listen-start', 'agent-roster']); + + const listenStart = h.events[1]; + expect(listenStart.type === 'listen-start' && listenStart.sttProviderId).toBe('fake-stt'); + }); + + /** + * Conversational mode: the Conductor answers instead of dispatching, until a + * doable thing has been described. Off by default, because it changes what a + * spoken sentence means. + */ + describe('talking it through before dispatching', () => { + function chatHarness() { + return makeHarness({ getConversationalMode: () => true }); + } + + it('speaks a reply and touches no agent', async () => { + const chat = chatHarness(); + chat.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: '', + confidence: 0.3, + reply: 'Is that failing on every load, or only the second?', + }; + await start(chat); + + chat.service.submitUtterance('the token refresh keeps breaking'); + await vi.waitFor(() => expect(chat.types()).toContain('speak-start')); + + expect(chat.executor).not.toHaveBeenCalled(); + expect(chat.types()).not.toContain('dispatch'); + }); + + it('hands the floor straight back, so the exchange continues', async () => { + const chat = chatHarness(); + chat.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: '', + confidence: 0.3, + reply: 'Which repository is that in?', + }; + await start(chat); + + chat.service.submitUtterance('something is wrong with auth'); + await vi.waitFor(() => expect(chat.service.getState()).toBe('listening')); + + expect(chat.service.getState()).toBe('listening'); + }); + + it('shows the Brain what has already been said', async () => { + // Without the history the second sentence is routed on its own, which is + // how a two-word answer becomes a tab called "the second one". + const chat = chatHarness(); + chat.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: '', + confidence: 0.3, + reply: 'Every load, or the second?', + }; + await start(chat); + + chat.service.submitUtterance('the refresh keeps failing'); + await vi.waitFor(() => expect(chat.service.getState()).toBe('listening')); + chat.service.submitUtterance('the second one'); + await vi.waitFor(() => expect(chat.brain.contexts.length).toBeGreaterThan(1)); + + const latest = chat.brain.contexts[chat.brain.contexts.length - 1]; + expect(latest.conversational).toBe(true); + expect(latest.conversation?.map((turn) => turn.text)).toEqual([ + 'the refresh keeps failing', + 'Every load, or the second?', + 'the second one', + ]); + }); + + it('dispatches once the Brain stops replying', async () => { + const chat = chatHarness(); + chat.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: '', + confidence: 0.3, + reply: 'Which part is failing?', + }; + await start(chat); + chat.service.submitUtterance('auth is broken'); + await vi.waitFor(() => expect(chat.service.getState()).toBe('listening')); + + // The Brain decides there is a task now. + chat.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: 'Fix the token refresh on the second load.', + confidence: 0.9, + }; + chat.service.submitUtterance('the token refresh on the second load'); + await vi.waitFor(() => expect(chat.types()).toContain('dispatch')); + + expect(chat.executor).toHaveBeenCalledTimes(1); + }); + + it('forgets the discussion once it has been sent', async () => { + // Otherwise the next request arrives wearing the last one's context. + const chat = chatHarness(); + await start(chat); + + chat.service.submitUtterance('run the tests'); + await vi.waitFor(() => expect(chat.types()).toContain('dispatch')); + chat.service.submitUtterance('now the other thing'); + await vi.waitFor(() => expect(chat.brain.contexts.length).toBeGreaterThan(1)); + + const latest = chat.brain.contexts[chat.brain.contexts.length - 1]; + expect(latest.conversation?.map((turn) => turn.text)).toEqual(['now the other thing']); + }); + + it('routes every utterance when the mode is off', async () => { + // The shipped default, and the behaviour anyone already using voice has. + await start(h); + + h.service.submitUtterance('run the tests'); + await vi.waitFor(() => expect(h.types()).toContain('dispatch')); + + expect(h.executor).toHaveBeenCalledTimes(1); + expect(h.brain.contexts[0].conversational).toBe(false); + expect(h.brain.contexts[0].conversation).toBeUndefined(); + }); + }); + + /** + * A pause mid-sentence is not the end of a request. Before the composer, the + * 700 ms endpoint in `audio/vad.ts` turned one thought into two dispatches and + * the agent answered half a sentence. + */ + describe('assembling one thought from several fragments', () => { + /** A recogniser that listens to a room, which is what triggers composing. */ + function listeningHarness(settleMs: number) { + const h = makeHarness({ getUtteranceComposerConfig: () => ({ settleMs }) }); + (h.stt as unknown as { acceptsAudio: boolean }).acceptsAudio = true; + return h; + } + + /** A recogniser final, bypassing the fake provider's own partial timing. */ + function final(h: Harness, text: string) { + (h.stt.callbacks as { onFinal: (t: string, c: number) => void }).onFinal(text, 0.9); + } + + it('dispatches ONE request for a sentence said in two halves', async () => { + const h2 = listeningHarness(40); + await start(h2); + + final(h2, 'look at the auth module'); + final(h2, 'and say why the refresh is failing'); + await vi.waitFor(() => expect(h2.types()).toContain('dispatch')); + + expect(h2.executor).toHaveBeenCalledTimes(1); + const finals = h2.events.filter((e) => e.type === 'final-transcript'); + expect(finals).toHaveLength(1); + expect(finals[0].type === 'final-transcript' && finals[0].text).toBe( + 'look at the auth module and say why the refresh is failing' + ); + }); + + it('leaves a text-in provider alone, since its utterance was already whole', async () => { + // `FakeStt` is text-in. Someone who typed a message and pressed send has + // delimited it themselves; holding it would be latency for nothing. + await start(h); + + h.service.submitUtterance('do the thing'); + await vi.waitFor(() => expect(h.types()).toContain('dispatch')); + + expect(h.executor).toHaveBeenCalledTimes(1); + }); + + it('sends on a release without waiting out the hold', async () => { + // Letting go of a push-to-talk key. The recogniser flush is a separate + // call; without `endUtteranceNow` the resulting final would be buffered + // and sit out a settle window the user already answered. + const h2 = listeningHarness(30_000); + await start(h2); + + final(h2, 'fix the auth bug'); + h2.service.endUtteranceNow(); + await vi.waitFor(() => expect(h2.types()).toContain('dispatch')); + + expect(h2.executor).toHaveBeenCalledTimes(1); + }); + + it('drops a half-collected thought when the floor closes', async () => { + // Otherwise it settles into a session nobody is in. + const h2 = listeningHarness(10_000); + await start(h2); + + final(h2, 'half a sentence'); + await h2.service.stopSession('user'); + await new Promise((resolve) => setTimeout(resolve, 30)); + + expect(h2.executor).not.toHaveBeenCalled(); + }); + }); + + /** + * A recogniser that hears but does not transcribe (the microphone check) is a + * meter, not a voice. Routing its measurement onward sent live agents prompts + * like "Echo utterance 4: 1.5s of speech." and billed the user for the reply. + */ + describe('a diagnostic recogniser', () => { + function deafHarness() { + const h = makeHarness(); + // Same object the service holds, so the flag is read at turn time. + (h.stt as unknown as { transcribesSpeech: boolean }).transcribesSpeech = false; + return h; + } + + it('shows the transcript but never dispatches it to an agent', async () => { + const deaf = deafHarness(); + await start(deaf); + + deaf.service.submitUtterance('Echo utterance 1: 1.5s of speech.'); + // Waiting for the FLOOR TO REOPEN, not for the transcript: the transcript + // is emitted before routing would happen, so asserting on it would pass + // even with the guard removed. Reopening is the end of the whole turn. + await vi.waitFor(() => expect(deaf.service.getState()).toBe('listening')); + + // The user still sees proof the microphone works... + expect(deaf.types()).toContain('final-transcript'); + // ...and no agent is told anything. + expect(deaf.executor).not.toHaveBeenCalled(); + expect(deaf.types()).not.toContain('route-decision'); + expect(deaf.types()).not.toContain('dispatch'); + }); + + it('reopens the floor, so the meter keeps working turn after turn', async () => { + const deaf = deafHarness(); + await start(deaf); + + deaf.service.submitUtterance('Echo utterance 1: 1.5s of speech.'); + await vi.waitFor(() => expect(deaf.service.getState()).toBe('listening')); + + expect(deaf.service.getState()).toBe('listening'); + }); + + it('still routes for an ordinary recogniser', async () => { + // The guard is one flag; a regression that read it backwards would make + // every real provider silent, which is worse than the bug it fixes. + await start(h); + + h.service.submitUtterance('do the thing'); + await vi.waitFor(() => expect(h.types()).toContain('dispatch')); + + expect(h.executor).toHaveBeenCalled(); + }); + }); + + it('stamps every event with the session id and a monotonic seq', async () => { + await start(h); + h.service.submitUtterance('do the thing'); + await vi.waitFor(() => expect(h.types()).toContain('dispatch')); + + const sessionId = h.service.getSnapshot().sessionId; + expect(new Set(h.events.map((e) => e.sessionId))).toEqual(new Set([sessionId])); + + const seqs = h.events.map((e) => e.seq); + expect(seqs).toEqual([...seqs].sort((a, b) => a - b)); + expect(new Set(seqs).size).toBe(seqs.length); + }); + + it('resets seq when a new session replaces the old one', async () => { + await start(h); + await h.service.stopSession('user'); + h.events.length = 0; + + await h.service.startSession({ scope: { kind: 'conductor' } }); + expect(h.events[0].seq).toBe(1); + }); + + it('replaces a running session rather than stacking one', async () => { + await start(h); + const first = h.service.getSnapshot().sessionId; + + await h.service.startSession({ scope: { kind: 'agent', sessionId: 'agent-backend' } }); + expect(h.service.getSnapshot().sessionId).not.toBe(first); + expect(h.service.getState()).toBe('listening'); + expect(h.types()).toContain('listen-stop'); + }); + + it('stops back to idle and releases the provider', async () => { + await start(h); + await h.service.stopSession('user'); + + expect(h.service.getState()).toBe('idle'); + expect(h.service.getSnapshot().sessionId).toBeNull(); + expect(h.stt.stopped).toBe(true); + expect(h.types()).toEqual(['listen-stop']); + }); + + it('ignores stopSession when already idle', async () => { + await h.service.stopSession('user'); + expect(h.events).toHaveLength(0); + }); +}); + +describe('VoiceSessionService turn pipeline', () => { + let h: Harness; + + beforeEach(async () => { + h = makeHarness(); + await start(h); + }); + + it('runs partials, final, routing, and dispatch in order', async () => { + h.service.submitUtterance('open the auth tab'); + await vi.waitFor(() => expect(h.types()).toContain('dispatch')); + + expect(h.types()).toEqual([ + 'partial-transcript', + 'partial-transcript', + 'final-transcript', + 'agent-roster', + 'route-decision', + 'dispatch', + ]); + expect(h.service.getState()).toBe('dispatching'); + }); + + it('passes the roster and recent utterances to the brain', async () => { + const spy = vi.spyOn(h.brain, 'route'); + h.service.submitUtterance('first thing'); + await vi.waitFor(() => expect(h.types()).toContain('dispatch')); + + const context = spy.mock.calls[0][1]; + expect(context.roster.map((agent) => agent.sessionId)).toEqual(['agent-backend']); + expect(context.recentUtterances).toEqual(['first thing']); + }); + + it('returns to listening on an empty utterance', async () => { + h.service.submitUtterance(' '); + await vi.waitFor(() => expect(h.types()).toContain('listen-start')); + + expect(h.service.getState()).toBe('listening'); + expect(h.types()).not.toContain('route-decision'); + }); + + it('refuses an utterance that arrives in a state that cannot take one', async () => { + await h.service.stopSession('user'); + expect(h.service.submitUtterance('too late')).toBe(false); + }); + + it('abandons a pending reply when the user speaks again', async () => { + h.service.submitUtterance('first'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + h.events.length = 0; + expect(h.service.submitUtterance('second')).toBe(true); + await vi.waitFor(() => expect(h.types()).toContain('dispatch')); + expect(h.events[0].type).toBe('listen-start'); + }); +}); + +describe('VoiceSessionService speech', () => { + let h: Harness; + + beforeEach(async () => { + h = makeHarness(); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + h.events.length = 0; + }); + + it('speaks a reply sentence by sentence and returns the floor', async () => { + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + expect(h.types()).toEqual([ + 'agent-reply', + 'speak-start', + 'speak-sentence', + 'speak-sentence', + 'speak-end', + 'listen-start', + ]); + + const start = h.events[1]; + expect(start.type === 'speak-start' && start.sentenceCount).toBe(2); + const sentences = h.events.filter((e) => e.type === 'speak-sentence'); + expect(sentences.map((e) => (e.type === 'speak-sentence' ? e.text : ''))).toEqual([ + 'All done.', + 'Two files changed.', + ]); + expect(h.service.getState()).toBe('listening'); + }); + + it('skips the speech run when there is nothing worth speaking', async () => { + h.brain.spoken = ' '; + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + expect(h.types()).toEqual(['agent-reply', 'listen-start']); + expect(h.service.getState()).toBe('listening'); + }); + + it('ignores a reply that arrives when nothing was dispatched', async () => { + await h.service.stopSession('user'); + const accepted = await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: 'late', + }); + expect(accepted).toBe(false); + }); +}); + +describe('VoiceSessionService barge-in versus stop', () => { + let h: Harness; + + beforeEach(async () => { + h = makeHarness(); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + h.events.length = 0; + }); + + it('barge-in cancels speech mid-run and keeps the floor', async () => { + // Triggered from the FIRST spoken sentence rather than from a TTS chunk: the + // scheduler synthesizes a sentence ahead of the one being heard, so a chunk + // arriving says nothing about what the user has actually listened to. + h.service.subscribe((event) => { + if (event.type === 'speak-sentence' && event.index === 0) h.service.interrupt('voice'); + }); + + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + expect(h.types()).toEqual([ + 'agent-reply', + 'speak-start', + 'speak-sentence', + 'barge-in', + 'speak-end', + 'listen-start', + ]); + const end = h.events.find((e) => e.type === 'speak-end'); + expect(end?.type === 'speak-end' && end.reason).toBe('cancelled'); + expect(h.tts.cancelled).toBe(true); + // The floor is retained: still in session, still listening. + expect(h.service.getState()).toBe('listening'); + expect(h.service.getSnapshot().sessionId).toBeTruthy(); + }); + + it('barge-in is a no-op when nothing is speaking', () => { + expect(h.service.interrupt('client-button')).toBe(false); + expect(h.events).toHaveLength(0); + }); + + it('the stop word ends the session from any state', async () => { + await h.service.hardStop('voice', 'never mind'); + + expect(h.types()).toEqual(['stop-word', 'listen-stop']); + expect(h.service.getState()).toBe('idle'); + expect(h.service.getSnapshot().sessionId).toBeNull(); + expect(h.stt.stopped).toBe(true); + }); + + it('the stop word cancels speech on the way out', async () => { + h.tts.onChunk = () => { + void h.service.hardStop('voice'); + }; + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + expect(h.tts.cancelled).toBe(true); + expect(h.service.getState()).toBe('idle'); + }); +}); + +describe('VoiceSessionService classified failures', () => { + it('reports a provider that cannot start', async () => { + const h = makeHarness(); + h.stt.startError = new Error('microphone busy'); + + const snapshot = await h.service.startSession({ scope: { kind: 'conductor' } }); + + expect(snapshot.state).toBe('error'); + expect(h.types()).toEqual(['wake', 'session-error']); + const error = h.events[1]; + expect(error.type === 'session-error' && error.code).toBe('provider-unavailable'); + expect(error.type === 'session-error' && error.providerId).toBe('fake-stt'); + }); + + it('refuses to start when the capability gate blocks a slot, and never substitutes', async () => { + const blocked: VoiceSlotReadiness = { + slot: 'stt', + providerId: 'whisper-local', + satisfied: false, + reason: 'model-not-installed', + detail: 'Speech-to-Text: Whisper Base (English) is not installed.', + suggestedAction: 'Download it in Settings > Plugins > A Cappella > Voice Setup.', + }; + const h = makeHarness({ + checkReadiness: () => ({ + canStartSession: false, + canRunHandsFree: false, + slots: [blocked], + blocking: [blocked], + }), + }); + + const snapshot = await h.service.startSession({ scope: { kind: 'conductor' } }); + + expect(snapshot.state).toBe('error'); + const error = h.events.find((event) => event.type === 'session-error'); + expect(error?.type === 'session-error' && error.code).toBe('provider-unavailable'); + // The missing piece AND the recovery are both named: a disabled voice mode + // with no stated reason is indistinguishable from a bug. + expect(error?.type === 'session-error' && error.message).toContain('is not installed'); + expect(error?.type === 'session-error' && error.message).toContain('Download it in Settings'); + // The gate ran BEFORE the device. Nothing was opened for a session that was + // never going to work. + expect(h.stt.started).toBe(false); + // And the blocked provider is reported as-is, never swapped for a working one. + expect(error?.type === 'session-error' && error.providerId).toBe('whisper-local'); + }); + + it('starts normally when the gate is satisfied', async () => { + const h = makeHarness({ + checkReadiness: () => ({ + canStartSession: true, + canRunHandsFree: true, + slots: [], + blocking: [], + }), + }); + + const snapshot = await h.service.startSession({ scope: { kind: 'conductor' } }); + + expect(snapshot.state).toBe('listening'); + expect(h.stt.started).toBe(true); + }); + + it('reports a decision that targets an agent which is not running', async () => { + const h = makeHarness(); + await start(h); + h.brain.decision = { + target: { sessionId: 'agent-ghost' }, + tabAction: 'current', + prompt: 'hi', + confidence: 0.5, + }; + + h.service.submitUtterance('talk to the ghost'); + await vi.waitFor(() => expect(h.types()).toContain('session-error')); + + const error = h.events.find((e) => e.type === 'session-error'); + expect(error?.type === 'session-error' && error.code).toBe('no-agent-matched'); + expect(h.service.getState()).toBe('error'); + expect(h.types()).not.toContain('dispatch'); + }); + + it('reports a known dispatch failure and swallows nothing else', async () => { + const h = makeHarness({ + executeRoute: async () => { + throw new VoiceDispatchError('renderer did not answer in time'); + }, + }); + await start(h); + + h.service.submitUtterance('open a new tab'); + await vi.waitFor(() => expect(h.types()).toContain('session-error')); + + const error = h.events.find((e) => e.type === 'session-error'); + expect(error?.type === 'session-error' && error.code).toBe('dispatch-failed'); + expect(h.service.getState()).toBe('error'); + }); + + it('reports a missing route executor rather than dispatching nowhere', async () => { + const stt = new FakeStt(); + const service = new VoiceSessionService({ + providers: { stt, tts: new FakeTts(), brain: new FakeBrain() }, + getRoster: () => makeRoster(), + }); + const events: VoiceEvent[] = []; + service.subscribe((e) => events.push(e)); + + await service.startSession({ scope: { kind: 'conductor' } }); + service.submitUtterance('do it'); + await vi.waitFor(() => expect(events.some((e) => e.type === 'session-error')).toBe(true)); + + expect(service.getState()).toBe('error'); + }); + + it('recovers from error only by stopping', async () => { + const h = makeHarness(); + h.stt.startError = new Error('nope'); + await h.service.startSession({ scope: { kind: 'conductor' } }); + expect(h.service.getState()).toBe('error'); + + await h.service.stopSession('error'); + expect(h.service.getState()).toBe('idle'); + }); + + it('reports an unexpected provider exception to Sentry and closes the floor', async () => { + const h = makeHarness(); + await start(h); + h.brain.routeError = new Error('brain exploded'); + + h.service.submitUtterance('boom'); + await vi.waitFor(() => expect(h.service.getState()).toBe('error')); + + const stop = h.events.find((e) => e.type === 'listen-stop'); + expect(stop?.type === 'listen-stop' && stop.reason).toBe('error'); + // Unexpected failures are never dressed up as a classified session-error. + expect(h.types()).not.toContain('session-error'); + expect(vi.mocked(captureException)).toHaveBeenCalledWith( + h.brain.routeError, + expect.objectContaining({ context: 'acappella.runTurn' }) + ); + }); +}); + +describe('VoiceSessionService audio telemetry', () => { + it('publishes a meter update on the one ordered stream', async () => { + const h = makeHarness(); + await start(h); + + h.service.publishAudioLevel(0.4, true); + const event = h.events.at(-1); + + expect(event?.type).toBe('audio-level'); + expect(event).toMatchObject({ level: 0.4, speech: true, sessionId: expect.any(String) }); + }); + + it('clamps a level rather than putting an impossible number on the wire', async () => { + const h = makeHarness(); + await start(h); + + h.service.publishAudioLevel(4, false); + h.service.publishAudioLevel(Number.NaN, false); + + const levels = h.events + .filter((e) => e.type === 'audio-level') + .map((e) => (e as Extract).level); + expect(levels).toEqual([1, 0]); + }); + + it('numbers audio events in the same seq space as the rest', async () => { + const h = makeHarness(); + await start(h); + + h.service.publishAudioLevel(0.1, false); + h.service.publishMicState({ + permission: 'granted', + capturing: true, + deviceId: 'default', + deviceLabel: 'Built-in Microphone', + issue: null, + deviceChanged: false, + }); + + const seqs = h.events.map((e) => e.seq); + expect(seqs).toEqual([...seqs].sort((a, b) => a - b)); + expect(new Set(seqs).size).toBe(seqs.length); + }); + + it('publishes the microphone state, including the benign transitions', async () => { + const h = makeHarness(); + await start(h); + + h.service.publishMicState({ + permission: 'denied', + capturing: false, + deviceId: null, + deviceLabel: null, + issue: 'permission-denied', + deviceChanged: false, + }); + + expect(h.events.at(-1)).toMatchObject({ + type: 'mic-state', + permission: 'denied', + issue: 'permission-denied', + }); + }); + + it('drops telemetry that belongs to no session', () => { + const h = makeHarness(); + + // A frame in flight when the session ended has no envelope to travel in. + h.service.publishAudioLevel(0.5, true); + expect(h.events).toHaveLength(0); + }); +}); + +describe('VoiceSessionService audio seams', () => { + it('exposes the recogniser only while a session exists', async () => { + const h = makeHarness(); + + // Audio that arrives with no session behind it has nowhere to go, and the + // pipeline reads this null to decide to drop it rather than buffer it. + expect(h.service.getActiveStt()).toBeNull(); + + await start(h); + expect(h.service.getActiveStt()).toBe(h.stt); + + await h.service.stopSession('user'); + expect(h.service.getActiveStt()).toBeNull(); + }); + + it('parks the session on a capture failure the user can fix, and says it is fixable', async () => { + const h = makeHarness(); + await start(h); + + h.service.reportAudioCaptureFailure('permission-denied', 'Microphone permission denied'); + + expect(h.events.at(-1)).toMatchObject({ + type: 'session-error', + code: 'audio-capture-failed', + recoverable: true, + }); + // A listening indicator over a microphone that will never produce a + // transcript is the worst outcome this feature has. + expect(h.service.getState()).toBe('error'); + }); + + it('reports an environment failure as unrecoverable, so no client offers a fix', async () => { + const h = makeHarness(); + await start(h); + + h.service.reportAudioCaptureFailure('audio-init-failed', 'AudioContext unavailable'); + + expect(h.events.at(-1)).toMatchObject({ + type: 'session-error', + code: 'audio-capture-failed', + recoverable: false, + }); + }); + + it('hands every spoken chunk to the audio sink, after its sentence event', async () => { + const seen: Array<{ index: number; eventsSoFar: number }> = []; + const h = makeHarness({ + onSpeechChunk: (chunk) => seen.push({ index: chunk.index, eventsSoFar: h.events.length }), + }); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + h.events.length = 0; + + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + // One per sentence, and each after the `speak-sentence` that announced it: + // the text should be on screen by the time it is audible. The chunk's own + // `index` is the PROVIDER's, and the scheduler synthesises one sentence per + // call, so it is 0 for both here - what is being asserted is the ordering + // against the events, not a counter the provider owns. + expect(seen).toHaveLength(2); + const sentenceEvents = h.events + .map((event, index) => ({ event, index })) + .filter(({ event }) => event.type === 'speak-sentence'); + expect(sentenceEvents).toHaveLength(2); + expect(seen[0].eventsSoFar).toBe(sentenceEvents[0].index + 1); + expect(seen[1].eventsSoFar).toBe(sentenceEvents[1].index + 1); + }); + + it('drops chunks from a run that was cancelled mid-sentence', async () => { + const chunks: TtsChunk[] = []; + const h = makeHarness({ onSpeechChunk: (chunk) => chunks.push(chunk) }); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + h.service.subscribe((event) => { + if (event.type === 'speak-sentence' && event.index === 0) h.service.interrupt('voice'); + }); + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + // The interrupt lands while the first sentence is being announced, so nothing + // reaches an output device the user has already talked over - including the + // second sentence, which the scheduler had synthesized ahead. + expect(chunks).toHaveLength(0); + }); +}); + +describe('VoiceSessionService subscribers', () => { + it('keeps delivering when one subscriber throws', async () => { + const h = makeHarness(); + const good = vi.fn(); + h.service.subscribe(() => { + throw new Error('bad client'); + }); + h.service.subscribe(good); + + await h.service.startSession({ scope: { kind: 'conductor' } }); + expect(good).toHaveBeenCalled(); + }); + + it('stops delivering after unsubscribe', async () => { + const h = makeHarness(); + const listener = vi.fn(); + const unsubscribe = h.service.subscribe(listener); + unsubscribe(); + + await h.service.startSession({ scope: { kind: 'conductor' } }); + expect(listener).not.toHaveBeenCalled(); + }); + + it('drops every subscriber on dispose', async () => { + const h = makeHarness(); + await start(h); + await h.service.dispose(); + + expect(h.service.getState()).toBe('idle'); + h.events.length = 0; + await h.service.startSession({ scope: { kind: 'conductor' } }); + expect(h.events).toHaveLength(0); + }); +}); + +// --------------------------------------------------------------------------- +// State machine coverage +// +// One test per edge the service can actually reach, each asserting the exact +// sequence of transitions it took. `DEFENSIVE_EDGES` names the rest, and the +// last test fails if the table grows an edge that appears in neither list. +// --------------------------------------------------------------------------- + +/** Edges driven end to end by the tests in this block. */ +const DRIVEN_EDGES = [ + 'idle -> arming', + 'arming -> listening', + 'arming -> error', + 'listening -> transcribing', + 'listening -> idle', + 'listening -> error', + 'transcribing -> routing', + 'transcribing -> listening', + 'routing -> dispatching', + 'routing -> speaking', + 'routing -> idle', + 'routing -> error', + 'dispatching -> speaking', + 'dispatching -> listening', + 'dispatching -> idle', + 'dispatching -> error', + 'speaking -> interrupted', + 'speaking -> listening', + 'speaking -> idle', + 'speaking -> error', + 'interrupted -> listening', + 'error -> idle', +]; + +/** + * Edges the table allows that nothing in Phase 01 can reach. Each is a guard + * against a shape a later phase adds, not dead weight: leaving them out of the + * table would turn that phase's first real failure into a thrown + * `InvalidVoiceStateTransitionError` instead of a clean teardown. + */ +const DEFENSIVE_EDGES = [ + // `arming` is only held across `stt.start()`, which no client can interrupt + // today. A real microphone permission prompt (Phase 05) is cancellable. + 'arming -> idle', + // `transcribing` and `interrupted` are both crossed synchronously, so nothing + // can stop or fail a session while it is in either one. + 'transcribing -> idle', + 'transcribing -> error', + 'interrupted -> idle', + 'interrupted -> error', +]; + +describe('VoiceSessionService state machine', () => { + let h: Harness; + + beforeEach(() => { + h = makeHarness(); + takeEdges(); + }); + + it('idle -> arming -> listening on wake', async () => { + await h.service.startSession({ scope: { kind: 'conductor' } }); + expect(takeEdges()).toEqual(['idle -> arming', 'arming -> listening']); + }); + + it('arming -> error when the speech provider will not open', async () => { + h.stt.startError = new Error('microphone busy'); + await h.service.startSession({ scope: { kind: 'conductor' } }); + expect(takeEdges()).toEqual(['idle -> arming', 'arming -> error']); + }); + + it('error -> idle is the only way out of error', async () => { + h.stt.startError = new Error('microphone busy'); + await h.service.startSession({ scope: { kind: 'conductor' } }); + takeEdges(); + + await h.service.stopSession('error'); + expect(takeEdges()).toEqual(['error -> idle']); + expect(h.service.getState()).toBe('idle'); + }); + + it('listening -> idle when the session is stopped', async () => { + await start(h); + await h.service.stopSession('user'); + expect(takeEdges()).toEqual(['listening -> idle']); + }); + + it('listening -> error when the speech provider drops out mid-session', async () => { + await start(h); + h.stt.callbacks?.onError(new Error('device disappeared')); + expect(takeEdges()).toEqual(['listening -> error']); + }); + + it('listening -> transcribing -> routing -> dispatching for a full utterance', async () => { + await start(h); + h.service.submitUtterance('open the auth tab'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + expect(takeEdges()).toEqual([ + 'listening -> transcribing', + 'transcribing -> routing', + 'routing -> dispatching', + ]); + }); + + it('transcribing -> listening when the utterance was empty', async () => { + await start(h); + h.service.submitUtterance(' '); + await vi.waitFor(() => expect(transitionLog.edges).toContain('transcribing -> listening')); + + expect(takeEdges()).toEqual(['listening -> transcribing', 'transcribing -> listening']); + }); + + it('routing -> idle when the stop word lands while the brain is still thinking', async () => { + await start(h); + const gate = deferred(); + h.brain.routeGate = gate.promise; + + h.service.submitUtterance('think about it'); + await vi.waitFor(() => expect(h.service.getState()).toBe('routing')); + + await h.service.hardStop('voice', 'never mind'); + expect(h.service.getState()).toBe('idle'); + + // The superseded turn resumes and drops itself rather than transitioning. + gate.resolve(); + await gate.promise; + expect(takeEdges()).toEqual([ + 'listening -> transcribing', + 'transcribing -> routing', + 'routing -> idle', + ]); + }); + + it('routing -> error when the decision names an agent that is gone', async () => { + await start(h); + h.brain.decision = { + target: { sessionId: 'agent-ghost' }, + tabAction: 'current', + prompt: 'hi', + confidence: 0.5, + }; + + h.service.submitUtterance('talk to the ghost'); + await vi.waitFor(() => expect(h.service.getState()).toBe('error')); + + expect(takeEdges()).toEqual([ + 'listening -> transcribing', + 'transcribing -> routing', + 'routing -> error', + ]); + }); + + it('dispatching -> error when the dispatch itself fails', async () => { + const failing = makeHarness({ + executeRoute: async () => { + throw new VoiceDispatchError('renderer did not answer in time'); + }, + }); + await start(failing); + + failing.service.submitUtterance('open a new tab'); + await vi.waitFor(() => expect(failing.service.getState()).toBe('error')); + + expect(takeEdges()).toEqual([ + 'listening -> transcribing', + 'transcribing -> routing', + 'routing -> dispatching', + 'dispatching -> error', + ]); + }); + + it('dispatching -> speaking -> listening for a spoken reply', async () => { + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + takeEdges(); + + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + expect(takeEdges()).toEqual(['dispatching -> speaking', 'speaking -> listening']); + }); + + it('dispatching -> listening when the reply is not worth speaking', async () => { + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + takeEdges(); + + h.brain.spoken = ' '; + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + expect(takeEdges()).toEqual(['dispatching -> listening']); + }); + + it('dispatching -> idle when the session is stopped before the reply lands', async () => { + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + takeEdges(); + + await h.service.stopSession('user'); + expect(takeEdges()).toEqual(['dispatching -> idle']); + }); + + it('speaking -> interrupted -> listening on barge-in, keeping the floor', async () => { + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + takeEdges(); + + h.tts.onChunk = () => { + h.service.interrupt('voice'); + }; + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + expect(takeEdges()).toEqual([ + 'dispatching -> speaking', + 'speaking -> interrupted', + 'interrupted -> listening', + ]); + expect(h.service.getSnapshot().sessionId).toBeTruthy(); + }); + + it('speaking -> idle when the stop word lands mid-sentence', async () => { + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + takeEdges(); + + h.tts.onChunk = () => { + void h.service.hardStop('voice'); + }; + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + await vi.waitFor(() => expect(h.service.getState()).toBe('idle')); + + expect(takeEdges()).toEqual(['dispatching -> speaking', 'speaking -> idle']); + }); + + it('speaking -> error when the voice throws mid-run, releasing the floor', async () => { + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + takeEdges(); + + h.tts.speakError = new Error('voice stream closed'); + // The rejection must not escape: a caller that only awaited this would + // otherwise leave the session in `speaking` with the floor held. + await expect( + h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }) + ).resolves.toBe(true); + + expect(takeEdges()).toEqual(['dispatching -> speaking', 'speaking -> error']); + expect(h.service.getState()).toBe('error'); + expect(h.events.filter((e) => e.type === 'listen-stop')).toEqual([ + expect.objectContaining({ reason: 'error' }), + ]); + }); + + it('never takes an edge the table does not name', async () => { + await start(h); + h.service.submitUtterance('open the auth tab'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: 'done', + }); + await h.service.stopSession('user'); + + // The wrapper delegates, so an illegal edge would already have thrown. This + // asserts the recorder is watching the same table the service asserts on. + for (const edge of takeEdges()) { + const [from, to] = edge.split(' -> ') as [VoiceSessionState, VoiceSessionState]; + expect(VOICE_STATE_TRANSITIONS[from]).toContain(to); + } + }); + + it('routing -> speaking when the router asks instead of guessing', async () => { + await start(h); + h.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: 'run it', + confidence: 0.3, + clarify: 'the backend agent or the API agent?', + }; + + h.service.submitUtterance('run it'); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + // Nothing was dispatched: the question IS the turn, and the answer arrives + // as the next utterance. + expect(h.executor).not.toHaveBeenCalled(); + expect(takeEdges()).toEqual([ + 'listening -> transcribing', + 'transcribing -> routing', + 'routing -> speaking', + 'speaking -> listening', + ]); + }); + + it('has a driving test or a documented reason for every edge in the table', () => { + const declared = Object.entries(VOICE_STATE_TRANSITIONS).flatMap(([from, targets]) => + targets.map((to) => `${from} -> ${to}`) + ); + + expect(DRIVEN_EDGES.filter((edge) => DEFENSIVE_EDGES.includes(edge))).toEqual([]); + expect([...DRIVEN_EDGES, ...DEFENSIVE_EDGES].sort()).toEqual([...declared].sort()); + }); +}); + +describe('illegal transitions throw', () => { + it('surfaces the offending edge', () => { + expect(() => { + throw new InvalidVoiceStateTransitionError('listening', 'speaking'); + }).toThrow(/listening -> speaking/); + }); +}); + +// --------------------------------------------------------------------------- +// Disambiguation and correction +// +// The two paths that exist so a low-confidence guess never becomes a spoken +// instruction in the wrong repository: asking before dispatching, and moving a +// dispatch the user says went to the wrong place. +// --------------------------------------------------------------------------- + +describe('spoken disambiguation', () => { + let h: Harness; + + beforeEach(() => { + h = makeHarness(); + }); + + afterEach(async () => { + await h.service.stopSession('user'); + }); + + function askAbout(question: string): void { + h.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: 'run it', + confidence: 0.3, + clarify: question, + }; + } + + it('speaks the question and hands the floor straight back', async () => { + await start(h); + askAbout('the backend agent or the API agent?'); + + h.service.submitUtterance('run it'); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + expect(h.types()).toContain('route-decision'); + expect(h.events.filter((e) => e.type === 'speak-sentence')).toEqual([ + expect.objectContaining({ text: 'the backend agent or the API agent?' }), + ]); + expect(h.types()).not.toContain('dispatch'); + }); + + it('routes the ORIGINAL request on the answer, not the fragment', async () => { + await start(h); + askAbout('the backend agent or the API agent?'); + h.service.submitUtterance('deploy the gateway'); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + const contexts: VoiceRouteContext[] = []; + h.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: 'deploy the gateway', + confidence: 0.9, + }; + const originalRoute = h.brain.route.bind(h.brain); + h.brain.route = async (input, context) => { + contexts.push(context); + return originalRoute(input, context); + }; + + h.service.submitUtterance('the backend one'); + await vi.waitFor(() => expect(h.executor).toHaveBeenCalled()); + + // "the backend one" routed on its own becomes a prompt, and the request it + // was answering is lost. + expect(contexts[0].clarification).toEqual({ + question: 'the backend agent or the API agent?', + utterance: 'deploy the gateway', + }); + }); + + it('forgets an abandoned question rather than reinterpreting a later sentence', async () => { + await start(h); + askAbout('the backend agent or the API agent?'); + h.service.submitUtterance('deploy the gateway'); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + const contexts: VoiceRouteContext[] = []; + h.brain.decision = { + target: 'conductor', + tabAction: 'current', + prompt: 'x', + confidence: 0.9, + }; + h.brain.route = async (_input, context) => { + contexts.push(context); + return h.brain.decision; + }; + + h.service.submitUtterance('the backend one'); + await vi.waitFor(() => expect(contexts).toHaveLength(1)); + h.service.submitUtterance('something else entirely'); + await vi.waitFor(() => expect(contexts).toHaveLength(2)); + + expect(contexts[1].clarification).toBeUndefined(); + }); +}); + +describe('wrong-tab correction', () => { + /** Two agents, so "the other one" has an unambiguous answer. */ + function twoAgentRoster(): RosterAgent[] { + return [ + ...makeRoster(), + { + sessionId: 'agent-api', + name: 'API', + agentType: 'codex', + cwd: '/repo/gateway', + tabs: [{ id: 'tab-gw', name: 'Gateway', lastActiveAt: 2 }], + }, + ]; + } + + function makeCorrectionHarness() { + const stt = new FakeStt(); + const tts = new FakeTts(); + const brain = new FakeBrain(); + brain.decision = { + target: { sessionId: 'agent-backend' }, + tabAction: 'current', + prompt: 'deploy the gateway', + confidence: 0.9, + }; + + const executed: RouteDecision[] = []; + const executor = vi.fn(async (decision: RouteDecision) => { + executed.push(decision); + const target = + typeof decision.target === 'string' ? 'agent-backend' : decision.target.sessionId; + return { + agentSessionId: target, + agentName: target === 'agent-api' ? 'API' : 'Backend', + tabId: target === 'agent-api' ? 'tab-gw' : 'tab-1', + action: 'focused' as const, + promptSent: true, + }; + }); + + const service = new VoiceSessionService({ + providers: { stt, tts, brain }, + getRoster: () => twoAgentRoster(), + executeRoute: executor as unknown as VoiceRouteExecutor, + }); + const events: VoiceEvent[] = []; + service.subscribe((event) => events.push(event)); + + return { service, stt, tts, brain, events, executor, executed }; + } + + it('moves the last prompt on a spoken "no, the other one"', async () => { + const h = makeCorrectionHarness(); + await h.service.startSession({ scope: { kind: 'conductor' } }); + h.service.submitUtterance('deploy the gateway'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + h.service.submitUtterance('no, the other one'); + await vi.waitFor(() => expect(h.executed).toHaveLength(2)); + + // The prompt that was actually sent, not the correction phrase. + expect(h.executed[1]).toMatchObject({ + target: { sessionId: 'agent-api' }, + prompt: 'deploy the gateway', + }); + const correction = h.events.find((event) => event.type === 'route-correction'); + expect(correction).toMatchObject({ + fromAgentSessionId: 'agent-backend', + agentSessionId: 'agent-api', + source: 'voice', + }); + await h.service.stopSession('user'); + }); + + it('never sends a correction phrase to an agent as a prompt', async () => { + const h = makeCorrectionHarness(); + await h.service.startSession({ scope: { kind: 'conductor' } }); + h.service.submitUtterance('deploy the gateway'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + h.service.submitUtterance('wrong agent'); + await vi.waitFor(() => expect(h.executed).toHaveLength(2)); + + expect(h.executed.map((decision) => decision.prompt)).toEqual([ + 'deploy the gateway', + 'deploy the gateway', + ]); + await h.service.stopSession('user'); + }); + + it('does nothing when there is no dispatch to correct', async () => { + const h = makeCorrectionHarness(); + await h.service.startSession({ scope: { kind: 'conductor' } }); + + await expect(h.service.correctLastDispatch('agent-api')).resolves.toBe(false); + await h.service.stopSession('user'); + }); + + it('takes a correction from a HUD control as well as from the voice', async () => { + const h = makeCorrectionHarness(); + await h.service.startSession({ scope: { kind: 'conductor' } }); + h.service.submitUtterance('deploy the gateway'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + await expect(h.service.correctLastDispatch('agent-api')).resolves.toBe(true); + + expect(h.events.find((event) => event.type === 'route-correction')).toMatchObject({ + source: 'client-button', + agentName: 'API', + }); + await h.service.stopSession('user'); + }); + + it('surfaces the last decision and dispatch in the snapshot', async () => { + const h = makeCorrectionHarness(); + await h.service.startSession({ scope: { kind: 'conductor' } }); + h.service.submitUtterance('deploy the gateway'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + const snapshot = h.service.getSnapshot(); + + expect(snapshot.lastDecision).toMatchObject({ confidence: 0.9 }); + expect(snapshot.lastDispatch).toMatchObject({ agentName: 'Backend', tabId: 'tab-1' }); + await h.service.stopSession('user'); + }); +}); + +// --------------------------------------------------------------------------- +// The Phase 08 speech layer, wired +// --------------------------------------------------------------------------- + +/** A fake tap: records what it was asked to follow, and lets a test push chunks. */ +function makeReplyStream(): AgentReplyStream & { watched: string[]; unwatched: string[] } { + const watched: string[] = []; + const unwatched: string[] = []; + return { + watched, + unwatched, + watch: ({ agentSessionId, tabId }) => watched.push(`${agentSessionId}/${tabId}`), + unwatch: ({ agentSessionId, tabId }) => unwatched.push(`${agentSessionId}/${tabId}`), + }; +} + +function chunk(overrides: Partial = {}): AgentOutputChunk { + return { + agentSessionId: 'agent-backend', + tabId: 'tab-1', + kind: 'text', + text: AGENT_REPLY, + ts: 1, + ...overrides, + }; +} + +describe('VoiceSessionService streamed agent output', () => { + /** Dispatch a turn and leave the session waiting on the tap. */ + async function dispatched(overrides: Parameters[0] = {}) { + const stream = overrides.agentReplyStream ?? makeReplyStream(); + const h = makeHarness({ ...overrides, agentReplyStream: stream }); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + h.events.length = 0; + return { h, stream: stream as ReturnType }; + } + + it('follows the dispatched tab and speaks a chunk before the turn is over', async () => { + const { h, stream } = await dispatched(); + expect(stream.watched).toEqual(['agent-backend/tab-1']); + + h.service.pushAgentOutput(chunk()); + await vi.waitFor(() => expect(h.types()).toContain('speak-sentence')); + + // Still speaking: the agent has not said it is finished, so the run stays + // open for the rest of the reply rather than closing after the first thought. + expect(h.service.getState()).toBe('speaking'); + expect(h.types()).not.toContain('speak-end'); + }); + + it('announces a streamed run as streaming, so the count is a lower bound', async () => { + const { h } = await dispatched(); + h.service.pushAgentOutput(chunk()); + await vi.waitFor(() => expect(h.types()).toContain('speak-start')); + + const started = h.events.find((event) => event.type === 'speak-start'); + expect(started).toMatchObject({ streaming: true, sentenceCount: 0 }); + }); + + it('closes the run and hands the floor back on the final chunk', async () => { + const { h, stream } = await dispatched(); + + h.service.pushAgentOutput(chunk({ kind: 'final' })); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + // The speech events in order. `agent-reply` is deliberately not pinned in + // among them: the record of the chunk is written when its rewrite finishes, + // while its last sentences are still being spoken, and holding it back until + // the audio caught up would put the transcript behind the voice. + expect(h.types().filter((type) => type !== 'agent-reply')).toEqual([ + 'speak-start', + 'speak-sentence', + 'speak-sentence', + 'speak-end', + 'listen-start', + ]); + expect(h.types()).toContain('agent-reply'); + expect(stream.unwatched).toEqual(['agent-backend/tab-1']); + }); + + it('hands the floor back when the whole turn produced nothing speakable', async () => { + const { h } = await dispatched(); + h.brain.spoken = ' '; + + h.service.pushAgentOutput(chunk({ kind: 'final' })); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + // No run was ever opened: a chunk the translator had nothing to say about + // must not strand the session in `speaking` with a silent floor. + expect(h.types()).toEqual(['listen-start']); + }); + + it('speaks a status chunk straight through, without a translation hop', async () => { + const { h } = await dispatched(); + + h.service.pushAgentOutput(chunk({ kind: 'status', text: 'It hit an error: exit code 1.' })); + await vi.waitFor(() => expect(h.types()).toContain('speak-sentence')); + + expect(h.events.filter((event) => event.type === 'speak-sentence')).toEqual([ + expect.objectContaining({ text: 'It hit an error: exit code 1.' }), + ]); + }); + + it('ignores output from a tab this turn is not about', async () => { + const { h } = await dispatched(); + + h.service.pushAgentOutput(chunk({ tabId: 'tab-other' })); + await Promise.resolve(); + + expect(h.events).toHaveLength(0); + expect(h.service.getState()).toBe('dispatching'); + }); + + it('stops following the tab when the session ends', async () => { + const { h, stream } = await dispatched(); + await h.service.stopSession('user'); + expect(stream.unwatched).toEqual(['agent-backend/tab-1']); + }); +}); + +describe('VoiceSessionService follow-ups', () => { + /** Speak one reply, so there is a retained turn to drill into. */ + async function afterAReply(overrides: Parameters[0] = {}) { + const h = makeHarness(overrides); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: `${AGENT_REPLY}\nThe stale check lived in src/main/auth/session.ts.`, + }); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + h.events.length = 0; + h.executor.mockClear(); + return h; + } + + it('serves "tell me more" from the retained output, with no agent turn', async () => { + const h = await afterAReply(); + + h.service.submitUtterance('tell me more'); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + expect(h.executor).not.toHaveBeenCalled(); + expect(h.types()).not.toContain('route-decision'); + expect(h.events.filter((event) => event.type === 'speak-sentence').length).toBeGreaterThan(0); + }); + + it('repeats what was actually said, not what was queued', async () => { + const h = await afterAReply(); + + h.service.submitUtterance('say that again'); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + const spoken = h.events + .filter((event) => event.type === 'speak-sentence') + .map((event) => (event.type === 'speak-sentence' ? event.text : '')); + expect(spoken.join(' ')).toContain('All done.'); + }); + + it('answers "show me" on screen and says nothing at all', async () => { + const focused: unknown[] = []; + const h = await afterAReply({ focusTarget: (target) => focused.push(target) }); + + h.service.submitUtterance('show me that file'); + await vi.waitFor(() => expect(h.service.getState()).toBe('listening')); + + expect(focused).toEqual([ + expect.objectContaining({ agentSessionId: 'agent-backend', tabId: 'tab-1' }), + ]); + expect(h.types()).not.toContain('speak-start'); + expect(h.executor).not.toHaveBeenCalled(); + }); + + it('still routes a fresh request that only looks like a follow-up', async () => { + const h = await afterAReply(); + + h.service.submitUtterance('open a new tab for the migration'); + await vi.waitFor(() => expect(h.executor).toHaveBeenCalled()); + + expect(h.types()).toContain('route-decision'); + }); +}); + +describe('VoiceSessionService barge-in guard window', () => { + it('refuses a voice barge-in inside the guard window', async () => { + const h = makeHarness({ bargeInGuardMs: 60_000 }); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + h.service.subscribe((event) => { + if (event.type === 'speak-sentence' && event.index === 0) h.service.interrupt('voice'); + }); + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + // The assistant's own leaked syllable must not interrupt the assistant. + expect(h.types()).not.toContain('barge-in'); + expect(h.events.filter((event) => event.type === 'speak-sentence')).toHaveLength(2); + }); + + it('still takes a button press inside the guard window', async () => { + const h = makeHarness({ bargeInGuardMs: 60_000 }); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + + h.service.subscribe((event) => { + if (event.type === 'speak-sentence' && event.index === 0) { + h.service.interrupt('client-button'); + } + }); + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + + expect(h.types()).toContain('barge-in'); + expect(h.service.getState()).toBe('listening'); + }); +}); + +describe('VoiceSessionService background completions', () => { + const completion = { + agentSessionId: 'agent-api', + agentName: 'API', + summary: 'the migration', + }; + + it('speaks a completion at a pause, naming the source', async () => { + const h = makeHarness({ getBackgroundAnnouncementSetting: () => 'on' }); + await start(h); + + expect(h.service.noteAgentCompletion(completion)).toBe(true); + await vi.waitFor(() => expect(h.types()).toContain('speak-sentence')); + + const spoken = h.events + .filter((event) => event.type === 'speak-sentence') + .map((event) => (event.type === 'speak-sentence' ? event.text : '')); + expect(spoken.join(' ')).toContain('the API agent finished the migration'); + }); + + it('waits for the pause rather than talking over the turn in progress', async () => { + const h = makeHarness({ getBackgroundAnnouncementSetting: () => 'on' }); + await start(h); + h.service.submitUtterance('what changed'); + await vi.waitFor(() => expect(h.service.getState()).toBe('dispatching')); + h.events.length = 0; + + expect(h.service.noteAgentCompletion(completion)).toBe(true); + expect(h.types()).toEqual([]); + + await h.service.submitAgentReply({ + agentSessionId: 'agent-backend', + tabId: 'tab-1', + text: AGENT_REPLY, + }); + await vi.waitFor(() => { + const spoken = h.events + .filter((event) => event.type === 'speak-sentence') + .map((event) => (event.type === 'speak-sentence' ? event.text : '')); + expect(spoken.join(' ')).toContain('the API agent finished'); + }); + }); + + it('says nothing when the setting is off', async () => { + const h = makeHarness({ getBackgroundAnnouncementSetting: () => 'off' }); + await start(h); + + expect(h.service.noteAgentCompletion(completion)).toBe(false); + expect(h.types()).toEqual([]); + }); +}); diff --git a/src/__tests__/main/acappella/wake/stop-word.test.ts b/src/__tests__/main/acappella/wake/stop-word.test.ts new file mode 100644 index 0000000000..792784ae05 --- /dev/null +++ b/src/__tests__/main/acappella/wake/stop-word.test.ts @@ -0,0 +1,297 @@ +/** + * @file stop-word.test.ts + * + * The stop word, and the one property this whole module exists to protect: + * **the stop word and barge-in are different things and produce different + * terminal states.** Stopping while the assistant is speaking must end the + * session; talking over it must not. A test that only checked "speech stopped" + * would pass for both and catch neither. + * + * The session is a fake with the same state machine shape the real one has, so + * the assertions are about the state the session ends in rather than about which + * methods were called. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/sentry', () => ({ captureException: vi.fn() })); +vi.mock('../../../../main/utils/logger', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); +vi.mock('../../../../main/acappella/models/model-store', () => ({ + modelFilePath: (id: string, file: string) => `/models/${id}/${file}`, +})); + +import type { AudioHostCommand } from '../../../../shared/acappella/audio-host'; +import type { InterruptSource } from '../../../../shared/acappella/protocol'; +import type { VoiceSessionState } from '../../../../shared/acappella/session-state'; +import { + DEFAULT_STOP_PHRASE, + FALLBACK_STOP_PHRASE, + FALLBACK_STOP_PHRASE_ID, + PRIMARY_STOP_PHRASE_ID, + StopWordController, + armedPhrases, + createStopWordController, + isStopPhraseId, + stopWordPhrases, + type StopWordEventInfo, + type StopWordSession, +} from '../../../../main/acappella/wake/stop-word'; +import { + globalWakePhrase, + type WakeDetection, +} from '../../../../main/acappella/wake/wake-detector'; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +/** + * A session with both behaviours on it, so a test can compare them. + * + * `interrupt` mirrors the real service: it cancels speech and returns the floor + * to `listening`. `hardStop` ends the session. The controller is only given the + * `hardStop` half, which is the point. + */ +class FakeSession implements StopWordSession { + state: VoiceSessionState = 'idle'; + speechCancelled = 0; + readonly hardStops: Array<{ source?: InterruptSource; phrase?: string }> = []; + + getState(): VoiceSessionState { + return this.state; + } + + async hardStop(source?: InterruptSource, phrase?: string): Promise { + this.hardStops.push({ source, phrase }); + this.speechCancelled += 1; + this.state = 'idle'; + } + + /** Barge-in, for the comparison test. Never reachable from the controller. */ + interrupt(): boolean { + if (this.state !== 'speaking') return false; + this.speechCancelled += 1; + this.state = 'listening'; + return true; + } +} + +function detection(phraseId: string, phrase: string): WakeDetection { + return { phraseId, phrase, scope: { kind: 'conductor' }, score: 0.9, at: 1234, preRoll: [] }; +} + +describe('stop phrases', () => { + it('always arms a second, non-editable phrase alongside the configured one', () => { + const phrases = stopWordPhrases({ phrase: 'that will do' }); + expect(phrases.map((p) => p.phrase)).toEqual(['that will do', FALLBACK_STOP_PHRASE]); + }); + + it('falls back to the default phrase when the setting is blank', () => { + expect(stopWordPhrases({ phrase: ' ' })[0].phrase).toBe(DEFAULT_STOP_PHRASE); + }); + + it('keeps "nevermind" armed even when the configured phrase is switched off', () => { + const phrases = stopWordPhrases({ enabled: false }); + expect(phrases[0].enabled).toBe(false); + expect(phrases[1].enabled).not.toBe(false); + }); + + it('tags stop phrase ids so a detection can be routed without a lookup', () => { + expect(isStopPhraseId(PRIMARY_STOP_PHRASE_ID)).toBe(true); + expect(isStopPhraseId(FALLBACK_STOP_PHRASE_ID)).toBe(true); + expect(isStopPhraseId('global')).toBe(false); + expect(isStopPhraseId('agent:seven')).toBe(false); + }); +}); + +describe('armedPhrases', () => { + const wake = [globalWakePhrase('hey maestro')]; + const stop = stopWordPhrases(); + + it('listens for the wake word only while the session is cold', () => { + for (const state of ['idle', 'error'] as VoiceSessionState[]) { + expect(armedPhrases(state, { wake, stop })).toEqual(wake); + } + }); + + it('listens for the stop word in every active state, including speaking', () => { + const active: VoiceSessionState[] = [ + 'arming', + 'listening', + 'transcribing', + 'routing', + 'dispatching', + 'speaking', + 'interrupted', + ]; + for (const state of active) { + expect(armedPhrases(state, { wake, stop })).toEqual(stop); + } + }); + + it('never arms both at once, so a wake phrase cannot stack a second session', () => { + const armed = armedPhrases('speaking', { wake, stop }); + expect(armed.some((phrase) => phrase.id === 'global')).toBe(false); + }); +}); + +describe('StopWordController', () => { + let session: FakeSession; + let commands: AudioHostCommand[]; + let stops: StopWordEventInfo[]; + let backToWakeOnly: number; + let controller: StopWordController; + + beforeEach(() => { + session = new FakeSession(); + commands = []; + stops = []; + backToWakeOnly = 0; + controller = createStopWordController({ + session, + sendCommand: (command) => commands.push(command), + onStopWord: (info) => stops.push(info), + onWakeWordOnly: () => { + backToWakeOnly += 1; + }, + }); + }); + + it('ignores a wake phrase', () => { + expect(controller.handleDetection(detection('global', 'hey maestro'))).toBe(false); + expect(session.hardStops).toHaveLength(0); + }); + + it('ends the session when heard while speaking', async () => { + session.state = 'speaking'; + expect(controller.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop'))).toBe( + true + ); + await controller.whenSettled(); + + expect(session.state).toBe('idle'); + expect(session.hardStops).toEqual([{ source: 'voice', phrase: 'maestro stop' }]); + }); + + it('ends the session when heard while listening', async () => { + session.state = 'listening'; + controller.handleDetection(detection(FALLBACK_STOP_PHRASE_ID, FALLBACK_STOP_PHRASE)); + await controller.whenSettled(); + + expect(session.state).toBe('idle'); + expect(stops[0].from).toBe('listening'); + }); + + it('flushes queued playback before stopping, then closes the microphone', async () => { + session.state = 'speaking'; + controller.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop')); + await controller.whenSettled(); + + expect(commands.map((command) => command.kind)).toEqual(['flush', 'stop-capture']); + }); + + it('goes back to wake-word-only listening', async () => { + session.state = 'speaking'; + controller.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop')); + await controller.whenSettled(); + + expect(backToWakeOnly).toBe(1); + }); + + it('does nothing when there is no session to stop', async () => { + controller.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop')); + await controller.whenSettled(); + + expect(session.hardStops).toHaveLength(0); + expect(commands).toHaveLength(0); + expect(stops).toHaveLength(0); + }); + + it('serialises two phrases heard back to back into one teardown', async () => { + session.state = 'speaking'; + controller.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop')); + controller.handleDetection(detection(FALLBACK_STOP_PHRASE_ID, FALLBACK_STOP_PHRASE)); + await controller.whenSettled(); + + expect(session.hardStops).toHaveLength(1); + }); + + it('still closes the microphone when the session teardown throws', async () => { + session.state = 'speaking'; + session.hardStop = async () => { + throw new Error('teardown exploded'); + }; + const errors: Error[] = []; + const failing = createStopWordController({ + session, + sendCommand: (command) => commands.push(command), + onError: (error) => errors.push(error), + }); + + failing.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop')); + await failing.whenSettled(); + + expect(errors).toHaveLength(1); + expect(commands.map((command) => command.kind)).toEqual(['flush', 'stop-capture']); + }); + + it('reads the stop phrase per call, so a settings change takes effect live', () => { + let phrase = 'first phrase'; + const live = createStopWordController({ session, getConfig: () => ({ phrase }) }); + expect(live.phrases()[0].phrase).toBe('first phrase'); + phrase = 'second phrase'; + expect(live.phrases()[0].phrase).toBe('second phrase'); + }); + + // ----------------------------------------------------------------------- + // The distinction + // ----------------------------------------------------------------------- + + describe('stop word versus barge-in', () => { + it('leaves the session in different terminal states from the same starting point', async () => { + const stopped = new FakeSession(); + stopped.state = 'speaking'; + const stopController = createStopWordController({ session: stopped }); + stopController.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop')); + await stopController.whenSettled(); + + const bargedIn = new FakeSession(); + bargedIn.state = 'speaking'; + bargedIn.interrupt(); + + // Both cancelled speech. Only one of them hung up. + expect(stopped.speechCancelled).toBe(1); + expect(bargedIn.speechCancelled).toBe(1); + expect(stopped.state).toBe('idle'); + expect(bargedIn.state).toBe('listening'); + expect(stopped.state).not.toBe(bargedIn.state); + }); + + it('cannot reach barge-in: the controller is handed no way to call it', async () => { + const spy = vi.spyOn(session, 'interrupt'); + session.state = 'speaking'; + controller.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop')); + await controller.whenSettled(); + + expect(spy).not.toHaveBeenCalled(); + }); + + it('reports the stop through its own seam, not a shared one', async () => { + session.state = 'speaking'; + controller.handleDetection(detection(PRIMARY_STOP_PHRASE_ID, 'maestro stop')); + await controller.whenSettled(); + + expect(stops).toEqual([ + { + phrase: 'maestro stop', + phraseId: PRIMARY_STOP_PHRASE_ID, + from: 'speaking', + score: 0.9, + at: 1234, + }, + ]); + }); + }); +}); diff --git a/src/__tests__/main/acappella/wake/wake-detector.test.ts b/src/__tests__/main/acappella/wake/wake-detector.test.ts new file mode 100644 index 0000000000..2b426bfb4b --- /dev/null +++ b/src/__tests__/main/acappella/wake/wake-detector.test.ts @@ -0,0 +1,410 @@ +/** + * @file wake-detector.test.ts + * + * The always-local wake word: phrase matching against a per-phrase sensitivity, + * the debounce that stops one spoken phrase becoming three sessions, per-agent + * scope resolution, pre-roll inclusion, and the invariant this whole subsystem + * exists to hold - **while only the wake detector is running, no audio frame + * reaches a hosted provider or leaves the process.** + * + * The scorer is injected, so the orchestration is tested with synthetic frames + * and deterministic scores rather than against a trained model. That is the + * right seam: everything above it is where a wake word gets a scope wrong or + * clips the user's first word, and none of that depends on ONNX. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../../main/utils/sentry', () => ({ captureException: vi.fn() })); +vi.mock('../../../../main/utils/logger', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); +// The detector reaches the model store only through the ONNX scorer, which no +// test here builds. Mocked so importing the module does not pull in `electron`. +vi.mock('../../../../main/acappella/models/model-store', () => ({ + modelFilePath: (id: string, file: string) => `/models/${id}/${file}`, +})); + +import { ACAPPELLA_AUDIO_FRAME_SAMPLES } from '../../../../shared/acappella/audio-host'; +import type { SttProvider } from '../../../../shared/acappella/providers'; +import { + DEFAULT_WAKE_DEBOUNCE_MS, + GLOBAL_WAKE_PHRASE_ID, + WAKE_HOP_SAMPLES, + WakeDetector, + agentWakePhrase, + assertWakeScorerLocal, + createWakeDetector, + globalWakePhrase, + wakeThresholdFor, + type WakeDetection, + type WakePhrase, + type WakePhraseScorer, +} from '../../../../main/acappella/wake/wake-detector'; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +/** A scorer whose answer is set by the test. Local by construction. */ +class ScriptedScorer implements WakePhraseScorer { + readonly tier = 'local' as const; + scores: Record = {}; + calls = 0; + disposed = 0; + throwOnce = false; + + score(hop: Float32Array, phrases: readonly WakePhrase[]): Record { + this.calls += 1; + if (this.throwOnce) { + this.throwOnce = false; + throw new Error('inference exploded'); + } + expect(hop.length).toBe(WAKE_HOP_SAMPLES); + expect(phrases.length).toBeGreaterThan(0); + return this.scores; + } + + dispose(): void { + this.disposed += 1; + } +} + +/** One 20 ms frame of non-silent audio. Content is irrelevant to the scorer. */ +function frame(fill = 1000): Int16Array { + return new Int16Array(ACAPPELLA_AUDIO_FRAME_SAMPLES).fill(fill); +} + +/** Push enough frames to complete `hops` scoring windows. */ +function pushHops(detector: WakeDetector, hops: number, fill?: number): void { + const framesPerHop = WAKE_HOP_SAMPLES / ACAPPELLA_AUDIO_FRAME_SAMPLES; + for (let i = 0; i < hops * framesPerHop; i++) detector.pushFrame(frame(fill)); +} + +describe('wakeThresholdFor', () => { + it('inverts sensitivity and clamps nonsense into the band', () => { + expect(wakeThresholdFor({ id: 'a', phrase: 'a', scope: { kind: 'conductor' } }, 0.5)).toBe(0.5); + expect( + wakeThresholdFor({ id: 'a', phrase: 'a', scope: { kind: 'conductor' }, sensitivity: 0.9 }) + ).toBeCloseTo(0.1); + // The most sensitive setting is still not a hair trigger. + expect( + wakeThresholdFor({ id: 'a', phrase: 'a', scope: { kind: 'conductor' }, sensitivity: 5 }) + ).toBeGreaterThan(0); + }); +}); + +describe('assertWakeScorerLocal', () => { + it('refuses a scorer that is not local', () => { + const hosted = { tier: 'cloud', score: () => ({}) } as unknown as WakePhraseScorer; + expect(() => assertWakeScorerLocal(hosted)).toThrow(/no audio may leave the machine/i); + }); +}); + +describe('WakeDetector', () => { + let scorer: ScriptedScorer; + let detections: WakeDetection[]; + let clock: number; + + const phrases: WakePhrase[] = [ + globalWakePhrase('hey maestro'), + agentWakePhrase('agent-7', 'hey scout'), + ]; + + function build(overrides: Partial[0]> = {}): WakeDetector { + return createWakeDetector({ + getPhrases: () => phrases, + scorer, + onWake: (detection) => detections.push(detection), + now: () => clock, + ...overrides, + }); + } + + beforeEach(() => { + scorer = new ScriptedScorer(); + detections = []; + clock = 1_000; + }); + + it('fires when a phrase clears its threshold', async () => { + const detector = build(); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.8 }; + + pushHops(detector, 1); + + expect(detections).toHaveLength(1); + expect(detections[0].phrase).toBe('hey maestro'); + expect(detections[0].scope).toEqual({ kind: 'conductor' }); + }); + + it('stays quiet below the threshold', async () => { + const detector = build(); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.3 }; + + pushHops(detector, 3); + + expect(detections).toHaveLength(0); + expect(detector.getStats().hopsScored).toBe(3); + }); + + it('resolves an agent phrase to that agent scope', async () => { + const detector = build(); + await detector.start(); + scorer.scores = { 'agent:agent-7': 0.9 }; + + pushHops(detector, 1); + + expect(detections[0].scope).toEqual({ kind: 'agent', sessionId: 'agent-7' }); + expect(detections[0].phrase).toBe('hey scout'); + }); + + it('fires the best match when two phrases clear at once', async () => { + const detector = build(); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.7, 'agent:agent-7': 0.95 }; + + pushHops(detector, 1); + + // One session per sentence, and it is the closer match that wins. + expect(detections).toHaveLength(1); + expect(detections[0].phraseId).toBe('agent:agent-7'); + }); + + it('debounces consecutive windows of one spoken phrase', async () => { + const detector = build(); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + + pushHops(detector, 4); + + expect(detections).toHaveLength(1); + expect(detector.getStats().debounced).toBe(3); + }); + + it('fires again once the debounce window has passed', async () => { + const detector = build(); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + + pushHops(detector, 1); + clock += DEFAULT_WAKE_DEBOUNCE_MS + 1; + pushHops(detector, 1); + + expect(detections).toHaveLength(2); + }); + + it('debounces per phrase, not globally', async () => { + const detector = build(); + await detector.start(); + + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + pushHops(detector, 1); + scorer.scores = { 'agent:agent-7': 0.9 }; + pushHops(detector, 1); + + expect(detections.map((d) => d.phraseId)).toEqual([GLOBAL_WAKE_PHRASE_ID, 'agent:agent-7']); + }); + + it('hands the pre-roll to the caller so the words after the phrase survive', async () => { + const detector = build({ preRollMs: 200 }); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + + pushHops(detector, 1); + + expect(detections[0].preRoll.length).toBeGreaterThan(0); + expect(detections[0].preRoll[0]).toBeInstanceOf(Int16Array); + }); + + it('drains the pre-roll on a hit, so the same audio is never replayed twice', async () => { + const detector = build({ preRollMs: 200 }); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + + pushHops(detector, 1); + const first = detections[0].preRoll; + clock += DEFAULT_WAKE_DEBOUNCE_MS + 1; + pushHops(detector, 1); + const second = detections[1].preRoll; + + expect(first.length).toBeGreaterThan(0); + expect(second.every((buffer) => !first.includes(buffer))).toBe(true); + }); + + it('uses an injected pre-roll ring, so the pipeline and the detector share one buffer', async () => { + const pushed: Int16Array[] = []; + const ring = { + push: (samples: Int16Array) => pushed.push(samples), + drain: () => pushed.splice(0, pushed.length), + clear: () => (pushed.length = 0), + }; + const detector = build({ preRoll: ring }); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + + pushHops(detector, 1); + + expect(detections[0].preRoll.length).toBe(WAKE_HOP_SAMPLES / ACAPPELLA_AUDIO_FRAME_SAMPLES); + expect(pushed).toHaveLength(0); + }); + + it('skips a phrase that is switched off', async () => { + const parked: WakePhrase[] = [{ ...globalWakePhrase('hey maestro'), enabled: false }]; + const detector = build({ getPhrases: () => parked }); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.99 }; + + pushHops(detector, 2); + + expect(detections).toHaveLength(0); + expect(scorer.calls).toBe(0); + }); + + it('reads the phrase list per hop, so a new agent phrase arms without a restart', async () => { + const live: WakePhrase[] = [globalWakePhrase('hey maestro')]; + const detector = build({ getPhrases: () => live }); + await detector.start(); + scorer.scores = { 'agent:late': 0.95 }; + + pushHops(detector, 1); + expect(detections).toHaveLength(0); + + live.push(agentWakePhrase('late', 'hey late')); + pushHops(detector, 1); + expect(detections).toHaveLength(1); + }); + + it('counts a scoring failure instead of throwing into the audio callback', async () => { + const detector = build(); + await detector.start(); + scorer.throwOnce = true; + + expect(() => pushHops(detector, 1)).not.toThrow(); + expect(detector.getStats().scoreErrors).toBe(1); + }); + + it('survives a wake handler that throws', async () => { + const detector = build({ + onWake: () => { + throw new Error('session exploded'); + }, + }); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + + expect(() => pushHops(detector, 1)).not.toThrow(); + }); + + it('runs inert rather than failing when no scorer can be built', async () => { + const detector = createWakeDetector({ + getPhrases: () => phrases, + createScorer: async () => null, + onWake: (detection) => detections.push(detection), + }); + await detector.start(); + + expect(detector.isRunning).toBe(true); + expect(detector.isArmed).toBe(false); + pushHops(detector, 3); + expect(detections).toHaveLength(0); + }); + + it('refuses a scorer that is not local', () => { + const hosted = { tier: 'cloud', score: () => ({}) } as unknown as WakePhraseScorer; + expect(() => + createWakeDetector({ getPhrases: () => phrases, scorer: hosted, onWake: vi.fn() }) + ).toThrow(/no audio may leave the machine/i); + }); + + it('consumes nothing before start and nothing after stop', async () => { + const detector = build(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + + pushHops(detector, 2); + expect(detector.getStats().framesReceived).toBe(0); + + await detector.start(); + await detector.stop(); + pushHops(detector, 2); + + expect(detections).toHaveLength(0); + // A scorer the caller supplied belongs to the caller, so stopping does not + // free it. Only one the detector built holds ONNX sessions to release. + expect(scorer.disposed).toBe(0); + }); + + it('disposes the scorer it built itself', async () => { + const built = new ScriptedScorer(); + const detector = createWakeDetector({ + getPhrases: () => phrases, + createScorer: async () => built, + onWake: vi.fn(), + }); + await detector.start(); + await detector.stop(); + + expect(built.disposed).toBe(1); + }); + + // ----------------------------------------------------------------------- + // The invariant + // ----------------------------------------------------------------------- + + describe('no audio leaves the process while only the wake word is running', () => { + it('never hands a frame to a hosted speech provider', async () => { + const feed = vi.fn(); + const hostedStt = { + id: 'openai-stt', + tier: 'cloud', + feed, + flush: vi.fn(), + start: vi.fn(), + stop: vi.fn(), + } as unknown as SttProvider; + + const detector = build(); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + + // 200 frames is four seconds of a room being listened to. + for (let i = 0; i < 200; i++) detector.pushFrame(frame(i * 7)); + + expect(feed).not.toHaveBeenCalled(); + // And nothing the detector produced references the provider at all: the + // only outward edge is `onWake`, which hands PCM to the CALLER. + expect(hostedStt).toBeDefined(); + expect(detector.getStats().framesReceived).toBe(200); + }); + + it('makes no network call of any kind', async () => { + const fetchSpy = vi.fn(); + const originalFetch = globalThis.fetch; + globalThis.fetch = fetchSpy as unknown as typeof fetch; + try { + const detector = build(); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + for (let i = 0; i < 200; i++) detector.pushFrame(frame(i)); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('clears the retained audio when it stops', async () => { + const detector = build({ preRollMs: 500 }); + await detector.start(); + pushHops(detector, 2, 5000); + await detector.stop(); + await detector.start(); + scorer.scores = { [GLOBAL_WAKE_PHRASE_ID]: 0.9 }; + pushHops(detector, 1); + + // Only this run's frames, never the previous run's. + expect(detections[0].preRoll.length).toBe(WAKE_HOP_SAMPLES / ACAPPELLA_AUDIO_FRAME_SAMPLES); + }); + }); +}); diff --git a/src/__tests__/main/cue/cue-executor.test.ts b/src/__tests__/main/cue/cue-executor.test.ts index dcee3c92f5..34254b784f 100644 --- a/src/__tests__/main/cue/cue-executor.test.ts +++ b/src/__tests__/main/cue/cue-executor.test.ts @@ -1492,6 +1492,49 @@ describe('cue-executor', () => { expect(result.providerSessionId).toBe('sess-final'); }); + it('still extracts the session id when the parser surfaces result text', async () => { + // Regression: `runProcess` hands back stdout that has ALREADY been + // through `extractCleanStdout`, which collapses a stream-json + // transcript down to its result text. Mining the session id out of + // that cleaned string finds nothing, because the JSON envelope + // carrying `session_id` is gone. The extraction must read the raw + // buffer instead. + // + // The sibling test above cannot catch this: its parser never sets + // `text`, so cleaning falls through and returns the raw NDJSON + // unchanged. A real agent parser DOES set `text` on the `result` + // event - which is exactly what erased the id in production and + // left `provider_session_id` NULL on every `cue_events` row. + mockGetOutputParser.mockReturnValue({ + parseJsonLine: (line: string) => { + try { + return JSON.parse(line); + } catch { + return null; + } + }, + extractSessionId: (event: any) => event?.session_id ?? null, + } as any); + + const ndjson = [ + JSON.stringify({ type: 'system', session_id: 'sess-init' }), + JSON.stringify({ type: 'result', session_id: 'sess-final', text: 'All done.' }), + ].join('\n'); + + const config = createExecutionConfig({ toolType: 'claude-code' }); + const resultPromise = executeCuePrompt(config); + await vi.advanceTimersByTimeAsync(0); + + mockChild.stdout.emit('data', ndjson); + mockChild.emit('close', 0); + const result = await resultPromise; + + // Cleaned stdout is just the result text - the id is only + // recoverable from the raw buffer. + expect(result.stdout).toBe('All done.'); + expect(result.providerSessionId).toBe('sess-final'); + }); + it('returns null provider session id when no parser is registered', async () => { // Plain-text agents / command runs have no parser to mine an id from. mockGetOutputParser.mockReturnValue(null); diff --git a/src/__tests__/main/debug-package/voice-runtime.test.ts b/src/__tests__/main/debug-package/voice-runtime.test.ts new file mode 100644 index 0000000000..ec805ad7b5 --- /dev/null +++ b/src/__tests__/main/debug-package/voice-runtime.test.ts @@ -0,0 +1,105 @@ +/** + * The A Cappella voice-runtime collector. + * + * Contracts defended: + * - With the Encore Feature OFF, the self-test does not run and NO native + * runtime is loaded. This is the collector's half of "off means off": the + * self-test `dlopen`s every declared inference engine, and a debug package is + * built by users who are usually reporting something else entirely. + * - "Skipped" and "errored" are different fields. A reader of a support package + * has to be able to tell "we chose not to run it" from "it ran and blew up". + * - The static runtime table is reported either way: which binaries this build + * expects, on this platform, is the half of the answer worth having with the + * feature off. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type Store from 'electron-store'; + +vi.mock('electron', () => ({ + app: { getPath: () => '/tmp/acappella-voice-runtime-test' }, + systemPreferences: { getMediaAccessStatus: () => 'granted', askForMediaAccess: vi.fn() }, +})); + +const selfTest = vi.hoisted(() => ({ run: vi.fn() })); + +vi.mock('../../../main/acappella/runtime/runtime-selftest', () => ({ + runSelfTest: selfTest.run, +})); + +import { collectVoiceRuntime } from '../../../main/debug-package/collectors/voice-runtime'; + +/** The one key this collector reads. */ +function storeWith(aCappella: unknown): Store> { + return { + get: (key: string, defaultValue?: unknown) => + key === 'encoreFeatures' ? { aCappella } : defaultValue, + } as unknown as Store>; +} + +const REPORT = { + ranAt: 1_700_000_000_000, + platform: 'darwin', + arch: 'arm64', + platformKey: 'darwin-arm64', + entries: [], + passed: true, + microphone: { permission: 'granted', canPrompt: false }, +}; + +beforeEach(() => { + vi.clearAllMocks(); + selfTest.run.mockResolvedValue(REPORT); +}); + +describe('collectVoiceRuntime', () => { + it('runs the self-test when the feature is on', async () => { + const info = await collectVoiceRuntime(storeWith(true)); + + expect(selfTest.run).toHaveBeenCalledTimes(1); + expect(info.enabled).toBe(true); + expect(info.selfTest).toEqual(REPORT); + expect(info.selfTestSkipped).toBeUndefined(); + expect(info.microphone.permission).toBe('granted'); + }); + + it('loads no native runtime when the feature is off', async () => { + const info = await collectVoiceRuntime(storeWith(false)); + + // The whole point. `runSelfTest` is what dlopens whisper.cpp, llama.cpp, and + // onnxruntime, and it also populates the loader's process-wide failure memo + // that the capability gate reads. + expect(selfTest.run).not.toHaveBeenCalled(); + expect(info.enabled).toBe(false); + expect(info.selfTest).toBeNull(); + expect(info.selfTestSkipped).toMatch(/Encore Features/); + expect(info.selfTestError).toBeUndefined(); + }); + + it('reports the static runtime table with the feature off', async () => { + const info = await collectVoiceRuntime(storeWith(false)); + + expect(info.runtimes.length).toBeGreaterThan(0); + for (const runtime of info.runtimes) { + expect(runtime.id).toEqual(expect.any(String)); + expect(runtime.moduleId).toEqual(expect.any(String)); + expect(runtime.declared).toEqual(expect.any(Boolean)); + } + }); + + it('reports an unknown microphone rather than a guess when the self-test did not run', async () => { + const info = await collectVoiceRuntime(storeWith(false)); + + expect(info.microphone).toEqual({ permission: 'unknown', canPrompt: false }); + }); + + it('separates a self-test that blew up from one that was skipped', async () => { + selfTest.run.mockRejectedValue(new Error('probe exploded')); + + const info = await collectVoiceRuntime(storeWith(true)); + + expect(info.selfTestError).toBe('probe exploded'); + expect(info.selfTestSkipped).toBeUndefined(); + expect(info.selfTest).toBeNull(); + }); +}); diff --git a/src/__tests__/main/global-hotkey-registry.test.ts b/src/__tests__/main/global-hotkey-registry.test.ts new file mode 100644 index 0000000000..868d305e67 --- /dev/null +++ b/src/__tests__/main/global-hotkey-registry.test.ts @@ -0,0 +1,253 @@ +/** + * @file global-hotkey-registry.test.ts + * + * The named global hotkey registry: several ids at once, per-id failure that + * does not take the others down, conflict detection between two Maestro + * hotkeys, rebinding, and that the migrated `showMaestro` hotkey still behaves + * exactly as it did when it was the only one. + * + * The Electron `globalShortcut` API is injected as a backend, so nothing here + * needs a main process. `keysToAccelerator` is exercised through the registry + * rather than in isolation, because the property that matters is what ends up + * bound. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../main/utils/logger', () => ({ + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock('electron', () => ({ + app: { show: vi.fn() }, + BrowserWindow: class {}, + globalShortcut: { register: vi.fn(() => true), unregister: vi.fn() }, +})); + +vi.mock('../../shared/platformDetection', () => ({ + isMacOS: () => true, + isWindows: () => false, + isLinux: () => false, +})); + +import { + GlobalHotkeyRegistry, + keysToAccelerator, + type GlobalShortcutBackend, +} from '../../main/global-hotkey-manager'; +import { SHOW_MAESTRO_HOTKEY_ID } from '../../shared/global-hotkeys'; + +// --------------------------------------------------------------------------- +// Fake backend +// --------------------------------------------------------------------------- + +class FakeBackend implements GlobalShortcutBackend { + readonly bound = new Map void>(); + /** Accelerators the "OS" refuses. */ + readonly refuse = new Set(); + /** Accelerators whose registration throws. */ + readonly explode = new Set(); + readonly unregistered: string[] = []; + + register(accelerator: string, callback: () => void): boolean { + if (this.explode.has(accelerator)) throw new Error('boom'); + if (this.refuse.has(accelerator)) return false; + this.bound.set(accelerator, callback); + return true; + } + + unregister(accelerator: string): void { + this.unregistered.push(accelerator); + this.bound.delete(accelerator); + } + + fire(accelerator: string): void { + this.bound.get(accelerator)?.(); + } +} + +describe('keysToAccelerator', () => { + it('translates modifiers and upper-cases single letters', () => { + expect(keysToAccelerator(['Meta', 'Shift', 'm'])).toBe('Command+Shift+M'); + expect(keysToAccelerator(['Alt', 'Ctrl', 'F5'])).toBe('Alt+Control+F5'); + }); + + it('returns null without a non-modifier key', () => { + expect(keysToAccelerator([])).toBeNull(); + expect(keysToAccelerator(['Meta', 'Shift'])).toBeNull(); + }); +}); + +describe('GlobalHotkeyRegistry', () => { + let backend: FakeBackend; + let registry: GlobalHotkeyRegistry; + + beforeEach(() => { + backend = new FakeBackend(); + registry = new GlobalHotkeyRegistry(backend); + }); + + it('registers several ids independently and routes each to its own handler', () => { + const show = vi.fn(); + const voice = vi.fn(); + registry.define(SHOW_MAESTRO_HOTKEY_ID, show); + registry.define('voiceConductor', voice); + + expect(registry.setKeys(SHOW_MAESTRO_HOTKEY_ID, ['Meta', 'Shift', 'm']).registered).toBe(true); + expect(registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']).registered).toBe(true); + + backend.fire('Command+Shift+M'); + backend.fire('Command+Alt+V'); + expect(show).toHaveBeenCalledTimes(1); + expect(voice).toHaveBeenCalledTimes(1); + }); + + it('fails one id without disturbing the others', () => { + backend.refuse.add('Command+Alt+V'); + registry.setKeys(SHOW_MAESTRO_HOTKEY_ID, ['Meta', 'Shift', 'm']); + const failed = registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + + expect(failed.registered).toBe(false); + expect(failed.reason).toBe('os-conflict'); + expect(registry.status(SHOW_MAESTRO_HOTKEY_ID)?.registered).toBe(true); + expect(backend.bound.has('Command+Shift+M')).toBe(true); + }); + + it('reports a registration that threw as its own reason', () => { + backend.explode.add('Command+Alt+V'); + const status = registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + expect(status.reason).toBe('register-error'); + expect(status.message).toContain('boom'); + }); + + it('detects a conflict between two Maestro hotkeys and names the holder', () => { + registry.setKeys(SHOW_MAESTRO_HOTKEY_ID, ['Meta', 'Alt', 'v']); + const clash = registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + + expect(clash.registered).toBe(false); + expect(clash.reason).toBe('maestro-conflict'); + expect(clash.conflictsWith).toBe(SHOW_MAESTRO_HOTKEY_ID); + // The first hotkey keeps the combo rather than silently losing it. + expect(registry.status(SHOW_MAESTRO_HOTKEY_ID)?.registered).toBe(true); + }); + + it('lets an id rebind onto the combo it already holds', () => { + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + const again = registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + expect(again.registered).toBe(true); + expect(again.reason).toBeUndefined(); + }); + + it('releases the previous combo on rebind', () => { + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'b']); + + expect(backend.unregistered).toContain('Command+Alt+V'); + expect(backend.bound.has('Command+Alt+V')).toBe(false); + expect(backend.bound.has('Command+Alt+B')).toBe(true); + }); + + it('frees a combo for another id once the first releases it', () => { + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + registry.setKeys('voiceConductor', []); + const second = registry.setKeys('voiceCurrentAgent', ['Meta', 'Alt', 'v']); + expect(second.registered).toBe(true); + }); + + it('treats an empty key array as a deliberate clear, not a failure', () => { + const failures: string[] = []; + registry.onFailure((status) => failures.push(status.id)); + const status = registry.setKeys('voiceConductor', []); + + expect(status.registered).toBe(false); + expect(status.reason).toBeUndefined(); + expect(failures).toEqual([]); + }); + + it('rejects a modifier-only combo with a distinct reason', () => { + const status = registry.setKeys('voiceConductor', ['Meta', 'Shift']); + expect(status.reason).toBe('invalid-accelerator'); + }); + + it('reports every failure through one listener with the failing id', () => { + backend.refuse.add('Command+Alt+V'); + const seen: Array<{ id: string; reason?: string }> = []; + registry.onFailure((status) => seen.push({ id: status.id, reason: status.reason })); + + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + expect(seen).toEqual([{ id: 'voiceConductor', reason: 'os-conflict' }]); + }); + + it('survives a failure listener that throws', () => { + backend.refuse.add('Command+Alt+V'); + registry.onFailure(() => { + throw new Error('window gone'); + }); + expect(() => registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v'])).not.toThrow(); + }); + + it('binds a handler defined after the keys were set', () => { + backend.refuse.add('Command+Alt+V'); + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + backend.refuse.clear(); + + const handler = vi.fn(); + registry.define('voiceConductor', handler); + + expect(registry.status('voiceConductor')?.registered).toBe(true); + backend.fire('Command+Alt+V'); + expect(handler).toHaveBeenCalledTimes(1); + }); + + it('routes to the handler live, so a redefine takes effect on the next press', () => { + const first = vi.fn(); + const second = vi.fn(); + registry.define('voiceConductor', first); + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + registry.define('voiceConductor', second); + + backend.fire('Command+Alt+V'); + expect(first).not.toHaveBeenCalled(); + expect(second).toHaveBeenCalledTimes(1); + }); + + it('disposeAll releases every combo', () => { + registry.setKeys(SHOW_MAESTRO_HOTKEY_ID, ['Meta', 'Shift', 'm']); + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + registry.disposeAll(); + + expect(backend.bound.size).toBe(0); + expect(registry.allStatuses().every((status) => !status.registered)).toBe(true); + }); + + it('remove forgets the id entirely', () => { + registry.define('voiceConductor', vi.fn()); + registry.setKeys('voiceConductor', ['Meta', 'Alt', 'v']); + registry.remove('voiceConductor'); + + expect(registry.status('voiceConductor')).toBeNull(); + expect(backend.bound.has('Command+Alt+V')).toBe(false); + }); + + describe('migrated showMaestro behaviour', () => { + it('binds, rebinds, and clears exactly as the singleton did', () => { + const summon = vi.fn(); + registry.define(SHOW_MAESTRO_HOTKEY_ID, summon); + + expect(registry.setKeys(SHOW_MAESTRO_HOTKEY_ID, ['Meta', 'Shift', 'm']).registered).toBe( + true + ); + backend.fire('Command+Shift+M'); + expect(summon).toHaveBeenCalledTimes(1); + + registry.setKeys(SHOW_MAESTRO_HOTKEY_ID, ['Meta', 'Shift', 'k']); + backend.fire('Command+Shift+M'); + expect(summon).toHaveBeenCalledTimes(1); + backend.fire('Command+Shift+K'); + expect(summon).toHaveBeenCalledTimes(2); + + registry.setKeys(SHOW_MAESTRO_HOTKEY_ID, []); + expect(backend.bound.size).toBe(0); + }); + }); +}); diff --git a/src/__tests__/main/ipc/handlers/acappella.test.ts b/src/__tests__/main/ipc/handlers/acappella.test.ts new file mode 100644 index 0000000000..3d85226cec --- /dev/null +++ b/src/__tests__/main/ipc/handlers/acappella.test.ts @@ -0,0 +1,873 @@ +/** + * @file acappella.test.ts + * + * Unit tests for the A Cappella IPC transport. + * + * Contracts defended: + * - Registering the handlers builds NOTHING. Enabling the Encore Feature must + * not open a device or construct a provider, so the service appears only on + * the first start-session. + * - The Encore gate rejects every channel with 'ACappellaDisabled' while the + * flag is off - except stop-session, which has to stay callable so toggling + * the feature off mid-session can still release the floor. + * - Every protocol event is broadcast on `acappella:event` exactly once. + * - A provider-selection change rebuilds the service; an unchanged one reuses + * it, so the fan-out is never registered twice. + * - Untrusted payloads are validated at the boundary: a malformed agent scope is + * an error rather than a silent fall back to whichever agent is active. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { app, ipcMain, shell, systemPreferences } from 'electron'; + +vi.mock('electron', () => ({ + ipcMain: { handle: vi.fn(), on: vi.fn() }, + // `getPath` and `getVersion` are what the paired-device transport reads at + // registration: the device file lives under userData and the app version goes + // in the Bonjour advert. + app: { + on: vi.fn(), + getPath: vi.fn(() => '/tmp/maestro-test-userdata'), + getVersion: vi.fn(() => '0.0.0-test'), + }, + shell: { openExternal: vi.fn().mockResolvedValue(undefined) }, + // Starting a session asks for the microphone. Granted here so these tests stay + // about the transport; the permission's own states are covered in + // mic-permission.test.ts. + systemPreferences: { + getMediaAccessStatus: vi.fn(() => 'granted'), + askForMediaAccess: vi.fn().mockResolvedValue(true), + }, +})); + +/** + * The audio host window is a real `BrowserWindow` in production. Here it is a + * webContents stand-in that records commands, plus a sender predicate a test can + * flip: "only the audio host may push PCM" is enforced at this boundary, so it + * has to be possible to fail it. + */ +const audioHost = vi.hoisted(() => ({ + webContents: { send: vi.fn(), isDestroyed: () => false }, + isHostContents: true, +})); + +vi.mock('../../../../main/acappella/audio-host-window', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + ensureAcappellaAudioHostWindow: vi.fn(() => audioHost), + closeAcappellaAudioHostWindow: vi.fn(), + getAcappellaAudioHostWindow: vi.fn(() => audioHost), + isAcappellaAudioHostContents: vi.fn(() => audioHost.isHostContents), + }; +}); +vi.mock('../../../../main/utils/sentry', () => ({ captureException: vi.fn() })); +vi.mock('../../../../main/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock('../../../../main/utils/safe-send', () => ({ + isWebContentsAvailable: (win: unknown) => !!win, +})); +vi.mock('../../../../main/web-server/callbacks/remoteRequest', () => ({ + requestFromRenderer: vi.fn(), +})); +vi.mock('../../../../main/stores/getters', () => ({ + getSessionsStore: vi.fn(), +})); + +import { getSessionsStore } from '../../../../main/stores/getters'; +import { + disposeACappellaAudioBridge, + registerACappellaHandlers, + resetACappellaHandlerState, + shutdownACappellaForDisable, + stopVoiceSessionForClosedWindow, + type VoiceStartSessionResult, +} from '../../../../main/ipc/handlers/acappella'; +import { + ACAPPELLA_AUDIO_FRAME_CHANNEL, + ACAPPELLA_AUDIO_FRAME_SAMPLES, + ACAPPELLA_AUDIO_SAMPLE_RATE, + ACAPPELLA_AUDIO_STATUS_CHANNEL, + type AudioFrame, + type AudioHostCommand, + type AudioHostStatus, +} from '../../../../shared/acappella/audio-host'; +import { ECHO_STT_PROVIDER_ID } from '../../../../main/acappella/providers/echo-stt'; +import { + disposeVoiceSessionService, + getACappellaTransport, + getVoiceSessionService, +} from '../../../../main/acappella'; +import type { RosterAgent, VoiceEvent } from '../../../../shared/acappella/protocol'; +import type { VoiceSessionSnapshot } from '../../../../main/acappella'; +import type { StoredSession } from '../../../../main/stores/types'; +import { createMockSession } from '../../../helpers/mockSession'; +import { createMockAITab } from '../../../helpers/mockTab'; + +// --------------------------------------------------------------------------- +// Harness +// --------------------------------------------------------------------------- + +type Handler = (event: unknown, ...args: unknown[]) => Promise; + +/** Settings the gate and the provider registry both read. */ +interface FakeSettings { + encoreFeatures?: Record; + acappella?: { providers?: Record }; +} + +let settings: FakeSettings; +let broadcasts: Array<{ channel: string; args: unknown[] }>; +let sessions: StoredSession[]; + +const settingsStore = { + get: (key: string, defaultValue?: unknown) => + (settings as Record)[key] ?? defaultValue, +}; + +const safeSend = vi.fn((channel: string, ...args: unknown[]) => { + broadcasts.push({ channel, args }); +}); + +/** A window stand-in - the executor only ever checks it is alive and sends. */ +const fakeWindow = { webContents: { send: vi.fn() } }; + +function handlerFor(channel: string): Handler { + const registration = vi + .mocked(ipcMain.handle) + .mock.calls.find(([registered]) => registered === channel); + expect(registration, `no handler registered for ${channel}`).toBeDefined(); + return registration?.[1] as unknown as Handler; +} + +function voiceEvents(): VoiceEvent[] { + return broadcasts + .filter((entry) => entry.channel === 'acappella:event') + .map((entry) => entry.args[0] as VoiceEvent); +} + +/** + * What `resolveVoiceWindowId` answers, per test. `undefined` leaves the dep off + * entirely, which is the single-window host: no window is ever named. + */ +let voiceWindowId: string | null | undefined; + +function register(options: { withAudio?: boolean } = {}): void { + registerACappellaHandlers({ + settingsStore, + getMainWindow: () => fakeWindow as never, + safeSend: safeSend as never, + // Absent by default, exactly as in a test process with no window: the + // session still runs, it is simply text-in. + audioHostDeps: options.withAudio ? ({} as never) : undefined, + resolveVoiceWindowId: + voiceWindowId === undefined ? undefined : () => voiceWindowId as string | null, + }); +} + +/** The `ipcMain.on` listener for one of the audio host's two channels. */ +function listenerFor(channel: string): (event: unknown, payload: unknown) => void { + const registration = vi.mocked(ipcMain.on).mock.calls.find(([name]) => name === channel); + expect(registration, `no listener registered for ${channel}`).toBeDefined(); + return registration?.[1] as unknown as (event: unknown, payload: unknown) => void; +} + +/** Commands pushed to the audio host renderer, in order. */ +function audioCommands(): AudioHostCommand[] { + return vi + .mocked(audioHost.webContents.send) + .mock.calls.map(([, command]) => command as AudioHostCommand); +} + +/** + * The real platform, restored after every test. Anything that reaches a + * platform-gated API (the macOS microphone prompt) has to say which platform it + * means; inheriting the host's is what makes a suite pass on a Mac and fail on + * both CI legs. + */ +const REAL_PLATFORM = process.platform; + +function setPlatform(platform: string): void { + Object.defineProperty(process, 'platform', { value: platform, configurable: true }); +} + +beforeEach(async () => { + vi.clearAllMocks(); + await disposeVoiceSessionService(); + resetACappellaHandlerState(); + + settings = { encoreFeatures: { aCappella: true } }; + broadcasts = []; + sessions = [ + createMockSession({ + id: 'agent-backend', + name: 'Backend', + toolType: 'claude-code', + cwd: '/repo/api', + activeTabId: 'tab-auth', + aiTabs: [createMockAITab({ id: 'tab-auth', name: 'Auth Refactor', createdAt: 1_000 })], + } as never) as unknown as StoredSession, + ]; + + vi.mocked(getSessionsStore).mockReturnValue({ + get: (key: string, fallback?: unknown) => (key === 'sessions' ? sessions : fallback), + } as never); + + register(); +}); + +afterEach(async () => { + setPlatform(REAL_PLATFORM); + await disposeVoiceSessionService(); + resetACappellaHandlerState(); +}); + +// --------------------------------------------------------------------------- +// Registration and laziness +// --------------------------------------------------------------------------- + +describe('A Cappella IPC handlers - registration', () => { + it('registers every documented channel', () => { + const channels = vi.mocked(ipcMain.handle).mock.calls.map(([channel]) => channel); + expect(channels).toEqual( + expect.arrayContaining([ + 'acappella:start-session', + 'acappella:stop-session', + 'acappella:submit-utterance', + 'acappella:interrupt', + 'acappella:stop-word', + 'acappella:get-roster', + 'acappella:get-state', + 'acappella:open-mic-settings', + 'acappella:mic-permission', + ]) + ); + }); + + it('asks for nothing before a session starts, on any platform', async () => { + // An app that prompts for the microphone at launch, or the moment an Encore + // Feature is switched on, is asking for a device to do something the user has + // not requested. Registering the handlers must ask for nothing. + expect(systemPreferences.askForMediaAccess).not.toHaveBeenCalled(); + + // Reading the permission is a pure query and must not prompt either: the + // capability gate calls it on every Settings render. + await handlerFor('acappella:mic-permission')({}); + expect(systemPreferences.askForMediaAccess).not.toHaveBeenCalled(); + }); + + it('asks for the microphone at the first session start on macOS', async () => { + // The platform is pinned rather than inherited from the host: `askForMediaAccess` + // is a macOS-only API, so a test that assumes the developer's Mac passes locally + // and fails on both CI legs. + setPlatform('darwin'); + vi.mocked(systemPreferences.getMediaAccessStatus).mockReturnValue('not-determined'); + await handlerFor('acappella:start-session')({}); + expect(systemPreferences.askForMediaAccess).toHaveBeenCalledWith('microphone'); + }); + + it('starts a session without prompting where there is no prompt to show', async () => { + // Linux and Windows have no in-app microphone prompt. Calling the macOS API + // there would either throw or silently do nothing, and either way a session + // must still start rather than be gated behind a permission that cannot be + // requested. + setPlatform('linux'); + vi.mocked(systemPreferences.getMediaAccessStatus).mockReturnValue('not-determined'); + await handlerFor('acappella:start-session')({}); + expect(systemPreferences.askForMediaAccess).not.toHaveBeenCalled(); + expect(getVoiceSessionService()).not.toBeNull(); + }); + + it('builds no session service until a session is started', async () => { + expect(getVoiceSessionService()).toBeNull(); + expect(await handlerFor('acappella:get-state')({})).toBeNull(); + expect(getVoiceSessionService()).toBeNull(); + + await handlerFor('acappella:start-session')({}); + expect(getVoiceSessionService()).not.toBeNull(); + }); + + it('disposes the live session on app quit', async () => { + await handlerFor('acappella:start-session')({}); + expect(getVoiceSessionService()).not.toBeNull(); + + // `app.on` is a union of ~40 per-event overloads, so the mock's call tuples + // narrow to the first one. Widen them before looking for our event. + const lifecycleCalls = vi.mocked(app.on).mock.calls as unknown as Array<[string, () => void]>; + const willQuit = lifecycleCalls.find(([event]) => event === 'will-quit')?.[1]; + expect(willQuit, 'no will-quit listener registered').toBeDefined(); + + willQuit?.(); + // The dispose is fire-and-forget from a synchronous lifecycle hook, so the + // singleton is cleared on the next tick rather than inline. + await vi.waitFor(() => expect(getVoiceSessionService()).toBeNull()); + }); +}); + +// --------------------------------------------------------------------------- +// Encore gate +// --------------------------------------------------------------------------- + +describe('A Cappella IPC handlers - Encore gate', () => { + beforeEach(() => { + settings.encoreFeatures = { aCappella: false }; + }); + + it.each([ + ['acappella:start-session'], + ['acappella:submit-utterance'], + ['acappella:interrupt'], + ['acappella:stop-word'], + ['acappella:get-roster'], + ['acappella:get-state'], + ])('%s rejects with ACappellaDisabled while the flag is off', async (channel) => { + await expect(handlerFor(channel)({}, 'anything')).rejects.toThrow('ACappellaDisabled'); + }); + + it('still allows stop-session so a live session can be released', async () => { + settings.encoreFeatures = { aCappella: true }; + await handlerFor('acappella:start-session')({}); + settings.encoreFeatures = { aCappella: false }; + + await expect(handlerFor('acappella:stop-session')({})).resolves.toBeUndefined(); + expect(getVoiceSessionService()?.getState()).toBe('idle'); + }); + + it('treats a missing encoreFeatures key as off', async () => { + settings = {}; + await expect(handlerFor('acappella:get-state')({})).rejects.toThrow('ACappellaDisabled'); + }); + + it('still allows open-mic-settings, the one recovery for a denied microphone', async () => { + // The value depends on the host platform; what matters is that the gate does + // not reject it, since a denied microphone is exactly the situation in which + // the feature may already have been switched back off. + await expect(handlerFor('acappella:open-mic-settings')({})).resolves.toEqual( + expect.any(Boolean) + ); + }); +}); + +// --------------------------------------------------------------------------- +// Switching the feature off +// --------------------------------------------------------------------------- + +/** + * The stand-down that `main/index.ts` runs from its `encoreFeatures` watcher. + * + * Rejecting new IPC calls is not the same as stopping: before this existed, + * turning A Cappella off released the microphone and closed the audio host, and + * left a live session, a loaded inference pipeline, and the paired-device + * transport running behind a switch its owner believed was off. + */ +describe('A Cappella IPC handlers - shutdownACappellaForDisable', () => { + it('returns a live session to idle', async () => { + await handlerFor('acappella:start-session')({}); + expect(getVoiceSessionService()?.getState()).not.toBe('idle'); + + settings.encoreFeatures = { aCappella: false }; + await shutdownACappellaForDisable(); + + expect(getVoiceSessionService()?.getState()).toBe('idle'); + }); + + it('stands the transport down, so no advert and no device outlive the switch', async () => { + const transport = getACappellaTransport(); + expect(transport, 'registration must have built a transport').not.toBeNull(); + const standDown = vi.spyOn(transport!, 'standDown'); + + await shutdownACappellaForDisable(); + + expect(standDown).toHaveBeenCalled(); + }); + + it('keeps the transport, so switching the feature back on needs no restart', async () => { + const before = getACappellaTransport(); + + await shutdownACappellaForDisable(); + + // Disposing it here would be the tempting move and the wrong one: it is + // built once, at handler registration, which only runs at boot. + expect(getACappellaTransport()).toBe(before); + }); + + it('drops the provider pipeline, so reclaiming disk is not blocked by an open model', async () => { + await handlerFor('acappella:start-session')({}); + settings.encoreFeatures = { aCappella: false }; + await shutdownACappellaForDisable(); + settings.encoreFeatures = { aCappella: true }; + + // A rebuild rather than a reuse is the observable proof the pipeline was + // disposed: the memo of what it was built from is cleared with it, so the + // next start cannot hand back a disposed pipeline. On Windows this is also + // what lets `fs.rm` delete a model directory whose files were mapped. + const result = (await handlerFor('acappella:start-session')({})) as VoiceStartSessionResult; + expect(result.snapshot.state).not.toBe('idle'); + }); + + it('is safe with nothing running at all', async () => { + vi.clearAllMocks(); + await disposeVoiceSessionService(); + resetACappellaHandlerState(); + register(); + + await expect(shutdownACappellaForDisable()).resolves.toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// Microphone settings +// --------------------------------------------------------------------------- + +describe('A Cappella IPC handlers - open-mic-settings', () => { + const realPlatform = process.platform; + + function setPlatform(platform: string): void { + Object.defineProperty(process, 'platform', { value: platform, configurable: true }); + } + + afterEach(() => { + setPlatform(realPlatform); + }); + + it.each([ + ['darwin', 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone'], + ['win32', 'ms-settings:privacy-microphone'], + ])('opens the %s privacy pane', async (platform, url) => { + setPlatform(platform); + await expect(handlerFor('acappella:open-mic-settings')({})).resolves.toBe(true); + expect(shell.openExternal).toHaveBeenCalledWith(url); + }); + + it('reports false and opens nothing where no deep link exists', async () => { + setPlatform('linux'); + await expect(handlerFor('acappella:open-mic-settings')({})).resolves.toBe(false); + expect(shell.openExternal).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// Session lifecycle +// --------------------------------------------------------------------------- + +describe('A Cappella IPC handlers - session lifecycle', () => { + it('starts a conductor session on the default trio with no substitutions', async () => { + const result = (await handlerFor('acappella:start-session')({})) as VoiceStartSessionResult; + + expect(result.snapshot.state).toBe('listening'); + expect(result.snapshot.scope).toEqual({ kind: 'conductor' }); + // STT defaults to the microphone check rather than the text-in mock, in + // every build: an unconfigured install must still be able to open a device. + expect(result.snapshot.providerIds).toEqual({ + stt: 'echo-stt', + tts: 'mock-tts', + brain: 'mock-brain', + }); + // Not configuring a provider is the documented default, not a downgrade. + expect(result.substitutions).toEqual([]); + }); + + it('reports a substitution when the configured provider is unknown', async () => { + settings.acappella = { providers: { stt: 'whisper-that-is-not-registered' } }; + + const result = (await handlerFor('acappella:start-session')({})) as VoiceStartSessionResult; + + expect(result.substitutions).toHaveLength(1); + expect(result.substitutions[0]).toMatchObject({ + role: 'stt', + requestedId: 'whisper-that-is-not-registered', + // Not the mock: an unbuildable slot refuses by name rather than quietly + // becoming a tier that transcribes nothing and looks healthy. + resolvedId: 'unresolved-stt', + reason: 'unknown-provider', + }); + expect(result.snapshot.providerIds.stt).toBe('unresolved-stt'); + }); + + it('binds an agent scope when one is given', async () => { + const result = (await handlerFor('acappella:start-session')( + {}, + { + kind: 'agent', + sessionId: 'agent-backend', + } + )) as VoiceStartSessionResult; + + expect(result.snapshot.scope).toEqual({ kind: 'agent', sessionId: 'agent-backend' }); + }); + + it('rejects an agent scope with no agent id rather than guessing one', async () => { + await expect(handlerFor('acappella:start-session')({}, { kind: 'agent' })).rejects.toThrow( + 'InvalidVoiceScope' + ); + expect(getVoiceSessionService()).toBeNull(); + }); + + it('returns a live snapshot from get-state once started', async () => { + await handlerFor('acappella:start-session')({}); + + const snapshot = (await handlerFor('acappella:get-state')({})) as VoiceSessionSnapshot; + expect(snapshot.state).toBe('listening'); + expect(snapshot.sessionId).toEqual(expect.any(String)); + expect(snapshot.seq).toBeGreaterThan(0); + }); + + it('stop-session returns the session to idle', async () => { + await handlerFor('acappella:start-session')({}); + await handlerFor('acappella:stop-session')({}); + + const snapshot = (await handlerFor('acappella:get-state')({})) as VoiceSessionSnapshot; + expect(snapshot.state).toBe('idle'); + expect(snapshot.sessionId).toBeNull(); + }); + + it('stop-session with no service is a no-op', async () => { + await expect(handlerFor('acappella:stop-session')({})).resolves.toBeUndefined(); + }); +}); + +/** + * Which window's HUD a session belongs to. Voice events are broadcast to every + * window, so this field is the only thing keeping a session opened in one window + * from drawing a HUD in all of them. + */ +describe('A Cappella IPC handlers - window scoping', () => { + afterEach(() => { + voiceWindowId = undefined; + }); + + it('stamps the session with the window the start came from', async () => { + voiceWindowId = 'window-2'; + vi.mocked(ipcMain.handle).mockClear(); + register(); + + const result = (await handlerFor('acappella:start-session')({})) as VoiceStartSessionResult; + + expect(result.snapshot.windowId).toBe('window-2'); + // On `wake`, the FIRST event: a window that had to wait for the catch-up + // snapshot would flash a HUD for a session that is not its own. + expect(voiceEvents()[0]).toMatchObject({ type: 'wake', windowId: 'window-2' }); + }); + + it('names no window when nothing resolves one', async () => { + // The single-window host, and every test host: null means the primary + // window shows it, which is the only window there is. + const result = (await handlerFor('acappella:start-session')({})) as VoiceStartSessionResult; + + expect(result.snapshot.windowId).toBeNull(); + }); + + it('clears the window when the session ends', async () => { + voiceWindowId = 'window-2'; + vi.mocked(ipcMain.handle).mockClear(); + register(); + await handlerFor('acappella:start-session')({}); + + await handlerFor('acappella:stop-session')({}); + + const snapshot = (await handlerFor('acappella:get-state')({})) as VoiceSessionSnapshot; + expect(snapshot.windowId).toBeNull(); + }); + + it('ends the session when its own window closes', async () => { + // Otherwise closing that window leaves an open microphone with no surface + // anywhere - the failure the HUD's close button exists to prevent, reached + // by a different route. + voiceWindowId = 'window-2'; + vi.mocked(ipcMain.handle).mockClear(); + register(); + await handlerFor('acappella:start-session')({}); + + await stopVoiceSessionForClosedWindow('window-2'); + + expect(getVoiceSessionService()?.getState()).toBe('idle'); + }); + + it('leaves a session alone when a DIFFERENT window closes', async () => { + voiceWindowId = 'window-2'; + vi.mocked(ipcMain.handle).mockClear(); + register(); + await handlerFor('acappella:start-session')({}); + + await stopVoiceSessionForClosedWindow('window-1'); + + expect(getVoiceSessionService()?.getState()).toBe('listening'); + }); +}); + +// --------------------------------------------------------------------------- +// Event fan-out +// --------------------------------------------------------------------------- + +describe('A Cappella IPC handlers - event fan-out', () => { + it('broadcasts every protocol event once on acappella:event', async () => { + await handlerFor('acappella:start-session')({}); + + const types = voiceEvents().map((event) => event.type); + expect(types).toEqual(['wake', 'listen-start', 'provider-state', 'agent-roster']); + expect(voiceEvents().map((event) => event.seq)).toEqual([1, 2, 3, 4]); + }); + + it('reuses the service across starts, so the fan-out is registered once', async () => { + await handlerFor('acappella:start-session')({}); + const first = getVoiceSessionService(); + broadcasts = []; + + await handlerFor('acappella:start-session')({}); + + expect(getVoiceSessionService()).toBe(first); + // One wake per start, not two: a second subscriber would double every event. + expect(voiceEvents().filter((event) => event.type === 'wake')).toHaveLength(1); + }); + + it('rebuilds the service when the provider selection changed', async () => { + await handlerFor('acappella:start-session')({}); + const first = getVoiceSessionService(); + + settings.acappella = { providers: { stt: 'whisper-that-is-not-registered' } }; + const result = (await handlerFor('acappella:start-session')({})) as VoiceStartSessionResult; + + expect(getVoiceSessionService()).not.toBe(first); + expect(result.substitutions).toHaveLength(1); + }); +}); + +// --------------------------------------------------------------------------- +// Input channels +// --------------------------------------------------------------------------- + +describe('A Cappella IPC handlers - input', () => { + it('accepts an utterance while listening', async () => { + await handlerFor('acappella:start-session')({}); + + await expect(handlerFor('acappella:submit-utterance')({}, 'hello there')).resolves.toBe(true); + + // Release the mock STT's pending partial timers. + await handlerFor('acappella:stop-session')({}); + }); + + it('rejects a non-string utterance at the boundary', async () => { + await handlerFor('acappella:start-session')({}); + + await expect(handlerFor('acappella:submit-utterance')({}, 42)).rejects.toThrow( + 'InvalidUtterance' + ); + }); + + it('reports false for an utterance with no session', async () => { + await expect(handlerFor('acappella:submit-utterance')({}, 'hello')).resolves.toBe(false); + }); + + it('reports false for an interrupt when nothing is speaking', async () => { + await handlerFor('acappella:start-session')({}); + await expect(handlerFor('acappella:interrupt')({}, 'client-button')).resolves.toBe(false); + }); + + it('stop-word ends the session and is distinct from barge-in', async () => { + await handlerFor('acappella:start-session')({}); + broadcasts = []; + + await handlerFor('acappella:stop-word')({}, { phrase: 'never mind', source: 'voice' }); + + const types = voiceEvents().map((event) => event.type); + expect(types).toContain('stop-word'); + expect(types).not.toContain('barge-in'); + expect(getVoiceSessionService()?.getState()).toBe('idle'); + }); + + it('stop-word with no payload is still accepted', async () => { + await handlerFor('acappella:start-session')({}); + await expect(handlerFor('acappella:stop-word')({})).resolves.toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// Audio host transport +// --------------------------------------------------------------------------- + +describe('A Cappella IPC handlers - audio host transport', () => { + let nodeEnv: string | undefined; + + /** One 20 ms frame of 200 Hz tone: voiced, and inside the detector's ZCR band. */ + function toneFrame(seq: number): AudioFrame { + const samples = new Int16Array(ACAPPELLA_AUDIO_FRAME_SAMPLES); + for (let i = 0; i < samples.length; i++) { + samples[i] = 0.4 * Math.sin((2 * Math.PI * 200 * i) / ACAPPELLA_AUDIO_SAMPLE_RATE) * 0x7fff; + } + return { seq, capturedAt: 1_000 + seq * 20, rms: 0.28, pcm: samples.buffer }; + } + + function pushStatus(status: AudioHostStatus): void { + listenerFor(ACAPPELLA_AUDIO_STATUS_CHANNEL)({ sender: audioHost.webContents }, status); + } + + function pushFrames(count: number): void { + const listener = listenerFor(ACAPPELLA_AUDIO_FRAME_CHANNEL); + for (let seq = 1; seq <= count; seq++) { + listener({ sender: audioHost.webContents }, toneFrame(seq)); + } + } + + beforeEach(async () => { + // The echo provider is the development default and the only registered STT + // that consumes audio, so the whole capture path hangs off this flag. + nodeEnv = process.env.NODE_ENV; + process.env.NODE_ENV = 'development'; + settings.acappella = { providers: { stt: ECHO_STT_PROVIDER_ID } }; + + vi.clearAllMocks(); + await disposeVoiceSessionService(); + resetACappellaHandlerState(); + audioHost.isHostContents = true; + register({ withAudio: true }); + }); + + afterEach(() => { + process.env.NODE_ENV = nodeEnv; + }); + + it('registers the two host channels as sends, not invokes', () => { + const channels = vi.mocked(ipcMain.on).mock.calls.map(([channel]) => channel); + expect(channels).toEqual( + expect.arrayContaining([ACAPPELLA_AUDIO_FRAME_CHANNEL, ACAPPELLA_AUDIO_STATUS_CHANNEL]) + ); + }); + + it('opens the microphone once the host reports ready', async () => { + await handlerFor('acappella:start-session')({}); + pushStatus({ kind: 'ready' }); + + expect(audioCommands().map((command) => command.kind)).toContain('start-capture'); + }); + + it('turns captured frames into meter events on the one ordered stream', async () => { + await handlerFor('acappella:start-session')({}); + pushStatus({ kind: 'ready' }); + broadcasts = []; + + pushFrames(9); + + const levels = voiceEvents().filter((event) => event.type === 'audio-level'); + expect(levels.length).toBeGreaterThan(0); + // Downsampled: nine 20 ms frames is well under nine updates. + expect(levels.length).toBeLessThan(9); + }); + + it('publishes the microphone state a capture start proves', async () => { + await handlerFor('acappella:start-session')({}); + broadcasts = []; + + pushStatus({ + kind: 'capture-start', + device: { deviceId: 'default', label: 'Built-in Microphone' }, + contextSampleRate: 48_000, + }); + + expect(voiceEvents().at(-1)).toMatchObject({ + type: 'mic-state', + permission: 'granted', + deviceLabel: 'Built-in Microphone', + }); + }); + + it('turns a capture failure into a session error rather than a quiet session', async () => { + await handlerFor('acappella:start-session')({}); + broadcasts = []; + + pushStatus({ kind: 'mic-error', code: 'permission-denied', message: 'Permission denied' }); + + expect(voiceEvents().map((event) => event.type)).toContain('session-error'); + expect(voiceEvents().find((event) => event.type === 'session-error')).toMatchObject({ + code: 'audio-capture-failed', + recoverable: true, + }); + }); + + it('ignores audio from a sender that is not the audio host', async () => { + await handlerFor('acappella:start-session')({}); + pushStatus({ kind: 'ready' }); + broadcasts = []; + + // A browser tab that found the channel must not be able to inject PCM into a + // live voice session. + audioHost.isHostContents = false; + pushFrames(9); + + expect(voiceEvents().filter((event) => event.type === 'audio-level')).toEqual([]); + }); + + it('ignores a malformed frame instead of throwing fifty times a second', async () => { + await handlerFor('acappella:start-session')({}); + pushStatus({ kind: 'ready' }); + const listener = listenerFor(ACAPPELLA_AUDIO_FRAME_CHANNEL); + + expect(() => listener({ sender: audioHost.webContents }, { seq: 1 })).not.toThrow(); + expect(() => listener({ sender: audioHost.webContents }, null)).not.toThrow(); + }); + + it('releases the microphone when the Encore Feature is switched off', async () => { + await handlerFor('acappella:start-session')({}); + pushStatus({ kind: 'ready' }); + vi.mocked(audioHost.webContents.send).mockClear(); + + disposeACappellaAudioBridge(); + + expect(audioCommands().map((command) => command.kind)).toContain('stop-capture'); + }); + + it('rewires audio when the Encore Feature is switched back on', async () => { + await handlerFor('acappella:start-session')({}); + pushStatus({ kind: 'ready' }); + + // Switching the feature off drops the bridge but deliberately leaves the + // session service alive, so the next start reuses it. Without rewiring, the + // host window opens and captures into nothing: no meter, no transcript, no + // barge-in, and nothing on screen to say the microphone is dead. + disposeACappellaAudioBridge(); + await handlerFor('acappella:stop-session')({}); + vi.mocked(audioHost.webContents.send).mockClear(); + broadcasts = []; + + await handlerFor('acappella:start-session')({}); + pushStatus({ kind: 'ready' }); + pushFrames(9); + + expect(audioCommands().map((command) => command.kind)).toContain('start-capture'); + expect(voiceEvents().filter((event) => event.type === 'audio-level').length).toBeGreaterThan(0); + }); + + it('wires no audio at all without a host window', async () => { + vi.clearAllMocks(); + await disposeVoiceSessionService(); + resetACappellaHandlerState(); + register(); + + await handlerFor('acappella:start-session')({}); + pushStatus({ kind: 'ready' }); + + // A session with no audio host is text-in, which is exactly what the mock + // tier promises; it must not reach for a device that is not there. + expect(audioCommands()).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Roster +// --------------------------------------------------------------------------- + +describe('A Cappella IPC handlers - roster', () => { + it('reads the roster straight from the sessions store', async () => { + const roster = (await handlerFor('acappella:get-roster')({})) as RosterAgent[]; + + expect(roster).toHaveLength(1); + expect(roster[0]).toMatchObject({ + sessionId: 'agent-backend', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo/api', + }); + expect(roster[0].tabs.map((tab) => tab.id)).toEqual(['tab-auth']); + }); +}); diff --git a/src/__tests__/main/preload/acappella.test.ts b/src/__tests__/main/preload/acappella.test.ts new file mode 100644 index 0000000000..d9fc81be03 --- /dev/null +++ b/src/__tests__/main/preload/acappella.test.ts @@ -0,0 +1,101 @@ +/** + * @file acappella.test.ts + * + * Unit tests for the `window.maestro.voice` preload bridge: each method has to + * reach the channel the main-process handler actually registered, and `onEvent` + * has to hand back a working unsubscribe (a leaked listener would keep feeding a + * torn-down HUD). + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockInvoke = vi.fn(); +const mockOn = vi.fn(); +const mockRemoveListener = vi.fn(); + +vi.mock('electron', () => ({ + ipcRenderer: { + invoke: (...args: unknown[]) => mockInvoke(...args), + on: (...args: unknown[]) => mockOn(...args), + removeListener: (...args: unknown[]) => mockRemoveListener(...args), + }, +})); + +import { createVoiceApi } from '../../../main/preload/acappella'; +import type { VoiceEvent } from '../../../shared/acappella/protocol'; + +describe('A Cappella Preload API', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('starts a session with the given scope', async () => { + mockInvoke.mockResolvedValue({ snapshot: { state: 'listening' }, substitutions: [] }); + const api = createVoiceApi(); + + const result = await api.start({ kind: 'agent', sessionId: 'agent-1' }); + + expect(mockInvoke).toHaveBeenCalledWith('acappella:start-session', { + kind: 'agent', + sessionId: 'agent-1', + }); + expect(result).toEqual({ snapshot: { state: 'listening' }, substitutions: [] }); + }); + + it('starts a conductor session when no scope is given', async () => { + createVoiceApi().start(); + expect(mockInvoke).toHaveBeenCalledWith('acappella:start-session', undefined); + }); + + it('stops a session', async () => { + createVoiceApi().stop(); + expect(mockInvoke).toHaveBeenCalledWith('acappella:stop-session'); + }); + + it('submits an utterance', async () => { + mockInvoke.mockResolvedValue(true); + await expect(createVoiceApi().submitUtterance('open the auth tab')).resolves.toBe(true); + expect(mockInvoke).toHaveBeenCalledWith('acappella:submit-utterance', 'open the auth tab'); + }); + + it('defaults an interrupt to a client button press', async () => { + createVoiceApi().interrupt(); + expect(mockInvoke).toHaveBeenCalledWith('acappella:interrupt', 'client-button'); + }); + + it('passes a spoken interrupt through as voice', async () => { + createVoiceApi().interrupt('voice'); + expect(mockInvoke).toHaveBeenCalledWith('acappella:interrupt', 'voice'); + }); + + it('sends the stop word with its phrase', async () => { + createVoiceApi().stopWord({ phrase: 'never mind', source: 'voice' }); + expect(mockInvoke).toHaveBeenCalledWith('acappella:stop-word', { + phrase: 'never mind', + source: 'voice', + }); + }); + + it('reads the roster and the state snapshot', async () => { + const api = createVoiceApi(); + api.getRoster(); + api.getState(); + expect(mockInvoke).toHaveBeenCalledWith('acappella:get-roster'); + expect(mockInvoke).toHaveBeenCalledWith('acappella:get-state'); + }); + + it('subscribes to the event stream and unsubscribes the same listener', () => { + const handler = vi.fn(); + const cleanup = createVoiceApi().onEvent(handler); + + expect(mockOn).toHaveBeenCalledWith('acappella:event', expect.any(Function)); + const [, wrapped] = mockOn.mock.calls[0] as [string, (...args: unknown[]) => void]; + + const event = { type: 'wake', sessionId: 'v1', seq: 1, ts: 0 } as unknown as VoiceEvent; + wrapped({}, event); + expect(handler).toHaveBeenCalledWith(event); + + cleanup(); + expect(mockRemoveListener).toHaveBeenCalledWith('acappella:event', wrapped); + }); +}); diff --git a/src/__tests__/main/preload/acappellaAudio.test.ts b/src/__tests__/main/preload/acappellaAudio.test.ts new file mode 100644 index 0000000000..462e0445e6 --- /dev/null +++ b/src/__tests__/main/preload/acappellaAudio.test.ts @@ -0,0 +1,93 @@ +/** + * @file acappellaAudio.test.ts + * + * Unit tests for the `window.maestro.voiceAudioHost` preload bridge. Two things + * matter here: frames go out on `send` (not `invoke` - a promise per 20 ms of + * audio is pure overhead), and `onCommand` hands back a working unsubscribe, or + * a torn-down audio host keeps receiving playback commands. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockSend = vi.fn(); +const mockInvoke = vi.fn(); +const mockOn = vi.fn(); +const mockRemoveListener = vi.fn(); + +vi.mock('electron', () => ({ + ipcRenderer: { + send: (...args: unknown[]) => mockSend(...args), + invoke: (...args: unknown[]) => mockInvoke(...args), + on: (...args: unknown[]) => mockOn(...args), + removeListener: (...args: unknown[]) => mockRemoveListener(...args), + }, +})); + +import { createVoiceAudioHostApi } from '../../../main/preload/acappellaAudio'; +import { + ACAPPELLA_AUDIO_COMMAND_CHANNEL, + ACAPPELLA_AUDIO_FRAME_CHANNEL, + ACAPPELLA_AUDIO_STATUS_CHANNEL, + type AudioHostCommand, +} from '../../../shared/acappella/audio-host'; + +describe('A Cappella audio host preload API', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('sends PCM frames on the dedicated frame channel without a round trip', () => { + const api = createVoiceAudioHostApi(); + const pcm = new ArrayBuffer(640); + + api.sendFrame({ seq: 7, capturedAt: 1234, rms: 0.25, pcm }); + + expect(mockSend).toHaveBeenCalledWith(ACAPPELLA_AUDIO_FRAME_CHANNEL, { + seq: 7, + capturedAt: 1234, + rms: 0.25, + pcm, + }); + expect(mockInvoke).not.toHaveBeenCalled(); + }); + + it('sends status on its own channel, so status is never buried under frames', () => { + const api = createVoiceAudioHostApi(); + + api.sendStatus({ kind: 'mic-error', code: 'permission-denied', message: 'nope' }); + + expect(mockSend).toHaveBeenCalledWith(ACAPPELLA_AUDIO_STATUS_CHANNEL, { + kind: 'mic-error', + code: 'permission-denied', + message: 'nope', + }); + }); + + it('delivers commands to the handler without the IpcRendererEvent', () => { + const api = createVoiceAudioHostApi(); + const handler = vi.fn(); + + api.onCommand(handler); + + expect(mockOn).toHaveBeenCalledWith(ACAPPELLA_AUDIO_COMMAND_CHANNEL, expect.any(Function)); + const registered = mockOn.mock.calls[0][1] as ( + event: unknown, + command: AudioHostCommand + ) => void; + registered({}, { kind: 'start-capture' }); + + expect(handler).toHaveBeenCalledWith({ kind: 'start-capture' }); + }); + + it('removes exactly the listener it registered on unsubscribe', () => { + const api = createVoiceAudioHostApi(); + + const unsubscribe = api.onCommand(vi.fn()); + unsubscribe(); + + expect(mockRemoveListener).toHaveBeenCalledWith( + ACAPPELLA_AUDIO_COMMAND_CHANNEL, + mockOn.mock.calls[0][1] + ); + }); +}); diff --git a/src/__tests__/main/web-server/handlers/acappellaSignal.test.ts b/src/__tests__/main/web-server/handlers/acappellaSignal.test.ts new file mode 100644 index 0000000000..0f0df7492c --- /dev/null +++ b/src/__tests__/main/web-server/handlers/acappellaSignal.test.ts @@ -0,0 +1,159 @@ +/** + * @file acappellaSignal.test.ts + * + * The WebSocket adapter that carries A Cappella signaling. + * + * Contracts defended: + * - A frame arriving with no transport, or with the Encore Feature switched off, + * gets a STATED refusal. A phone that gets no answer cannot tell "the feature is + * off" from "the network ate it", and only one of those is worth retrying. + * - Feature-off is checked separately from transport-absent. The transport is + * built once at boot and deliberately kept alive so the feature can be switched + * back on without a restart, so "a transport exists" does NOT mean "the feature + * is on" - and a device that was already connected when the user unticked the + * box would otherwise keep signaling into a live transport. + * - Registration is lazy and idempotent, and a refused frame registers nothing: + * a browser client that never speaks A Cappella must not cost a signaling + * session. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const transportRef = vi.hoisted(() => ({ current: null as unknown })); + +vi.mock('../../../../main/acappella', () => ({ + getACappellaTransport: () => transportRef.current, +})); + +import { + ACAPPELLA_SIGNAL_MESSAGE, + handleACappellaSignal, + handleACappellaSignalDisconnect, +} from '../../../../main/web-server/handlers/messageHandlers/acappellaSignal'; +import type { + MessageHandlerContext, + WebClient, + WebClientMessage, +} from '../../../../main/web-server/handlers/messageHandlers/types'; + +interface FakeTransport { + featureEnabled: () => boolean; + registerClient: ReturnType; + handleSignalMessage: ReturnType; + handleClientDisconnect: ReturnType; +} + +let sent: Array>; +let transport: FakeTransport; +let enabled: boolean; + +const client = { id: 'client-1' } as unknown as WebClient; + +const ctx = { + send: (_client: WebClient, data: Record) => sent.push(data), +} as unknown as MessageHandlerContext; + +const frame = { + type: ACAPPELLA_SIGNAL_MESSAGE, + payload: { op: 'auth', deviceId: 'device-1', token: 'secret' }, +} as unknown as WebClientMessage; + +/** The refusal both gated branches share. */ +function expectRefusal(): void { + expect(sent).toHaveLength(1); + expect(sent[0]).toMatchObject({ + type: ACAPPELLA_SIGNAL_MESSAGE, + payload: { + op: 'error', + code: 'not-authenticated', + message: expect.stringContaining('Encore Features'), + }, + }); +} + +beforeEach(() => { + sent = []; + enabled = true; + transport = { + featureEnabled: () => enabled, + registerClient: vi.fn(), + handleSignalMessage: vi.fn().mockResolvedValue(undefined), + handleClientDisconnect: vi.fn(), + }; + transportRef.current = transport; +}); + +describe('handleACappellaSignal', () => { + it('registers the client and hands over the payload when the feature is on', () => { + handleACappellaSignal(ctx, client, frame); + + expect(transport.registerClient).toHaveBeenCalledWith( + expect.objectContaining({ clientId: 'client-1' }) + ); + expect(transport.handleSignalMessage).toHaveBeenCalledWith('client-1', { + op: 'auth', + deviceId: 'device-1', + token: 'secret', + }); + expect(sent).toEqual([]); + }); + + it('writes back through the socket it was given', () => { + handleACappellaSignal(ctx, client, frame); + + const [params] = transport.registerClient.mock.calls[0] as [ + { send: (message: unknown) => void }, + ]; + params.send({ op: 'authenticated', deviceId: 'device-1' }); + + expect(sent[0]).toEqual({ + type: ACAPPELLA_SIGNAL_MESSAGE, + payload: { op: 'authenticated', deviceId: 'device-1' }, + }); + }); + + it('refuses in a sentence when no transport has ever been built', () => { + transportRef.current = null; + + handleACappellaSignal(ctx, client, frame); + + expectRefusal(); + }); + + it('refuses while the Encore Feature is off, even with a live transport', () => { + // The regression this exists for: switching the feature off used to leave the + // transport serving, so a phone that was connected at the moment of the + // toggle kept holding a signaling session against a desktop whose owner + // believed voice was off. + enabled = false; + + handleACappellaSignal(ctx, client, frame); + + expectRefusal(); + expect(transport.registerClient).not.toHaveBeenCalled(); + expect(transport.handleSignalMessage).not.toHaveBeenCalled(); + }); + + it('serves again once the feature comes back on, with no restart', () => { + enabled = false; + handleACappellaSignal(ctx, client, frame); + enabled = true; + handleACappellaSignal(ctx, client, frame); + + expect(transport.handleSignalMessage).toHaveBeenCalledTimes(1); + }); +}); + +describe('handleACappellaSignalDisconnect', () => { + it('tears down the signaling session for a socket that went away', () => { + handleACappellaSignalDisconnect('client-1'); + + expect(transport.handleClientDisconnect).toHaveBeenCalledWith('client-1'); + }); + + it('is a no-op with no transport', () => { + transportRef.current = null; + + expect(() => handleACappellaSignalDisconnect('client-1')).not.toThrow(); + }); +}); diff --git a/src/__tests__/main/web-server/routes/staticRoutes.test.ts b/src/__tests__/main/web-server/routes/staticRoutes.test.ts index d601ab0499..b7d5411d30 100644 --- a/src/__tests__/main/web-server/routes/staticRoutes.test.ts +++ b/src/__tests__/main/web-server/routes/staticRoutes.test.ts @@ -73,9 +73,9 @@ describe('StaticRoutes', () => { describe('Route Registration', () => { it('should register all static routes', () => { - // 10 routes: /, /health, manifest.json, sw.js, token root, token root/, - // /desktop, /desktop/, session/:id, /:token - expect(mockFastify.get).toHaveBeenCalledTimes(10); + // 12 routes: /, /health, manifest.json, sw.js, token root, token root/, + // /desktop, /desktop/, /acappella, /acappella/, session/:id, /:token + expect(mockFastify.get).toHaveBeenCalledTimes(12); }); it('should register routes with correct paths', () => { @@ -87,6 +87,8 @@ describe('StaticRoutes', () => { expect(mockFastify.routes.has(`GET:/${securityToken}/`)).toBe(true); expect(mockFastify.routes.has(`GET:/${securityToken}/desktop`)).toBe(true); expect(mockFastify.routes.has(`GET:/${securityToken}/desktop/`)).toBe(true); + expect(mockFastify.routes.has(`GET:/${securityToken}/acappella`)).toBe(true); + expect(mockFastify.routes.has(`GET:/${securityToken}/acappella/`)).toBe(true); expect(mockFastify.routes.has(`GET:/${securityToken}/session/:sessionId`)).toBe(true); expect(mockFastify.routes.has('GET:/:token')).toBe(true); }); @@ -267,4 +269,56 @@ describe('StaticRoutes', () => { } }); }); + + describe('GET /$TOKEN/acappella (reference client)', () => { + it('serves the reference client page with its assets repointed', async () => { + const tempRoot = mkdtempSync(path.join(tmpdir(), 'maestro-static-routes-')); + const tempDesktopPath = path.join(tempRoot, 'web-desktop'); + const clientDir = path.join(tempDesktopPath, 'acappella-client'); + + mkdirSync(clientDir, { recursive: true }); + + try { + // Vite emits `../assets/` for a page one directory down from the bundle + // root, which is why the desktop index's `./assets/` rewrite alone was + // not enough. + writeFileSync( + path.join(clientDir, 'index.html'), + '', + 'utf8' + ); + + const freshRoutes = new StaticRoutes(securityToken, webAssetsPath, tempDesktopPath); + const freshFastify = createMockFastify(); + freshRoutes.registerRoutes(freshFastify as any); + + const route = freshFastify.getRoute('GET', `/${securityToken}/acappella`); + const reply = createMockReply(); + await route!.handler({}, reply); + + expect(reply.type).toHaveBeenCalledWith('text/html'); + expect(reply.send).toHaveBeenCalledWith( + expect.stringContaining(`/${securityToken}/desktop/assets/acappella-client.js`) + ); + // No config injection: the reference client pairs with a code like any + // other device, and handing it the token would skip the one flow it + // exists to exercise. + expect(reply.send).not.toHaveBeenCalledWith(expect.stringContaining('__MAESTRO_CONFIG__')); + } finally { + rmSync(tempRoot, { recursive: true, force: true }); + } + }); + + it('returns 503 when the bundle has not been built', async () => { + const noDesktopRoutes = new StaticRoutes(securityToken, webAssetsPath, null); + const noDesktopFastify = createMockFastify(); + noDesktopRoutes.registerRoutes(noDesktopFastify as any); + + const route = noDesktopFastify.getRoute('GET', `/${securityToken}/acappella`); + const reply = createMockReply(); + await route!.handler({}, reply); + + expect(reply.code).toHaveBeenCalledWith(503); + }); + }); }); diff --git a/src/__tests__/renderer/acappella-audio/AudioHostRoot.test.tsx b/src/__tests__/renderer/acappella-audio/AudioHostRoot.test.tsx new file mode 100644 index 0000000000..9e127346a9 --- /dev/null +++ b/src/__tests__/renderer/acappella-audio/AudioHostRoot.test.tsx @@ -0,0 +1,299 @@ +/** + * @file AudioHostRoot.test.tsx + * + * The audio host controller and its React shell. + * + * Two properties matter beyond the plumbing. The `AudioContext` must be created + * lazily, because the window is built when a session starts but a session the + * user abandons should never open an audio device. And dispose must actually + * release everything: a hidden window holding a live microphone is invisible, so + * nobody would ever notice it. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { render } from '@testing-library/react'; + +import { createFakeAudioContext } from '../../helpers/mockWebAudio'; +import type { + AudioHostCommand, + AudioHostStatus, + AudioFrame, +} from '../../../shared/acappella/audio-host'; +import type { WebRtcHostCommand, WebRtcHostEvent } from '../../../shared/acappella/webrtc-host'; +import { DEFAULT_REMOTE_AUDIO_CONFIG } from '../../../shared/acappella/webrtc-host'; +import type { AudioHostBridge } from '../../../renderer/acappella-audio/bridge'; + +// The real module resolves a Vite `?worker&url` import, which has no meaning +// outside a browser bundle. +vi.mock('../../../renderer/acappella-audio/worklet-url', () => ({ + pcmWorkletUrl: '/assets/pcm-worklet.js', +})); + +import { + AudioHostRoot, + createAudioHostController, +} from '../../../renderer/acappella-audio/AudioHostRoot'; + +interface Harness { + bridge: AudioHostBridge; + statuses: AudioHostStatus[]; + frames: AudioFrame[]; + /** Peer events the controller pushed back to main. */ + peerEvents: WebRtcHostEvent[]; + send(command: AudioHostCommand): void; + sendWebRtc(command: WebRtcHostCommand): void; + unsubscribed(): boolean; +} + +function createHarness(): Harness { + const statuses: AudioHostStatus[] = []; + const frames: AudioFrame[] = []; + const peerEvents: WebRtcHostEvent[] = []; + let handler: ((command: AudioHostCommand) => void) | null = null; + let peerHandler: ((command: WebRtcHostCommand) => void) | null = null; + let unsubscribed = false; + + return { + statuses, + frames, + peerEvents, + send: (command) => handler?.(command), + sendWebRtc: (command) => peerHandler?.(command), + unsubscribed: () => unsubscribed, + bridge: { + sendFrame: (frame) => frames.push(frame), + sendStatus: (status) => statuses.push(status), + onCommand: (next) => { + handler = next; + return () => { + unsubscribed = true; + handler = null; + }; + }, + sendWebRtcEvent: (event) => peerEvents.push(event), + onWebRtcCommand: (next) => { + peerHandler = next; + return () => { + peerHandler = null; + }; + }, + }, + }; +} + +describe('createAudioHostController', () => { + it('announces readiness so main knows the hidden window booted', () => { + const harness = createHarness(); + + const controller = createAudioHostController({ + bridge: harness.bridge, + createContext: () => createFakeAudioContext() as unknown as AudioContext, + }); + + expect(harness.statuses).toEqual([{ kind: 'ready' }]); + controller.dispose(); + }); + + it('does not build an AudioContext until a command needs one', () => { + const harness = createHarness(); + const createContext = vi.fn(() => createFakeAudioContext() as unknown as AudioContext); + + const controller = createAudioHostController({ bridge: harness.bridge, createContext }); + + // The window exists from the first session start; opening an audio device + // for a session the user never speaks into would be a wasted permission + // prompt and a lit recording indicator. + expect(createContext).not.toHaveBeenCalled(); + controller.handleCommand({ kind: 'stop-capture' }); + controller.handleCommand({ kind: 'flush' }); + controller.handleCommand({ kind: 'duck', gain: 0, ms: 10 }); + expect(createContext).not.toHaveBeenCalled(); + + controller.handleCommand({ + kind: 'play', + utteranceId: 'u1', + format: 'encoded', + data: new ArrayBuffer(8), + }); + expect(createContext).toHaveBeenCalledTimes(1); + controller.dispose(); + }); + + it('shares one context between capture and playback, so AEC has a reference signal', async () => { + const harness = createHarness(); + const context = createFakeAudioContext(); + const createContext = vi.fn(() => context as unknown as AudioContext); + const controller = createAudioHostController({ bridge: harness.bridge, createContext }); + + controller.handleCommand({ + kind: 'play', + utteranceId: 'u1', + format: 'encoded', + data: new ArrayBuffer(8), + }); + controller.handleCommand({ kind: 'start-capture' }); + await vi.waitFor(() => + expect(context.addedModules.length + context.gains.length).toBeGreaterThan(0) + ); + + // Two contexts would mean the echo canceller never sees what we played, and + // the assistant would hear itself and barge in on its own voice. + expect(createContext).toHaveBeenCalledTimes(1); + controller.dispose(); + }); + + it('routes playback commands to the playback and releases everything on dispose', async () => { + const harness = createHarness(); + const context = createFakeAudioContext(); + const controller = createAudioHostController({ + bridge: harness.bridge, + createContext: () => context as unknown as AudioContext, + }); + + controller.handleCommand({ + kind: 'play', + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: new Int16Array(16000).buffer, + }); + await vi.waitFor(() => expect(context.sources).toHaveLength(1)); + + controller.handleCommand({ kind: 'duck', gain: 0.1, ms: 40 }); + expect(context.gains[0].gain.value).toBeCloseTo(0.1, 6); + + controller.handleCommand({ kind: 'flush' }); + expect(context.sources[0].stopped).toBe(true); + + controller.dispose(); + expect(harness.unsubscribed()).toBe(true); + expect(context.close).toHaveBeenCalled(); + }); + + it('ignores commands that arrive after dispose', () => { + const harness = createHarness(); + const createContext = vi.fn(() => createFakeAudioContext() as unknown as AudioContext); + const controller = createAudioHostController({ bridge: harness.bridge, createContext }); + + controller.dispose(); + controller.handleCommand({ + kind: 'play', + utteranceId: 'u1', + format: 'encoded', + data: new ArrayBuffer(8), + }); + + expect(createContext).not.toHaveBeenCalled(); + }); + + it('answers a peer offer and reports the answer back to main', async () => { + const harness = createHarness(); + const controller = createAudioHostController({ + bridge: harness.bridge, + createContext: () => createFakeAudioContext() as unknown as AudioContext, + createPeerConnection: () => createFakePeerConnection() as unknown as RTCPeerConnection, + }); + + harness.sendWebRtc({ + kind: 'accept-offer', + deviceId: 'phone', + offer: { type: 'offer', sdp: 'v=0\r\na=rtpmap:111 opus/48000/2' }, + iceServers: [], + audio: DEFAULT_REMOTE_AUDIO_CONFIG, + }); + + await vi.waitFor(() => + expect(harness.peerEvents.some((event) => event.kind === 'answer')).toBe(true) + ); + controller.dispose(); + }); + + it('taps the shared playback output for the outbound voice track', async () => { + const harness = createHarness(); + const context = createFakeAudioContext(); + const peer = createFakePeerConnection(); + const controller = createAudioHostController({ + bridge: harness.bridge, + createContext: () => context as unknown as AudioContext, + createPeerConnection: () => peer as unknown as RTCPeerConnection, + }); + + harness.sendWebRtc({ + kind: 'accept-offer', + deviceId: 'phone', + offer: { type: 'offer', sdp: 'v=0\r\na=rtpmap:111 opus/48000/2' }, + iceServers: [], + audio: DEFAULT_REMOTE_AUDIO_CONFIG, + }); + + // One synthesis, two places it comes out: the phone hears the configured + // voice at the configured volume rather than a second rendering of it. + await vi.waitFor(() => expect(peer.addedTracks).toHaveLength(1)); + controller.dispose(); + }); + + it('ignores peer commands that arrive after dispose', () => { + const harness = createHarness(); + const createPeerConnection = vi.fn( + () => createFakePeerConnection() as unknown as RTCPeerConnection + ); + const controller = createAudioHostController({ + bridge: harness.bridge, + createContext: () => createFakeAudioContext() as unknown as AudioContext, + createPeerConnection, + }); + + controller.dispose(); + harness.sendWebRtc({ kind: 'set-floor-holder', deviceId: 'phone' }); + expect(createPeerConnection).not.toHaveBeenCalled(); + }); + + it('is safe to dispose twice', () => { + const harness = createHarness(); + const controller = createAudioHostController({ + bridge: harness.bridge, + createContext: () => createFakeAudioContext() as unknown as AudioContext, + }); + + controller.dispose(); + expect(() => controller.dispose()).not.toThrow(); + }); +}); + +describe('AudioHostRoot', () => { + it('renders nothing at all', () => { + const { container } = render(); + + // The window is never painted; anything rendered here would be a bug that + // nobody could see. + expect(container.innerHTML).toBe(''); + }); +}); + +/** + * The smallest `RTCPeerConnection` the controller can answer an offer with. + * jsdom has none, and the peer's own behaviour is covered in + * `peer-connection.test.ts`. + */ +function createFakePeerConnection() { + const peer = { + connectionState: 'new', + localDescription: null as { type: string; sdp?: string } | null, + addedTracks: [] as unknown[], + onicecandidate: null, + onconnectionstatechange: null, + ontrack: null, + ondatachannel: null, + setRemoteDescription: vi.fn(async () => {}), + setLocalDescription: vi.fn(async (description: { type: string; sdp?: string }) => { + peer.localDescription = description; + }), + createAnswer: vi.fn(async () => ({ type: 'answer', sdp: 'v=0' })), + addTrack: vi.fn((track: unknown) => peer.addedTracks.push(track)), + getSenders: vi.fn(() => []), + addIceCandidate: vi.fn(async () => {}), + getStats: vi.fn(async () => ({ forEach: () => {} })), + close: vi.fn(), + }; + return peer; +} diff --git a/src/__tests__/renderer/acappella-audio/bridge.test.ts b/src/__tests__/renderer/acappella-audio/bridge.test.ts new file mode 100644 index 0000000000..c25c9ec613 --- /dev/null +++ b/src/__tests__/renderer/acappella-audio/bridge.test.ts @@ -0,0 +1,63 @@ +/** + * @file bridge.test.ts + * + * The audio host's link to main. The only interesting case is the missing one: + * without a preload bridge the audio host must degrade to a no-op instead of + * throwing during boot, because a throw here takes down the whole hidden window + * and there is no UI in it to show what happened. + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createAudioHostBridge } from '../../../renderer/acappella-audio/bridge'; + +vi.mock('../../../renderer/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +// The shared jsdom setup installs a `window.maestro`; swap the field rather than +// the object so the rest of it survives. +type MaestroWindow = { maestro: { voiceAudioHost?: unknown } }; + +const originalApi = (window as unknown as MaestroWindow).maestro?.voiceAudioHost; + +function installApi(api: unknown): void { + (window as unknown as MaestroWindow).maestro.voiceAudioHost = api; +} + +describe('createAudioHostBridge', () => { + afterEach(() => { + (window as unknown as MaestroWindow).maestro.voiceAudioHost = originalApi; + }); + + it('forwards frames, status, and commands to the preload API', () => { + const unsubscribe = vi.fn(); + const api = { + sendFrame: vi.fn(), + sendStatus: vi.fn(), + onCommand: vi.fn(() => unsubscribe), + }; + installApi(api); + + const bridge = createAudioHostBridge(); + const pcm = new ArrayBuffer(640); + bridge.sendFrame({ seq: 1, capturedAt: 5, rms: 0.3, pcm }); + bridge.sendStatus({ kind: 'ready' }); + const handler = vi.fn(); + expect(bridge.onCommand(handler)).toBe(unsubscribe); + + expect(api.sendFrame).toHaveBeenCalledWith({ seq: 1, capturedAt: 5, rms: 0.3, pcm }); + expect(api.sendStatus).toHaveBeenCalledWith({ kind: 'ready' }); + expect(api.onCommand).toHaveBeenCalledWith(handler); + }); + + it('degrades to a no-op bridge when there is no preload', () => { + const bridge = createAudioHostBridge(undefined); + + expect(() => { + bridge.sendFrame({ seq: 1, capturedAt: 0, rms: 0, pcm: new ArrayBuffer(2) }); + bridge.sendStatus({ kind: 'ready' }); + bridge.onCommand(vi.fn())(); + }).not.toThrow(); + }); +}); diff --git a/src/__tests__/renderer/acappella-audio/capture.test.ts b/src/__tests__/renderer/acappella-audio/capture.test.ts new file mode 100644 index 0000000000..208a72e664 --- /dev/null +++ b/src/__tests__/renderer/acappella-audio/capture.test.ts @@ -0,0 +1,280 @@ +/** + * @file capture.test.ts + * + * Microphone capture for the A Cappella audio host. + * + * The behaviour worth defending is that no failure is silent. A user whose + * microphone permission was denied, whose device vanished, or whose worklet + * failed to load must get a classified `mic-error` - not a session that sits in + * "listening" forever producing nothing. Every failure path here asserts on the + * status that reaches the bridge, not just on the absence of a throw. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + createFakeAudioContext, + installAudioWorkletNodeMock, + installMediaDevicesMock, + type FakeAudioContext, + type FakeAudioWorkletNode, + type MediaDevicesMock, +} from '../../helpers/mockWebAudio'; +import { + ACAPPELLA_AUDIO_SAMPLE_RATE, + ACAPPELLA_PCM_WORKLET_NAME, + type AudioFrame, + type AudioHostStatus, +} from '../../../shared/acappella/audio-host'; +import { classifyCaptureError, MicCapture } from '../../../renderer/acappella-audio/capture'; + +const WORKLET_URL = '/assets/pcm-worklet.js'; + +describe('classifyCaptureError', () => { + it.each([ + ['NotAllowedError', 'permission-denied'], + ['SecurityError', 'permission-denied'], + ['NotFoundError', 'no-device'], + ['OverconstrainedError', 'no-device'], + ['NotReadableError', 'device-lost'], + ['AbortError', 'device-lost'], + ['TypeError', 'audio-init-failed'], + ])('maps %s to %s', (name, expected) => { + const error = new Error('boom'); + error.name = name; + expect(classifyCaptureError(error)).toBe(expected); + }); + + it('treats a non-Error rejection as an init failure rather than crashing', () => { + expect(classifyCaptureError('something odd')).toBe('audio-init-failed'); + }); +}); + +describe('MicCapture', () => { + let context: FakeAudioContext; + let media: MediaDevicesMock; + let worklet: { nodes: FakeAudioWorkletNode[]; restore(): void }; + let frames: AudioFrame[]; + let statuses: AudioHostStatus[]; + + const build = (ctx: FakeAudioContext = context) => + new MicCapture({ + context: ctx as unknown as AudioContext, + workletUrl: WORKLET_URL, + onFrame: (frame) => frames.push(frame), + onStatus: (status) => statuses.push(status), + }); + + beforeEach(() => { + context = createFakeAudioContext(); + media = installMediaDevicesMock(); + worklet = installAudioWorkletNodeMock(); + frames = []; + statuses = []; + }); + + afterEach(() => { + media.restore(); + worklet.restore(); + vi.restoreAllMocks(); + }); + + it('opens the mic with echo cancellation, noise suppression, and auto gain', async () => { + const capture = build(); + + await expect(capture.start()).resolves.toBe(true); + + expect(media.getUserMedia).toHaveBeenCalledWith({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + channelCount: 1, + }, + video: false, + }); + expect(statuses).toContainEqual({ + kind: 'capture-start', + device: { deviceId: 'default', label: 'MacBook Pro Microphone' }, + contextSampleRate: 48000, + }); + }); + + it('builds the graph into a muted sink so the mic is never routed to the speakers', async () => { + const capture = build(); + await capture.start(); + + const node = worklet.nodes[0]; + expect(node.name).toBe(ACAPPELLA_PCM_WORKLET_NAME); + expect(context.addedModules).toEqual([WORKLET_URL]); + + // The terminating gain node exists only to keep the graph pulled; audible + // gain here would be direct microphone feedback. + const sink = context.gains[0]; + expect(sink.gain.value).toBe(0); + expect(node.connectedTo).toContain(sink); + expect(sink.connectedTo).toContain(context.destination); + }); + + it('resumes a suspended context, which a hidden window never gets a gesture for', async () => { + const suspended = createFakeAudioContext({ state: 'suspended' }); + const capture = build(suspended); + + await capture.start(); + + expect(suspended.resume).toHaveBeenCalled(); + }); + + it('stamps frames from the audio clock rather than from wall time per frame', async () => { + vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + context.currentTime = 4; + const capture = build(); + await capture.start(); + + const pcm = new Int16Array([1, 2, 3]).buffer; + worklet.nodes[0].emit({ pcm, rms: 0.4, t: 4.5 }); + worklet.nodes[0].emit({ pcm, rms: 0.6, t: 4.52 }); + + // Context time 0 was 4 seconds before "now", so t=4.5 is 500 ms after now. + expect(frames).toEqual([ + { seq: 1, capturedAt: 1_000_500, rms: 0.4, pcm }, + { seq: 2, capturedAt: 1_000_520, rms: 0.6, pcm }, + ]); + }); + + it('reports a denied permission instead of stalling', async () => { + media.failWith('NotAllowedError', 'Permission dismissed'); + const capture = build(); + + await expect(capture.start()).resolves.toBe(false); + + expect(statuses).toEqual([ + { kind: 'mic-error', code: 'permission-denied', message: 'Permission dismissed' }, + ]); + expect(capture.active).toBe(false); + }); + + it('reports a missing device instead of stalling', async () => { + media.failWith('NotFoundError', 'No audio input'); + const capture = build(); + + await expect(capture.start()).resolves.toBe(false); + + expect(statuses).toEqual([{ kind: 'mic-error', code: 'no-device', message: 'No audio input' }]); + }); + + it('releases the stream and reports when the worklet module fails to load', async () => { + const broken = createFakeAudioContext({ addModuleError: new Error('bad chunk') }); + const capture = build(broken); + + await expect(capture.start()).resolves.toBe(false); + + // The mic was already open when the worklet failed; leaving it open would + // light the OS recording indicator with nothing listening. + expect(media.stream.track.stop).toHaveBeenCalled(); + expect(statuses).toEqual([ + { kind: 'mic-error', code: 'audio-init-failed', message: 'bad chunk' }, + ]); + }); + + it('reports a device disappearing mid-capture as recoverable and stops cleanly', async () => { + const capture = build(); + await capture.start(); + statuses.length = 0; + + media.stream.track.end(); + + expect(statuses).toEqual([ + { kind: 'mic-error', code: 'device-lost', message: 'The microphone was disconnected.' }, + { kind: 'capture-stop', reason: 'device-lost' }, + ]); + expect(capture.active).toBe(false); + }); + + it('reports device changes so the UI can name the microphone in use', async () => { + const capture = build(); + + media.emitDeviceChange(); + + expect(statuses).toEqual([{ kind: 'device-change' }]); + capture.dispose(); + }); + + it('is idempotent: a second start reuses the open device and adds the module once', async () => { + const capture = build(); + + const [first, second] = await Promise.all([capture.start(), capture.start()]); + + expect(first).toBe(true); + expect(second).toBe(true); + expect(media.getUserMedia).toHaveBeenCalledTimes(1); + expect(context.addedModules).toHaveLength(1); + }); + + it('releases the microphone and tears down the graph on stop', async () => { + const capture = build(); + await capture.start(); + const node = worklet.nodes[0]; + + capture.stop(); + + expect(media.stream.track.stop).toHaveBeenCalled(); + expect(node.disconnectCount).toBe(1); + expect(node.port.onmessage).toBeNull(); + expect(capture.active).toBe(false); + expect(statuses).toContainEqual({ kind: 'capture-stop', reason: 'requested' }); + }); + + it('does not emit a stop status when it was never capturing', () => { + const capture = build(); + + capture.stop(); + + expect(statuses).toEqual([]); + }); + + it('drops a device that arrived after dispose rather than leaking it', async () => { + const capture = build(); + const pending = capture.start(); + capture.dispose(); + + await expect(pending).resolves.toBe(false); + expect(media.stream.track.stop).toHaveBeenCalled(); + expect(media.listenerCount()).toBe(0); + }); + + it('restarts after a stop, and frame numbering restarts with it', async () => { + const capture = build(); + await capture.start(); + worklet.nodes[0].emit({ pcm: new ArrayBuffer(2), rms: 0.1, t: 0 }); + capture.stop(); + + await capture.start(); + worklet.nodes[1].emit({ pcm: new ArrayBuffer(2), rms: 0.1, t: 0 }); + + // Sequence numbers are per capture run: main counts gaps to detect drops, + // so continuing the old numbering across a restart would read as a gap. + expect(frames.map((frame) => frame.seq)).toEqual([1, 1]); + }); + + it('reports unsupported when the build has no getUserMedia at all', async () => { + media.restore(); + Object.defineProperty(navigator, 'mediaDevices', { value: undefined, configurable: true }); + const capture = build(); + + await expect(capture.start()).resolves.toBe(false); + + expect(statuses[0]).toMatchObject({ kind: 'mic-error', code: 'unsupported' }); + }); + + it('asks the worklet for 20 ms frames at the STT sample rate', async () => { + const capture = build(); + await capture.start(); + + expect(worklet.nodes[0].options).toMatchObject({ + numberOfInputs: 1, + outputChannelCount: [1], + processorOptions: { targetSampleRate: ACAPPELLA_AUDIO_SAMPLE_RATE, frameSamples: 320 }, + }); + }); +}); diff --git a/src/__tests__/renderer/acappella-audio/pcm-worklet.test.ts b/src/__tests__/renderer/acappella-audio/pcm-worklet.test.ts new file mode 100644 index 0000000000..486e8f5a97 --- /dev/null +++ b/src/__tests__/renderer/acappella-audio/pcm-worklet.test.ts @@ -0,0 +1,194 @@ +/** + * @file pcm-worklet.test.ts + * + * The PCM worklet is the one piece of A Cappella that no integration test can + * cover: it only ever runs inside `AudioWorkletGlobalScope`, on the audio + * thread, in a hidden window. So it is tested the only way it can be - by + * standing up the three globals it reads, importing the module, and driving the + * processor with synthetic render quanta. + * + * The load-bearing case is the ramp test. Resampling block by block, with the + * read position reset each time, produces output that looks right in aggregate + * but has a discontinuity every 128 samples: a 375 Hz buzz on top of the user's + * voice at 48 kHz. Feeding a perfect ramp and checking every output sample + * against the position it should have been read from is what catches it. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { ACAPPELLA_AUDIO_FRAME_SAMPLES } from '../../../shared/acappella/audio-host'; + +interface WorkletFrame { + pcm: ArrayBuffer; + rms: number; + t: number; +} + +interface ProcessorLike { + process(inputs: Float32Array[][]): boolean; +} + +const RENDER_QUANTUM = 128; + +let posted: WorkletFrame[] = []; + +/** Stand up `AudioWorkletGlobalScope` and load the module fresh under it. */ +async function loadProcessor( + contextSampleRate: number +): Promise ProcessorLike> { + posted = []; + let registered: (new (options?: unknown) => ProcessorLike) | null = null; + + class FakeAudioWorkletProcessor { + readonly port = { + postMessage: (message: WorkletFrame) => { + posted.push(message); + }, + }; + } + + vi.stubGlobal('AudioWorkletProcessor', FakeAudioWorkletProcessor); + vi.stubGlobal('sampleRate', contextSampleRate); + vi.stubGlobal('currentTime', 1.5); + vi.stubGlobal('registerProcessor', (_name: string, ctor: unknown) => { + registered = ctor as new (options?: unknown) => ProcessorLike; + }); + + vi.resetModules(); + await import('../../../renderer/acappella-audio/pcm-worklet'); + + if (!registered) throw new Error('worklet did not register a processor'); + return registered; +} + +/** Feed `sampleCount` samples as consecutive 128-sample render quanta. */ +function drive( + processor: ProcessorLike, + sampleCount: number, + valueAt: (globalIndex: number) => number, + channels = 1 +): void { + for (let offset = 0; offset < sampleCount; offset += RENDER_QUANTUM) { + const block: Float32Array[] = []; + for (let c = 0; c < channels; c++) { + const channel = new Float32Array(RENDER_QUANTUM); + for (let i = 0; i < RENDER_QUANTUM; i++) { + // Channel 1 is inverted so a stereo downmix cancels to silence. + channel[i] = c === 0 ? valueAt(offset + i) : -valueAt(offset + i); + } + block.push(channel); + } + processor.process([block]); + } +} + +function framesToFloat(frames: WorkletFrame[]): number[] { + const out: number[] = []; + for (const frame of frames) { + const samples = new Int16Array(frame.pcm); + for (const sample of samples) out.push(sample < 0 ? sample / 0x8000 : sample / 0x7fff); + } + return out; +} + +describe('A Cappella PCM worklet', () => { + beforeEach(() => { + posted = []; + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('emits 20 ms frames of 320 samples at 16 kHz from a 48 kHz context', async () => { + const Processor = await loadProcessor(48000); + const processor = new Processor(); + + // One second of audio at 48 kHz -> 16000 output samples -> 50 frames. + drive(processor, 48000, () => 0.5); + + expect(posted).toHaveLength(50); + for (const frame of posted) { + expect(new Int16Array(frame.pcm)).toHaveLength(ACAPPELLA_AUDIO_FRAME_SAMPLES); + } + }); + + it('resamples continuously across render quanta, with no seam at the block boundary', async () => { + const Processor = await loadProcessor(48000); + const processor = new Processor(); + + // A perfect ramp: linear interpolation of a line is exact, so every output + // sample must equal the input at the position it was read from. Any per-block + // reset of the read position shows up immediately as a step. + const slope = 1e-5; + drive(processor, 48000, (index) => index * slope); + + const output = framesToFloat(posted); + expect(output.length).toBe(16000); + // Tolerance is one int16 quantisation step, not an approximation of the maths. + const tolerance = 2 / 0x7fff; + for (let k = 0; k < output.length; k++) { + expect(Math.abs(output[k] - k * 3 * slope)).toBeLessThan(tolerance); + } + }); + + it('downmixes stereo to mono by averaging the channels', async () => { + const Processor = await loadProcessor(48000); + const processor = new Processor(); + + // Channel 1 is the inverse of channel 0, so the average is silence. + drive(processor, 48000, () => 0.8, 2); + + expect(posted.length).toBeGreaterThan(0); + for (const value of framesToFloat(posted)) expect(Math.abs(value)).toBeLessThan(1e-6); + }); + + it('handles a non-integer resample ratio (44.1 kHz)', async () => { + const Processor = await loadProcessor(44100); + const processor = new Processor(); + + drive(processor, 44100, () => 0.25); + + // 44100 in at 44.1 kHz is one second, so ~16000 samples out (50 frames), + // minus whatever is still short of a full frame. + expect(posted.length).toBeGreaterThanOrEqual(49); + expect(posted.length).toBeLessThanOrEqual(50); + }); + + it('reports RMS per frame and clamps out-of-range samples', async () => { + const Processor = await loadProcessor(16000); + const processor = new Processor(); + + // Deliberately over full scale: an auto-gained mic can overshoot, and + // wrapping instead of clamping turns a loud vowel into a burst of noise. + drive(processor, 16000, () => 2); + + expect(posted.length).toBeGreaterThan(0); + for (const frame of posted) { + expect(frame.rms).toBeCloseTo(1, 5); + for (const sample of new Int16Array(frame.pcm)) expect(sample).toBe(0x7fff); + } + }); + + it('passes 16 kHz input through unchanged and stamps the context clock', async () => { + const Processor = await loadProcessor(16000); + const processor = new Processor(); + + drive(processor, 3200, () => 0.5); + + expect(posted).toHaveLength(10); + expect(posted[0].t).toBe(1.5); + for (const value of framesToFloat(posted)) expect(value).toBeCloseTo(0.5, 4); + }); + + it('stays alive when the graph delivers no input', async () => { + const Processor = await loadProcessor(48000); + const processor = new Processor(); + + // Returning false here would retire the processor for good, so a momentary + // gap while the mic connects would silently kill capture. + expect(processor.process([])).toBe(true); + expect(processor.process([[]])).toBe(true); + expect(posted).toHaveLength(0); + }); +}); diff --git a/src/__tests__/renderer/acappella-audio/peer-connection.test.ts b/src/__tests__/renderer/acappella-audio/peer-connection.test.ts new file mode 100644 index 0000000000..6c3617bced --- /dev/null +++ b/src/__tests__/renderer/acappella-audio/peer-connection.test.ts @@ -0,0 +1,477 @@ +/** + * The WebRTC peer, as terminated in the hidden audio window. + * + * `RTCPeerConnection` is mocked, so this runs in jsdom with no network and no + * audio device. What is asserted is the contract the rest of the feature leans + * on: + * + * - Opus is negotiated with in-band FEC and DTX at a voice bitrate, because + * that is what makes 5% packet loss sound like nothing and stops a phone in + * a pocket transmitting silence over a metered radio; + * - exactly one device's microphone reaches the capture pipeline, and a + * takeover is a graph reconnection rather than a renegotiation; + * - a stats reading collapses to the four numbers a signal bar needs, with the + * WORSE end of the candidate pair deciding what to call the path. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + PeerRegistry, + applyOpusPreferences, + summarizeStats, + type PeerAudioBinding, +} from '../../../renderer/acappella-audio/peer-connection'; +import { DEFAULT_REMOTE_AUDIO_CONFIG } from '../../../shared/acappella/webrtc-host'; +import { + RELIABLE_CHANNEL_LABEL, + UNRELIABLE_CHANNEL_LABEL, + encodeDeviceMessage, +} from '../../../shared/acappella/device-protocol'; + +vi.mock('../../../renderer/utils/logger', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +const OFFER_SDP = [ + 'v=0', + 'm=audio 9 UDP/TLS/RTP/SAVPF 111', + 'a=rtpmap:111 opus/48000/2', + 'a=fmtp:111 minptime=10;useinbandfec=0', +].join('\r\n'); + +class FakeDataChannel { + readyState = 'open'; + sent: string[] = []; + onmessage: ((event: { data: string }) => void) | null = null; + closed = false; + + constructor(public label: string) {} + + send(data: string): void { + this.sent.push(data); + } + + close(): void { + this.closed = true; + } +} + +class FakePeerConnection { + static instances: FakePeerConnection[] = []; + + connectionState = 'new'; + localDescription: { type: string; sdp?: string } | null = null; + remoteDescription: { type: string; sdp?: string } | null = null; + addedTracks: unknown[] = []; + candidates: unknown[] = []; + closed = false; + statsReports: Array> = []; + senderParameters: Record = {}; + + onicecandidate: ((event: { candidate: unknown }) => void) | null = null; + onconnectionstatechange: (() => void) | null = null; + ontrack: ((event: { streams: MediaStream[]; track: unknown }) => void) | null = null; + ondatachannel: ((event: { channel: FakeDataChannel }) => void) | null = null; + + constructor(public config: RTCConfiguration) { + FakePeerConnection.instances.push(this); + } + + async setRemoteDescription(description: { type: string; sdp?: string }): Promise { + this.remoteDescription = description; + } + + async setLocalDescription(description: { type: string; sdp?: string }): Promise { + this.localDescription = description; + } + + async createAnswer(): Promise<{ type: string; sdp: string }> { + return { type: 'answer', sdp: OFFER_SDP }; + } + + async createOffer(): Promise<{ type: string; sdp: string }> { + return { type: 'offer', sdp: OFFER_SDP }; + } + + addTrack(track: unknown): void { + this.addedTracks.push(track); + } + + getSenders(): Array<{ + track: { kind: string } | null; + getParameters: () => Record; + setParameters: (parameters: Record) => Promise; + }> { + return this.addedTracks.map((track) => ({ + track: track as { kind: string }, + getParameters: () => ({ encodings: [{}] }), + setParameters: async (parameters: Record) => { + this.senderParameters = parameters; + }, + })); + } + + async addIceCandidate(candidate: unknown): Promise { + this.candidates.push(candidate); + } + + createDataChannel(label: string): FakeDataChannel { + return new FakeDataChannel(label); + } + + async getStats(): Promise<{ forEach: (fn: (value: unknown) => void) => void }> { + const reports = this.statsReports; + return { forEach: (fn) => reports.forEach(fn) }; + } + + close(): void { + this.closed = true; + } + + /** Open both protocol channels, as a real client would on connect. */ + openChannels(): { reliable: FakeDataChannel; unreliable: FakeDataChannel } { + const reliable = new FakeDataChannel(RELIABLE_CHANNEL_LABEL); + const unreliable = new FakeDataChannel(UNRELIABLE_CHANNEL_LABEL); + this.ondatachannel?.({ channel: reliable }); + this.ondatachannel?.({ channel: unreliable }); + return { reliable, unreliable }; + } +} + +/** The shared outbound voice track, tapped off playback in production. */ +const outboundTrack = { kind: 'audio' } as unknown as MediaStreamTrack; + +let audio: PeerAudioBinding & { + attachRemoteStream: ReturnType; + detachRemoteStream: ReturnType; +}; +let registry: PeerRegistry; +let callbacks: Record>; + +function createRegistry(): PeerRegistry { + return new PeerRegistry({ + audio, + callbacks: callbacks as never, + createPeerConnection: (config) => + new FakePeerConnection(config) as unknown as RTCPeerConnection, + // Long enough that no timer fires during a test; polling is driven by hand. + statsIntervalMs: 1_000_000, + }); +} + +async function acceptOffer(deviceId: string): Promise { + await registry.acceptOffer({ + deviceId, + offer: { type: 'offer', sdp: OFFER_SDP }, + iceServers: [{ urls: 'stun:stun.example.com:3478' }], + audio: DEFAULT_REMOTE_AUDIO_CONFIG, + }); + return FakePeerConnection.instances[FakePeerConnection.instances.length - 1]; +} + +beforeEach(() => { + FakePeerConnection.instances = []; + audio = { + attachRemoteStream: vi.fn(), + detachRemoteStream: vi.fn(), + // The same track object every time, as the real binding returns: one + // destination node tapped off playback, shared by every peer. + getOutboundTrack: () => outboundTrack, + } as unknown as typeof audio; + callbacks = { + onAnswer: vi.fn(), + onIceCandidate: vi.fn(), + onConnectionState: vi.fn(), + onStats: vi.fn(), + onMessage: vi.fn(), + onError: vi.fn(), + }; + registry = createRegistry(); +}); + +describe('applyOpusPreferences', () => { + it('turns FEC and DTX on and sets a voice bitrate', () => { + const sdp = applyOpusPreferences(OFFER_SDP, DEFAULT_REMOTE_AUDIO_CONFIG); + expect(sdp).toContain('useinbandfec=1'); + expect(sdp).toContain('usedtx=1'); + expect(sdp).toContain(`maxaveragebitrate=${DEFAULT_REMOTE_AUDIO_CONFIG.maxAverageBitrate}`); + // Mono: the pipeline downmixes anyway, and stereo doubles the bitrate to + // carry a duplicate of the same voice. + expect(sdp).toContain('stereo=0'); + }); + + it('overwrites a value the far end asked for rather than appending a second one', () => { + const sdp = applyOpusPreferences(OFFER_SDP, DEFAULT_REMOTE_AUDIO_CONFIG); + expect(sdp).not.toContain('useinbandfec=0'); + expect(sdp.match(/useinbandfec/g)).toHaveLength(1); + // And keeps parameters it does not own. + expect(sdp).toContain('minptime=10'); + }); + + it('adds an fmtp line when the offer names Opus without one', () => { + const bare = 'v=0\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=rtpmap:111 opus/48000/2'; + expect(applyOpusPreferences(bare, DEFAULT_REMOTE_AUDIO_CONFIG)).toContain('a=fmtp:111 '); + }); + + it('leaves an SDP with no Opus in it alone rather than throwing', () => { + const noOpus = 'v=0\r\nm=audio 9 UDP/TLS/RTP/SAVPF 8\r\na=rtpmap:8 PCMA/8000'; + expect(applyOpusPreferences(noOpus, DEFAULT_REMOTE_AUDIO_CONFIG)).toBe(noOpus); + }); + + it('honours a configuration with FEC and DTX off', () => { + const sdp = applyOpusPreferences(OFFER_SDP, { + ...DEFAULT_REMOTE_AUDIO_CONFIG, + fec: false, + dtx: false, + }); + expect(sdp).toContain('useinbandfec=0'); + expect(sdp).toContain('usedtx=0'); + }); +}); + +describe('answering an offer', () => { + it('answers with the tuned SDP and caps the sender bitrate', async () => { + const pc = await acceptOffer('phone'); + + expect(callbacks.onAnswer).toHaveBeenCalledWith( + 'phone', + expect.objectContaining({ type: 'answer' }) + ); + expect(pc.localDescription?.sdp).toContain('usedtx=1'); + // Both the encoder target and the sender cap, because either one alone is + // routinely ignored depending on which end negotiated what. + expect(pc.senderParameters.encodings).toEqual([ + expect.objectContaining({ maxBitrate: DEFAULT_REMOTE_AUDIO_CONFIG.maxAverageBitrate }), + ]); + }); + + it('adds the shared outbound voice track exactly once across renegotiations', async () => { + const pc = await acceptOffer('phone'); + await acceptOffer('phone'); + expect(pc.addedTracks).toHaveLength(1); + expect(FakePeerConnection.instances).toHaveLength(1); + }); + + it('reuses the peer on a renegotiation, so a network change is not a new call', async () => { + await acceptOffer('phone'); + const before = FakePeerConnection.instances.length; + await acceptOffer('phone'); + expect(FakePeerConnection.instances).toHaveLength(before); + }); + + it('trickles local candidates out as they are gathered', async () => { + const pc = await acceptOffer('phone'); + pc.onicecandidate?.({ + candidate: { + candidate: 'candidate:1 1 udp 1 10.0.0.1 1 typ host', + sdpMid: '0', + sdpMLineIndex: 0, + usernameFragment: 'abc', + }, + }); + expect(callbacks.onIceCandidate).toHaveBeenCalledWith( + 'phone', + expect.objectContaining({ sdpMid: '0' }) + ); + }); + + it('reports a connection state change', async () => { + const pc = await acceptOffer('phone'); + pc.connectionState = 'connected'; + pc.onconnectionstatechange?.(); + expect(callbacks.onConnectionState).toHaveBeenCalledWith('phone', 'connected'); + }); +}); + +describe('one floor across several peers', () => { + it('only routes the holder"s microphone into the capture pipeline', async () => { + const phone = await acceptOffer('phone'); + const laptop = await acceptOffer('laptop'); + const stream = { id: 'remote' } as unknown as MediaStream; + + registry.setFloorHolder('phone'); + phone.ontrack?.({ streams: [stream], track: {} }); + laptop.ontrack?.({ streams: [stream], track: {} }); + + expect(audio.attachRemoteStream).toHaveBeenCalledTimes(1); + expect(audio.attachRemoteStream).toHaveBeenCalledWith(stream, 'phone'); + }); + + it('makes a takeover a graph reconnection rather than a renegotiation', async () => { + const phone = await acceptOffer('phone'); + const laptop = await acceptOffer('laptop'); + const stream = { id: 'remote' } as unknown as MediaStream; + registry.setFloorHolder('phone'); + phone.ontrack?.({ streams: [stream], track: {} }); + laptop.ontrack?.({ streams: [stream], track: {} }); + + registry.setFloorHolder('laptop'); + + expect(audio.detachRemoteStream).toHaveBeenCalledWith('phone'); + expect(audio.attachRemoteStream).toHaveBeenLastCalledWith(stream, 'laptop'); + // Nothing was renegotiated: the tracks were already flowing. + expect(FakePeerConnection.instances).toHaveLength(2); + }); + + it('releases the capture when the holder is closed', async () => { + const phone = await acceptOffer('phone'); + registry.setFloorHolder('phone'); + phone.ontrack?.({ streams: [{ id: 'r' } as unknown as MediaStream], track: {} }); + + registry.close('phone', 'revoked'); + expect(audio.detachRemoteStream).toHaveBeenCalledWith('phone'); + expect(phone.closed).toBe(true); + }); +}); + +describe('data channels', () => { + it('routes each message onto the channel the protocol table names', async () => { + const pc = await acceptOffer('phone'); + const { reliable, unreliable } = pc.openChannels(); + + registry.send('phone', { type: 'floor-state', holder: 'phone', isSelf: true }); + registry.send('phone', { type: 'audio-level', level: 0.5, speech: true }); + + expect(reliable.sent).toHaveLength(1); + expect(JSON.parse(reliable.sent[0]).type).toBe('floor-state'); + expect(unreliable.sent).toHaveLength(1); + expect(JSON.parse(unreliable.sent[0]).type).toBe('audio-level'); + }); + + it('hands an inbound message to the callback, decoded', async () => { + const pc = await acceptOffer('phone'); + const { unreliable } = pc.openChannels(); + unreliable.onmessage?.({ data: encodeDeviceMessage({ type: 'floor', action: 'press' }) }); + expect(callbacks.onMessage).toHaveBeenCalledWith( + 'phone', + expect.objectContaining({ type: 'floor', action: 'press' }) + ); + }); + + it('drops a malformed frame rather than throwing inside a channel handler', async () => { + const pc = await acceptOffer('phone'); + const { reliable } = pc.openChannels(); + expect(() => reliable.onmessage?.({ data: 'not json' })).not.toThrow(); + expect(callbacks.onMessage).not.toHaveBeenCalled(); + }); + + it('closes a channel it did not name', async () => { + const pc = await acceptOffer('phone'); + const rogue = new FakeDataChannel('exfiltrate'); + pc.ondatachannel?.({ channel: rogue }); + expect(rogue.closed).toBe(true); + }); + + it('broadcasts to every peer', async () => { + const phone = await acceptOffer('phone'); + const laptop = await acceptOffer('laptop'); + const a = phone.openChannels(); + const b = laptop.openChannels(); + + registry.broadcast({ type: 'revoked', message: 'all devices disconnected' }); + expect(a.reliable.sent).toHaveLength(1); + expect(b.reliable.sent).toHaveLength(1); + }); +}); + +describe('summarizeStats', () => { + it('reduces a report to the four numbers a signal bar needs', () => { + const stats = summarizeStats('phone', [ + { + type: 'candidate-pair', + selected: true, + currentRoundTripTime: 0.042, + localCandidateId: 'L', + remoteCandidateId: 'R', + }, + { type: 'local-candidate', id: 'L', candidateType: 'host' }, + { type: 'remote-candidate', id: 'R', candidateType: 'host' }, + { + type: 'inbound-rtp', + kind: 'audio', + jitter: 0.005, + packetsReceived: 990, + packetsLost: 10, + bytesReceived: 1000, + }, + ]); + + expect(stats.rttMs).toBe(42); + expect(stats.jitterMs).toBe(5); + expect(stats.packetLoss).toBeCloseTo(0.01); + expect(stats.candidateType).toBe('lan'); + }); + + it('calls the path relayed when EITHER end is a relay', () => { + const stats = summarizeStats('phone', [ + { + type: 'candidate-pair', + selected: true, + localCandidateId: 'L', + remoteCandidateId: 'R', + }, + // Our end gathered a host candidate; the phone is on a relay. Saying + // "direct" here would describe a path the audio is not taking. + { type: 'local-candidate', id: 'L', candidateType: 'host' }, + { type: 'remote-candidate', id: 'R', candidateType: 'relay' }, + ]); + expect(stats.candidateType).toBe('relay'); + }); + + it('folds prflx in with srflx, because the difference is ICE trivia', () => { + const stats = summarizeStats('phone', [ + { type: 'candidate-pair', selected: true, localCandidateId: 'L', remoteCandidateId: 'R' }, + { type: 'local-candidate', id: 'L', candidateType: 'prflx' }, + { type: 'remote-candidate', id: 'R', candidateType: 'srflx' }, + ]); + expect(stats.candidateType).toBe('stun'); + }); + + it('reports unknown before a pair has been selected', () => { + const stats = summarizeStats('phone', []); + expect(stats.candidateType).toBe('unknown'); + expect(stats.rttMs).toBeNull(); + expect(stats.packetLoss).toBe(0); + expect(stats.inboundBitrate).toBeNull(); + }); + + it('derives a bitrate from the delta between two readings', () => { + const first = summarizeStats('phone', [ + { type: 'inbound-rtp', kind: 'audio', bytesReceived: 1000 }, + ]); + const second = summarizeStats( + 'phone', + [{ type: 'inbound-rtp', kind: 'audio', bytesReceived: 4000 }], + { bytesReceived: first.bytesReceived, at: performance.now() - 1000 } + ); + // 3000 bytes in about a second is about 24 kbps, which is the voice target. + expect(second.inboundBitrate).toBeGreaterThan(20_000); + expect(second.inboundBitrate).toBeLessThan(28_000); + }); +}); + +describe('probeIce', () => { + it('reports a relay as the best path and stops there', async () => { + const probe = registry.probeIce([{ urls: 'turn:relay.example.com' }], 50); + const pc = FakePeerConnection.instances[FakePeerConnection.instances.length - 1]; + await Promise.resolve(); + pc.onicecandidate?.({ candidate: { candidate: 'candidate:1 1 udp 1 1.2.3.4 1 typ host' } }); + pc.onicecandidate?.({ candidate: { candidate: 'candidate:2 1 udp 1 5.6.7.8 1 typ relay' } }); + + const result = await probe; + expect(result).toMatchObject({ host: true, relay: true, best: 'relay' }); + expect(pc.closed).toBe(true); + }); + + it('reports what it got when gathering finishes without a relay', async () => { + const probe = registry.probeIce([{ urls: 'stun:stun.example.com' }], 50); + const pc = FakePeerConnection.instances[FakePeerConnection.instances.length - 1]; + await Promise.resolve(); + pc.onicecandidate?.({ candidate: { candidate: 'candidate:1 1 udp 1 1.2.3.4 1 typ srflx' } }); + // A null candidate is the end of gathering. + pc.onicecandidate?.({ candidate: null }); + + expect(await probe).toMatchObject({ stun: true, relay: false, best: 'stun' }); + }); +}); diff --git a/src/__tests__/renderer/acappella-audio/playback.test.ts b/src/__tests__/renderer/acappella-audio/playback.test.ts new file mode 100644 index 0000000000..d20d6a65ca --- /dev/null +++ b/src/__tests__/renderer/acappella-audio/playback.test.ts @@ -0,0 +1,342 @@ +/** + * @file playback.test.ts + * + * TTS playback for the A Cappella audio host. + * + * Most of these tests are really barge-in tests. Interrupting the assistant has + * to be felt as instant, and there are exactly two ways to get that wrong: + * leaving already-scheduled buffers running (the assistant keeps talking for the + * length of the last chunk), and letting a decode that was in flight during the + * flush schedule itself afterwards (the assistant starts again from nowhere). + * Both have their own test below. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createFakeAudioContext, type FakeAudioContext } from '../../helpers/mockWebAudio'; +import type { AudioHostStatus } from '../../../shared/acappella/audio-host'; +import { pcm16ToFloat32, TtsPlayback } from '../../../renderer/acappella-audio/playback'; + +/** 16 kHz mono PCM16 of `samples` length, so duration is samples/16000 seconds. */ +function pcmChunk(samples: number): ArrayBuffer { + return new Int16Array(samples).buffer; +} + +describe('pcm16ToFloat32', () => { + it('round-trips the worklet quantisation exactly at both extremes', () => { + const out = pcm16ToFloat32(new Int16Array([0, 0x7fff, -0x8000, 0x4000]).buffer); + + expect(out[0]).toBe(0); + expect(out[1]).toBeCloseTo(1, 6); + expect(out[2]).toBeCloseTo(-1, 6); + expect(out[3]).toBeCloseTo(0.5, 4); + }); +}); + +describe('TtsPlayback', () => { + let context: FakeAudioContext; + let statuses: AudioHostStatus[]; + + const build = () => + new TtsPlayback({ + context: context as unknown as AudioContext, + onStatus: (status) => statuses.push(status), + }); + + beforeEach(() => { + context = createFakeAudioContext(); + statuses = []; + }); + + const playbackStates = () => + statuses.filter( + (status): status is Extract => + status.kind === 'playback-state' + ); + + it('plays a PCM16 chunk through its own gain node into the destination', async () => { + const playback = build(); + + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(16000), + }); + + const gain = context.gains[0]; + expect(gain.connectedTo).toContain(context.destination); + expect(context.sources).toHaveLength(1); + expect(context.sources[0].connectedTo).toContain(gain); + expect(context.sources[0].startedAt).toBe(0); + }); + + it('schedules consecutive chunks gaplessly rather than all at once', async () => { + const playback = build(); + + // Two half-second chunks: the second must start where the first ends, or a + // streamed sentence comes out as overlapping speech. + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(8000), + }); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(8000), + }); + + expect(context.sources[0].startedAt).toBe(0); + expect(context.sources[1].startedAt).toBeCloseTo(0.5, 6); + expect(playback.queuedMs).toBeCloseTo(1000, 3); + }); + + it('never schedules in the past when the queue has already drained', async () => { + const playback = build(); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + context.advance(5); + + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + + expect(context.sources[1].startedAt).toBe(5); + }); + + it('decodes encoded chunks through the Web Audio decoder', async () => { + const playback = build(); + + await playback.enqueue({ utteranceId: 'u1', format: 'encoded', data: new ArrayBuffer(2048) }); + + expect(context.decodeAudioData).toHaveBeenCalledTimes(1); + expect(context.sources).toHaveLength(1); + }); + + it('stops every scheduled source on flush, not just future ones', async () => { + const playback = build(); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(80000), + }); + + playback.flush(); + + // A five-second chunk left running is five seconds of the assistant talking + // over a user who already interrupted. + expect(context.sources[0].stopped).toBe(true); + expect(context.sources[0].disconnectCount).toBe(1); + expect(playback.playing).toBe(false); + expect(playback.queuedMs).toBe(0); + }); + + it('drops a chunk whose decode finishes after a flush', async () => { + const playback = build(); + let release!: (buffer: unknown) => void; + context.decodeAudioData.mockImplementationOnce( + () => new Promise((resolve) => (release = resolve)) + ); + + const pending = playback.enqueue({ + utteranceId: 'u1', + format: 'encoded', + data: new ArrayBuffer(64), + }); + playback.flush(); + release({ duration: 1, length: 24000, sampleRate: 24000, numberOfChannels: 1 }); + await pending; + + // Scheduling this would restart speech the user already talked over. + expect(context.sources).toHaveLength(0); + expect(playback.playing).toBe(false); + }); + + it('restores gain on flush so the next utterance is not silently ducked', async () => { + const playback = build(); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + playback.duck(0, 30); + + playback.flush(); + + expect(context.gains[0].gain.value).toBe(1); + }); + + it('ramps gain from its current value, so a duck during a duck does not jump', () => { + const playback = build(); + context.advance(2); + + playback.duck(0.2, 120); + + const gain = context.gains[0]; + expect(gain.automations).toEqual([ + { kind: 'cancel', value: 1, time: 2 }, + { kind: 'set', value: 1, time: 2 }, + { kind: 'ramp', value: 0.2, time: 2.12 }, + ]); + }); + + it('clamps duck gain into range', () => { + const playback = build(); + + playback.duck(5, -10); + + expect(context.gains[0].gain.value).toBe(1); + playback.duck(-3, 0); + expect(context.gains[0].gain.value).toBe(0); + }); + + it('applies the user volume as the base gain', () => { + const playback = build(); + + playback.setVolume(0.4); + + expect(context.gains[0].gain.value).toBeCloseTo(0.4, 6); + }); + + it('ducks RELATIVE to the user volume, so a quiet session does not get louder', () => { + const playback = build(); + playback.setVolume(0.5); + + playback.duck(0.2, 0); + + expect(context.gains[0].gain.value).toBeCloseTo(0.1, 6); + }); + + it('flush restores the user volume, not full output', async () => { + // The other order of this bug is the one that matters: a flush that + // restored gain to 1 would silently un-mute a muted session on the first + // barge-in, and there is nothing on screen to explain the noise. + const playback = build(); + playback.setVolume(0.3); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + playback.duck(0, 30); + + playback.flush(); + + expect(context.gains[0].gain.value).toBeCloseTo(0.3, 6); + }); + + it('clamps a nonsensical volume rather than passing it to the gain node', () => { + const playback = build(); + + playback.setVolume(Number.NaN); + expect(context.gains[0].gain.value).toBe(1); + + playback.setVolume(9); + expect(context.gains[0].gain.value).toBe(1); + + playback.setVolume(-1); + expect(context.gains[0].gain.value).toBe(0); + }); + + it('keeps reporting the utterance while chunks may still be coming', async () => { + const playback = build(); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + statuses.length = 0; + + context.sources[0].finish(); + + // Drained between sentences of a streaming run. Reporting idle here would + // let the pipeline close a speech run that is still mid-utterance. + expect(playbackStates().at(-1)).toEqual({ + kind: 'playback-state', + playing: false, + utteranceId: 'u1', + queuedMs: 0, + }); + }); + + it('reports idle once the utterance is ended and drained', async () => { + const playback = build(); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + playback.endUtterance('u1'); + statuses.length = 0; + + context.sources[0].finish(); + + expect(playbackStates().at(-1)).toEqual({ + kind: 'playback-state', + playing: false, + utteranceId: null, + queuedMs: 0, + }); + }); + + it('closes out an utterance whose end marker arrives after it drained', async () => { + const playback = build(); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + context.sources[0].finish(); + statuses.length = 0; + + playback.endUtterance('u1'); + + expect(playbackStates().at(-1)).toMatchObject({ playing: false, utteranceId: null }); + }); + + it('ignores work queued after dispose', async () => { + const playback = build(); + playback.dispose(); + + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + + expect(context.sources).toHaveLength(0); + expect(context.gains[0].disconnectCount).toBe(1); + }); + + it('does not warn when stop() throws on a source that never started', async () => { + const playback = build(); + await playback.enqueue({ + utteranceId: 'u1', + format: 'pcm16', + sampleRate: 16000, + data: pcmChunk(1600), + }); + context.sources[0].stop = vi.fn(() => { + throw new Error('InvalidStateError'); + }); + + expect(() => playback.flush()).not.toThrow(); + }); +}); diff --git a/src/__tests__/renderer/components/ACappella/VoiceAccessibility.test.tsx b/src/__tests__/renderer/components/ACappella/VoiceAccessibility.test.tsx new file mode 100644 index 0000000000..b578e87ea0 --- /dev/null +++ b/src/__tests__/renderer/components/ACappella/VoiceAccessibility.test.tsx @@ -0,0 +1,297 @@ +/** + * The voice UI, for people who are not looking at it. + * + * A HUD that is designed to sit on screen all day has two obligations most + * widgets do not: it must stop moving when the user has asked for less motion, + * and it must say what it is doing in words, because "the ring is pulsing" is + * not information a screen reader can convey. Both are covered here, along with + * keyboard reachability and the colour contrast of every state against every + * shipped theme. + */ + +import React from 'react'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, cleanup, act } from '@testing-library/react'; +import { VoiceHud } from '../../../../renderer/components/ACappella'; +import { VoiceIndicator } from '../../../../renderer/components/ACappella/VoiceIndicator'; +import { VoicePillMenu } from '../../../../renderer/components/VoicePillMenu'; +import { LayerStackProvider } from '../../../../renderer/contexts/LayerStackContext'; +import { useVoiceSessionStore } from '../../../../renderer/stores/voiceSessionStore'; +import { useVoiceUiStore } from '../../../../renderer/stores/voiceUiStore'; +import { contrastRatio, readableTextOn } from '../../../../shared/colorContrast'; +import { THEMES, type Theme } from '../../../../shared/themes'; +import { + VOICE_HUD_STATE_LABELS, + type VoiceHudVisualState, +} from '../../../../shared/acappella/hud-state'; +import { mockTheme } from '../../../helpers/mockTheme'; +import type { VoiceEvent } from '../../../../shared/acappella/protocol'; + +const SESSION = 'voice-1'; +let seq = 0; + +function event( + type: T, + body: Omit, 'type' | 'sessionId' | 'seq' | 'ts'> +): VoiceEvent { + seq += 1; + return { + type, + sessionId: SESSION, + seq, + ts: 1_700_000_000_000 + seq, + ...body, + } as unknown as VoiceEvent; +} + +function emit(...events: VoiceEvent[]): void { + const calls = vi.mocked(window.maestro.voice.onEvent).mock.calls; + const push = calls[calls.length - 1][0]; + act(() => { + for (const e of events) push(e); + }); +} + +function renderHud() { + return render( + + + + ); +} + +function startSession(): void { + emit( + event('wake', { source: 'client-button', scope: { kind: 'conductor' } }), + event('listen-start', { scope: { kind: 'conductor' }, sttProviderId: 'mock-stt' }) + ); +} + +/** Force the `prefers-reduced-motion` answer for this test. */ +function setReducedMotion(reduce: boolean): void { + window.matchMedia = vi.fn().mockImplementation((query: string) => ({ + matches: query.includes('prefers-reduced-motion') ? reduce : false, + media: query, + onchange: null, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + addListener: vi.fn(), + removeListener: vi.fn(), + dispatchEvent: vi.fn(), + })) as unknown as typeof window.matchMedia; +} + +beforeEach(() => { + seq = 0; + vi.clearAllMocks(); + vi.mocked(window.maestro.voice.onEvent).mockReturnValue(() => {}); + vi.mocked(window.maestro.voice.getState).mockResolvedValue(null); + useVoiceSessionStore.getState().reset(); + useVoiceUiStore.setState({ + transcriptVisible: false, + hudPosition: null, + minimizeBehavior: 'manual', + minimized: false, + muted: false, + loaded: true, + }); + setReducedMotion(false); +}); + +afterEach(() => { + cleanup(); +}); + +describe('Reduced motion', () => { + const STATES: VoiceHudVisualState[] = [ + 'idle-armed', + 'listening', + 'thinking', + 'speaking', + 'error', + ]; + + it('animates the live states by default', () => { + render(); + expect(screen.getByTestId('voice-indicator-listening').getAttribute('data-motion')).toBe( + 'animated' + ); + }); + + it.each(STATES)('renders %s without animation under reduced motion', (state) => { + setReducedMotion(true); + const { container } = render(); + + const indicator = container.querySelector('[data-motion]'); + expect(indicator?.getAttribute('data-motion')).toBe('static'); + // No canned animation classes anywhere in the subtree: an always-animating + // widget is a real problem for people with vestibular disorders, and this + // one is designed to be left on screen all day. + expect(container.querySelector('.animate-pulse')).toBeNull(); + expect(container.querySelector('.animate-spin')).toBeNull(); + }); + + it('stops the level disc tracking the microphone under reduced motion', () => { + setReducedMotion(true); + render(); + act(() => { + useVoiceSessionStore.setState({ audioLevel: 0.25 }); + }); + + const disc = screen.getByTestId('voice-hud-level') as HTMLElement; + expect(disc.style.transform).toBe(''); + expect(disc.style.transition).toBe(''); + }); +}); + +describe('Screen reader announcements', () => { + it('has a polite live region that names the state and the bound scope', () => { + renderHud(); + startSession(); + + const region = screen.getByTestId('voice-hud-live-region'); + expect(region.getAttribute('role')).toBe('status'); + expect(region.getAttribute('aria-live')).toBe('polite'); + expect(region.textContent).toContain('Listening'); + expect(region.textContent).toContain('microphone is open'); + expect(region.textContent).toContain('Conductor'); + }); + + it('announces the change when the state moves on', () => { + renderHud(); + startSession(); + emit(event('speak-start', { utteranceId: 'u1', sentenceCount: 1, ttsProviderId: 'mock-tts' })); + expect(screen.getByTestId('voice-hud-live-region').textContent).toContain('Speaking'); + }); + + it('keeps announcing while minimized, because the microphone is still open', async () => { + renderHud(); + startSession(); + await act(async () => { + useVoiceUiStore.getState().setMinimized(true); + }); + + expect(screen.queryByTestId('voice-hud')).toBeNull(); + expect(screen.getByTestId('voice-hud-live-region').textContent).toContain('Listening'); + }); + + it('gives every state a text label rather than only a shape', () => { + for (const state of Object.keys(VOICE_HUD_STATE_LABELS) as VoiceHudVisualState[]) { + const { container, unmount } = render(); + const indicator = container.querySelector('[data-motion]'); + expect(indicator?.getAttribute('aria-label')).toBe(VOICE_HUD_STATE_LABELS[state]); + unmount(); + } + }); +}); + +describe('Keyboard reachability', () => { + it('gives every HUD control a real button with an accessible name', () => { + renderHud(); + startSession(); + + for (const testId of [ + 'voice-hud-talk', + 'voice-hud-interrupt', + 'voice-hud-stop', + 'voice-hud-transcript-toggle', + 'voice-hud-mute', + 'voice-hud-minimize', + 'voice-hud-close', + ]) { + const control = screen.getByTestId(testId); + expect(control.tagName).toBe('BUTTON'); + expect(control.getAttribute('aria-label')).toBeTruthy(); + // A control reachable by Tab but with no visible focus ring is reachable + // only in theory. + expect(control.className).toContain('focus-visible:ring'); + } + }); + + it('does not trap focus: the HUD registers as a non-blocking layer', () => { + renderHud(); + startSession(); + // Nothing in the HUD steals focus on mount, so a user mid-sentence in the + // composer keeps typing while a voice session runs. + expect(document.activeElement).toBe(document.body); + }); + + it('toggles the session from the keyboard alone', () => { + renderHud(); + startSession(); + const talk = screen.getByTestId('voice-hud-talk'); + act(() => { + talk.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })); + }); + expect(window.maestro.voice.stop).toHaveBeenCalled(); + }); +}); + +describe('Header-anchored menu placement', () => { + it('portals out of the header subtree rather than rendering inside it', () => { + // jsdom has no layout engine, so `toBeInTheDocument()` passes on an element + // that is clipped to invisibility. The check that actually means something + // is that the menu is NOT a descendant of the header: `absolute top-full` + // inside `.header-container` is silently cut off, and so is bare `fixed`, + // because the header is a containing block for fixed descendants. + const header = document.createElement('div'); + header.className = 'header-container'; + header.style.overflow = 'hidden'; + document.body.appendChild(header); + const anchor = document.createElement('div'); + header.appendChild(anchor); + + render( + + + + ); + + const menu = screen.getByTestId('voice-pill-menu'); + expect(header.contains(menu)).toBe(false); + expect(menu.parentElement).toBe(document.body); + + header.remove(); + }); +}); + +describe('Contrast against every shipped theme', () => { + /** WCAG AA for the small text and icons this widget is made of. */ + const AA = 4.5; + + it.each(Object.entries(THEMES))( + '%s clears AA for every voice state', + (_id: string, theme: Theme) => { + // Every colour the HUD derives runs through `readableTextOn` against the + // surfaces it is actually painted on. This asserts the guarantee holds + // for the real theme values rather than trusting it by inspection. + const surfaces = [theme.colors.bgSidebar, theme.colors.bgMain]; + const derived = { + accent: readableTextOn(theme.colors.accent, surfaces), + warning: readableTextOn(theme.colors.warning, [theme.colors.bgSidebar]), + error: readableTextOn(theme.colors.error, [theme.colors.bgSidebar]), + onAccent: readableTextOn(theme.colors.accentForeground, [theme.colors.accent]), + }; + + expect(contrastRatio(derived.accent, theme.colors.bgSidebar)).toBeGreaterThanOrEqual(AA); + expect(contrastRatio(derived.accent, theme.colors.bgMain)).toBeGreaterThanOrEqual(AA); + expect(contrastRatio(derived.warning, theme.colors.bgSidebar)).toBeGreaterThanOrEqual(AA); + expect(contrastRatio(derived.error, theme.colors.bgSidebar)).toBeGreaterThanOrEqual(AA); + // The speaking indicator is a FILLED accent disc, so its glyph is read + // against the accent rather than against the panel. + expect(contrastRatio(derived.onAccent, theme.colors.accent)).toBeGreaterThanOrEqual(AA); + } + ); +}); diff --git a/src/__tests__/renderer/components/ACappella/VoiceHud.test.tsx b/src/__tests__/renderer/components/ACappella/VoiceHud.test.tsx new file mode 100644 index 0000000000..7b1705f677 --- /dev/null +++ b/src/__tests__/renderer/components/ACappella/VoiceHud.test.tsx @@ -0,0 +1,877 @@ +/** + * VoiceHud - the A Cappella overlay. + * + * What matters here: it is invisible and inert while the Encore Feature is off, + * it renders what the event stream says (not what the caller hoped), listening + * and speaking are distinguishable, and closing it ENDS the session rather than + * leaving an open floor behind an invisible surface. + */ + +import React from 'react'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, cleanup, act, fireEvent } from '@testing-library/react'; +import { + DEV_HARNESS_STORAGE_KEY, + VoiceHud, + VoiceStatusIndicator, +} from '../../../../renderer/components/ACappella'; +import { useVoiceSessionStore } from '../../../../renderer/stores/voiceSessionStore'; +import { useVoiceUiStore } from '../../../../renderer/stores/voiceUiStore'; +import { useSettingsStore } from '../../../../renderer/stores/settingsStore'; +import { LayerStackProvider } from '../../../../renderer/contexts/LayerStackContext'; +import type { VoiceEvent } from '../../../../shared/acappella/protocol'; +import { mockTheme } from '../../../helpers/mockTheme'; + +const SESSION = 'voice-1'; + +let seq = 0; + +function event( + type: T, + body: Omit, 'type' | 'sessionId' | 'seq' | 'ts'> +): VoiceEvent { + seq += 1; + return { + type, + sessionId: SESSION, + seq, + ts: 1_700_000_000_000 + seq, + ...body, + } as unknown as VoiceEvent; +} + +/** The handler the HUD registered with `voice.onEvent`. */ +function emitter(): (event: VoiceEvent) => void { + const calls = vi.mocked(window.maestro.voice.onEvent).mock.calls; + return calls[calls.length - 1][0]; +} + +function emit(...events: VoiceEvent[]): void { + const push = emitter(); + act(() => { + for (const e of events) push(e); + }); +} + +/** + * Let the mocked ResizeObserver deliver its measurement. + * + * jsdom has no layout engine, so the transcript's virtualizer sees a zero + * viewport and renders no rows until `setup.ts`'s observer fires - and it fires + * on a `setTimeout(0)`. A test that reads the scrollback without this asserts + * against an empty list and passes for the wrong reason. + */ +async function flushLayout(): Promise { + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)); + }); +} + +function renderHud( + props: { + enabled?: boolean; + showDevHarness?: boolean; + transcript?: boolean; + /** + * Also mount the Left Bar indicator, which is where a minimized HUD lives. + * Off by default so the HUD's own tests render only the HUD. + */ + withStatusIndicator?: boolean; + } = {} +) { + // The transcript is off by default and lives behind the toggle, so a test + // that wants to read the scrollback has to open it - exactly as a user does. + useVoiceUiStore.setState({ transcriptVisible: props.transcript === true, loaded: true }); + // The HUD takes the Encore flag as a prop; the Left Bar indicator reads it + // from the store, the way every other Left Bar surface does. + if (props.withStatusIndicator) { + useSettingsStore.setState((state) => ({ + encoreFeatures: { ...state.encoreFeatures, aCappella: props.enabled ?? true }, + })); + } + return render( + + + {props.withStatusIndicator && } + + ); +} + +/** Drive the session to `listening`, which is where every turn starts. */ +function startSession(): void { + emit( + event('wake', { source: 'client-button', scope: { kind: 'conductor' } }), + event('listen-start', { scope: { kind: 'conductor' }, sttProviderId: 'mock-stt' }) + ); +} + +beforeEach(() => { + seq = 0; + vi.clearAllMocks(); + vi.mocked(window.maestro.voice.onEvent).mockReturnValue(() => {}); + vi.mocked(window.maestro.voice.getState).mockResolvedValue(null); + useVoiceSessionStore.getState().reset(); + useVoiceUiStore.setState({ + transcriptVisible: false, + hudPosition: null, + minimizeBehavior: 'manual', + minimized: false, + muted: false, + loaded: true, + }); +}); + +afterEach(() => { + cleanup(); +}); + +describe('VoiceHud gating', () => { + it('renders nothing and subscribes to nothing when the Encore flag is off', () => { + renderHud({ enabled: false, showDevHarness: true }); + expect(screen.queryByTestId('voice-hud')).toBeNull(); + expect(window.maestro.voice.onEvent).not.toHaveBeenCalled(); + }); + + it('renders nothing while idle in a build without the dev harness', () => { + renderHud(); + expect(screen.queryByTestId('voice-hud')).toBeNull(); + }); + + /** + * The harness is a reason the widget renders, so its DEFAULT decides whether + * A Cappella opens itself. It used to default to the development build, which + * put a type-an-utterance box over every dev workspace at startup with no + * session, no microphone, and nobody having asked for one. + */ + describe('the dev harness does not opt itself in', () => { + // This jsdom has no localStorage at all, which is itself the first case: + // `devHarnessOptedIn()` has to read "not opted in" from a window with no + // storage rather than throw on the way to rendering the app. + const store = new Map(); + let original: Storage | undefined; + + beforeEach(() => { + store.clear(); + original = (window as { localStorage?: Storage }).localStorage; + Object.defineProperty(window, 'localStorage', { + configurable: true, + value: { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => void store.set(key, value), + removeItem: (key: string) => void store.delete(key), + }, + }); + }); + + afterEach(() => { + Object.defineProperty(window, 'localStorage', { + configurable: true, + value: original, + }); + }); + + /** No `showDevHarness` prop at all, which is how AppShell mounts it. */ + function renderWithDefaults() { + return render( + + + + ); + } + + it('renders nothing while idle when the prop is left to its default', () => { + renderWithDefaults(); + + expect(screen.queryByTestId('voice-hud')).toBeNull(); + expect(screen.queryByTestId('voice-dev-harness')).toBeNull(); + }); + + it('opens for someone who explicitly opted in', () => { + // An opt-in IS a trigger: whoever set the key wants the box to type into. + window.localStorage.setItem(DEV_HARNESS_STORAGE_KEY, 'true'); + + renderWithDefaults(); + + expect(screen.getByTestId('voice-dev-harness')).toBeTruthy(); + }); + + it('ignores a key set to anything other than true', () => { + window.localStorage.setItem(DEV_HARNESS_STORAGE_KEY, 'false'); + + renderWithDefaults(); + + expect(screen.queryByTestId('voice-hud')).toBeNull(); + }); + }); + + it('shows the harness controls only in a dev build', () => { + const { unmount } = renderHud(); + expect(screen.queryByTestId('voice-dev-harness')).toBeNull(); + unmount(); + + renderHud({ showDevHarness: true }); + expect(screen.getByTestId('voice-dev-harness')).toBeTruthy(); + }); + + it('clears mirrored state when the flag is turned off', () => { + const { rerender } = renderHud(); + startSession(); + expect(useVoiceSessionStore.getState().state).toBe('listening'); + + rerender( + + + + ); + expect(useVoiceSessionStore.getState().state).toBe('idle'); + expect(screen.queryByTestId('voice-hud')).toBeNull(); + }); +}); + +describe('VoiceHud rendering', () => { + it('appears with a listening indicator once a session opens', () => { + renderHud(); + startSession(); + + expect(screen.getByTestId('voice-hud')).toBeTruthy(); + expect(screen.getByTestId('voice-indicator-listening')).toBeTruthy(); + expect(screen.queryByTestId('voice-indicator-speaking')).toBeNull(); + expect(screen.getByTestId('voice-hud-scope').textContent).toBe('Conductor'); + }); + + it('shows the live hypothesis on the collapsed HUD without opening the transcript', () => { + renderHud(); + startSession(); + + emit(event('partial-transcript', { text: 'start a new', stability: 0.4 })); + expect(screen.getByTestId('voice-hud-latest').textContent).toBe('start a new'); + + // The settled utterance belongs to the scrollback, which is closed. The + // one-line readout is for what is happening NOW, not a running history. + emit(event('final-transcript', { text: 'start a new tab', confidence: 1 })); + expect(screen.queryByTestId('voice-hud-latest')).toBeNull(); + }); + + it('streams the partial transcript and settles it into the transcript', async () => { + renderHud({ transcript: true }); + startSession(); + + emit(event('partial-transcript', { text: 'start a new', stability: 0.4 })); + expect(screen.getByTestId('voice-transcript-partial').textContent).toBe('start a new'); + + emit(event('final-transcript', { text: 'start a new tab', confidence: 1 })); + await flushLayout(); + expect(screen.queryByTestId('voice-transcript-partial')).toBeNull(); + expect(screen.getByText('start a new tab')).toBeTruthy(); + }); + + it('names the agent and tab a dispatch landed on', async () => { + renderHud({ transcript: true }); + startSession(); + emit( + event('dispatch', { + agentSessionId: 'agent-1', + agentName: 'Backend', + tabId: 'tab-1', + tabName: 'Auth Refactor', + action: 'created', + promptSent: true, + }) + ); + await flushLayout(); + expect(screen.getByText('Opened a new tab named Auth Refactor on Backend')).toBeTruthy(); + // The bound scope is the prominent line, and a dispatch names the tab on it. + expect(screen.getByTestId('voice-hud-scope').textContent).toContain('Auth Refactor'); + }); + + it('switches the indicator and counts sentences while speaking', () => { + renderHud({ transcript: true }); + startSession(); + emit( + event('speak-start', { utteranceId: 'u1', sentenceCount: 2, ttsProviderId: 'mock-tts' }), + event('speak-sentence', { utteranceId: 'u1', index: 0, text: 'The tests pass.' }) + ); + + expect(screen.getByTestId('voice-indicator-speaking')).toBeTruthy(); + expect(screen.queryByTestId('voice-indicator-listening')).toBeNull(); + expect(screen.getByTestId('voice-hud-speech-progress').textContent).toBe('1 of 2'); + expect(screen.getByTestId('voice-transcript-spoken').textContent).toContain('The tests pass.'); + }); + + it('marks the total provisional while the reply is still being written', () => { + // A streamed reply starts speaking before the sentence count is known, so + // the count at `speak-start` is a lower bound the delivered index runs past. + // The live app printed "1 of 0" here. + renderHud({ transcript: true }); + startSession(); + emit( + event('speak-start', { + utteranceId: 'u1', + sentenceCount: 0, + ttsProviderId: 'mock-tts', + streaming: true, + }), + event('speak-sentence', { utteranceId: 'u1', index: 0, text: 'On it.' }) + ); + + expect(screen.getByTestId('voice-hud-speech-progress').textContent).toBe('1 of 1+'); + }); + + it('marks a barge-in as a cut and hands the floor back', () => { + renderHud({ transcript: true }); + startSession(); + emit( + event('speak-start', { utteranceId: 'u1', sentenceCount: 3, ttsProviderId: 'mock-tts' }), + event('speak-sentence', { utteranceId: 'u1', index: 0, text: 'The tests pass.' }), + event('barge-in', { source: 'client-button', cancelledUtteranceId: 'u1' }), + event('speak-end', { utteranceId: 'u1', reason: 'cancelled' }), + event('listen-start', { scope: { kind: 'conductor' }, sttProviderId: 'mock-stt' }) + ); + + expect(screen.getByTestId('voice-transcript-spoken').textContent).toContain('(cut off)'); + // Barge-in keeps the floor: the session is listening, not gone. + expect(screen.getByTestId('voice-indicator-listening')).toBeTruthy(); + }); + + it('shows a thinking state for the whole route-and-dispatch stretch', () => { + renderHud(); + startSession(); + emit(event('final-transcript', { text: 'refactor auth', confidence: 1 })); + expect(screen.getByTestId('voice-indicator-thinking')).toBeTruthy(); + expect(screen.getByText('Thinking')).toBeTruthy(); + }); + + it('shows an error state rather than pretending the session is idle', () => { + renderHud(); + startSession(); + emit( + event('session-error', { + code: 'provider-unavailable', + message: 'The speech engine is not installed', + recoverable: false, + }) + ); + expect(screen.getByTestId('voice-indicator-error')).toBeTruthy(); + expect(screen.getByTestId('voice-hud-error')).toBeTruthy(); + }); + + it('surfaces a provider substitution rather than running the mock quietly', async () => { + vi.mocked(window.maestro.voice.start).mockResolvedValue({ + snapshot: { + sessionId: SESSION, + state: 'listening', + scope: { kind: 'conductor' }, + seq: 2, + startedAt: 0, + providerIds: { stt: 'mock-stt', tts: 'mock-tts', brain: 'mock-brain' }, + }, + substitutions: [ + { + role: 'stt', + requestedId: 'whisper-local', + resolvedId: 'mock-stt', + reason: 'unavailable', + message: "Voice provider 'whisper-local' is not available; using 'mock-stt'", + }, + ], + }); + + renderHud({ showDevHarness: true }); + const input = screen.getByTestId('voice-dev-harness-input'); + fireEvent.change(input, { target: { value: 'hello' } }); + await act(async () => { + fireEvent.click(screen.getByTestId('voice-dev-harness-send')); + }); + + expect(screen.getByTestId('voice-hud-substitutions').textContent).toContain('whisper-local'); + }); + + it('warns when the event stream skipped a seq', () => { + renderHud(); + startSession(); + seq = 42; + emit(event('partial-transcript', { text: 'hello', stability: 0.5 })); + expect(screen.getByTestId('voice-hud-gap')).toBeTruthy(); + }); +}); + +describe('VoiceHud audio', () => { + function micEvent( + overrides: Partial, 'type'>> = {} + ): VoiceEvent { + return event('mic-state', { + permission: 'granted', + capturing: true, + deviceId: 'default', + deviceLabel: 'MacBook Pro Microphone', + issue: null, + deviceChanged: false, + ...overrides, + } as never); + } + + function level(): HTMLElement { + return screen.getByTestId('voice-indicator-listening'); + } + + it('drives the listening indicator from real level values', () => { + renderHud(); + startSession(); + const quiet = level().getAttribute('data-level'); + + emit(event('audio-level', { level: 0.25, speech: true })); + const loud = level().getAttribute('data-level'); + + expect(Number(quiet)).toBe(0); + expect(Number(loud)).toBe(1); + expect(screen.getByTestId('voice-hud-level')).toBeTruthy(); + }); + + it('falls back to rest when the floor closes rather than freezing the last level', () => { + renderHud(); + startSession(); + emit(event('audio-level', { level: 0.25, speech: true })); + expect(Number(level().getAttribute('data-level'))).toBe(1); + + emit( + event('listen-stop', { reason: 'endpoint' }), + event('listen-start', { scope: { kind: 'conductor' }, sttProviderId: 'mock-stt' }) + ); + expect(Number(level().getAttribute('data-level'))).toBe(0); + }); + + it('names the microphone in use on the indicator', () => { + renderHud(); + startSession(); + emit(micEvent()); + // The device sits alongside the state sentence rather than replacing it: + // the tooltip has to answer "is it listening" as well as "through what". + expect(level().getAttribute('title')).toContain('MacBook Pro Microphone'); + expect(level().getAttribute('title')).toContain('microphone is open'); + }); + + it('explains a denied microphone and offers the system settings', async () => { + renderHud(); + startSession(); + emit(micEvent({ permission: 'denied', capturing: false, issue: 'permission-denied' })); + + expect(screen.getByTestId('voice-hud-mic').textContent).toContain( + 'does not have microphone access' + ); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-mic-settings')); + }); + expect(window.maestro.voice.openMicSettings).toHaveBeenCalledTimes(1); + }); + + it('offers no settings button for a problem the settings cannot fix', () => { + renderHud(); + startSession(); + emit(micEvent({ capturing: false, issue: 'no-device' })); + + expect(screen.getByTestId('voice-hud-mic').textContent).toContain('No microphone was found'); + expect(screen.queryByTestId('voice-hud-mic-settings')).toBeNull(); + }); + + it('says it once: the calm mic notice replaces the red capture error', () => { + renderHud(); + startSession(); + emit( + event('session-error', { + code: 'audio-capture-failed', + message: 'Microphone permission was denied', + recoverable: true, + }), + micEvent({ permission: 'denied', capturing: false, issue: 'permission-denied' }) + ); + + expect(screen.getByTestId('voice-hud-mic')).toBeTruthy(); + expect(screen.queryByTestId('voice-hud-error')).toBeNull(); + }); + + it('still shows unrelated errors next to a mic problem', () => { + renderHud(); + startSession(); + emit( + micEvent({ capturing: false, issue: 'device-lost' }), + event('session-error', { + code: 'no-agent-matched', + message: "No agent with id 'agent-9' is running", + recoverable: true, + }) + ); + + expect(screen.getByTestId('voice-hud-mic')).toBeTruthy(); + expect(screen.getByTestId('voice-hud-error')).toBeTruthy(); + }); + + it('stays on screen for a denied microphone after the session parks', () => { + renderHud(); + startSession(); + emit( + micEvent({ permission: 'denied', capturing: false, issue: 'permission-denied' }), + event('listen-stop', { reason: 'stopped' }) + ); + + // The session is over, but the reason it produced nothing is still worth + // reading: a HUD that vanishes here leaves the user with silence and no + // explanation. + expect(screen.getByTestId('voice-hud')).toBeTruthy(); + expect(screen.getByTestId('voice-hud-mic')).toBeTruthy(); + }); +}); + +describe('VoiceHud dismissal', () => { + it('ends the session and hides when the ESC pill is clicked', async () => { + renderHud(); + startSession(); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-close')); + }); + + expect(window.maestro.voice.stop).toHaveBeenCalledTimes(1); + expect(screen.queryByTestId('voice-hud')).toBeNull(); + }); + + it('comes back for the next session rather than staying dismissed forever', async () => { + renderHud(); + startSession(); + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-close')); + }); + expect(screen.queryByTestId('voice-hud')).toBeNull(); + + act(() => { + emitter()({ + type: 'wake', + sessionId: 'voice-2', + seq: 1, + ts: 1_700_000_100_000, + source: 'client-button', + scope: { kind: 'conductor' }, + }); + }); + expect(screen.getByTestId('voice-hud')).toBeTruthy(); + }); +}); + +describe('VoiceHud minimize versus close', () => { + it('minimize leaves the session running behind a restore affordance', async () => { + renderHud({ withStatusIndicator: true }); + startSession(); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-minimize')); + }); + + // The widget is gone from the workspace and nothing touched the floor. The + // restore affordance is the Left Bar indicator, not a floating pill parked + // over the work the user just asked to see. + expect(screen.queryByTestId('voice-hud')).toBeNull(); + expect(screen.getByTestId('voice-status-indicator')).toBeTruthy(); + expect(window.maestro.voice.stop).not.toHaveBeenCalled(); + expect(useVoiceSessionStore.getState().state).toBe('listening'); + }); + + it('restores from the Left Bar indicator', async () => { + renderHud({ withStatusIndicator: true }); + startSession(); + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-minimize')); + }); + await act(async () => { + fireEvent.click(screen.getByTestId('voice-status-indicator')); + }); + expect(screen.getByTestId('voice-hud')).toBeTruthy(); + }); + + it('leaves no indicator anywhere once the session is closed', async () => { + // The pair that matters: minimize must leave something visible, and close + // must not. An indicator that outlived the session would claim an open + // microphone that is not there. + renderHud({ withStatusIndicator: true }); + startSession(); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-close')); + }); + + expect(screen.queryByTestId('voice-hud')).toBeNull(); + expect(screen.queryByTestId('voice-status-indicator')).toBeNull(); + }); + + it('close ends the session, unlike minimize', async () => { + renderHud(); + startSession(); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-close')); + }); + + expect(window.maestro.voice.stop).toHaveBeenCalledTimes(1); + expect(screen.queryByTestId('voice-hud')).toBeNull(); + expect(screen.queryByTestId('voice-hud-minimized')).toBeNull(); + }); + + it('Escape and the ESC pill do exactly the same thing', async () => { + const first = renderHud(); + startSession(); + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-close')); + }); + const viaPill = { + stopCalls: vi.mocked(window.maestro.voice.stop).mock.calls.length, + dismissed: useVoiceSessionStore.getState().dismissed, + rendered: screen.queryByTestId('voice-hud') !== null, + }; + first.unmount(); + + vi.clearAllMocks(); + vi.mocked(window.maestro.voice.onEvent).mockReturnValue(() => {}); + useVoiceSessionStore.getState().reset(); + seq = 0; + + renderHud(); + startSession(); + await act(async () => { + fireEvent.keyDown(document, { key: 'Escape' }); + }); + const viaEscape = { + stopCalls: vi.mocked(window.maestro.voice.stop).mock.calls.length, + dismissed: useVoiceSessionStore.getState().dismissed, + rendered: screen.queryByTestId('voice-hud') !== null, + }; + + expect(viaEscape).toEqual(viaPill); + expect(viaEscape.stopCalls).toBe(1); + }); + + describe('a recogniser that cannot hear', () => { + /** Report the live providers, the way `provider-state` does mid-session. */ + function reportStt(hearsAudio: boolean) { + emit( + event('provider-state', { + pipeline: 'cascade', + slots: [ + { + role: 'stt', + providerId: hearsAudio ? 'echo-stt' : 'mock-stt', + label: 'Test STT', + tier: 'mock', + hearsAudio, + }, + ], + egressStatement: 'Nothing leaves this machine.', + audioLeavesMachine: false, + } as never) + ); + } + + it('says so, because "Listening" cannot', () => { + // The whole point: the floor IS open and the state machine is telling the + // truth, but a text-in recogniser opens no capture device, so speaking can + // never produce a transcript. Six rebuilds were spent on that gap. + renderHud(); + startSession(); + reportStt(false); + + expect(screen.getByTestId('voice-hud-deaf').textContent).toContain( + 'does not listen to the microphone' + ); + }); + + it('stays quiet for a recogniser that does hear', () => { + renderHud(); + startSession(); + reportStt(true); + + expect(screen.queryByTestId('voice-hud-deaf')).toBeNull(); + }); + + it('stays quiet before the providers are known', () => { + // Absent is not the same as false. Warning on "not yet reported" would + // put the notice on screen at the start of every healthy session. + renderHud(); + startSession(); + + expect(screen.queryByTestId('voice-hud-deaf')).toBeNull(); + }); + }); + + it('lets Escape close a HUD that is only showing a refusal', async () => { + // The Escape layer was registered for an active session and the dev harness + // only, while the widget also renders for an error - so the HUD explaining + // why voice would not start drew an ESC pill that Escape never reached. + renderHud(); + emit(event('wake', { source: 'client-button', scope: { kind: 'conductor' } })); + emit( + event('session-error', { + code: 'provider-unavailable', + message: 'whisper.cpp is not part of this build yet.', + recoverable: false, + }) + ); + expect(screen.getByTestId('voice-hud-error')).toBeTruthy(); + + await act(async () => { + fireEvent.keyDown(document, { key: 'Escape' }); + }); + + expect(screen.queryByTestId('voice-hud')).toBeNull(); + }); +}); + +describe('VoiceHud controls', () => { + it('toggles the transcript and remembers the answer', async () => { + renderHud(); + startSession(); + expect(screen.queryByTestId('voice-transcript')).toBeNull(); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-transcript-toggle')); + }); + + expect(useVoiceUiStore.getState().transcriptVisible).toBe(true); + expect(screen.getByTestId('voice-transcript')).toBeTruthy(); + expect(useVoiceUiStore.getState().transcriptVisible).toBe(true); + expect(window.maestro.settings.set).toHaveBeenCalledWith( + 'acappella', + expect.objectContaining({ ui: expect.objectContaining({ transcriptVisible: true }) }) + ); + }); + + it('mutes the live output without persisting the mute', async () => { + renderHud(); + startSession(); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-mute')); + }); + + expect(window.maestro.voice.setVolume).toHaveBeenCalledWith(0); + expect(useVoiceUiStore.getState().muted).toBe(true); + // A mute that survived a restart is a voice assistant that has silently + // stopped talking to you. + expect(window.maestro.settings.set).not.toHaveBeenCalledWith( + 'acappella', + expect.objectContaining({ ui: expect.objectContaining({ muted: true }) }) + ); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-mute')); + }); + expect(window.maestro.voice.setVolume).toHaveBeenLastCalledWith(1); + }); + + it('offers Interrupt only while something is being said', async () => { + renderHud(); + startSession(); + expect(screen.getByTestId('voice-hud-interrupt')).toHaveProperty('disabled', true); + + emit(event('speak-start', { utteranceId: 'u1', sentenceCount: 1, ttsProviderId: 'mock-tts' })); + expect(screen.getByTestId('voice-hud-interrupt')).toHaveProperty('disabled', false); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-hud-interrupt')); + }); + expect(window.maestro.voice.interrupt).toHaveBeenCalledWith('client-button'); + // Interrupting keeps the floor. It is not a stop. + expect(window.maestro.voice.stop).not.toHaveBeenCalled(); + }); + + it('the talk button toggles on a tap', async () => { + renderHud({ showDevHarness: true }); + const talk = screen.getByTestId('voice-hud-talk'); + + await act(async () => { + fireEvent.pointerDown(talk, { button: 0 }); + fireEvent.pointerUp(talk); + }); + expect(window.maestro.voice.start).toHaveBeenCalledTimes(1); + + startSession(); + await act(async () => { + fireEvent.pointerDown(talk, { button: 0 }); + fireEvent.pointerUp(talk); + }); + expect(window.maestro.voice.stop).toHaveBeenCalledTimes(1); + }); +}); + +describe('VoiceDevHarness', () => { + it('starts a session on the first Send, so enabling the feature opens nothing', async () => { + renderHud({ showDevHarness: true }); + expect(window.maestro.voice.start).not.toHaveBeenCalled(); + + fireEvent.change(screen.getByTestId('voice-dev-harness-input'), { + target: { value: 'open a new tab on backend' }, + }); + await act(async () => { + fireEvent.click(screen.getByTestId('voice-dev-harness-send')); + }); + + expect(window.maestro.voice.start).toHaveBeenCalledTimes(1); + expect(window.maestro.voice.submitUtterance).toHaveBeenCalledWith('open a new tab on backend'); + }); + + it('keeps Interrupt and Stop as different actions', async () => { + renderHud({ showDevHarness: true }); + startSession(); + + // Nothing is speaking yet, so barge-in is not offered. + expect(screen.getByTestId('voice-dev-harness-interrupt')).toHaveProperty('disabled', true); + expect(screen.getByTestId('voice-dev-harness-stop')).toHaveProperty('disabled', false); + + emit(event('speak-start', { utteranceId: 'u1', sentenceCount: 1, ttsProviderId: 'mock-tts' })); + await act(async () => { + fireEvent.click(screen.getByTestId('voice-dev-harness-interrupt')); + }); + expect(window.maestro.voice.interrupt).toHaveBeenCalledWith('client-button'); + expect(window.maestro.voice.stop).not.toHaveBeenCalled(); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-dev-harness-stop')); + }); + expect(window.maestro.voice.stop).toHaveBeenCalledTimes(1); + }); + + it('offers Reply only while the session waits on one, and addresses it to the dispatched tab', async () => { + renderHud({ showDevHarness: true }); + startSession(); + expect(screen.getByTestId('voice-dev-harness-reply')).toHaveProperty('disabled', true); + + emit( + event('route-decision', { + decision: { + target: { sessionId: 'agent-1' }, + tabAction: 'new', + prompt: 'refactor auth', + confidence: 0.8, + }, + brainProviderId: 'mock-brain', + latencyMs: 2, + }), + event('dispatch', { + agentSessionId: 'agent-1', + agentName: 'Backend', + tabId: 'tab-7', + tabName: 'Auth Refactor', + action: 'created', + promptSent: true, + }) + ); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-dev-harness-reply')); + }); + expect(window.maestro.voice.submitAgentReply).toHaveBeenCalledWith( + expect.objectContaining({ agentSessionId: 'agent-1', tabId: 'tab-7' }) + ); + }); +}); diff --git a/src/__tests__/renderer/components/ACappella/VoiceStatusIndicator.test.tsx b/src/__tests__/renderer/components/ACappella/VoiceStatusIndicator.test.tsx new file mode 100644 index 0000000000..5bbf29a977 --- /dev/null +++ b/src/__tests__/renderer/components/ACappella/VoiceStatusIndicator.test.tsx @@ -0,0 +1,157 @@ +/** + * VoiceStatusIndicator - the minimized HUD's home in the Left Bar header. + * + * The invariant worth a test file: a minimized voice session must be visible + * somewhere, and a session that is NOT running must not be. Both halves matter. + * An indicator that renders too eagerly claims an open microphone that is not + * there; one that renders too rarely leaves a real one with no surface at all, + * which is the whole reason minimize is allowed to hide the widget. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { render, screen, fireEvent, cleanup } from '@testing-library/react'; + +// Controlled WindowContext. `null` is "no WindowProvider", which permits +// everything - the shape every other test in this file runs under. +let mockWindow: { windowId: string | null; isMainWindow: boolean } | null = null; +vi.mock('../../../../renderer/contexts/WindowContext', () => ({ + useWindowContextOptional: () => mockWindow, +})); + +import { VoiceStatusIndicator } from '../../../../renderer/components/ACappella'; +import { useVoiceSessionStore } from '../../../../renderer/stores/voiceSessionStore'; +import { useVoiceUiStore } from '../../../../renderer/stores/voiceUiStore'; +import { useSettingsStore } from '../../../../renderer/stores/settingsStore'; +import type { VoiceSessionState } from '../../../../shared/acappella/session-state'; +import { mockTheme } from '../../../helpers/mockTheme'; + +function seed( + options: { + enabled?: boolean; + state?: VoiceSessionState; + minimized?: boolean; + scope?: { kind: 'conductor' } | { kind: 'agent'; sessionId: string }; + } = {} +): void { + useSettingsStore.setState((prev) => ({ + encoreFeatures: { ...prev.encoreFeatures, aCappella: options.enabled ?? true }, + })); + useVoiceSessionStore.setState({ + sessionId: 'voice-1', + state: options.state ?? 'listening', + scope: options.scope ?? { kind: 'conductor' }, + roster: [ + { + sessionId: 'agent-1', + name: 'Backend', + agentType: 'claude-code', + cwd: '/repo', + tabs: [], + }, + ], + }); + useVoiceUiStore.setState({ minimized: options.minimized ?? true }); +} + +describe('VoiceStatusIndicator', () => { + beforeEach(() => { + cleanup(); + vi.clearAllMocks(); + mockWindow = null; + useVoiceSessionStore.getState().reset(); + seed(); + }); + + it('shows the bound scope while the HUD is minimized', () => { + render(); + expect(screen.getByTestId('voice-status-indicator').textContent).toContain('Conductor'); + }); + + it('names the agent when the session is bound to one', () => { + seed({ scope: { kind: 'agent', sessionId: 'agent-1' } }); + render(); + expect(screen.getByTestId('voice-status-indicator').textContent).toContain('Backend'); + }); + + it('stays out of the way while the HUD is on screen', () => { + seed({ minimized: false }); + render(); + expect(screen.queryByTestId('voice-status-indicator')).toBeNull(); + }); + + it('renders nothing when no session is running', () => { + // The dangerous direction: a leftover pill implying an open microphone. + seed({ state: 'idle' }); + render(); + expect(screen.queryByTestId('voice-status-indicator')).toBeNull(); + }); + + it('renders nothing when the Encore Feature is off', () => { + seed({ enabled: false }); + render(); + expect(screen.queryByTestId('voice-status-indicator')).toBeNull(); + }); + + it('restores the HUD without touching the session', () => { + render(); + fireEvent.click(screen.getByTestId('voice-status-indicator')); + + expect(useVoiceUiStore.getState().minimized).toBe(false); + // The floor is untouched: this control shows a window, it does not speak. + expect(useVoiceSessionStore.getState().state).toBe('listening'); + }); + + it('keeps the glyph but drops the label in compact mode', () => { + // The collapsed rail has no room for a name, and losing the glyph there + // would make the narrow sidebar the one place a live microphone is silent. + render(); + const pill = screen.getByTestId('voice-status-indicator'); + expect(pill.textContent).not.toContain('Conductor'); + expect(pill.querySelector('svg')).toBeTruthy(); + }); + + it('reports the live state to assistive tech, not just in colour', () => { + seed({ state: 'speaking' }); + render(); + expect(screen.getByTestId('voice-status-indicator').getAttribute('aria-label')).toContain( + 'Speaking' + ); + }); + + /** + * The minimized HUD is still the session's surface, so it follows the HUD's + * window. This is the same `useOwnsVoiceSession` rule the HUD itself uses, and + * it is tested on BOTH surfaces on purpose: a session hidden in one and shown + * in the other is exactly the drift the shared hook exists to prevent. + */ + describe('window scoping', () => { + it('shows a session this window owns', () => { + mockWindow = { windowId: 'w2', isMainWindow: false }; + useVoiceSessionStore.setState({ windowId: 'w2' }); + + render(); + + expect(screen.getByTestId('voice-status-indicator')).toBeTruthy(); + }); + + it('renders nothing for a session another window owns', () => { + // Voice events reach every window, so without the gate this window's Left + // Bar would claim an open microphone that belongs to a different one. + mockWindow = { windowId: 'w2', isMainWindow: false }; + useVoiceSessionStore.setState({ windowId: 'w1' }); + + render(); + + expect(screen.queryByTestId('voice-status-indicator')).toBeNull(); + }); + + it('shows a session that names no window on the primary', () => { + mockWindow = { windowId: 'w1', isMainWindow: true }; + useVoiceSessionStore.setState({ windowId: null }); + + render(); + + expect(screen.getByTestId('voice-status-indicator')).toBeTruthy(); + }); + }); +}); diff --git a/src/__tests__/renderer/components/ACappella/VoiceTranscript.test.tsx b/src/__tests__/renderer/components/ACappella/VoiceTranscript.test.tsx new file mode 100644 index 0000000000..63ecff150c --- /dev/null +++ b/src/__tests__/renderer/components/ACappella/VoiceTranscript.test.tsx @@ -0,0 +1,184 @@ +/** + * VoiceTranscript - the scrollback. + * + * Three behaviours are the point of the component and are what these cover: a + * partial settles into a final rather than piling up, the sentence currently + * coming out of the speakers is the one highlighted, and a route chip is a + * working address rather than a label. + */ + +import React from 'react'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, cleanup, act, fireEvent } from '@testing-library/react'; +import { VoiceTranscript } from '../../../../renderer/components/ACappella/VoiceTranscript'; +import { useVoiceSessionStore } from '../../../../renderer/stores/voiceSessionStore'; +import { useSessionStore } from '../../../../renderer/stores/sessionStore'; +import { mockTheme } from '../../../helpers/mockTheme'; +import type { VoiceEvent } from '../../../../shared/acappella/protocol'; +import type { Session } from '../../../../renderer/types'; + +const SESSION = 'voice-1'; +let seq = 0; + +function event( + type: T, + body: Omit, 'type' | 'sessionId' | 'seq' | 'ts'> +): VoiceEvent { + seq += 1; + return { + type, + sessionId: SESSION, + seq, + ts: 1_700_000_000_000 + seq, + ...body, + } as unknown as VoiceEvent; +} + +function apply(...events: VoiceEvent[]): void { + act(() => { + for (const e of events) useVoiceSessionStore.getState().applyEvent(e); + }); +} + +/** jsdom has no layout; let the mocked ResizeObserver deliver its measurement. */ +async function flushLayout(): Promise { + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)); + }); +} + +function renderTranscript() { + return render(); +} + +beforeEach(() => { + seq = 0; + vi.clearAllMocks(); + useVoiceSessionStore.getState().reset(); +}); + +afterEach(() => { + cleanup(); +}); + +describe('VoiceTranscript partials', () => { + it('shows a live hypothesis and drops it when the utterance settles', async () => { + renderTranscript(); + apply(event('partial-transcript', { text: 'open the auth', stability: 0.4 })); + expect(screen.getByTestId('voice-transcript-partial').textContent).toBe('open the auth'); + + apply(event('final-transcript', { text: 'open the auth tab', confidence: 1 })); + await flushLayout(); + + expect(screen.queryByTestId('voice-transcript-partial')).toBeNull(); + // The settled line is in the scrollback exactly once - the partial was + // replaced, not appended alongside. + expect(screen.getAllByText('open the auth tab')).toHaveLength(1); + }); + + it('says so when nothing has been said', () => { + renderTranscript(); + expect(screen.getByText('Nothing said yet.')).toBeTruthy(); + }); +}); + +describe('VoiceTranscript spoken sentences', () => { + function speakTwo(): void { + apply( + event('speak-start', { utteranceId: 'u1', sentenceCount: 2, ttsProviderId: 'mock-tts' }), + event('speak-sentence', { utteranceId: 'u1', index: 0, text: 'The tests pass.' }), + event('speak-sentence', { utteranceId: 'u1', index: 1, text: 'Nothing else changed.' }) + ); + } + + it('highlights the sentence being spoken, which is the newest one', () => { + renderTranscript(); + speakTwo(); + + const current = screen.getByTestId('voice-transcript-speaking'); + expect(current.textContent).toContain('Nothing else changed.'); + expect(current.getAttribute('aria-current')).toBe('true'); + }); + + it('highlights nothing once the run ends, because nothing is being spoken', () => { + renderTranscript(); + speakTwo(); + apply(event('speak-end', { utteranceId: 'u1', reason: 'complete' })); + + expect(screen.queryByTestId('voice-transcript-speaking')).toBeNull(); + expect(screen.getByTestId('voice-transcript-spoken').textContent).toContain('The tests pass.'); + }); + + it('marks a cancelled run as cut off rather than complete', () => { + renderTranscript(); + speakTwo(); + apply(event('speak-end', { utteranceId: 'u1', reason: 'cancelled' })); + expect(screen.getByTestId('voice-transcript-spoken').textContent).toContain('(cut off)'); + }); +}); + +describe('VoiceTranscript route chips', () => { + const dispatch = () => + event('dispatch', { + agentSessionId: 'agent-1', + agentName: 'Backend', + tabId: 'tab-7', + tabName: 'Auth Refactor', + action: 'created', + promptSent: true, + }); + + it('says where the turn went, including what it did to the tab', async () => { + renderTranscript(); + apply(dispatch()); + await flushLayout(); + + const chip = screen.getByTestId('voice-route-chip'); + expect(chip.textContent).toContain('Backend / Auth Refactor'); + expect(chip.textContent).toContain('new tab'); + }); + + it('jumps to the agent and tab when clicked', async () => { + const session = { id: 'agent-1', name: 'Backend', aiTabs: [{ id: 'tab-7' }] } as Session; + const setSessions = vi.fn((updater: (prev: Session[]) => Session[]) => { + updater([session]); + }); + const setActiveSessionId = vi.fn(); + useSessionStore.setState({ + sessions: [session], + setSessions, + setActiveSessionId, + } as never); + + renderTranscript(); + apply(dispatch()); + await flushLayout(); + + await act(async () => { + fireEvent.click(screen.getByTestId('voice-route-chip')); + }); + + expect(setActiveSessionId).toHaveBeenCalledWith('agent-1'); + expect(setSessions).toHaveBeenCalled(); + }); + + it('leaves the address on screen even when the agent is gone', async () => { + useSessionStore.setState({ sessions: [] } as never); + renderTranscript(); + apply(dispatch()); + await flushLayout(); + + // The chip still names where the turn went. Hiding it would erase the one + // record of a prompt that landed somewhere the user can no longer find. + expect(screen.getByTestId('voice-route-chip').textContent).toContain('Backend'); + }); + + it('puts a chip only on the line that narrates a dispatch', async () => { + renderTranscript(); + apply(event('final-transcript', { text: 'refactor auth', confidence: 1 }), dispatch()); + await flushLayout(); + + expect(screen.getAllByTestId('voice-route-chip')).toHaveLength(1); + expect(screen.getAllByTestId('voice-transcript-entry').length).toBeGreaterThan(1); + }); +}); diff --git a/src/__tests__/renderer/components/ACappella/useOwnsVoiceSession.test.tsx b/src/__tests__/renderer/components/ACappella/useOwnsVoiceSession.test.tsx new file mode 100644 index 0000000000..a0aa6c831d --- /dev/null +++ b/src/__tests__/renderer/components/ACappella/useOwnsVoiceSession.test.tsx @@ -0,0 +1,99 @@ +/** + * @file useOwnsVoiceSession.test.tsx + * + * One voice session, several windows. Every window receives the whole + * `acappella:event` stream (the multi-window broadcast invariant), so each one + * has to decide for itself whether the session is its own. Getting this wrong + * draws the same HUD in every window and makes one microphone look like several. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { renderHook } from '@testing-library/react'; + +// Controlled WindowContext: `null` is "no WindowProvider" (a single-window host +// or an isolation test), which must permit everything rather than hide the HUD. +let mockWindow: { windowId: string | null; isMainWindow: boolean } | null = null; +vi.mock('../../../../renderer/contexts/WindowContext', () => ({ + useWindowContextOptional: () => mockWindow, +})); + +let mockIsWebDesktop = false; +vi.mock('../../../../renderer/utils/runtimeContext', () => ({ + isWebDesktop: () => mockIsWebDesktop, +})); + +import { useOwnsVoiceSession } from '../../../../renderer/components/ACappella/useOwnsVoiceSession'; +import { useVoiceSessionStore } from '../../../../renderer/stores/voiceSessionStore'; + +/** Put the mirrored session in the state a window would see after a `wake`. */ +function sessionOwnedBy(windowId: string | null): void { + useVoiceSessionStore.setState({ windowId }); +} + +function owns(): boolean { + return renderHook(() => useOwnsVoiceSession()).result.current; +} + +describe('useOwnsVoiceSession', () => { + beforeEach(() => { + mockWindow = null; + mockIsWebDesktop = false; + useVoiceSessionStore.getState().reset(); + }); + + it('shows a session started in THIS window', () => { + mockWindow = { windowId: 'w2', isMainWindow: false }; + sessionOwnedBy('w2'); + + expect(owns()).toBe(true); + }); + + it('hides a session started in ANOTHER window', () => { + // The bug this hook exists for: opening voice in one window drew an + // identical HUD in every other one. + mockWindow = { windowId: 'w2', isMainWindow: false }; + sessionOwnedBy('w1'); + + expect(owns()).toBe(false); + }); + + it('hides another window’s session from the primary too', () => { + // The primary window is not a catch-all here. It is the fallback for a + // session that names NO window, not for one that names a different window. + mockWindow = { windowId: 'w1', isMainWindow: true }; + sessionOwnedBy('w2'); + + expect(owns()).toBe(false); + }); + + describe('a session that names no window', () => { + it('lands on the primary, so it has exactly one surface', () => { + mockWindow = { windowId: 'w1', isMainWindow: true }; + sessionOwnedBy(null); + + expect(owns()).toBe(true); + }); + + it('does not also land on a secondary window', () => { + mockWindow = { windowId: 'w2', isMainWindow: false }; + sessionOwnedBy(null); + + expect(owns()).toBe(false); + }); + }); + + it('permits everything on web-desktop, which is not one of several windows', () => { + mockIsWebDesktop = true; + mockWindow = { windowId: null, isMainWindow: true }; + sessionOwnedBy('w2'); + + expect(owns()).toBe(true); + }); + + it('permits everything with no WindowProvider, where no window can be the wrong one', () => { + mockWindow = null; + sessionOwnedBy('w2'); + + expect(owns()).toBe(true); + }); +}); diff --git a/src/__tests__/renderer/components/ACappella/useVoiceInputDevices.test.tsx b/src/__tests__/renderer/components/ACappella/useVoiceInputDevices.test.tsx new file mode 100644 index 0000000000..4104ae58d7 --- /dev/null +++ b/src/__tests__/renderer/components/ACappella/useVoiceInputDevices.test.tsx @@ -0,0 +1,97 @@ +/** + * @file useVoiceInputDevices.test.tsx + * + * The microphone picker's data source. The behaviour worth pinning is that the + * list is PUSHED as well as pulled: Chromium redacts device labels until a + * capture has been granted once, so the first read routinely returns nameless + * entries and the real names arrive later on the subscription. A picker built on + * a single read shows "Microphone 1 / Microphone 2" forever. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { renderHook, act, waitFor } from '@testing-library/react'; +import { + deviceLabel, + useVoiceInputDevices, +} from '../../../../renderer/components/ACappella/useVoiceInputDevices'; + +/** The pushed-update handler the hook registered, so a test can drive it. */ +function pushHandler(): (devices: Array<{ deviceId: string; label: string }>) => void { + const calls = vi.mocked(window.maestro.voice.onInputDevices).mock.calls; + return calls[calls.length - 1][0]; +} + +describe('useVoiceInputDevices', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(window.maestro.voice.onInputDevices).mockReturnValue(() => {}); + vi.mocked(window.maestro.voice.inputDevices).mockResolvedValue({ + devices: [{ deviceId: 'usb-mic', label: 'Yeti' }], + selectedId: 'system-default', + }); + vi.mocked(window.maestro.voice.setInputDevice).mockResolvedValue(true); + }); + + it('reads the devices and the current selection', async () => { + const { result } = renderHook(() => useVoiceInputDevices(true)); + + await waitFor(() => expect(result.current.loading).toBe(false)); + expect(result.current.devices).toEqual([{ deviceId: 'usb-mic', label: 'Yeti' }]); + expect(result.current.selectedId).toBe('system-default'); + }); + + it('reads nothing when the Encore Feature is off', async () => { + const { result } = renderHook(() => useVoiceInputDevices(false)); + + await waitFor(() => expect(result.current.loading).toBe(false)); + expect(window.maestro.voice.inputDevices).not.toHaveBeenCalled(); + }); + + it('takes a pushed list, which is how redacted labels are ever filled in', async () => { + const { result } = renderHook(() => useVoiceInputDevices(true)); + await waitFor(() => expect(result.current.loading).toBe(false)); + + act(() => { + pushHandler()([{ deviceId: 'usb-mic', label: 'Blue Yeti (USB)' }]); + }); + + expect(result.current.devices[0].label).toBe('Blue Yeti (USB)'); + }); + + it('persists a choice and shows it immediately', async () => { + const { result } = renderHook(() => useVoiceInputDevices(true)); + await waitFor(() => expect(result.current.loading).toBe(false)); + + await act(async () => { + await result.current.select('usb-mic'); + }); + + expect(window.maestro.voice.setInputDevice).toHaveBeenCalledWith('usb-mic'); + expect(result.current.selectedId).toBe('usb-mic'); + }); + + it('puts the selection back when the write fails', async () => { + // Otherwise the dropdown claims a microphone that was never saved, and the + // next session opens a different one than the UI is showing. + vi.mocked(window.maestro.voice.setInputDevice).mockRejectedValueOnce(new Error('nope')); + const { result } = renderHook(() => useVoiceInputDevices(true)); + await waitFor(() => expect(result.current.loading).toBe(false)); + + await act(async () => { + await result.current.select('usb-mic'); + }); + + expect(result.current.selectedId).toBe('system-default'); + }); + + describe('deviceLabel', () => { + it('uses the OS label when there is one', () => { + expect(deviceLabel({ deviceId: 'x', label: 'Yeti' }, 0)).toBe('Yeti'); + }); + + it('never renders an unclickable blank row for a redacted label', () => { + expect(deviceLabel({ deviceId: 'abc', label: '' }, 1)).toBe('Microphone 2'); + expect(deviceLabel({ deviceId: 'default', label: '' }, 0)).toBe('System default'); + }); + }); +}); diff --git a/src/__tests__/renderer/components/InputArea/components/NotificationSendControls.test.tsx b/src/__tests__/renderer/components/InputArea/components/NotificationSendControls.test.tsx index a318d8fed6..d5ef514597 100644 --- a/src/__tests__/renderer/components/InputArea/components/NotificationSendControls.test.tsx +++ b/src/__tests__/renderer/components/InputArea/components/NotificationSendControls.test.tsx @@ -38,4 +38,56 @@ describe('NotificationSendControls', () => { expect(processInput).toHaveBeenCalled(); }); + + describe('the A Cappella microphone', () => { + it('is absent unless A Cappella owns the composer microphone', () => { + // The default. With the Encore Feature off the button stays the Web Speech + // one in the toolbar row, so this column must not draw a second mic. + render( + + ); + + expect(screen.queryByTestId('composer-voice-button')).toBeNull(); + }); + + it('starts a voice session when clicked', () => { + const onToggleVoice = vi.fn(); + render( + + ); + + fireEvent.click(screen.getByTestId('composer-voice-button')); + + expect(onToggleVoice).toHaveBeenCalledTimes(1); + }); + + it('offers to END the session while the floor is open', () => { + // The one button here that can leave a microphone running, so it must not + // read the same open as shut. + render( + + ); + + const button = screen.getByTestId('composer-voice-button'); + expect(button.getAttribute('aria-pressed')).toBe('true'); + expect(button.getAttribute('aria-label')).toBe('End the voice session'); + }); + }); }); diff --git a/src/__tests__/renderer/components/InputArea/components/ToolbarControls.test.tsx b/src/__tests__/renderer/components/InputArea/components/ToolbarControls.test.tsx index f44fa8e589..cc83f81548 100644 --- a/src/__tests__/renderer/components/InputArea/components/ToolbarControls.test.tsx +++ b/src/__tests__/renderer/components/InputArea/components/ToolbarControls.test.tsx @@ -178,6 +178,20 @@ describe('ToolbarControls', () => { expect(screen.queryByRole('button', { name: /voice input/ })).not.toBeInTheDocument(); }); + it('hides the mic when A Cappella owns it, so there is never a second one', () => { + // A Cappella's microphone lives under Send, where it shows on every + // pointer type. Without this gate a touch device with the Encore Feature + // on would draw two microphones wired to the same toggle. + setCoarsePointer(true); + renderToolbar({ + voiceSupported: true, + onToggleVoiceInput: vi.fn(), + voiceHandledElsewhere: true, + }); + + expect(screen.queryByRole('button', { name: /voice input/ })).not.toBeInTheDocument(); + }); + it('hides the mic in terminal mode', () => { setCoarsePointer(true); renderToolbar({ @@ -190,4 +204,16 @@ describe('ToolbarControls', () => { expect(screen.queryByRole('button', { name: /voice input/ })).not.toBeInTheDocument(); }); }); + + describe('layout', () => { + it('pins the pill row to the bottom of the composer box', () => { + // The A Cappella microphone makes the Send column taller than the + // textarea, so the composer box stretches and the pills would otherwise + // float mid-height with dead space beneath them. jsdom has no layout + // engine, so the margin class is the only thing there is to assert. + const { container } = renderToolbar(); + + expect(container.firstElementChild).toHaveClass('mt-auto'); + }); + }); }); diff --git a/src/__tests__/renderer/components/MainPanel/MainPanelHeader.test.tsx b/src/__tests__/renderer/components/MainPanel/MainPanelHeader.test.tsx index 03d33489e2..5119d31213 100644 --- a/src/__tests__/renderer/components/MainPanel/MainPanelHeader.test.tsx +++ b/src/__tests__/renderer/components/MainPanel/MainPanelHeader.test.tsx @@ -18,8 +18,13 @@ vi.mock('../../../../renderer/stores/settingsStore', () => ({ showAgentName: true, showSessionIdPill: true, showSessionCostPill: true, + encoreFeatures: {}, }) ), + // ON, so the "no microphone in the header" test below is not vacuous: with the + // flag off, an assertion that no voice surface renders would pass even if the + // header still had one. + selectACappellaEnabled: () => true, })); // Mutable UI state + stable setters so tests can drive the sidebar opener. @@ -768,4 +773,14 @@ describe('MainPanelHeader', () => { // Just verify the header renders without errors expect(screen.getByText('main')).toBeInTheDocument(); }); + + it('has no voice microphone, even with A Cappella switched on', () => { + // The composer's Send column owns that button. A second microphone here was + // the same action twice on one screen, in the busiest row in the app. + // `selectACappellaEnabled` is mocked ON above, so this fails if the header + // ever grows one back. + render(); + + expect(screen.queryByTestId('header-voice-pill')).not.toBeInTheDocument(); + }); }); diff --git a/src/__tests__/renderer/components/QuickActionsModal/commands/voiceCommands.test.ts b/src/__tests__/renderer/components/QuickActionsModal/commands/voiceCommands.test.ts new file mode 100644 index 0000000000..c1162c901e --- /dev/null +++ b/src/__tests__/renderer/components/QuickActionsModal/commands/voiceCommands.test.ts @@ -0,0 +1,132 @@ +/** + * The command palette is how a feature gets found by someone who does not know + * its hotkey, so A Cappella has to be reachable from it - and reachable by the + * word a user would actually type, which is "voice", not "A Cappella". + * + * The load-bearing case is the Encore gate: an install that never turned the + * feature on must have no voice entries at all, rather than entries that fail. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { buildVoiceCommands } from '../../../../../renderer/components/QuickActionsModal/commands/voiceCommands'; +import { filterAndSortQuickActions } from '../../../../../renderer/components/QuickActionsModal/utils/quickActionSorting'; +import type { VoiceAgentActions } from '../../../../../renderer/hooks/voice/useVoiceAgentActions'; +import type { Session } from '../../../../../renderer/types'; + +function voiceActions(overrides: Partial = {}): VoiceAgentActions { + return { + enabled: true, + hasVoiceFloor: false, + isSpeaking: false, + wakePhrase: null, + talkToAgent: vi.fn().mockResolvedValue(undefined), + talkToConductor: vi.fn().mockResolvedValue(undefined), + endVoiceSession: vi.fn().mockResolvedValue(undefined), + showHud: vi.fn(), + hudHidden: false, + ...overrides, + }; +} + +function harness( + overrides: Partial = {}, + // `noActiveSession` rather than an optional session: an `activeSession?` field + // cannot express "deliberately none" once the harness supplies a default. + options: { noActiveSession?: boolean } = {} +) { + const actions = voiceActions(overrides); + const setQuickActionOpen = vi.fn(); + const toggleTranscript = vi.fn().mockResolvedValue(undefined); + const commands = buildVoiceCommands({ + activeSession: options.noActiveSession ? undefined : ({ id: 'a1', name: 'Backend' } as Session), + voiceActions: actions, + transcriptVisible: false, + toggleTranscript, + setQuickActionOpen, + }); + return { commands, actions, setQuickActionOpen, toggleTranscript }; +} + +const idsOf = (commands: ReturnType['commands']) => commands.map((c) => c.id); + +describe('buildVoiceCommands', () => { + it('offers nothing at all when the Encore Feature is off', () => { + expect(harness({ enabled: false }).commands).toEqual([]); + }); + + it('offers the conductor and the active agent', () => { + expect(idsOf(harness().commands)).toEqual( + expect.arrayContaining(['voiceTalkToAgent', 'voiceTalkToConductor']) + ); + }); + + it('drops the per-agent entry when no agent is active', () => { + const { commands } = harness({}, { noActiveSession: true }); + expect(idsOf(commands)).not.toContain('voiceTalkToAgent'); + expect(idsOf(commands)).toContain('voiceTalkToConductor'); + }); + + it('starts the conductor through the shared action, which un-hides the HUD', () => { + // Not `window.maestro.voice.start()` directly: that was how asking for the + // conductor from the palette opened a microphone behind a minimized HUD. + const { commands, actions, setQuickActionOpen } = harness(); + commands.find((c) => c.id === 'voiceTalkToConductor')!.action(); + + expect(actions.talkToConductor).toHaveBeenCalledOnce(); + expect(setQuickActionOpen).toHaveBeenCalledWith(false); + }); + + it('teaches the wake phrase on the agent entry when there is one', () => { + const { commands } = harness({ wakePhrase: 'hey backend' }); + const talk = commands.find((c) => c.id === 'voiceTalkToAgent'); + expect(talk?.subtext).toContain('hey backend'); + }); + + it('hides the restore entry while the HUD is on screen', () => { + expect(idsOf(harness().commands)).not.toContain('voiceShowHud'); + }); + + it('offers to restore a minimized HUD, and does it', () => { + const { commands, actions, setQuickActionOpen } = harness({ hudHidden: true }); + const show = commands.find((c) => c.id === 'voiceShowHud'); + + expect(show).toBeTruthy(); + show!.action(); + expect(actions.showHud).toHaveBeenCalledOnce(); + expect(setQuickActionOpen).toHaveBeenCalledWith(false); + }); + + it('offers to end a session only while one is running', () => { + expect(idsOf(harness().commands)).not.toContain('voiceEndSession'); + expect(idsOf(harness({ hasVoiceFloor: true }).commands)).toContain('voiceEndSession'); + }); + + // Through the real filter, not an approximation of it: palette search reads + // LABELS, so "Talk to Backend" was invisible to someone typing "voice" - the + // palette's only voice hit was the transcript toggle, which cannot start + // anything. These pin the entries to the words a user actually types. + describe('discoverability', () => { + const everyCommand = () => harness({ hudHidden: true, hasVoiceFloor: true }).commands; + const search = (term: string) => + filterAndSortQuickActions(everyCommand(), term, 'main').map((c) => c.id); + + it.each(['voice', 'acappella', 'a cappella', 'mic'])( + 'surfaces a way to START talking when searching "%s"', + (term) => { + // Not just "returns something": an entry that only toggles a transcript + // is exactly the dead end this replaced. + expect(search(term)).toEqual( + expect.arrayContaining(['voiceTalkToAgent', 'voiceTalkToConductor']) + ); + } + ); + + it('still matches on the label itself', () => { + expect(search('conductor')).toEqual(['voiceTalkToConductor']); + }); + + it('does not match an unrelated search', () => { + expect(search('worktree')).toEqual([]); + }); + }); +}); diff --git a/src/__tests__/renderer/components/SessionList/AgentVoiceIndicator.test.tsx b/src/__tests__/renderer/components/SessionList/AgentVoiceIndicator.test.tsx new file mode 100644 index 0000000000..75d5436d98 --- /dev/null +++ b/src/__tests__/renderer/components/SessionList/AgentVoiceIndicator.test.tsx @@ -0,0 +1,170 @@ +/** + * AgentVoiceIndicator - the Left Bar's voice glyphs. + * + * The invariant worth protecting: these COMPOSE with the status dot rather than + * replacing it. An agent that is busy and holding the voice floor has to still + * read as busy, because "is it working" is the question the colour answers and + * the voice glyph answers a different one. + */ + +import React from 'react'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, cleanup, act } from '@testing-library/react'; +import { AgentVoiceIndicator } from '../../../../renderer/components/SessionList/AgentVoiceIndicator'; +import { getEnhancedStatusColor } from '../../../../renderer/components/SessionItem'; +import { useVoiceSessionStore } from '../../../../renderer/stores/voiceSessionStore'; +import { useVoiceUiStore } from '../../../../renderer/stores/voiceUiStore'; +import { useSettingsStore } from '../../../../renderer/stores/settingsStore'; +import { mockTheme } from '../../../helpers/mockTheme'; +import type { VoiceEvent } from '../../../../shared/acappella/protocol'; +import type { Session } from '../../../../renderer/types'; + +const AGENT = 'agent-1'; +const VOICE_SESSION = 'voice-1'; +let seq = 0; + +function event( + type: T, + body: Omit, 'type' | 'sessionId' | 'seq' | 'ts'> +): VoiceEvent { + seq += 1; + return { + type, + sessionId: VOICE_SESSION, + seq, + ts: 1_700_000_000_000 + seq, + ...body, + } as unknown as VoiceEvent; +} + +function apply(...events: VoiceEvent[]): void { + act(() => { + for (const e of events) useVoiceSessionStore.getState().applyEvent(e); + }); +} + +function renderIndicator(sessionId = AGENT) { + return render(); +} + +/** Open a session bound to this agent. */ +function holdFloor(sessionId = AGENT): void { + apply( + event('wake', { source: 'hotkey', scope: { kind: 'agent', sessionId } }), + event('listen-start', { + scope: { kind: 'agent', sessionId }, + sttProviderId: 'mock-stt', + }) + ); +} + +beforeEach(() => { + seq = 0; + vi.clearAllMocks(); + useVoiceSessionStore.getState().reset(); + useVoiceUiStore.setState({ wakePhrases: {}, loaded: true }); + useSettingsStore.setState({ encoreFeatures: { aCappella: true } } as never); +}); + +afterEach(() => { + cleanup(); +}); + +describe('AgentVoiceIndicator gating', () => { + it('renders nothing when the Encore Feature is off', () => { + useSettingsStore.setState({ encoreFeatures: { aCappella: false } } as never); + renderIndicator(); + holdFloor(); + expect(screen.queryByTestId('agent-voice-floor')).toBeNull(); + }); + + it('renders nothing for an agent with no voice and no phrase', () => { + const { container } = renderIndicator(); + expect(container.textContent).toBe(''); + expect(screen.queryByTestId('agent-voice-floor')).toBeNull(); + expect(screen.queryByTestId('agent-voice-wake-phrase')).toBeNull(); + }); +}); + +describe('AgentVoiceIndicator states', () => { + it('marks the agent holding the voice floor', () => { + renderIndicator(); + holdFloor(); + expect(screen.getByTestId('agent-voice-floor')).toBeTruthy(); + }); + + it('does not mark a different agent', () => { + renderIndicator('agent-2'); + holdFloor(AGENT); + expect(screen.queryByTestId('agent-voice-floor')).toBeNull(); + }); + + it('marks the agent whose reply is being spoken, even without the floor', () => { + // A Conductor session speaks replies from whichever agent it routed to. + // That agent never holds the floor, and it is still the one talking. + renderIndicator(); + apply( + event('wake', { source: 'wake-word', scope: { kind: 'conductor' } }), + event('listen-start', { scope: { kind: 'conductor' }, sttProviderId: 'mock-stt' }), + event('dispatch', { + agentSessionId: AGENT, + agentName: 'Backend', + tabId: 'tab-1', + action: 'focused', + promptSent: true, + }), + event('speak-start', { utteranceId: 'u1', sentenceCount: 1, ttsProviderId: 'mock-tts' }) + ); + + expect(screen.getByTestId('agent-voice-speaking')).toBeTruthy(); + expect(screen.queryByTestId('agent-voice-floor')).toBeNull(); + }); + + it('badges an agent that has a wake phrase, so the mapping is discoverable', () => { + useVoiceUiStore.setState({ wakePhrases: { [AGENT]: 'hey backend' }, loaded: true }); + renderIndicator(); + expect(screen.getByTestId('agent-voice-wake-phrase').getAttribute('aria-label')).toContain( + 'hey backend' + ); + }); +}); + +describe('AgentVoiceIndicator composes with the status colours', () => { + const busy = { id: AGENT, state: 'busy', toolType: 'codex' } as unknown as Session; + + it('leaves the busy colour alone while the agent holds the floor', () => { + // The voice glyph is additive. Nothing about it touches the status dot's + // colour, its animation, or its label - which is the whole point of + // rendering a separate element rather than recolouring the dot. + const before = getEnhancedStatusColor(busy, mockTheme, false); + renderIndicator(); + holdFloor(); + const after = getEnhancedStatusColor(busy, mockTheme, false); + + expect(after).toEqual(before); + expect(after.color).toBe(mockTheme.colors.warning); + expect(after.label).toBe('Thinking'); + expect(screen.getByTestId('agent-voice-floor')).toBeTruthy(); + }); + + it('leaves an error colour alone while the agent is being spoken', () => { + const errored = { id: AGENT, state: 'error', toolType: 'codex' } as unknown as Session; + renderIndicator(); + apply( + event('wake', { source: 'wake-word', scope: { kind: 'conductor' } }), + event('listen-start', { scope: { kind: 'conductor' }, sttProviderId: 'mock-stt' }), + event('dispatch', { + agentSessionId: AGENT, + agentName: 'Backend', + tabId: 'tab-1', + action: 'focused', + promptSent: true, + }), + event('speak-start', { utteranceId: 'u1', sentenceCount: 1, ttsProviderId: 'mock-tts' }) + ); + + const status = getEnhancedStatusColor(errored, mockTheme, false); + expect(status.color).toBe(mockTheme.colors.error); + expect(screen.getByTestId('agent-voice-speaking')).toBeTruthy(); + }); +}); diff --git a/src/__tests__/renderer/components/Settings/ACappella/VoiceControlsPanel.test.tsx b/src/__tests__/renderer/components/Settings/ACappella/VoiceControlsPanel.test.tsx new file mode 100644 index 0000000000..7eb0b35d50 --- /dev/null +++ b/src/__tests__/renderer/components/Settings/ACappella/VoiceControlsPanel.test.tsx @@ -0,0 +1,117 @@ +/** + * @file VoiceControlsPanel.test.tsx + * + * The panel's defining property for the hotkey rows: **what they show is what is + * actually bound.** + * + * The failure this suite exists to prevent was real and silent. Main registers a + * voice hotkey from `defaultGlobalHotkeyKeys(id)` whenever the stored shortcuts + * map has no entry, which is the ordinary state of any profile that existed + * before these hotkeys did. The panel read only that map, so it told those users + * "Click to set" and "Registered as (none)" about a combo that was live, working, + * and holding a system-wide accelerator. A settings row that disagrees with the + * registry is worse than no row: it invites the user to bind a second combo for a + * hotkey they already have. + * + * The other half of the same rule is that an explicitly CLEARED binding still + * reads as cleared, which is why the fallback is nullish rather than truthy: an + * entry with an empty key array is a decision, not an absence. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; + +import { VoiceControlsPanel } from '../../../../../renderer/components/Settings/ACappella/VoiceControlsPanel'; +import type { Shortcut } from '../../../../../renderer/types'; +import { mockTheme } from '../../../../helpers/mockTheme'; + +const voice = () => window.maestro.voice; + +/** The stored `shortcuts` map, swapped per test before render. */ +let mockShortcuts: Record = {}; +const mockSetShortcuts = vi.fn(); + +vi.mock('../../../../../renderer/hooks/settings/useSettings', () => ({ + useSettings: () => ({ + shortcuts: mockShortcuts, + setShortcuts: mockSetShortcuts, + tabShortcuts: {}, + setTabShortcuts: vi.fn(), + }), +})); + +/** What main reports for a hotkey it has actually bound. */ +function registered(id: string, keys: string[], accelerator: string) { + return { id, keys, accelerator, registered: true }; +} + +describe('VoiceControlsPanel hotkey rows', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockShortcuts = {}; + vi.mocked(window.maestro.settings.get).mockResolvedValue({}); + vi.mocked(voice().getRoster).mockResolvedValue([]); + vi.mocked(voice().hotkeyStatus).mockResolvedValue({ + statuses: [ + registered('voiceConductor', ['Meta', 'Alt', 'v'], 'Command+Alt+V'), + registered('voiceCurrentAgent', ['Meta', 'Alt', 'a'], 'Command+Alt+A'), + ], + note: 'Tap to toggle the microphone.', + }); + }); + + it('shows the combo main actually registered when the stored map has no entry', async () => { + render(); + + // Two rows, both bound, and neither of them inviting the user to set a + // hotkey that is already holding a system-wide accelerator. + await waitFor(() => { + expect(screen.queryAllByText('(none)')).toHaveLength(0); + }); + expect(screen.queryAllByText('Click to set')).toHaveLength(0); + }); + + it('still reads as unset when the user has explicitly cleared the binding', async () => { + mockShortcuts = { + voiceConductor: { id: 'voiceConductor', label: 'Talk to Maestro', keys: [] }, + }; + vi.mocked(voice().hotkeyStatus).mockResolvedValue({ + statuses: [ + { + id: 'voiceConductor', + keys: [], + accelerator: null, + registered: false, + reason: 'invalid-accelerator' as const, + message: 'No key is bound.', + }, + registered('voiceCurrentAgent', ['Meta', 'Alt', 'a'], 'Command+Alt+A'), + ], + note: 'Tap to toggle the microphone.', + }); + + render(); + + // Exactly one row is empty: the cleared one. An empty array is a decision, + // so it must not fall through to the shipped default. + await waitFor(() => { + expect(screen.getAllByText('Click to set')).toHaveLength(1); + }); + }); + + it('prefers a user rebinding over both the registry and the default', async () => { + mockShortcuts = { + voiceConductor: { + id: 'voiceConductor', + label: 'Talk to Maestro', + keys: ['Meta', 'Alt', 'j'], + }, + }; + + render(); + + await waitFor(() => { + expect(screen.getAllByText(/J$/).length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/__tests__/renderer/components/Settings/ACappella/VoiceDiagnosticsCard.test.tsx b/src/__tests__/renderer/components/Settings/ACappella/VoiceDiagnosticsCard.test.tsx new file mode 100644 index 0000000000..53bbea3eef --- /dev/null +++ b/src/__tests__/renderer/components/Settings/ACappella/VoiceDiagnosticsCard.test.tsx @@ -0,0 +1,84 @@ +/** + * @file VoiceDiagnosticsCard.test.tsx + * + * The panel someone opens after voice did nothing. It is only worth having if it + * distinguishes the failures that look identical from the outside: a microphone + * producing no signal, a microphone producing signal nobody classifies as + * speech, and a recogniser that was never going to transcribe anything. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { render, screen, cleanup } from '@testing-library/react'; +import { VoiceDiagnosticsCard } from '../../../../../renderer/components/Settings/ACappella/VoiceDiagnosticsCard'; +import { useVoiceDiagnosticsStore } from '../../../../../renderer/stores/voiceDiagnosticsStore'; +import { useVoiceSessionStore } from '../../../../../renderer/stores/voiceSessionStore'; +import { mockTheme } from '../../../../helpers/mockTheme'; + +function renderCard() { + return render(); +} + +beforeEach(() => { + vi.clearAllMocks(); + useVoiceDiagnosticsStore.getState().clear(); + useVoiceSessionStore.getState().reset(); +}); + +afterEach(() => cleanup()); + +describe('VoiceDiagnosticsCard', () => { + it('says plainly when no audio has arrived', () => { + renderCard(); + + expect(screen.getByText(/No audio frames yet/)).toBeTruthy(); + }); + + it('reports frames, peak and speech count once audio flows', () => { + useVoiceDiagnosticsStore.setState({ + audioLevelCount: 71, + audioLevelPeak: 0.42, + speechFrames: 12, + }); + + renderCard(); + + expect(screen.getByText(/71 frames/)).toBeTruthy(); + expect(screen.getByText(/12 classified as speech/)).toBeTruthy(); + }); + + it('names a recogniser that does not listen to the microphone', () => { + // The case that cost six rebuilds: everything else looks healthy. + useVoiceSessionStore.setState({ + providerIds: { stt: 'mock-stt', tts: 'mock-tts', brain: 'mock-brain' }, + sttHearsAudio: false, + }); + + renderCard(); + + expect(screen.getByText(/does not listen to the microphone/)).toBeTruthy(); + }); + + it('renders the recorded event log', () => { + useVoiceDiagnosticsStore.getState().record({ + type: 'session-error', + sessionId: 'voice-1', + seq: 1, + ts: 1_700_000_000_000, + code: 'provider-unavailable', + message: 'whisper.cpp is not part of this build yet.', + recoverable: false, + } as never); + + renderCard(); + + expect(screen.getByTestId('voice-diagnostics-log').textContent).toContain('whisper.cpp'); + }); + + it('offers nothing to copy before anything has been recorded', () => { + // A copy button that yields an empty report is a support thread with no + // information in it. + renderCard(); + + expect(screen.getByText('Copy diagnostics')).toHaveProperty('disabled', true); + }); +}); diff --git a/src/__tests__/renderer/components/Settings/ACappella/VoiceOutputPanel.test.tsx b/src/__tests__/renderer/components/Settings/ACappella/VoiceOutputPanel.test.tsx new file mode 100644 index 0000000000..d325023bd0 --- /dev/null +++ b/src/__tests__/renderer/components/Settings/ACappella/VoiceOutputPanel.test.tsx @@ -0,0 +1,132 @@ +/** + * @file VoiceOutputPanel.test.tsx + * + * Voice and Speed. The defining property is that everything here applies to the + * NEXT SPOKEN SENTENCE rather than the next session, so the tests assert on the + * calls that make that true: the volume is pushed at the live audio host as well + * as saved, and a Preview can audition a voice that has not been selected. + * + * The second property is the audio-destination statement, repeated here from + * Voice Providers and computed from the live TTS slot rather than written. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; + +import { VoiceOutputPanel } from '../../../../../renderer/components/Settings/ACappella/VoiceOutputPanel'; +import { useVoiceUiStore } from '../../../../../renderer/stores/voiceUiStore'; +import { mockTheme } from '../../../../helpers/mockTheme'; + +const voice = () => window.maestro.voice; + +const PREVIEW_LINE = 'Backend agent finished the migration and all tests pass.'; + +describe('VoiceOutputPanel', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(window.maestro.settings.get).mockResolvedValue({}); + vi.mocked(voice().listVoices).mockResolvedValue([ + { id: 'af_heart', name: 'Heart' }, + { id: 'am_puck', name: 'Puck' }, + ]); + useVoiceUiStore.setState({ + transcriptVisible: false, + hudPosition: null, + minimizeBehavior: 'manual', + minimized: false, + muted: false, + loaded: true, + }); + }); + + it('says where audio goes under the current text-to-speech provider', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue({ + providers: { tts: 'elevenlabs-tts' }, + }); + + render(); + + expect(await screen.findByText(/ElevenLabs/)).toBeInTheDocument(); + }); + + it('auditions a voice that has not been selected', async () => { + render(); + + fireEvent.click(await screen.findByTestId('voice-preview-am_puck')); + + // The voice id is passed explicitly. Without it, hearing a voice would mean + // selecting it first and undoing the ones you did not want. + await waitFor(() => expect(voice().previewVoice).toHaveBeenCalledWith(PREVIEW_LINE, 'am_puck')); + }); + + it('previews one fixed line so two voices can be compared on the same words', async () => { + render(); + + fireEvent.click(await screen.findByTestId('voice-preview-current')); + + await waitFor(() => expect(voice().previewVoice).toHaveBeenCalledWith(PREVIEW_LINE, undefined)); + }); + + it('applies a volume change to the live output as well as saving it', async () => { + render(); + + const slider = await screen.findByLabelText('Volume'); + fireEvent.change(slider, { target: { value: '0.5' } }); + + await waitFor(() => expect(voice().setVolume).toHaveBeenCalledWith(0.5)); + expect(window.maestro.settings.set).toHaveBeenCalledWith( + 'acappella', + expect.objectContaining({ voice: expect.objectContaining({ volume: 0.5 }) }) + ); + }); + + it('saves the speed without needing a session restart', async () => { + render(); + + fireEvent.change(await screen.findByLabelText('Speed'), { target: { value: '1.2' } }); + + await waitFor(() => + expect(window.maestro.settings.set).toHaveBeenCalledWith( + 'acappella', + expect.objectContaining({ voice: expect.objectContaining({ rate: 1.2 }) }) + ) + ); + }); + + it('persists the transcript toggle', async () => { + render(); + + fireEvent.click(await screen.findByLabelText('Live transcript')); + + await waitFor(() => expect(useVoiceUiStore.getState().transcriptVisible).toBe(true)); + expect(window.maestro.settings.set).toHaveBeenCalledWith( + 'acappella', + expect.objectContaining({ ui: expect.objectContaining({ transcriptVisible: true }) }) + ); + }); + + it('puts the HUD back in its default corner', async () => { + useVoiceUiStore.setState({ hudPosition: { top: 12, left: 34 }, loaded: true }); + render(); + + expect(await screen.findByText(/Currently at 34, 12/)).toBeInTheDocument(); + fireEvent.click(screen.getByTestId('voice-hud-reset-position')); + + await waitFor(() => expect(useVoiceUiStore.getState().hudPosition).toBeNull()); + expect(screen.getByText(/default, bottom right/)).toBeInTheDocument(); + }); + + it('says plainly that minimize and close are different actions', async () => { + render(); + expect( + await screen.findByText(/Minimizing collapses the HUD .* Closing it ends the session\./) + ).toBeInTheDocument(); + }); + + it('offers nothing to choose between for an engine with one voice', async () => { + vi.mocked(voice().listVoices).mockResolvedValue([]); + render(); + + expect(await screen.findByText(/This engine has one voice/)).toBeInTheDocument(); + }); +}); diff --git a/src/__tests__/renderer/components/Settings/ACappella/VoiceProvidersPanel.test.tsx b/src/__tests__/renderer/components/Settings/ACappella/VoiceProvidersPanel.test.tsx new file mode 100644 index 0000000000..5954759008 --- /dev/null +++ b/src/__tests__/renderer/components/Settings/ACappella/VoiceProvidersPanel.test.tsx @@ -0,0 +1,207 @@ +/** + * @file VoiceProvidersPanel.test.tsx + * + * The panel's defining property: **the sentence about where audio goes is + * computed from the current selection and is always visible.** + * + * That line is the one fact a person configuring a voice assistant actually + * needs, and the failure mode this suite exists to prevent is it being written as + * copy that drifts from the engines that are really running. So the tests change + * a slot and assert the sentence changes with it. + * + * The second property, checked here rather than trusted: a stored API key is + * never read back into the renderer. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; + +import { VoiceProvidersPanel } from '../../../../../renderer/components/Settings/ACappella/VoiceProvidersPanel'; +import { mockTheme } from '../../../../helpers/mockTheme'; + +const voice = () => window.maestro.voice; + +/** The stored `acappella` blob, as the settings store would return it. */ +function storedBlob(providers: Record, extra: Record = {}) { + return { providers, ...extra }; +} + +describe('VoiceProvidersPanel', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(window.maestro.settings.get).mockResolvedValue({}); + vi.mocked(voice().models.list).mockResolvedValue([]); + vi.mocked(voice().models.footprint).mockResolvedValue({ bytes: 0, models: [] }); + vi.mocked(voice().models.readiness).mockResolvedValue({ + canStartSession: true, + canRunHandsFree: true, + slots: [], + blocking: [], + }); + vi.mocked(voice().credentials.list).mockResolvedValue([ + { service: 'openai', label: 'OpenAI', configured: false, keyringAvailable: true }, + { service: 'elevenlabs', label: 'ElevenLabs', configured: false, keyringAvailable: true }, + { service: 'anthropic', label: 'Anthropic', configured: false, keyringAvailable: true }, + ]); + }); + + it('says audio stays on this machine for a local configuration', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue( + storedBlob({ stt: 'whisper-local', tts: 'kokoro-local', brain: 'qwen3-local' }) + ); + + render(); + + expect(await screen.findByText('Audio stays on this machine.')).toBeInTheDocument(); + }); + + it('names the service the moment the recogniser is hosted', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue( + storedBlob({ stt: 'openai-stt', tts: 'kokoro-local', brain: 'qwen3-local' }) + ); + + render(); + + expect(await screen.findByText('Audio is sent to OpenAI.')).toBeInTheDocument(); + }); + + it('distinguishes text leaving from audio leaving', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue( + storedBlob({ stt: 'whisper-local', tts: 'elevenlabs-tts', brain: 'anthropic-brain' }) + ); + + render(); + + // The microphone samples never leave, and saying otherwise would be as wrong + // as hiding that the transcripts do. + expect( + await screen.findByText( + 'Audio stays on this machine. Text is sent to ElevenLabs and Anthropic.' + ) + ).toBeInTheDocument(); + }); + + it('updates the statement when a slot changes', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue( + storedBlob({ stt: 'whisper-local', tts: 'kokoro-local', brain: 'qwen3-local' }) + ); + + render(); + await screen.findByText('Audio stays on this machine.'); + + fireEvent.change(screen.getByLabelText('Speech-to-Text provider'), { + target: { value: 'openai-stt' }, + }); + + expect(await screen.findByText('Audio is sent to OpenAI.')).toBeInTheDocument(); + }); + + it('persists a slot change and applies it to the running app', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue(storedBlob({})); + + render(); + await screen.findByLabelText('Conductor Brain provider'); + + fireEvent.change(screen.getByLabelText('Conductor Brain provider'), { + target: { value: 'anthropic-brain' }, + }); + + await waitFor(() => { + expect(window.maestro.settings.set).toHaveBeenCalledWith( + 'acappella', + expect.objectContaining({ + providers: expect.objectContaining({ brain: 'anthropic-brain' }), + }) + ); + }); + // Without this the change would not take effect until the next app start. + await waitFor(() => expect(voice().applyProviders).toHaveBeenCalled()); + }); + + it('shows the capability gate verdict for an unsatisfied slot, with its recovery', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue(storedBlob({ stt: 'whisper-local' })); + vi.mocked(voice().models.readiness).mockResolvedValue({ + canStartSession: false, + canRunHandsFree: false, + slots: [ + { + slot: 'stt', + providerId: 'whisper-local', + satisfied: false, + reason: 'model-not-installed', + requiredModelId: 'whisper-base-en', + detail: 'Speech-to-Text: Whisper Base (English) is not installed.', + suggestedAction: 'Download it in Settings.', + }, + ], + blocking: [], + }); + + render(); + + expect( + await screen.findByText('Speech-to-Text: Whisper Base (English) is not installed.') + ).toBeInTheDocument(); + // A link to the fix, not just a complaint. + fireEvent.click(screen.getByText('Download the model')); + expect(voice().models.download).toHaveBeenCalledWith('whisper-base-en'); + }); + + it('offers a masked key field that never shows a stored key', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue(storedBlob({ stt: 'openai-stt' })); + vi.mocked(voice().credentials.list).mockResolvedValue([ + { service: 'openai', label: 'OpenAI', configured: true, keyringAvailable: true }, + ]); + + render(); + + const input = (await screen.findByPlaceholderText( + 'A key is stored. Type a new one to replace it.' + )) as HTMLInputElement; + expect(input.type).toBe('password'); + // The value is not fetched at all: nothing in the renderer needs it, and a + // channel that returned one would put it in a heap and in any crash dump. + expect(input.value).toBe(''); + }); + + it('tests a typed key without storing it', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue(storedBlob({ stt: 'openai-stt' })); + vi.mocked(voice().credentials.validate).mockResolvedValue({ + service: 'openai', + status: 'rate-limited', + message: 'OpenAI is rate limiting this key right now.', + }); + + render(); + + fireEvent.change(await screen.findByPlaceholderText('Paste your key'), { + target: { value: 'sk-typed-but-not-saved' }, + }); + fireEvent.click(screen.getByText('Test')); + + expect( + await screen.findByText('OpenAI is rate limiting this key right now.') + ).toBeInTheDocument(); + expect(voice().credentials.validate).toHaveBeenCalledWith('openai', 'sk-typed-but-not-saved'); + expect(voice().credentials.set).not.toHaveBeenCalled(); + }); + + it('states the realtime tradeoff where the choice is made', async () => { + render(); + + expect( + await screen.findByText(/Realtime is the lowest latency, but it speaks in that provider/) + ).toBeInTheDocument(); + }); + + it('replaces the three slots with one provider in realtime mode', async () => { + vi.mocked(window.maestro.settings.get).mockResolvedValue( + storedBlob({}, { pipeline: 'realtime' }) + ); + + render(); + + expect(await screen.findByText('Audio is sent to OpenAI.')).toBeInTheDocument(); + expect(screen.queryByLabelText('Speech-to-Text provider')).not.toBeInTheDocument(); + }); +}); diff --git a/src/__tests__/renderer/components/Settings/ACappella/VoiceSetupPanel.test.tsx b/src/__tests__/renderer/components/Settings/ACappella/VoiceSetupPanel.test.tsx new file mode 100644 index 0000000000..0f447b5bc8 --- /dev/null +++ b/src/__tests__/renderer/components/Settings/ACappella/VoiceSetupPanel.test.tsx @@ -0,0 +1,171 @@ +/** + * @file VoiceSetupPanel.test.tsx + * + * The panel's defining property: **opening it downloads nothing.** + * + * A Cappella asks the user for up to 1.4 GB of disk and bandwidth, and the whole + * consent story rests on the bill of materials being visible BEFORE anything is + * fetched. So this suite asserts that mounting the panel issues zero network + * calls (no `fetch`, no download channel), that it still renders the full + * catalog detail, and that the Download button is the only thing that starts a + * transfer. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; + +import { VoiceSetupPanel } from '../../../../../renderer/components/Settings/ACappella/VoiceSetupPanel'; +import { + KOKORO_82M_ID, + OPENWAKEWORD_BASE_ID, + QWEN3_1_7B_ID, + VOICE_MODEL_CATALOG, + WHISPER_BASE_EN_ID, + getVoiceModel, +} from '../../../../../shared/acappella/model-catalog'; +import { mockTheme } from '../../../../helpers/mockTheme'; + +/** Every catalog model, reported as not installed. */ +function notInstalledListings() { + return VOICE_MODEL_CATALOG.map((entry) => ({ + entry, + status: { + id: entry.id, + status: 'not-installed' as const, + manifest: null, + detail: 'Not installed', + bytesOnDisk: 0, + }, + installPaths: entry.files.map((file) => `/tmp/models/acappella/${entry.id}/${file.path}`), + })); +} + +const voiceModels = () => window.maestro.voice.models; + +describe('VoiceSetupPanel', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(voiceModels().list).mockResolvedValue(notInstalledListings()); + vi.mocked(voiceModels().footprint).mockResolvedValue({ bytes: 0, models: [] }); + vi.mocked(voiceModels().readiness).mockResolvedValue({ + canStartSession: false, + canRunHandsFree: false, + slots: [], + blocking: [ + { + slot: 'stt', + providerId: 'whisper-local', + satisfied: false, + reason: 'model-not-installed', + detail: 'Speech-to-Text: Whisper Base (English) is not installed.', + suggestedAction: 'Download it in Settings.', + }, + ], + }); + vi.mocked(window.maestro.settings.get).mockResolvedValue({}); + }); + + it('issues zero network calls when mounted', async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal('fetch', fetchSpy); + + render(); + + await waitFor(() => { + expect(voiceModels().list).toHaveBeenCalled(); + }); + + // The panel reads the catalog and the disk. It does not open a socket, and + // it does not ask the main process to open one. + expect(fetchSpy).not.toHaveBeenCalled(); + expect(voiceModels().download).not.toHaveBeenCalled(); + expect(voiceModels().resume).not.toHaveBeenCalled(); + + vi.unstubAllGlobals(); + }); + + it('renders the bill of materials from the frozen catalog', async () => { + render(); + + const whisper = getVoiceModel(WHISPER_BASE_EN_ID)!; + await screen.findByText(whisper.displayName); + + // Name, license, and install path are all on screen before anything is + // downloaded. That is the consent surface. + expect(screen.getAllByText(whisper.license).length).toBeGreaterThan(0); + expect( + screen.getByText(`/tmp/models/acappella/${whisper.id}/${whisper.files[0].path}`) + ).toBeInTheDocument(); + expect(screen.getByText(getVoiceModel(KOKORO_82M_ID)!.displayName)).toBeInTheDocument(); + expect(screen.getByText(getVoiceModel(QWEN3_1_7B_ID)!.displayName)).toBeInTheDocument(); + expect(screen.getByText(getVoiceModel(OPENWAKEWORD_BASE_ID)!.displayName)).toBeInTheDocument(); + }); + + it('states why voice mode is not ready', async () => { + render(); + + await screen.findByText(/Whisper Base \(English\) is not installed/); + }); + + it('starts a download only when the Download button is pressed', async () => { + const { container } = render(); + + // The set button, not one of the per-model rows: this is the primary + // "Download (~N MB)" affordance for the whole selection. + const button = await waitFor(() => { + const found = container.querySelector( + '[data-setting-id="encore-a-cappella-download-set"]' + ); + if (!found || found.disabled) throw new Error('download button not ready'); + return found; + }); + expect(voiceModels().download).not.toHaveBeenCalled(); + + fireEvent.click(button); + + await waitFor(() => { + expect(voiceModels().download).toHaveBeenCalled(); + }); + // The fully-local set: everything is missing, so everything is requested. + const requested = vi + .mocked(voiceModels().download) + .mock.calls.map((call: unknown[]) => call[0]); + expect(requested).toEqual( + expect.arrayContaining([WHISPER_BASE_EN_ID, OPENWAKEWORD_BASE_ID, KOKORO_82M_ID]) + ); + }); + + it('offers Re-verify and Re-download for a corrupt model', async () => { + const listings = notInstalledListings().map((listing) => + listing.entry.id === WHISPER_BASE_EN_ID + ? { + ...listing, + status: { + ...listing.status, + status: 'corrupt' as const, + detail: 'ggml-base.en.bin is 10 bytes, expected 147964211', + manifest: { + id: WHISPER_BASE_EN_ID, + revision: 'abc', + sha256: 'abc', + bytes: 1, + sourceUrl: '', + license: 'MIT', + files: [], + installedAt: 1, + verifiedAt: 1, + }, + bytesOnDisk: 10, + }, + } + : listing + ); + vi.mocked(voiceModels().list).mockResolvedValue(listings); + + render(); + + await screen.findByText(/Re-verify to confirm/); + expect(screen.getByRole('button', { name: /Re-verify/ })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: /^Re-download/ })).toBeInTheDocument(); + }); +}); diff --git a/src/__tests__/renderer/components/Settings/Extensions/extensionModel.test.ts b/src/__tests__/renderer/components/Settings/Extensions/extensionModel.test.ts index 1064eb6b24..0467668865 100644 --- a/src/__tests__/renderer/components/Settings/Extensions/extensionModel.test.ts +++ b/src/__tests__/renderer/components/Settings/Extensions/extensionModel.test.ts @@ -96,6 +96,7 @@ describe('extensionModel first-party projection (all Encore features)', () => { 'opencodeServer', 'concerto', 'groupsPlus', + 'aCappella', ]); for (const def of BUILTIN_FEATURES) { @@ -134,6 +135,26 @@ describe('extensionModel first-party projection (all Encore features)', () => { expect(builtinExtension(groupsPlus!, flags({ groupsPlus: true })).state).toBe('enabled'); }); + it('projects A Cappella as a beta, disabled-by-default plugin-backed UI extension', () => { + const aCappella = BUILTIN_FEATURES.find((def) => def.flag === 'aCappella'); + expect(aCappella).toBeDefined(); + expect(aCappella!.beta).toBe(true); + + expect(builtinExtension(aCappella!, flags())).toMatchObject({ + key: 'builtin:aCappella', + state: 'not-installed', + name: 'A Cappella', + category: 'ui', + pluginId: 'com.maestro.acappella', + settingsNamespace: 'aCappella', + }); + expect(builtinExtension(aCappella!, flags({ aCappella: true })).state).toBe('enabled'); + + // Enabling voice must not imply a running background service: nothing + // listens until the user explicitly starts a session. + expect(builtinExtension(aCappella!, flags()).backgroundServiceId).toBeUndefined(); + }); + it('surfaces the plan-table identities on the details pane fields', () => { const byFlag = (flag: keyof EncoreFeatureFlags) => builtinExtension(BUILTIN_FEATURES.find((d) => d.flag === flag)!, flags()); diff --git a/src/__tests__/renderer/hooks/useRemoteIntegration.test.ts b/src/__tests__/renderer/hooks/useRemoteIntegration.test.ts index 47d9b85a0e..b4a9d120b0 100644 --- a/src/__tests__/renderer/hooks/useRemoteIntegration.test.ts +++ b/src/__tests__/renderer/hooks/useRemoteIntegration.test.ts @@ -61,6 +61,9 @@ describe('useRemoteIntegration', () => { let onRemoteSelectSessionHandler: ((sessionId: string, tabId?: string) => void) | undefined; let onRemoteSelectTabHandler: ((sessionId: string, tabId: string) => void) | undefined; let onRemoteNewTabHandler: ((sessionId: string, responseChannel: string) => void) | undefined; + let onRemoteFocusAiTabHandler: + | ((sessionId: string, tabId: string, responseChannel: string) => void) + | undefined; let onRemoteCloseTabHandler: ((sessionId: string, tabId: string) => void) | undefined; let onRemoteRenameTabHandler: | ((sessionId: string, tabId: string, newName: string) => void) @@ -143,6 +146,11 @@ describe('useRemoteIntegration', () => { onRemoteNewTabHandler = handler; return () => {}; }), + onRemoteFocusAiTab: vi.fn().mockImplementation((handler) => { + onRemoteFocusAiTabHandler = handler; + return () => {}; + }), + sendRemoteFocusAiTabResponse: vi.fn(), onRemoteCloseTab: vi.fn().mockImplementation((handler) => { onRemoteCloseTabHandler = handler; return () => {}; @@ -381,6 +389,7 @@ describe('useRemoteIntegration', () => { onRemoteSelectSessionHandler = undefined; onRemoteSelectTabHandler = undefined; onRemoteNewTabHandler = undefined; + onRemoteFocusAiTabHandler = undefined; onRemoteCloseTabHandler = undefined; onRemoteRenameTabHandler = undefined; onRemoteStarTabHandler = undefined; @@ -928,6 +937,86 @@ describe('useRemoteIntegration', () => { }); }); + describe('remote focus AI tab', () => { + // A Cappella's spoken recall lands here. It answers on a response channel + // because the caller narrates the result out loud, and main cannot see + // whether the tab was focused, woken out of a snooze, or reopened. + it('focuses an open tab and reports what that took', () => { + const session = createMockSession({ + id: 'session-1', + aiTabs: [createMockAITab({ id: 'tab-1' })], + }); + const deps = createDeps({ sessions: [session] }); + + renderHook(() => useRemoteIntegration(deps)); + + act(() => { + onRemoteFocusAiTabHandler?.('session-1', 'tab-1', 'chan-1'); + }); + + expect(deps.setActiveSessionId).toHaveBeenCalledWith('session-1'); + expect(mockProcess.sendRemoteFocusAiTabResponse).toHaveBeenCalledWith('chan-1', { + ok: true, + tabId: 'tab-1', + action: 'focused', + }); + }); + + it('wakes a snoozed tab rather than landing on a tab that is not on screen', () => { + const snoozedTab = createMockAITab({ id: 'tab-snoozed' }); + const session = createMockSession({ + id: 'session-1', + aiTabs: [createMockAITab({ id: 'tab-1' })], + snoozedTabs: [ + { + // `type` is the discriminant on the snooze union - a snooze can + // now hold a file, browser, terminal, or whole group, and the + // wake path switches on it. Omitting it leaves the entry + // matching no branch, so the tab reads as simply not there. + type: 'ai', + id: 'snooze-1', + tab: snoozedTab, + unifiedIndex: 0, + snoozedAt: 1, + wakeAt: Date.now() + 100_000, + }, + ], + }); + const deps = createDeps({ sessions: [session] }); + + renderHook(() => useRemoteIntegration(deps)); + + act(() => { + onRemoteFocusAiTabHandler?.('session-1', 'tab-snoozed', 'chan-1'); + }); + + expect(mockProcess.sendRemoteFocusAiTabResponse).toHaveBeenCalledWith('chan-1', { + ok: true, + tabId: 'tab-snoozed', + action: 'woke', + }); + }); + + it('says so rather than silently landing somewhere else when the tab is gone', () => { + const session = createMockSession({ + id: 'session-1', + aiTabs: [createMockAITab({ id: 'tab-1' })], + }); + const deps = createDeps({ sessions: [session] }); + + renderHook(() => useRemoteIntegration(deps)); + + act(() => { + onRemoteFocusAiTabHandler?.('session-1', 'tab-gone', 'chan-1'); + }); + + expect(mockProcess.sendRemoteFocusAiTabResponse).toHaveBeenCalledWith('chan-1', { + ok: false, + reason: 'tab-not-found', + }); + }); + }); + describe('remote new AI tab with prompt', () => { it('creates tab, dispatches remoteCommand, and acks true with the new tab id on idle session', () => { const session = createMockSession({ id: 'session-1', state: 'idle' }); diff --git a/src/__tests__/renderer/hooks/voice/useComposerVoice.test.tsx b/src/__tests__/renderer/hooks/voice/useComposerVoice.test.tsx new file mode 100644 index 0000000000..d858c8d235 --- /dev/null +++ b/src/__tests__/renderer/hooks/voice/useComposerVoice.test.tsx @@ -0,0 +1,146 @@ +/** + * The composer microphone button, and which of the two voice stacks it drives. + * + * The behaviour that has to survive: turning the A Cappella Encore Feature ON + * must not open two microphones, and leaving it OFF must not take Web Speech + * dictation away from the people who have never turned it on - which is + * everyone, by default. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { renderHook, act, cleanup } from '@testing-library/react'; +import { useComposerVoice } from '../../../../renderer/hooks/voice/useComposerVoice'; +import { useSettingsStore } from '../../../../renderer/stores/settingsStore'; +import { useVoiceSessionStore } from '../../../../renderer/stores/voiceSessionStore'; +import { useVoiceUiStore } from '../../../../renderer/stores/voiceUiStore'; +import type { Session } from '../../../../renderer/types'; + +const session = { id: 'agent-1', name: 'Backend' } as Session; + +const startVoiceInput = vi.fn(); +const stopVoiceInput = vi.fn(); +const toggleVoiceInput = vi.fn(); +let webSpeechDisabled = false; + +vi.mock('../../../../renderer/hooks/utils/useVoiceInput', () => ({ + useVoiceInput: (options: { disabled?: boolean }) => { + webSpeechDisabled = options.disabled === true; + return { + isListening: false, + voiceSupported: true, + startVoiceInput, + stopVoiceInput, + toggleVoiceInput, + }; + }, +})); + +function render(enabled: boolean) { + useSettingsStore.setState({ encoreFeatures: { aCappella: enabled } } as never); + return renderHook(() => + useComposerVoice({ + session, + currentValue: '', + onTranscriptionChange: vi.fn(), + }) + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + webSpeechDisabled = false; + useVoiceSessionStore.getState().reset(); + useVoiceUiStore.setState({ wakePhrases: {}, loaded: true }); +}); + +afterEach(() => { + cleanup(); +}); + +describe('with A Cappella off', () => { + it('keeps the Web Speech dictation exactly as it was', async () => { + const { result } = render(false); + expect(result.current.usesACappella).toBe(false); + expect(webSpeechDisabled).toBe(false); + + await act(async () => { + result.current.toggle(); + }); + + expect(toggleVoiceInput).toHaveBeenCalledTimes(1); + expect(window.maestro.voice.start).not.toHaveBeenCalled(); + }); +}); + +describe('with A Cappella on', () => { + it('opens a voice session bound to this agent instead of dictating', async () => { + const { result } = render(true); + expect(result.current.usesACappella).toBe(true); + + await act(async () => { + result.current.toggle(); + }); + + expect(window.maestro.voice.start).toHaveBeenCalledWith({ + kind: 'agent', + sessionId: 'agent-1', + }); + expect(toggleVoiceInput).not.toHaveBeenCalled(); + }); + + it('surfaces a provider downgrade reported by the start', async () => { + // The failure this guards: the composer microphone called + // `window.maestro.voice.start()` for its side effect and threw the result + // away, so a build whose speech-to-text had fallen back to the mock tier + // said "Listening" and explained nothing. The registry refuses to + // substitute silently; dropping the report here undid that at the last hop. + const substitution = { + role: 'stt' as const, + requestedId: 'echo-stt', + resolvedId: 'mock-stt', + reason: 'unavailable' as const, + }; + vi.mocked(window.maestro.voice.start).mockResolvedValueOnce({ + snapshot: { + sessionId: 'voice-1', + state: 'listening', + scope: { kind: 'agent', sessionId: 'agent-1' }, + seq: 1, + startedAt: 0, + providerIds: { stt: 'mock-stt', tts: 'mock-tts', brain: 'mock-brain' }, + }, + substitutions: [substitution], + } as never); + + const { result } = render(true); + await act(async () => { + result.current.toggle(); + }); + + expect(useVoiceSessionStore.getState().substitutions).toEqual([substitution]); + }); + + it('hard-disables the Web Speech path, so one button cannot open two microphones', () => { + render(true); + expect(webSpeechDisabled).toBe(true); + }); + + it('ends the session on a second press rather than starting a second one', async () => { + const { result, rerender } = render(true); + act(() => { + useVoiceSessionStore.setState({ + state: 'listening', + scope: { kind: 'agent', sessionId: 'agent-1' }, + }); + }); + rerender(); + + expect(result.current.isListening).toBe(true); + await act(async () => { + result.current.toggle(); + }); + + expect(window.maestro.voice.stop).toHaveBeenCalledTimes(1); + expect(window.maestro.voice.start).not.toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/renderer/stores/voiceDiagnosticsStore.test.ts b/src/__tests__/renderer/stores/voiceDiagnosticsStore.test.ts new file mode 100644 index 0000000000..f0e627f7dc --- /dev/null +++ b/src/__tests__/renderer/stores/voiceDiagnosticsStore.test.ts @@ -0,0 +1,103 @@ +/** + * @file voiceDiagnosticsStore.test.ts + * + * The recorder exists for one situation: the pipeline appeared to do nothing and + * someone has to say where it stopped. Two properties make it useful for that, + * and both are easy to lose - it must survive a busy session without evicting + * the events that explain anything, and it must not fill up with meter ticks. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import type { VoiceEvent } from '../../../shared/acappella/protocol'; +import { + useVoiceDiagnosticsStore, + VOICE_DIAGNOSTIC_LIMIT, +} from '../../../renderer/stores/voiceDiagnosticsStore'; + +let seq = 0; + +function event( + type: T, + body: Omit, 'type' | 'sessionId' | 'seq' | 'ts'> +): VoiceEvent { + seq += 1; + return { type, sessionId: 'voice-1', seq, ts: 1_700_000_000_000 + seq, ...body } as VoiceEvent; +} + +function record(...events: VoiceEvent[]): void { + for (const e of events) useVoiceDiagnosticsStore.getState().record(e); +} + +beforeEach(() => { + seq = 0; + useVoiceDiagnosticsStore.getState().clear(); +}); + +describe('voiceDiagnosticsStore', () => { + it('records what an event carried, not the whole payload', () => { + record( + event('listen-start', { scope: { kind: 'conductor' }, sttProviderId: 'echo-stt' }), + event('final-transcript', { text: 'open the auth tab', confidence: 0.9 }) + ); + + const entries = useVoiceDiagnosticsStore.getState().entries; + expect(entries.map((entry) => entry.type)).toEqual(['listen-start', 'final-transcript']); + expect(entries[0].detail).toContain('echo-stt'); + expect(entries[1].detail).toContain('open the auth tab'); + }); + + it('tallies audio levels instead of storing them', () => { + // Twenty a second: stored individually, ten seconds of silence would evict + // every event that explains anything. + record( + event('audio-level', { level: 0.2, speech: false }), + event('audio-level', { level: 0.7, speech: true }), + event('audio-level', { level: 0.1, speech: false }) + ); + + const state = useVoiceDiagnosticsStore.getState(); + expect(state.entries).toHaveLength(0); + expect(state.audioLevelCount).toBe(3); + expect(state.audioLevelPeak).toBeCloseTo(0.7); + expect(state.speechFrames).toBe(1); + }); + + it('keeps the most recent entries once it is full', () => { + for (let i = 0; i < VOICE_DIAGNOSTIC_LIMIT + 10; i += 1) { + record(event('listen-stop', { reason: 'stopped' })); + } + + const entries = useVoiceDiagnosticsStore.getState().entries; + expect(entries).toHaveLength(VOICE_DIAGNOSTIC_LIMIT); + // Ids stay strictly increasing across an eviction, so the log cannot appear + // to go backwards in time after the buffer wraps. + expect(entries[0].id).toBeLessThan(entries[entries.length - 1].id); + }); + + it('records the error that explains a dead session', () => { + record( + event('session-error', { + code: 'provider-unavailable', + message: 'whisper.cpp is not part of this build yet.', + recoverable: false, + }) + ); + + expect(useVoiceDiagnosticsStore.getState().entries[0].detail).toContain('whisper.cpp'); + }); + + it('clears both the log and the tallies', () => { + record( + event('audio-level', { level: 0.5, speech: true }), + event('listen-stop', { reason: 'stopped' }) + ); + + useVoiceDiagnosticsStore.getState().clear(); + + const state = useVoiceDiagnosticsStore.getState(); + expect(state.entries).toEqual([]); + expect(state.audioLevelCount).toBe(0); + expect(state.audioLevelPeak).toBe(0); + expect(state.speechFrames).toBe(0); + }); +}); diff --git a/src/__tests__/renderer/stores/voiceSessionStore.test.ts b/src/__tests__/renderer/stores/voiceSessionStore.test.ts new file mode 100644 index 0000000000..669518a3c2 --- /dev/null +++ b/src/__tests__/renderer/stores/voiceSessionStore.test.ts @@ -0,0 +1,398 @@ +/** + * voiceSessionStore - projection of the A Cappella event stream. + * + * The store owns no truth, so these tests are all about faithfulness: the state + * it derives, the transcript it builds, what it drops, and what it refuses to + * rewind. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import type { VoiceEvent } from '../../../shared/acappella/protocol'; +import { + selectVoiceAudioLevel, + selectVoiceMicIssue, + selectVoiceScopeLabel, + useVoiceSessionStore, + VOICE_FEED_LIMIT, +} from '../../../renderer/stores/voiceSessionStore'; + +const SESSION = 'voice-session-1'; + +let seq = 0; + +/** Build the next event in the stream, stamping a contiguous `seq`. */ +function event( + type: T, + body: Omit, 'type' | 'sessionId' | 'seq' | 'ts'>, + overrides: { sessionId?: string; seq?: number } = {} +): VoiceEvent { + seq += 1; + return { + type, + sessionId: overrides.sessionId ?? SESSION, + seq: overrides.seq ?? seq, + ts: 1_700_000_000_000 + seq, + ...body, + } as unknown as VoiceEvent; +} + +function apply(...events: VoiceEvent[]): void { + for (const e of events) useVoiceSessionStore.getState().applyEvent(e); +} + +function wake(): VoiceEvent { + return event('wake', { source: 'client-button', scope: { kind: 'conductor' } }); +} + +function listenStart(): VoiceEvent { + return event('listen-start', { scope: { kind: 'conductor' }, sttProviderId: 'mock-stt' }); +} + +beforeEach(() => { + seq = 0; + useVoiceSessionStore.getState().reset(); +}); + +describe('voiceSessionStore projection', () => { + it('derives the state the service is in after each event', () => { + apply(wake()); + expect(useVoiceSessionStore.getState().state).toBe('arming'); + + apply(listenStart()); + expect(useVoiceSessionStore.getState().state).toBe('listening'); + + apply(event('final-transcript', { text: 'open a new tab', confidence: 1 })); + expect(useVoiceSessionStore.getState().state).toBe('transcribing'); + + apply( + event('route-decision', { + decision: { + target: 'conductor', + tabAction: 'new', + prompt: 'open a new tab', + confidence: 0.9, + }, + brainProviderId: 'mock-brain', + latencyMs: 3, + }) + ); + expect(useVoiceSessionStore.getState().state).toBe('dispatching'); + + apply(event('speak-start', { utteranceId: 'u1', sentenceCount: 2, ttsProviderId: 'mock-tts' })); + expect(useVoiceSessionStore.getState().state).toBe('speaking'); + }); + + it('returns to idle on a listen-stop that ends the session', () => { + apply(wake(), listenStart(), event('listen-stop', { reason: 'stopped' })); + expect(useVoiceSessionStore.getState().state).toBe('idle'); + }); + + it('parks in error on a session-error and keeps the message', () => { + apply( + wake(), + event('session-error', { + code: 'no-agent-matched', + message: 'No agent named Backend is running', + recoverable: true, + }) + ); + const state = useVoiceSessionStore.getState(); + expect(state.state).toBe('error'); + expect(state.error?.message).toContain('Backend'); + }); + + it('keeps a refusal on screen while the session is still parked in error', () => { + // No `listen-stop` is emitted for a session that failed, which is how the + // user gets to read why it would not start. + apply( + wake(), + event('session-error', { + code: 'provider-unavailable', + message: 'whisper.cpp is not part of this build yet.', + recoverable: false, + }) + ); + + expect(useVoiceSessionStore.getState().error).not.toBeNull(); + }); + + it('records which window owns the session, from the very first event', () => { + // On `wake` rather than only in the catch-up snapshot: a window that waited + // for the snapshot would flash a HUD for a session belonging to another one. + apply(event('wake', { source: 'client-button', scope: { kind: 'conductor' }, windowId: 'w2' })); + + expect(useVoiceSessionStore.getState().windowId).toBe('w2'); + }); + + it('takes the owning window from a catch-up snapshot after a reload', () => { + // A window that reloaded mid-session never saw the `wake`, and without this + // it would decide it does not own a session that is in fact its own. + useVoiceSessionStore.getState().applySnapshot({ + sessionId: 'reloaded-session', + state: 'listening', + scope: { kind: 'conductor' }, + seq: 9, + providerIds: { stt: 'mock-stt', tts: 'mock-tts', brain: 'mock-brain' }, + windowId: 'w2', + }); + + expect(useVoiceSessionStore.getState().windowId).toBe('w2'); + }); + + it('clears the error once the user ends the session', () => { + // The HUD renders whenever `error` is set, so an error outliving its + // session left a voice panel on screen indefinitely that nobody opened. + apply( + wake(), + event('session-error', { + code: 'provider-unavailable', + message: 'whisper.cpp is not part of this build yet.', + recoverable: false, + }), + event('listen-stop', { reason: 'stopped' }) + ); + + const state = useVoiceSessionStore.getState(); + expect(state.state).toBe('idle'); + expect(state.error).toBeNull(); + }); + + it('streams partials and clears them when the utterance settles', () => { + apply( + wake(), + listenStart(), + event('partial-transcript', { text: 'open a', stability: 0.3 }), + event('partial-transcript', { text: 'open a new tab', stability: 0.8 }) + ); + expect(useVoiceSessionStore.getState().partialTranscript).toBe('open a new tab'); + + apply(event('final-transcript', { text: 'open a new tab', confidence: 1 })); + const state = useVoiceSessionStore.getState(); + expect(state.partialTranscript).toBe(''); + expect(state.utterance).toBe('open a new tab'); + expect(state.feed[state.feed.length - 1]).toMatchObject({ + kind: 'you', + text: 'open a new tab', + }); + }); + + it('narrates a dispatch in words and keeps the reply address', () => { + apply( + wake(), + listenStart(), + event('dispatch', { + agentSessionId: 'agent-1', + agentName: 'Backend', + tabId: 'tab-9', + tabName: 'Auth Refactor', + action: 'created', + promptSent: true, + }) + ); + const state = useVoiceSessionStore.getState(); + expect(state.feed[state.feed.length - 1].text).toBe( + 'Opened a new tab named Auth Refactor on Backend' + ); + expect(state.lastDispatch).toMatchObject({ agentSessionId: 'agent-1', tabId: 'tab-9' }); + }); + + it('collects spoken sentences and drops stragglers from a cancelled run', () => { + apply( + wake(), + listenStart(), + event('speak-start', { utteranceId: 'u1', sentenceCount: 3, ttsProviderId: 'mock-tts' }), + event('speak-sentence', { utteranceId: 'u1', index: 0, text: 'Done.' }), + event('barge-in', { source: 'client-button', cancelledUtteranceId: 'u1' }), + event('speak-end', { utteranceId: 'u1', reason: 'cancelled' }), + // A late chunk from the cancelled run. It must not extend the transcript. + event('speak-sentence', { utteranceId: 'u1', index: 1, text: 'Also this.' }), + event('speak-sentence', { utteranceId: 'u-old', index: 0, text: 'Wrong run.' }) + ); + const state = useVoiceSessionStore.getState(); + expect(state.speech?.sentences).toEqual(['Done.', 'Also this.']); + expect(state.speech?.endedReason).toBe('cancelled'); + + // The straggler from a DIFFERENT run is dropped entirely. + expect(state.speech?.sentences).not.toContain('Wrong run.'); + }); + + it('flags a seq gap instead of smoothing over it', () => { + apply(wake(), listenStart()); + expect(useVoiceSessionStore.getState().lostEvents).toBe(false); + + apply(event('partial-transcript', { text: 'hello', stability: 0.5 }, { seq: 99 })); + expect(useVoiceSessionStore.getState().lostEvents).toBe(true); + }); + + it('starts the projection over when a new session id appears', () => { + apply( + wake(), + listenStart(), + event('final-transcript', { text: 'first session', confidence: 1 }) + ); + expect(useVoiceSessionStore.getState().feed).toHaveLength(1); + + apply( + event( + 'wake', + { source: 'hotkey', scope: { kind: 'agent', sessionId: 'agent-1' } }, + { sessionId: 'voice-session-2', seq: 1 } + ) + ); + const state = useVoiceSessionStore.getState(); + expect(state.sessionId).toBe('voice-session-2'); + expect(state.feed).toHaveLength(0); + expect(state.lostEvents).toBe(false); + }); + + it('caps the transcript so a long conversation cannot grow without bound', () => { + apply(wake(), listenStart()); + for (let i = 0; i < VOICE_FEED_LIMIT + 10; i++) { + apply(event('final-transcript', { text: `line ${i}`, confidence: 1 })); + } + const feed = useVoiceSessionStore.getState().feed; + expect(feed).toHaveLength(VOICE_FEED_LIMIT); + expect(feed[feed.length - 1].text).toBe(`line ${VOICE_FEED_LIMIT + 9}`); + }); + + it('names the bound agent once the roster arrives', () => { + apply( + event('wake', { + source: 'hotkey', + scope: { kind: 'agent', sessionId: 'agent-1' }, + }), + event('agent-roster', { + agents: [ + { sessionId: 'agent-1', name: 'Backend', agentType: 'claude-code', cwd: '/p', tabs: [] }, + ], + }) + ); + expect(selectVoiceScopeLabel(useVoiceSessionStore.getState())).toBe('Backend'); + }); +}); + +describe('voiceSessionStore audio projection', () => { + function micState( + overrides: Partial, 'type'>> = {} + ): VoiceEvent { + return event('mic-state', { + permission: 'granted', + capturing: true, + deviceId: 'default', + deviceLabel: 'MacBook Pro Microphone', + issue: null, + deviceChanged: false, + ...overrides, + } as never); + } + + it('tracks the meter level and whether the window was speech', () => { + apply(wake(), listenStart(), event('audio-level', { level: 0.42, speech: true })); + + const state = useVoiceSessionStore.getState(); + expect(state.audioLevel).toBeCloseTo(0.42); + expect(state.speechDetected).toBe(true); + expect(selectVoiceAudioLevel(state)).toBeCloseTo(0.42); + }); + + it('does not let a level move the session state or reach the transcript', () => { + apply(wake(), listenStart(), event('audio-level', { level: 0.4, speech: true })); + + const state = useVoiceSessionStore.getState(); + expect(state.state).toBe('listening'); + // 20 lines a second of "the meter moved" would bury the conversation. + expect(state.feed).toHaveLength(0); + }); + + it('drops the meter to rest when the floor closes', () => { + apply( + wake(), + listenStart(), + event('audio-level', { level: 0.4, speech: true }), + event('listen-stop', { reason: 'endpoint' }) + ); + + expect(useVoiceSessionStore.getState().audioLevel).toBe(0); + expect(useVoiceSessionStore.getState().speechDetected).toBe(false); + }); + + it('drops the meter to rest when the microphone stops capturing', () => { + apply(wake(), listenStart(), event('audio-level', { level: 0.4, speech: true })); + apply(micState({ capturing: false })); + + // A bar left standing over a closed device is the same lie as a listening + // indicator over a denied one. + expect(useVoiceSessionStore.getState().audioLevel).toBe(0); + }); + + it('projects the microphone state, issue and all', () => { + apply(wake(), micState({ permission: 'denied', capturing: false, issue: 'permission-denied' })); + + const state = useVoiceSessionStore.getState(); + expect(state.mic?.permission).toBe('denied'); + expect(selectVoiceMicIssue(state)).toBe('permission-denied'); + }); + + it('reports no issue before anything has been attempted', () => { + expect(selectVoiceMicIssue(useVoiceSessionStore.getState())).toBeNull(); + }); + + it('keeps the microphone state across a session restart', () => { + apply(wake(), micState({ permission: 'denied', capturing: false, issue: 'permission-denied' })); + + // A permission the user denied is still denied on the next attempt, and + // forgetting it would leave the new session unable to explain its silence. + apply( + event('wake', { source: 'hotkey', scope: { kind: 'conductor' } }, { sessionId: 'voice-2' }) + ); + + const state = useVoiceSessionStore.getState(); + expect(state.sessionId).toBe('voice-2'); + expect(state.mic?.issue).toBe('permission-denied'); + expect(state.audioLevel).toBe(0); + }); +}); + +describe('voiceSessionStore snapshot catch-up', () => { + it('adopts a snapshot when the client has no session', () => { + useVoiceSessionStore.getState().applySnapshot({ + sessionId: SESSION, + state: 'speaking', + scope: { kind: 'conductor' }, + seq: 7, + providerIds: { stt: 'mock-stt', tts: 'mock-tts', brain: 'mock-brain' }, + }); + expect(useVoiceSessionStore.getState().state).toBe('speaking'); + expect(useVoiceSessionStore.getState().lostEvents).toBe(false); + }); + + it('does not rewind a projection the stream has already carried past', () => { + apply( + wake(), + listenStart(), + event('speak-start', { + utteranceId: 'u1', + sentenceCount: 1, + ttsProviderId: 'mock-tts', + }) + ); + expect(useVoiceSessionStore.getState().state).toBe('speaking'); + + // A catch-up read that resolved late, describing an earlier moment. + useVoiceSessionStore.getState().applySnapshot({ + sessionId: SESSION, + state: 'arming', + scope: { kind: 'conductor' }, + seq: 1, + providerIds: { stt: 'mock-stt', tts: 'mock-tts', brain: 'mock-brain' }, + }); + expect(useVoiceSessionStore.getState().state).toBe('speaking'); + expect(useVoiceSessionStore.getState().providerIds?.tts).toBe('mock-tts'); + }); + + it('ignores a null snapshot while a session is live', () => { + apply(wake(), listenStart()); + useVoiceSessionStore.getState().applySnapshot(null); + expect(useVoiceSessionStore.getState().state).toBe('listening'); + }); +}); diff --git a/src/__tests__/setup.ts b/src/__tests__/setup.ts index 5c2fbe457e..c867220aa8 100644 --- a/src/__tests__/setup.ts +++ b/src/__tests__/setup.ts @@ -601,6 +601,86 @@ const mockMaestro = { onTtsCompleted: vi.fn().mockReturnValue(() => {}), // Legacy alias show: vi.fn().mockResolvedValue(undefined), }, + voice: { + start: vi.fn().mockResolvedValue({ + snapshot: { + sessionId: 'voice-1', + state: 'listening', + scope: { kind: 'conductor' }, + seq: 2, + startedAt: 0, + providerIds: { stt: 'mock-stt', tts: 'mock-tts', brain: 'mock-brain' }, + }, + substitutions: [], + }), + stop: vi.fn().mockResolvedValue(undefined), + submitUtterance: vi.fn().mockResolvedValue(true), + interrupt: vi.fn().mockResolvedValue(true), + stopWord: vi.fn().mockResolvedValue(undefined), + // The microphone picker. `system-default` is the shipped state: no explicit + // choice, follow the OS. + inputDevices: vi.fn().mockResolvedValue({ devices: [], selectedId: 'system-default' }), + setInputDevice: vi.fn().mockResolvedValue(true), + onInputDevices: vi.fn().mockReturnValue(() => {}), + submitAgentReply: vi.fn().mockResolvedValue(true), + getRoster: vi.fn().mockResolvedValue([]), + getState: vi.fn().mockResolvedValue(null), + openMicSettings: vi.fn().mockResolvedValue(true), + onEvent: vi.fn().mockReturnValue(() => {}), + applyProviders: vi.fn().mockResolvedValue({ status: 'swapped' }), + lastTurn: vi.fn().mockResolvedValue(null), + listVoices: vi.fn().mockResolvedValue([]), + previewVoice: vi.fn().mockResolvedValue(true), + setVolume: vi.fn().mockResolvedValue(true), + // Bound by default, because that is what a fresh install really does: main + // falls back to `defaultGlobalHotkeyKeys(id)` whenever the stored shortcuts + // map has no entry. A mock that reported nothing registered would let the + // settings row drift back to claiming an unbound hotkey. + hotkeyStatus: vi.fn().mockResolvedValue({ + statuses: [ + { + id: 'voiceConductor', + keys: ['Meta', 'Alt', 'v'], + accelerator: 'Command+Alt+V', + registered: true, + }, + { + id: 'voiceCurrentAgent', + keys: ['Meta', 'Alt', 'a'], + accelerator: 'Command+Alt+A', + registered: true, + }, + ], + capability: 'tap-only', + note: 'Tap to toggle the microphone.', + }), + wakeTest: vi.fn().mockResolvedValue(true), + wakeTestStop: vi.fn().mockResolvedValue(undefined), + onWakeTest: vi.fn().mockReturnValue(() => {}), + credentials: { + list: vi.fn().mockResolvedValue([]), + set: vi.fn().mockResolvedValue({ ok: true }), + validate: vi.fn().mockResolvedValue({ service: 'openai', status: 'valid', message: 'ok' }), + }, + models: { + list: vi.fn().mockResolvedValue([]), + download: vi.fn().mockResolvedValue({ modelId: '', status: 'complete' }), + pause: vi.fn().mockResolvedValue(true), + resume: vi.fn().mockResolvedValue({ modelId: '', status: 'complete' }), + cancel: vi.fn().mockResolvedValue(true), + verify: vi.fn().mockResolvedValue({ modelId: '', ok: true, status: 'installed' }), + remove: vi.fn().mockResolvedValue(0), + removeAll: vi.fn().mockResolvedValue(0), + footprint: vi.fn().mockResolvedValue({ bytes: 0, models: [] }), + readiness: vi.fn().mockResolvedValue({ + canStartSession: true, + canRunHandsFree: true, + slots: [], + blocking: [], + }), + onProgress: vi.fn().mockReturnValue(() => {}), + }, + }, dialog: { selectFolder: vi.fn().mockResolvedValue(null), saveFile: vi.fn().mockResolvedValue(null), diff --git a/src/__tests__/shared/acappella-device-protocol.test.ts b/src/__tests__/shared/acappella-device-protocol.test.ts new file mode 100644 index 0000000000..26bb6e2015 --- /dev/null +++ b/src/__tests__/shared/acappella-device-protocol.test.ts @@ -0,0 +1,222 @@ +/** + * The A Cappella data-channel protocol. + * + * Two properties are load-bearing and both are tested here rather than + * discovered on a phone: + * + * - **Version negotiation refuses loudly.** An old client against a new + * desktop must fail with a sentence that names which end has to update. The + * failure this prevents is a client that half works, where a feature quietly + * does nothing and nobody connects it to a version. + * - **The reliable/unreliable split is total.** Every message type has exactly + * one channel, decided in one table. A `revoked` sent lossy is a device that + * keeps its microphone; an `audio-level` sent reliably is a meter that lags + * a walk down the street. + */ + +import { describe, expect, it } from 'vitest'; + +import { + DEVICE_ORIGINATED_MESSAGES, + DEVICE_PROTOCOL_VERSION, + MIN_SUPPORTED_DEVICE_PROTOCOL_VERSION, + RELIABLE_CHANNEL_LABEL, + UNRELIABLE_CHANNEL_INIT, + UNRELIABLE_CHANNEL_LABEL, + decodeDeviceMessage, + deviceChannelForMessage, + deviceChannelForVoiceEvent, + deviceChannelLabel, + encodeDeviceMessage, + isDeviceOriginatedMessage, + negotiateProtocolVersion, + type DeviceMessage, +} from '../../shared/acappella/device-protocol'; +import type { VoiceEvent, VoiceEventType } from '../../shared/acappella/protocol'; +import { VOICE_EVENT_DIRECTIONS } from '../../shared/acappella/protocol'; + +describe('negotiateProtocolVersion', () => { + it('accepts the current version', () => { + const result = negotiateProtocolVersion(DEVICE_PROTOCOL_VERSION); + expect(result).toEqual({ ok: true, version: DEVICE_PROTOCOL_VERSION }); + }); + + it('tells an old client to update the DEVICE', () => { + // The shipped floor and ceiling are the same number today, so the range is + // passed explicitly: the branch has to keep working for the release where + // they diverge, and that is not the release to discover it on a phone. + const result = negotiateProtocolVersion(1, { min: 2, max: 3 }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe('client-too-old'); + expect(result.message).toMatch(/update the app on the device/i); + }); + + it('accepts a client below the ceiling but at or above the floor', () => { + expect(negotiateProtocolVersion(2, { min: 1, max: 3 })).toEqual({ ok: true, version: 2 }); + expect(MIN_SUPPORTED_DEVICE_PROTOCOL_VERSION).toBeLessThanOrEqual(DEVICE_PROTOCOL_VERSION); + }); + + it('tells a newer client to update the DESKTOP', () => { + const result = negotiateProtocolVersion(DEVICE_PROTOCOL_VERSION + 1); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe('client-too-new'); + expect(result.message).toMatch(/update maestro on the desktop/i); + }); + + it.each([undefined, null, 'one', 1.5, 0, -3, Number.NaN])( + 'rejects a malformed version (%s)', + (value) => { + const result = negotiateProtocolVersion(value); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe('malformed'); + } + ); +}); + +describe('channel routing', () => { + const message = (type: DeviceMessage['type']): DeviceMessage => { + switch (type) { + case 'hello': + return { type: 'hello', identity: { deviceId: 'd', name: 'n', platform: 'ios' } }; + case 'welcome': + return { type: 'welcome', version: 1, appVersion: '1.0.0', sessionId: null }; + case 'version-rejected': + return { + type: 'version-rejected', + reason: 'client-too-old', + message: 'nope', + desktopVersion: 1, + minimumVersion: 1, + }; + case 'voice-event': + return { type: 'voice-event', event: voiceEvent('listen-stop') }; + case 'floor': + return { type: 'floor', action: 'press' }; + case 'interrupt': + return { type: 'interrupt', kind: 'barge-in' }; + case 'audio-level': + return { type: 'audio-level', level: 0.4, speech: true }; + case 'link-quality': + return { + type: 'link-quality', + rttMs: 20, + jitterMs: 3, + packetLoss: 0, + candidateType: 'lan', + }; + case 'floor-state': + return { type: 'floor-state', holder: null, isSelf: false }; + case 'revoked': + return { type: 'revoked', message: 'gone' }; + } + }; + + it('sends state on the reliable channel and chatter on the lossy one', () => { + expect(deviceChannelForMessage(message('welcome'))).toBe('reliable'); + expect(deviceChannelForMessage(message('floor-state'))).toBe('reliable'); + expect(deviceChannelForMessage(message('revoked'))).toBe('reliable'); + expect(deviceChannelForMessage(message('audio-level'))).toBe('unreliable'); + expect(deviceChannelForMessage(message('floor'))).toBe('unreliable'); + expect(deviceChannelForMessage(message('interrupt'))).toBe('unreliable'); + expect(deviceChannelForMessage(message('link-quality'))).toBe('unreliable'); + }); + + it('routes a wrapped voice event by the event type, not the envelope', () => { + expect(deviceChannelForMessage({ type: 'voice-event', event: voiceEvent('audio-level') })).toBe( + 'unreliable' + ); + expect( + deviceChannelForMessage({ type: 'voice-event', event: voiceEvent('partial-transcript') }) + ).toBe('unreliable'); + expect(deviceChannelForMessage({ type: 'voice-event', event: voiceEvent('dispatch') })).toBe( + 'reliable' + ); + expect( + deviceChannelForMessage({ type: 'voice-event', event: voiceEvent('session-error') }) + ).toBe('reliable'); + }); + + it('assigns every message type a channel', () => { + const types: Array = [ + 'hello', + 'welcome', + 'version-rejected', + 'voice-event', + 'floor', + 'interrupt', + 'audio-level', + 'link-quality', + 'floor-state', + 'revoked', + ]; + for (const type of types) { + expect(['reliable', 'unreliable']).toContain(deviceChannelForMessage(message(type))); + } + }); + + it('assigns every protocol event a channel, so a new one cannot be unrouted', () => { + for (const type of Object.keys(VOICE_EVENT_DIRECTIONS) as VoiceEventType[]) { + expect(['reliable', 'unreliable']).toContain(deviceChannelForVoiceEvent(type)); + } + }); + + it('maps a channel kind onto its label', () => { + expect(deviceChannelLabel('reliable')).toBe(RELIABLE_CHANNEL_LABEL); + expect(deviceChannelLabel('unreliable')).toBe(UNRELIABLE_CHANNEL_LABEL); + }); + + it('configures the lossy channel as send-once, unordered', () => { + // Partial reliability is the whole point: a retransmitted meter reading is + // worse than the loss it repaired. + expect(UNRELIABLE_CHANNEL_INIT).toEqual({ ordered: false, maxRetransmits: 0 }); + }); +}); + +describe('encode and decode', () => { + it('stamps the negotiated version on the way out', () => { + const raw = encodeDeviceMessage({ type: 'floor', action: 'press' }, 7); + expect(JSON.parse(raw)).toMatchObject({ v: 7, type: 'floor', action: 'press' }); + }); + + it('round trips', () => { + const original: DeviceMessage = { type: 'audio-level', level: 0.25, speech: false }; + const decoded = decodeDeviceMessage(encodeDeviceMessage(original)); + expect(decoded).toMatchObject({ type: 'audio-level', level: 0.25, speech: false }); + }); + + it.each([ + ['not a string', 42], + ['broken json', '{'], + ['an array', '[]'], + ['no type', '{"v":1}'], + ['no version', '{"type":"floor","action":"press"}'], + ['an unknown type', '{"v":1,"type":"pwn"}'], + ['a voice-event with no event', '{"v":1,"type":"voice-event"}'], + ['a floor with a bad action', '{"v":1,"type":"floor","action":"wiggle"}'], + ])('returns null for %s rather than throwing', (_label, raw) => { + expect(decodeDeviceMessage(raw)).toBeNull(); + }); +}); + +describe('device-originated messages', () => { + it('lets a device originate only its own five', () => { + expect([...DEVICE_ORIGINATED_MESSAGES].sort()).toEqual( + ['audio-level', 'floor', 'hello', 'interrupt', 'link-quality'].sort() + ); + }); + + it('refuses a device that tries to originate desktop state', () => { + expect(isDeviceOriginatedMessage({ type: 'revoked', message: 'nice try' })).toBe(false); + expect(isDeviceOriginatedMessage({ type: 'floor-state', holder: 'x', isSelf: true })).toBe( + false + ); + expect(isDeviceOriginatedMessage({ type: 'floor', action: 'press' })).toBe(true); + }); +}); + +/** A minimally valid event of `type`. Only the discriminant is read by routing. */ +function voiceEvent(type: VoiceEventType): VoiceEvent { + return { type, sessionId: 's', seq: 1, ts: 0 } as unknown as VoiceEvent; +} diff --git a/src/__tests__/shared/acappella-feature-flag.test.ts b/src/__tests__/shared/acappella-feature-flag.test.ts new file mode 100644 index 0000000000..eb2c262846 --- /dev/null +++ b/src/__tests__/shared/acappella-feature-flag.test.ts @@ -0,0 +1,79 @@ +/** + * @file acappella-feature-flag.test.ts + * + * The one reader of the A Cappella Encore flag. + * + * Contracts defended: + * - Only the literal `true` counts. This is the whole reason the helper exists: + * five hand-rolled copies of `flags.aCappella === true` is five chances for one + * of them to drift into truthiness, and the surfaces that read it (IPC, hotkeys, + * the signaling adapter, the transport, the debug collector) each control a real + * resource - a microphone, a global shortcut, a Bonjour advert. + * - A missing, null, or malformed `encoreFeatures` blob reads as OFF rather than + * throwing. It is read on paths that run before any settings have been written. + * - The gate throws a stable error string, not a sentence. + */ + +import { describe, it, expect } from 'vitest'; + +import { + ACAPPELLA_DISABLED_ERROR, + isACappellaEnabled, + requireACappellaEnabled, +} from '../../shared/acappella/feature-flag'; + +/** A settings store that answers with whatever it was handed. */ +function storeOf(encoreFeatures: unknown) { + return { + get: (key: string, defaultValue?: unknown) => + key === 'encoreFeatures' ? encoreFeatures : defaultValue, + }; +} + +describe('isACappellaEnabled', () => { + it('is true only for the literal true', () => { + expect(isACappellaEnabled(storeOf({ aCappella: true }))).toBe(true); + }); + + it.each([ + ['string "true"', 'true'], + ['number 1', 1], + ['an object', {}], + ['the string "on"', 'on'], + ])('reads a truthy %s as OFF', (_label, value) => { + // The safe direction for a flag whose "on" state opens a capture device and + // puts the machine on the network. A hand-edited settings file must not be + // able to half-enable it. + expect(isACappellaEnabled(storeOf({ aCappella: value }))).toBe(false); + }); + + it.each([ + ['false', false], + ['undefined', undefined], + ['null', null], + ])('reads %s as off', (_label, value) => { + expect(isACappellaEnabled(storeOf({ aCappella: value }))).toBe(false); + }); + + it.each([ + ['an absent key', {}], + ['null', null], + ['a string', 'nonsense'], + ['a number', 7], + ])('survives %s where the flag blob should be', (_label, blob) => { + expect(isACappellaEnabled(storeOf(blob))).toBe(false); + }); +}); + +describe('requireACappellaEnabled', () => { + it('passes through when the flag is on', () => { + expect(() => requireACappellaEnabled(storeOf({ aCappella: true }))).not.toThrow(); + }); + + it('throws the stable error code, not prose', () => { + // The renderer maps this string. A channel that answered with a sentence + // would make the copy a wire contract. + expect(() => requireACappellaEnabled(storeOf({}))).toThrow(ACAPPELLA_DISABLED_ERROR); + expect(ACAPPELLA_DISABLED_ERROR).toBe('ACappellaDisabled'); + }); +}); diff --git a/src/__tests__/shared/acappella-model-catalog.test.ts b/src/__tests__/shared/acappella-model-catalog.test.ts new file mode 100644 index 0000000000..b807b62996 --- /dev/null +++ b/src/__tests__/shared/acappella-model-catalog.test.ts @@ -0,0 +1,121 @@ +/** + * @file acappella-model-catalog.test.ts + * + * The catalog is a promise to the user about exactly which bytes will be + * fetched. These tests guard the properties that make that promise checkable: + * pinned revisions (never `main`), real 64-hex SHA-256s, computed totals, and a + * frozen table nothing downstream can edit in place. + */ + +import { describe, it, expect } from 'vitest'; + +import { + KOKORO_82M_ID, + MODEL_SETS, + OPENWAKEWORD_BASE_ID, + QWEN3_1_7B_ID, + VOICE_MODEL_CATALOG, + WHISPER_BASE_EN_ID, + formatModelSetSize, + getModelSetEntries, + getVoiceModel, + isVoiceModelId, + sumModelBytes, +} from '../../shared/acappella/model-catalog'; +import { formatSize } from '../../shared/formatters'; + +describe('voice model catalog', () => { + it('contains the four models the phase specifies', () => { + expect(VOICE_MODEL_CATALOG.map((entry) => entry.id).sort()).toEqual( + [KOKORO_82M_ID, OPENWAKEWORD_BASE_ID, QWEN3_1_7B_ID, WHISPER_BASE_EN_ID].sort() + ); + }); + + it('pins every revision to a commit, never a moving ref', () => { + for (const entry of VOICE_MODEL_CATALOG) { + expect(entry.revision).toMatch(/^[0-9a-f]{40}$/); + expect(entry.revision).not.toBe('main'); + for (const file of entry.files) { + // A `/main/` URL would make the hash below meaningless: the bytes + // behind it could change without the catalog knowing. + expect(file.sourceUrl).toContain(`/resolve/${entry.revision}/`); + expect(file.sourceUrl).not.toContain('/resolve/main/'); + } + } + }); + + it('carries a real SHA-256 and a positive size for every file', () => { + for (const entry of VOICE_MODEL_CATALOG) { + expect(entry.files.length).toBeGreaterThan(0); + for (const file of entry.files) { + expect(file.sha256).toMatch(/^[0-9a-f]{64}$/); + expect(file.bytes).toBeGreaterThan(0); + // Relative, POSIX, and never escaping the install root. + expect(file.path.startsWith('/')).toBe(false); + expect(file.path).not.toContain('..'); + expect(file.path).not.toContain('\\'); + } + } + }); + + it('names a license and a license URL for every model', () => { + for (const entry of VOICE_MODEL_CATALOG) { + expect(entry.license).toBeTruthy(); + expect(entry.licenseUrl).toMatch(/^https:\/\//); + expect(entry.requiredFor).toBeTruthy(); + } + }); + + it('computes each model total from its files', () => { + for (const entry of VOICE_MODEL_CATALOG) { + expect(entry.bytes).toBe(entry.files.reduce((total, file) => total + file.bytes, 0)); + } + }); + + it('computes set totals rather than hard-coding them', () => { + for (const set of Object.values(MODEL_SETS)) { + expect(set.bytes).toBe(sumModelBytes(set.modelIds)); + expect(set.bytes).toBeGreaterThan(0); + } + // The fully-local set is the hands-free set plus the Brain, so it must be + // strictly larger. A copy-paste that left both lists the same fails here. + expect(MODEL_SETS['fully-local'].bytes).toBeGreaterThan(MODEL_SETS['hands-free-local'].bytes); + }); + + it('formats set sizes through the shared formatter', () => { + expect(formatModelSetSize('fully-local')).toBe(formatSize(MODEL_SETS['fully-local'].bytes)); + }); + + it('returns set entries in catalog order', () => { + const ids = getModelSetEntries('fully-local').map((entry) => entry.id); + expect(ids).toEqual(VOICE_MODEL_CATALOG.map((entry) => entry.id)); + }); + + it('excludes the Brain from the hands-free set', () => { + expect(MODEL_SETS['hands-free-local'].modelIds).not.toContain(QWEN3_1_7B_ID); + expect(MODEL_SETS['hands-free-local'].modelIds).toContain(OPENWAKEWORD_BASE_ID); + }); + + it('is frozen all the way down', () => { + expect(Object.isFrozen(VOICE_MODEL_CATALOG)).toBe(true); + for (const entry of VOICE_MODEL_CATALOG) { + expect(Object.isFrozen(entry)).toBe(true); + expect(Object.isFrozen(entry.files)).toBe(true); + for (const file of entry.files) expect(Object.isFrozen(file)).toBe(true); + } + }); + + it('looks models up by id and rejects anything else', () => { + expect(getVoiceModel(WHISPER_BASE_EN_ID)?.role).toBe('stt'); + expect(getVoiceModel('../../etc/passwd')).toBeUndefined(); + expect(isVoiceModelId(KOKORO_82M_ID)).toBe(true); + expect(isVoiceModelId('nope')).toBe(false); + }); + + it('ignores unknown ids when summing', () => { + expect(sumModelBytes(['nope'])).toBe(0); + expect(sumModelBytes([WHISPER_BASE_EN_ID, 'nope'])).toBe( + getVoiceModel(WHISPER_BASE_EN_ID)!.bytes + ); + }); +}); diff --git a/src/__tests__/shared/acappella-native-runtimes.test.ts b/src/__tests__/shared/acappella-native-runtimes.test.ts new file mode 100644 index 0000000000..a202d30050 --- /dev/null +++ b/src/__tests__/shared/acappella-native-runtimes.test.ts @@ -0,0 +1,129 @@ +/** + * @file acappella-native-runtimes.test.ts + * + * The registry is read by four consumers that cannot see each other: the lazy + * loader, the self-test, the capability gate, and a packaging script that runs + * after electron-builder on a machine nobody is watching. Every test here exists + * because a mismatch between the table and reality shows up as a signed release + * that dies on launch, which is the most expensive failure mode this codebase + * has. + */ + +import { describe, it, expect } from 'vitest'; +import fs from 'fs'; +import path from 'path'; + +import { + NATIVE_PLATFORM_KEYS, + NATIVE_RUNTIMES, + getNativeRuntime, + nativeAsarUnpackGlobs, + nativePlatformKey, + runtimesForSlot, +} from '../../shared/acappella/native-runtimes'; + +function readPackageJson(): { + dependencies?: Record; + scripts?: Record; + build?: { asarUnpack?: string[]; mac?: { extendInfo?: Record } }; +} { + return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), 'package.json'), 'utf8')); +} + +describe('native runtime registry', () => { + it('pins an exact version for every runtime', () => { + // A range is how a native dependency silently changes its ABI, its prebuild + // matrix, or its binary layout between two builds of the same tag. + for (const runtime of NATIVE_RUNTIMES) { + expect(runtime.versionPin).toMatch(/^\d+\.\d+\.\d+/); + expect(runtime.versionPin).not.toMatch(/[\^~*x]/); + } + }); + + it('covers every shipped platform for every runtime', () => { + for (const runtime of NATIVE_RUNTIMES) { + for (const key of NATIVE_PLATFORM_KEYS) { + expect(runtime.prebuilds[key]).toBeDefined(); + expect(runtime.packagedBinaries[key]).toBeDefined(); + } + } + }); + + it('keeps `declared` in step with package.json dependencies', () => { + // The one fact this table gets wrong most easily. A runtime marked declared + // that is not installed makes the loader report a broken install; an + // installed runtime marked undeclared makes the loader refuse to load + // something that is sitting right there. + const deps = readPackageJson().dependencies ?? {}; + for (const runtime of NATIVE_RUNTIMES) { + expect( + runtime.declared, + `${runtime.moduleId}: registry says declared=${runtime.declared}, package.json says ${ + runtime.moduleId in deps ? 'present' : 'absent' + }` + ).toBe(runtime.moduleId in deps); + } + }); + + it('pins the version package.json actually installs, when it installs one', () => { + const deps = readPackageJson().dependencies ?? {}; + for (const runtime of NATIVE_RUNTIMES) { + const installed = deps[runtime.moduleId]; + if (!installed) continue; + expect(installed).toBe(runtime.versionPin); + } + }); + + it('has every asarUnpack glob present in the electron-builder config', () => { + // A .node file inside app.asar cannot be dlopen'd. This is the check that + // catches a runtime added to the registry but never added to the build. + const configured = readPackageJson().build?.asarUnpack ?? []; + for (const glob of nativeAsarUnpackGlobs()) { + expect(configured, `missing asarUnpack entry: ${glob}`).toContain(glob); + } + }); + + it('marks a runtime as needing electron-rebuild only if the postinstall rebuilds it', () => { + const postinstall = readPackageJson().scripts?.postinstall ?? ''; + for (const runtime of NATIVE_RUNTIMES) { + if (!runtime.requiresElectronRebuild) continue; + expect( + postinstall, + `${runtime.moduleId} needs an Electron ABI rebuild but is not in the postinstall list` + ).toContain(runtime.moduleId); + } + }); + + it('records a microphone usage description that names the feature and the local path', () => { + // Apple requires the string; a user deciding whether to grant the microphone + // requires it to say something true. "Maestro would like to access the + // microphone" answers neither question. + const description = + readPackageJson().build?.mac?.extendInfo?.NSMicrophoneUsageDescription ?? ''; + expect(description).toContain('A Cappella'); + expect(description.toLowerCase()).toContain('local'); + }); + + it('resolves platform keys, and rejects platforms with no installer', () => { + expect(nativePlatformKey('darwin', 'arm64')).toBe('darwin-arm64'); + expect(nativePlatformKey('win32', 'x64')).toBe('win32-x64'); + expect(nativePlatformKey('linux', 'arm64')).toBeNull(); + expect(nativePlatformKey('sunos', 'sparc')).toBeNull(); + }); + + it('maps every voice slot that has a local tier to a runtime', () => { + expect(runtimesForSlot('brain').map((runtime) => runtime.id)).toEqual(['llama']); + expect(runtimesForSlot('stt').map((runtime) => runtime.id)).toEqual(['whisper']); + // One ONNX Runtime serves both, which is the point of picking it. + expect(runtimesForSlot('tts').map((runtime) => runtime.id)).toEqual(['onnx']); + expect(runtimesForSlot('wake-word').map((runtime) => runtime.id)).toEqual(['onnx']); + expect(runtimesForSlot('microphone')).toEqual([]); + }); + + it('is addressable by id and frozen against mutation', () => { + expect(getNativeRuntime('llama')?.moduleId).toBe('node-llama-cpp'); + expect(getNativeRuntime('nope' as 'llama')).toBeUndefined(); + expect(Object.isFrozen(NATIVE_RUNTIMES)).toBe(true); + expect(Object.isFrozen(NATIVE_RUNTIMES[0])).toBe(true); + }); +}); diff --git a/src/__tests__/shared/acappella-protocol.test.ts b/src/__tests__/shared/acappella-protocol.test.ts new file mode 100644 index 0000000000..1a507ea2d7 --- /dev/null +++ b/src/__tests__/shared/acappella-protocol.test.ts @@ -0,0 +1,260 @@ +/** + * Tests for the A Cappella shared protocol module (src/shared/acappella/). + * + * These cover the runtime pieces only: the state transition table, the event + * direction map, and the route decision helpers plus JSON Schema. The session + * service's use of them is tested separately in the main-process suites. + */ + +import { describe, it, expect } from 'vitest'; +import { + VOICE_EVENT_DIRECTIONS, + isClientVoiceEvent, + isContiguousVoiceSeq, + type VoiceEvent, + type VoiceEventType, +} from '../../shared/acappella/protocol'; +import { + VOICE_SESSION_STATES, + VOICE_STATE_TRANSITIONS, + assertVoiceStateTransition, + canTransitionVoiceState, + isVoiceSessionActive, + InvalidVoiceStateTransitionError, + type VoiceSessionState, +} from '../../shared/acappella/session-state'; +import { + ROUTE_DECISION_JSON_SCHEMA, + ROUTE_TAB_ACTIONS, + isConductorTarget, + routeTargetSessionId, +} from '../../shared/acappella/route-decision'; +import { + audioHostErrorToMicIssue, + audioHostErrorToSessionError, + isRecoverableAudioHostError, + type AudioHostErrorCode, +} from '../../shared/acappella/audio-host'; +import { micSettingsLabel, micSettingsUrl } from '../../shared/acappella/mic-settings'; + +const ALL_EVENT_TYPES: VoiceEventType[] = [ + 'wake', + 'listen-start', + 'listen-stop', + 'partial-transcript', + 'final-transcript', + 'route-decision', + 'dispatch', + 'route-correction', + 'agent-reply', + 'speak-start', + 'speak-sentence', + 'speak-end', + 'barge-in', + 'stop-word', + 'session-error', + 'audio-level', + 'mic-state', + 'provider-state', + 'tab-state', + 'agent-roster', +]; + +function makeEvent(type: VoiceEventType): VoiceEvent { + return { type, sessionId: 'voice-1', seq: 1, ts: 0 } as VoiceEvent; +} + +describe('VOICE_STATE_TRANSITIONS', () => { + it('should cover every state as a source', () => { + expect(Object.keys(VOICE_STATE_TRANSITIONS).sort()).toEqual([...VOICE_SESSION_STATES].sort()); + }); + + it('should only name known states as targets', () => { + for (const targets of Object.values(VOICE_STATE_TRANSITIONS)) { + for (const target of targets) { + expect(VOICE_SESSION_STATES).toContain(target); + } + } + }); + + it('should let every non-idle state stop to idle', () => { + for (const state of VOICE_SESSION_STATES) { + if (state === 'idle') continue; + expect(canTransitionVoiceState(state, 'idle')).toBe(true); + } + }); + + it('should keep the floor on barge-in: speaking -> interrupted -> listening', () => { + expect(canTransitionVoiceState('speaking', 'interrupted')).toBe(true); + expect(canTransitionVoiceState('interrupted', 'listening')).toBe(true); + }); + + it('should reject skipping the pipeline', () => { + expect(canTransitionVoiceState('idle', 'speaking')).toBe(false); + expect(canTransitionVoiceState('listening', 'dispatching')).toBe(false); + expect(canTransitionVoiceState('error', 'listening')).toBe(false); + }); +}); + +describe('assertVoiceStateTransition', () => { + it('should not throw on a legal edge', () => { + expect(() => assertVoiceStateTransition('idle', 'arming')).not.toThrow(); + }); + + it('should accept every edge the table names and reject every pair it does not', () => { + const rejected: string[] = []; + for (const from of VOICE_SESSION_STATES) { + for (const to of VOICE_SESSION_STATES) { + if (VOICE_STATE_TRANSITIONS[from].includes(to)) { + expect(() => assertVoiceStateTransition(from, to)).not.toThrow(); + continue; + } + expect(() => assertVoiceStateTransition(from, to)).toThrow( + InvalidVoiceStateTransitionError + ); + rejected.push(`${from} -> ${to}`); + } + } + + // A state is never a legal target of itself: re-entering `speaking` would + // silently orphan the utterance already on the floor. + for (const state of VOICE_SESSION_STATES) { + expect(rejected).toContain(`${state} -> ${state}`); + } + }); + + it('should throw a typed error naming both states', () => { + let caught: unknown; + try { + assertVoiceStateTransition('idle', 'speaking'); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(InvalidVoiceStateTransitionError); + const typed = caught as InvalidVoiceStateTransitionError; + expect(typed.from).toBe('idle'); + expect(typed.to).toBe('speaking'); + expect(typed.message).toContain('idle -> speaking'); + }); +}); + +describe('isVoiceSessionActive', () => { + it('should treat idle and error as not holding resources', () => { + const inactive: VoiceSessionState[] = ['idle', 'error']; + for (const state of VOICE_SESSION_STATES) { + expect(isVoiceSessionActive(state)).toBe(!inactive.includes(state)); + } + }); +}); + +describe('VOICE_EVENT_DIRECTIONS', () => { + it('should have an entry for every event type', () => { + expect(Object.keys(VOICE_EVENT_DIRECTIONS).sort()).toEqual([...ALL_EVENT_TYPES].sort()); + }); + + it('should mark exactly the four client-originated events as both-way', () => { + const both = ALL_EVENT_TYPES.filter((type) => VOICE_EVENT_DIRECTIONS[type] === 'both'); + expect(both.sort()).toEqual(['barge-in', 'final-transcript', 'stop-word', 'wake']); + }); + + it('should classify events through isClientVoiceEvent', () => { + expect(isClientVoiceEvent(makeEvent('barge-in'))).toBe(true); + expect(isClientVoiceEvent(makeEvent('speak-sentence'))).toBe(false); + }); +}); + +describe('audio host error translation', () => { + const ALL_CODES: AudioHostErrorCode[] = [ + 'permission-denied', + 'no-device', + 'device-lost', + 'unsupported', + 'audio-init-failed', + ]; + + it('gives every capture failure the same session error code', () => { + for (const code of ALL_CODES) { + const error = audioHostErrorToSessionError({ kind: 'mic-error', code, message: 'x' }); + expect(error.code).toBe('audio-capture-failed'); + expect(error.recoverable).toBe(isRecoverableAudioHostError(code)); + } + }); + + it('keeps the three user-fixable failures apart and collapses the rest', () => { + expect(audioHostErrorToMicIssue('permission-denied')).toBe('permission-denied'); + expect(audioHostErrorToMicIssue('no-device')).toBe('no-device'); + expect(audioHostErrorToMicIssue('device-lost')).toBe('device-lost'); + expect(audioHostErrorToMicIssue('unsupported')).toBe('unavailable'); + expect(audioHostErrorToMicIssue('audio-init-failed')).toBe('unavailable'); + }); + + it('agrees with the session error about which failures the user can act on', () => { + for (const code of ALL_CODES) { + expect(audioHostErrorToMicIssue(code) === 'unavailable').toBe( + !isRecoverableAudioHostError(code) + ); + } + }); +}); + +describe('micSettingsUrl', () => { + it('knows where the microphone permission lives on macOS and Windows', () => { + expect(micSettingsUrl('darwin')).toContain('Privacy_Microphone'); + expect(micSettingsUrl('win32')).toBe('ms-settings:privacy-microphone'); + }); + + it('returns null where there is no reliable deep link', () => { + // A button that opens the wrong window is worse than a sentence saying + // where to look, so Linux gets no URL rather than a guess. + expect(micSettingsUrl('linux')).toBeNull(); + expect(micSettingsUrl('')).toBeNull(); + }); + + it('names the place the user is being sent', () => { + expect(micSettingsLabel('win32')).toBe('Open Microphone Settings'); + expect(micSettingsLabel('darwin')).toBe('Open Privacy Settings'); + }); +}); + +describe('isContiguousVoiceSeq', () => { + it('should accept the next sequence number and reject gaps or replays', () => { + expect(isContiguousVoiceSeq(4, 5)).toBe(true); + expect(isContiguousVoiceSeq(4, 6)).toBe(false); + expect(isContiguousVoiceSeq(4, 4)).toBe(false); + }); +}); + +describe('route decision helpers', () => { + it('should identify the conductor target', () => { + expect(isConductorTarget('conductor')).toBe(true); + expect(isConductorTarget({ sessionId: 'agent-1' })).toBe(false); + }); + + it('should extract the agent id, or null for the conductor', () => { + expect(routeTargetSessionId({ sessionId: 'agent-1' })).toBe('agent-1'); + expect(routeTargetSessionId('conductor')).toBeNull(); + }); +}); + +describe('ROUTE_DECISION_JSON_SCHEMA', () => { + it('should describe the same tab actions as the type', () => { + expect(ROUTE_DECISION_JSON_SCHEMA.properties.tabAction.enum).toEqual(ROUTE_TAB_ACTIONS); + }); + + it('should require the fields a dispatch cannot run without', () => { + expect(ROUTE_DECISION_JSON_SCHEMA.required).toEqual([ + 'target', + 'tabAction', + 'prompt', + 'confidence', + ]); + }); + + it('should stay GBNF-friendly: closed object, no $ref, bounded confidence', () => { + const serialized = JSON.stringify(ROUTE_DECISION_JSON_SCHEMA); + expect(serialized).not.toContain('$ref'); + expect(ROUTE_DECISION_JSON_SCHEMA.additionalProperties).toBe(false); + expect(ROUTE_DECISION_JSON_SCHEMA.properties.confidence.minimum).toBe(0); + expect(ROUTE_DECISION_JSON_SCHEMA.properties.confidence.maximum).toBe(1); + }); +}); diff --git a/src/__tests__/shared/acappella-provider-catalog.test.ts b/src/__tests__/shared/acappella-provider-catalog.test.ts new file mode 100644 index 0000000000..413e2c2a55 --- /dev/null +++ b/src/__tests__/shared/acappella-provider-catalog.test.ts @@ -0,0 +1,137 @@ +/** + * @file acappella-provider-catalog.test.ts + * + * The catalog is the one table four subsystems read: the capability gate, the + * provider registry, the credential layer, and the settings panel. Two things + * have to stay true of it, and both are easy to break in a change that looks + * unrelated: + * + * 1. Every provider declares what it needs and what it sends. A provider with + * no `egress` declared would silently be summarised as private. + * 2. The privacy statement is DERIVED. Nobody writes "audio stays on this + * machine" as copy anywhere, because copy cannot be kept in step with a + * selection the user changes at runtime. + */ + +import { describe, it, expect } from 'vitest'; + +import { + ELEVENLABS_TTS_PROVIDER_ID, + HOSTED_PROVIDER_IDS, + LOCAL_PROVIDER_IDS, + OPENAI_REALTIME_PROVIDER_ID, + VOICE_CREDENTIALS, + VOICE_CREDENTIAL_SERVICES, + VOICE_PROVIDER_CATALOG, + getVoiceProvider, + summariseVoiceEgress, + voiceProviderCredential, + voiceProviderRequirement, + voiceProvidersForRole, +} from '../../shared/acappella/provider-catalog'; + +describe('the provider catalog', () => { + it('has a unique id per provider', () => { + const ids = VOICE_PROVIDER_CATALOG.map((entry) => entry.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + it('offers at least one provider for every slot', () => { + for (const role of ['stt', 'tts', 'brain', 'realtime'] as const) { + expect(voiceProvidersForRole(role).length).toBeGreaterThan(0); + } + }); + + it('names a real credential for every provider that needs one', () => { + for (const entry of VOICE_PROVIDER_CATALOG) { + if (entry.requires.kind !== 'api-key') continue; + expect(VOICE_CREDENTIAL_SERVICES).toContain(entry.requires.service); + expect(VOICE_CREDENTIALS[entry.requires.service].label).toBeTruthy(); + } + }); + + it('keeps egress and its service consistent', () => { + for (const entry of VOICE_PROVIDER_CATALOG) { + // A provider that sends something has to say where, or the privacy + // summary would report a service-less egress as nothing at all. + if (entry.egress === 'none') expect(entry.egressService).toBeNull(); + else expect(entry.egressService).not.toBeNull(); + } + }); + + it('says every local provider keeps its data here', () => { + for (const id of Object.values(LOCAL_PROVIDER_IDS)) { + expect(getVoiceProvider(id)?.egress).toBe('none'); + expect(getVoiceProvider(id)?.requires.kind).toBe('model'); + } + }); + + it('says every hosted provider sends something and needs a key', () => { + for (const id of Object.values(HOSTED_PROVIDER_IDS)) { + expect(getVoiceProvider(id)?.egress).not.toBe('none'); + expect(voiceProviderCredential(id)).not.toBeNull(); + } + }); + + it('treats an unknown id as needing nothing', () => { + // The mock tier's contract, and the honest answer for an id the catalog has + // never heard of: it is the registry, not this table, that refuses to run it. + expect(voiceProviderRequirement('not-a-provider')).toEqual({ kind: 'none' }); + expect(voiceProviderCredential('not-a-provider')).toBeNull(); + }); +}); + +describe('summariseVoiceEgress', () => { + it('states plainly that nothing leaves for a fully local trio', () => { + const summary = summariseVoiceEgress(Object.values(LOCAL_PROVIDER_IDS)); + + expect(summary).toMatchObject({ + audioLeaves: false, + textLeaves: false, + services: [], + statement: 'Audio stays on this machine.', + }); + }); + + it('names the service when audio leaves', () => { + const summary = summariseVoiceEgress(['openai-stt', 'kokoro-local', 'qwen3-local']); + + expect(summary.audioLeaves).toBe(true); + expect(summary.statement).toBe('Audio is sent to OpenAI.'); + }); + + it('separates text leaving from audio leaving', () => { + const summary = summariseVoiceEgress([ + LOCAL_PROVIDER_IDS.stt, + ELEVENLABS_TTS_PROVIDER_ID, + 'anthropic-brain', + ]); + + expect(summary.audioLeaves).toBe(false); + expect(summary.textLeaves).toBe(true); + expect(summary.statement).toBe( + 'Audio stays on this machine. Text is sent to ElevenLabs and Anthropic.' + ); + }); + + it('lists three services readably', () => { + const summary = summariseVoiceEgress(['openai-stt', ELEVENLABS_TTS_PROVIDER_ID]); + expect(summary.services).toEqual(['openai', 'elevenlabs']); + expect(summary.statement).toBe('Audio is sent to OpenAI and ElevenLabs.'); + }); + + it('reports the realtime tier as audio leaving', () => { + expect(summariseVoiceEgress([OPENAI_REALTIME_PROVIDER_ID]).statement).toBe( + 'Audio is sent to OpenAI.' + ); + }); + + it('counts an unknown or unresolved id as sending nothing', () => { + // A slot that could not be built sends nothing anywhere, whatever it was + // configured with. Reporting otherwise would be the one sentence in this + // feature that must never be wrong. + expect(summariseVoiceEgress(['unresolved-stt', 'mock-tts']).statement).toBe( + 'Audio stays on this machine.' + ); + }); +}); diff --git a/src/__tests__/shared/acappella-runtime-artifacts.test.ts b/src/__tests__/shared/acappella-runtime-artifacts.test.ts new file mode 100644 index 0000000000..a6169b490b --- /dev/null +++ b/src/__tests__/shared/acappella-runtime-artifacts.test.ts @@ -0,0 +1,247 @@ +/** + * @file acappella-runtime-artifacts.test.ts + * + * The artifact table is a set of promises about bytes that will be downloaded + * onto a user's machine and then dlopen'd. Every test here guards a failure that + * only shows up on someone else's computer: + * + * - A hash that is not a hash, or was pasted one character short, fails AFTER + * the user has waited through a 101 MB download. + * - A URL whose version has drifted from `versionPin` downloads a runtime the + * rest of the build was not written against. + * - A `keep` list that does not include the artifact's own `binary` extracts + * cleanly and produces an install with nothing in it. + * + * The last one is the reason `shouldKeepArchiveEntry` is exported and pure: it is + * the only logic in the installer that can be wrong in two opposite directions, + * and neither is visible without running an extraction. + */ + +import { describe, it, expect } from 'vitest'; + +import { + NATIVE_PLATFORM_KEYS, + NATIVE_RUNTIMES, + getNativeRuntime, + type NativePlatformKey, +} from '../../shared/acappella/native-runtimes'; +import { + NATIVE_RUNTIME_ARTIFACTS, + isNativeRuntimeDownloadable, + nativeRuntimeArtifact, + nativeRuntimeDownloadBytes, +} from '../../shared/acappella/runtime-artifacts'; +import { shouldKeepArchiveEntry } from '../../main/acappella/runtime/runtime-installer'; + +const SHA256_PATTERN = /^[0-9a-f]{64}$/; + +describe('native runtime artifacts', () => { + it('records a real, full-length SHA-256 for every artifact', () => { + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + expect( + SHA256_PATTERN.test(artifact.sha256), + `${artifact.runtimeId}/${artifact.platform} has a malformed sha256: ${artifact.sha256}` + ).toBe(true); + } + }); + + it('pins a version in every URL and never a tag or a range', () => { + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + expect(artifact.url).toMatch(/^https:\/\//); + expect( + /-\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?\.tgz$/.test(artifact.url), + `${artifact.runtimeId}/${artifact.platform} is not pinned to a version: ${artifact.url}` + ).toBe(true); + } + }); + + it('downloads the exact version the runtime registry pins', () => { + // The registry's `versionPin` is what the rest of the build was written + // against. A URL that has drifted from it is how a runtime gets upgraded + // without anyone deciding to upgrade it. + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + const descriptor = getNativeRuntime(artifact.runtimeId); + expect(descriptor).toBeTruthy(); + expect( + artifact.url.includes(descriptor!.versionPin), + `${artifact.runtimeId} pins ${descriptor!.versionPin} but downloads ${artifact.url}` + ).toBe(true); + } + }); + + it('states a positive compressed size for every artifact', () => { + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + expect(artifact.bytes).toBeGreaterThan(0); + } + }); + + it('strips exactly one component, because npm tarballs are rooted at package/', () => { + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + expect(artifact.stripComponents).toBe(1); + } + }); + + it('keeps the subtree its own binary lives in', () => { + // The failure this catches: an artifact that extracts successfully and is + // then missing the one file the loader needs. + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + const kept = shouldKeepArchiveEntry( + `package/${artifact.binary}`, + artifact.stripComponents, + artifact.keep + ); + expect( + kept, + `${artifact.runtimeId}/${artifact.platform} discards its own binary: ${artifact.binary}` + ).toBe(artifact.binary); + } + }); + + it('keeps the subtree its own entry point lives in', () => { + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + const kept = shouldKeepArchiveEntry( + `package/${artifact.entry}`, + artifact.stripComponents, + artifact.keep + ); + expect( + kept, + `${artifact.runtimeId}/${artifact.platform} discards its own entry: ${artifact.entry}` + ).toBe(artifact.entry); + } + }); + + it('offers a payload for every supported platform of the runtimes it covers', () => { + for (const runtimeId of ['llama', 'onnx'] as const) { + for (const platform of NATIVE_PLATFORM_KEYS) { + expect( + isNativeRuntimeDownloadable(runtimeId, platform), + `${runtimeId} has no payload for ${platform}` + ).toBe(true); + } + } + }); + + it('offers no payload for whisper, which publishes no prebuilt binary', () => { + // Deliberate and load-bearing: `smart-whisper` runs node-gyp at install + // time, so there is nothing to download. If this ever starts passing as + // downloadable, someone has added a binary distribution we now maintain. + for (const platform of NATIVE_PLATFORM_KEYS) { + expect(isNativeRuntimeDownloadable('whisper', platform)).toBe(false); + expect(nativeRuntimeArtifact('whisper', platform)).toBeNull(); + } + }); + + it('describes only runtimes that exist in the registry', () => { + const known = new Set(NATIVE_RUNTIMES.map((runtime) => runtime.id)); + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + expect(known.has(artifact.runtimeId)).toBe(true); + } + }); + + it('gives one runtime and platform exactly one payload', () => { + const seen = new Set(); + for (const artifact of NATIVE_RUNTIME_ARTIFACTS) { + const key = `${artifact.runtimeId}/${artifact.platform}`; + expect(seen.has(key), `duplicate artifact for ${key}`).toBe(false); + seen.add(key); + } + }); + + it('serves every ONNX platform from one tarball, with one hash', () => { + // Not a coincidence to be preserved for its own sake: it is why `keep` + // exists, and why the download total has to deduplicate. + const onnx = NATIVE_RUNTIME_ARTIFACTS.filter((entry) => entry.runtimeId === 'onnx'); + expect(onnx.length).toBeGreaterThan(1); + expect(new Set(onnx.map((entry) => entry.url)).size).toBe(1); + expect(new Set(onnx.map((entry) => entry.sha256)).size).toBe(1); + }); + + it('gives each ONNX platform a different subtree to keep', () => { + const onnx = NATIVE_RUNTIME_ARTIFACTS.filter((entry) => entry.runtimeId === 'onnx'); + const binaries = onnx.map((entry) => entry.binary); + expect(new Set(binaries).size).toBe(binaries.length); + }); +}); + +describe('nativeRuntimeDownloadBytes', () => { + const platform: NativePlatformKey = 'darwin-arm64'; + + it('sums the payloads a set of runtimes needs', () => { + const llama = nativeRuntimeArtifact('llama', platform)!; + const onnx = nativeRuntimeArtifact('onnx', platform)!; + expect(nativeRuntimeDownloadBytes(['llama', 'onnx'], platform)).toBe(llama.bytes + onnx.bytes); + }); + + it('counts a shared tarball once', () => { + // Both ONNX slots come from the same 101 MB download. Counting it twice + // would promise a wait that never happens, which reads as a stalled + // progress bar when the second half completes instantly. + const onnx = nativeRuntimeArtifact('onnx', platform)!; + expect(nativeRuntimeDownloadBytes(['onnx', 'onnx'], platform)).toBe(onnx.bytes); + }); + + it('ignores runtimes with no payload rather than throwing', () => { + expect(nativeRuntimeDownloadBytes(['whisper'], platform)).toBe(0); + }); + + it('is zero for an empty selection', () => { + expect(nativeRuntimeDownloadBytes([], platform)).toBe(0); + }); +}); + +describe('shouldKeepArchiveEntry', () => { + const keep = ['dist', 'package.json', 'bin/napi-v6/darwin/arm64']; + + it('strips the leading package/ that every npm tarball carries', () => { + expect(shouldKeepArchiveEntry('package/dist/index.js', 1, keep)).toBe('dist/index.js'); + }); + + it('keeps a file that is exactly a kept prefix', () => { + expect(shouldKeepArchiveEntry('package/package.json', 1, keep)).toBe('package.json'); + }); + + it('keeps everything under a kept directory, however deep', () => { + expect(shouldKeepArchiveEntry('package/bin/napi-v6/darwin/arm64/lib.dylib', 1, keep)).toBe( + 'bin/napi-v6/darwin/arm64/lib.dylib' + ); + }); + + it('discards another platform, which is the whole point', () => { + expect(shouldKeepArchiveEntry('package/bin/napi-v6/win32/x64/onnxruntime.dll', 1, keep)).toBe( + null + ); + expect( + shouldKeepArchiveEntry('package/bin/napi-v6/linux/x64/libonnxruntime.so.1', 1, keep) + ).toBe(null); + }); + + it('matches on whole segments, not string prefixes', () => { + // `arm64-extra` starts with `arm64`. A `startsWith` implementation would + // admit it, and admitting a sibling directory is how the size argument + // quietly stops being true. + expect(shouldKeepArchiveEntry('package/bin/napi-v6/darwin/arm64-extra/x.node', 1, keep)).toBe( + null + ); + expect(shouldKeepArchiveEntry('package/distraction/x.js', 1, keep)).toBe(null); + }); + + it('refuses an entry that climbs out of the root', () => { + expect(shouldKeepArchiveEntry('package/../evil.node', 1, keep)).toBe(null); + expect(shouldKeepArchiveEntry('package/dist/../../evil.node', 1, keep)).toBe(null); + }); + + it('discards an entry with nothing left after stripping', () => { + expect(shouldKeepArchiveEntry('package', 1, keep)).toBe(null); + expect(shouldKeepArchiveEntry('package/', 1, keep)).toBe(null); + }); + + it('discards anything outside every kept prefix', () => { + expect(shouldKeepArchiveEntry('package/README.md', 1, keep)).toBe(null); + expect(shouldKeepArchiveEntry('package/LICENSE', 1, keep)).toBe(null); + }); + + it('keeps nothing when the keep list is empty', () => { + expect(shouldKeepArchiveEntry('package/dist/index.js', 1, [])).toBe(null); + }); +}); diff --git a/src/__tests__/shared/acappella-sentences.test.ts b/src/__tests__/shared/acappella-sentences.test.ts new file mode 100644 index 0000000000..942bcbe58b --- /dev/null +++ b/src/__tests__/shared/acappella-sentences.test.ts @@ -0,0 +1,129 @@ +/** + * Tests for the single spoken-sentence splitter shared by the session service + * (which announces `sentenceCount` up front) and every TTS provider (which + * emits one chunk per sentence). If these two ever disagree, a client's + * "sentence 3 of 5" progress never completes, so the splitter is pinned here. + */ + +import { describe, it, expect } from 'vitest'; +import { + MAX_SPOKEN_SENTENCE_LENGTH, + countSpokenSentences, + splitCompleteSentences, + splitIntoSpokenSentences, +} from '../../shared/acappella/sentences'; + +describe('splitIntoSpokenSentences', () => { + it('returns nothing for empty or whitespace-only text', () => { + expect(splitIntoSpokenSentences('')).toEqual([]); + expect(splitIntoSpokenSentences(' \n\t ')).toEqual([]); + }); + + it('splits on terminal punctuation and keeps it', () => { + expect(splitIntoSpokenSentences('All done. Two files changed! Ready?')).toEqual([ + 'All done.', + 'Two files changed!', + 'Ready?', + ]); + }); + + it('normalizes whitespace so wrapped agent output has no embedded newlines', () => { + expect(splitIntoSpokenSentences('Fixed the\n auth bug.\n\nTests pass.')).toEqual([ + 'Fixed the auth bug.', + 'Tests pass.', + ]); + }); + + it('keeps abbreviations intact', () => { + expect(splitIntoSpokenSentences('Ask Dr. Kim about it.')).toEqual(['Ask Dr. Kim about it.']); + expect(splitIntoSpokenSentences('Check a store, e.g. the session one.')).toEqual([ + 'Check a store, e.g. the session one.', + ]); + expect(splitIntoSpokenSentences('It ships in the U.S. only.')).toEqual([ + 'It ships in the U.S. only.', + ]); + }); + + it('splits after an acronym, which agents write constantly', () => { + // A `(? { + expect(splitIntoSpokenSentences('Coverage is 99.5 percent now.')).toEqual([ + 'Coverage is 99.5 percent now.', + ]); + expect(splitIntoSpokenSentences('Bumped it to v1.2.3 this morning.')).toEqual([ + 'Bumped it to v1.2.3 this morning.', + ]); + expect(splitIntoSpokenSentences('The fix is in src/main/index.ts near the top.')).toEqual([ + 'The fix is in src/main/index.ts near the top.', + ]); + }); + + it('treats an unterminated tail as its own sentence', () => { + expect(splitIntoSpokenSentences('Done. Now the tests')).toEqual(['Done.', 'Now the tests']); + }); + + it('collapses a run of terminal punctuation into one boundary', () => { + expect(splitIntoSpokenSentences('Wow!!! Really?!')).toEqual(['Wow!!!', 'Really?!']); + }); + + it('hard-wraps punctuation-free text at a word boundary', () => { + const long = 'word '.repeat(120).trim(); + const sentences = splitIntoSpokenSentences(long); + + expect(sentences.length).toBeGreaterThan(1); + for (const sentence of sentences) { + expect(sentence.length).toBeLessThanOrEqual(MAX_SPOKEN_SENTENCE_LENGTH); + expect(sentence).not.toMatch(/^\s|\s$/); + } + expect(sentences.join(' ')).toBe(long); + }); + + it('counts what it splits', () => { + const text = 'One. Two. Three.'; + expect(countSpokenSentences(text)).toBe(splitIntoSpokenSentences(text).length); + expect(countSpokenSentences(' ')).toBe(0); + }); +}); + +describe('splitCompleteSentences', () => { + it('holds back the fragment still being written', () => { + expect(splitCompleteSentences('All done. Now the te')).toEqual({ + sentences: ['All done.'], + rest: 'Now the te', + }); + }); + + it('holds back a token that ends in a period, because the next character decides', () => { + // "index." becomes "index.ts" one token later. A sentence already synthesized + // cannot be taken back. + expect(splitCompleteSentences('The fix is in index.')).toEqual({ + sentences: [], + rest: 'The fix is in index.', + }); + }); + + it('preserves the separator so the next delta does not fuse onto the tail', () => { + let buffer = ''; + const spoken: string[] = []; + for (const delta of ['Done, ', 'the auth bug ', 'was stale. ', 'Two files changed.']) { + buffer += delta; + const result = splitCompleteSentences(buffer); + buffer = result.rest; + spoken.push(...result.sentences); + } + + expect(spoken).toEqual(['Done, the auth bug was stale.']); + expect(buffer).toBe('Two files changed.'); + }); + + it('has nothing to say about an empty buffer', () => { + expect(splitCompleteSentences('')).toEqual({ sentences: [], rest: '' }); + }); +}); diff --git a/src/__tests__/shared/acappella-ui-prefs.test.ts b/src/__tests__/shared/acappella-ui-prefs.test.ts new file mode 100644 index 0000000000..b9acfab06d --- /dev/null +++ b/src/__tests__/shared/acappella-ui-prefs.test.ts @@ -0,0 +1,161 @@ +/** + * The HUD's remembered preferences, and where the widget is allowed to sit. + * + * The placement math is the part worth testing without a DOM: the cases that + * matter are a position saved on a monitor that is no longer attached and a + * window the user just made smaller, both of which leave a live microphone on + * screen at coordinates nobody can see. + */ + +import { describe, it, expect } from 'vitest'; +import { + DEFAULT_VOICE_UI_PREFS, + VOICE_HUD_EDGE_MARGIN, + clampVoiceHudPosition, + defaultVoiceHudPosition, + readVoiceHudPosition, + readVoiceUiPrefs, +} from '../../shared/acappella/ui-prefs'; +import { + VOICE_HUD_STATE_LABELS, + voiceHudIsHotMic, + voiceHudVisualState, +} from '../../shared/acappella/hud-state'; +import { + clampTtsRate, + clampTtsVolume, + DEFAULT_TTS_RATE, + DEFAULT_TTS_VOLUME, + MAX_TTS_RATE, + MIN_TTS_RATE, + MIN_TTS_VOLUME, + resolveHoldThresholdMs, + DEFAULT_HOLD_THRESHOLD_MS, + MAX_HOLD_THRESHOLD_MS, + MIN_HOLD_THRESHOLD_MS, +} from '../../shared/acappella/voice-controls'; + +const SIZE = { width: 340, height: 160 }; +const VIEWPORT = { width: 1440, height: 900 }; + +describe('readVoiceUiPrefs', () => { + it('reads an empty blob as the shipped defaults', () => { + expect(readVoiceUiPrefs(undefined)).toEqual(DEFAULT_VOICE_UI_PREFS); + expect(readVoiceUiPrefs({})).toEqual(DEFAULT_VOICE_UI_PREFS); + }); + + it('keeps the transcript off unless it was explicitly turned on', () => { + expect(readVoiceUiPrefs({ transcriptVisible: 'yes' }).transcriptVisible).toBe(false); + expect(readVoiceUiPrefs({ transcriptVisible: true }).transcriptVisible).toBe(true); + }); + + it('rejects a minimize behaviour it does not recognise', () => { + expect(readVoiceUiPrefs({ minimizeBehavior: 'close' }).minimizeBehavior).toBe('manual'); + expect(readVoiceUiPrefs({ minimizeBehavior: 'auto-idle' }).minimizeBehavior).toBe('auto-idle'); + }); +}); + +describe('readVoiceHudPosition', () => { + it('takes a complete position', () => { + expect(readVoiceHudPosition({ top: 10, left: 20 })).toEqual({ top: 10, left: 20 }); + }); + + it.each([ + ['half a position', { top: 10 }], + ['a string coordinate', { top: '10', left: 20 }], + ['a NaN coordinate', { top: Number.NaN, left: 20 }], + ['not an object', 'bottom-right'], + ['null', null], + ])('reads %s as no position at all', (_label, value) => { + expect(readVoiceHudPosition(value)).toBeNull(); + }); +}); + +describe('clampVoiceHudPosition', () => { + it('leaves an on-screen position alone', () => { + expect(clampVoiceHudPosition({ top: 100, left: 200 }, SIZE, VIEWPORT)).toEqual({ + top: 100, + left: 200, + }); + }); + + it('pulls back a position saved on a monitor that is no longer there', () => { + const rescued = clampVoiceHudPosition({ top: 400, left: 3000 }, SIZE, VIEWPORT); + expect(rescued.left).toBe(VIEWPORT.width - SIZE.width); + expect(rescued.top).toBe(400); + }); + + it('pulls back a position the window just shrank past', () => { + const rescued = clampVoiceHudPosition({ top: 880, left: 100 }, SIZE, { + width: 800, + height: 600, + }); + expect(rescued.top).toBe(600 - SIZE.height); + expect(rescued.left).toBe(100); + }); + + it('never goes negative, even in a viewport smaller than the widget', () => { + const rescued = clampVoiceHudPosition({ top: -50, left: -50 }, SIZE, { + width: 100, + height: 80, + }); + expect(rescued).toEqual({ top: 0, left: 0 }); + }); +}); + +describe('defaultVoiceHudPosition', () => { + it('parks the widget bottom-right, inset by the edge margin', () => { + expect(defaultVoiceHudPosition(SIZE, VIEWPORT)).toEqual({ + left: VIEWPORT.width - SIZE.width - VOICE_HUD_EDGE_MARGIN, + top: VIEWPORT.height - SIZE.height - VOICE_HUD_EDGE_MARGIN, + }); + }); +}); + +describe('voiceHudVisualState', () => { + it('collapses the three working states into one readable "thinking"', () => { + expect(voiceHudVisualState('transcribing')).toBe('thinking'); + expect(voiceHudVisualState('routing')).toBe('thinking'); + expect(voiceHudVisualState('dispatching')).toBe('thinking'); + }); + + it('reports an interruption as listening, because barge-in keeps the floor', () => { + expect(voiceHudVisualState('interrupted')).toBe('listening'); + }); + + it('never claims a hot microphone for a state that has none', () => { + expect(voiceHudIsHotMic(voiceHudVisualState('listening'))).toBe(true); + for (const state of ['idle', 'arming', 'speaking', 'routing', 'error'] as const) { + expect(voiceHudIsHotMic(voiceHudVisualState(state))).toBe(false); + } + }); + + it('has a label for every visual state', () => { + for (const label of Object.values(VOICE_HUD_STATE_LABELS)) { + expect(label.length).toBeGreaterThan(0); + } + }); +}); + +describe('voice output clamps', () => { + it('keeps the rate inside the shipped window', () => { + expect(clampTtsRate(5)).toBe(MAX_TTS_RATE); + expect(clampTtsRate(0.1)).toBe(MIN_TTS_RATE); + expect(clampTtsRate('fast')).toBe(DEFAULT_TTS_RATE); + expect(clampTtsRate(Number.NaN)).toBe(DEFAULT_TTS_RATE); + }); + + it('floors the volume above silence, so a slider cannot become a silent mute', () => { + expect(clampTtsVolume(0)).toBe(MIN_TTS_VOLUME); + expect(clampTtsVolume(-1)).toBe(MIN_TTS_VOLUME); + expect(clampTtsVolume(4)).toBe(1); + expect(clampTtsVolume(undefined)).toBe(DEFAULT_TTS_VOLUME); + }); + + it('clamps a hold threshold rather than rejecting it', () => { + expect(resolveHoldThresholdMs(10)).toBe(MIN_HOLD_THRESHOLD_MS); + expect(resolveHoldThresholdMs(99_999)).toBe(MAX_HOLD_THRESHOLD_MS); + expect(resolveHoldThresholdMs('slow')).toBe(DEFAULT_HOLD_THRESHOLD_MS); + expect(resolveHoldThresholdMs(450)).toBe(450); + }); +}); diff --git a/src/__tests__/shared/pianola/pianola-first-party-plugin.test.ts b/src/__tests__/shared/pianola/pianola-first-party-plugin.test.ts index 32a3047418..545338c2fd 100644 --- a/src/__tests__/shared/pianola/pianola-first-party-plugin.test.ts +++ b/src/__tests__/shared/pianola/pianola-first-party-plugin.test.ts @@ -147,6 +147,7 @@ describe('first-party plugin registry', () => { ['opencodeServer', 'com.maestro.opencode-server'], ['concerto', 'com.maestro.concerto'], ['groupsPlus', 'com.maestro.groups-plus'], + ['aCappella', 'com.maestro.acappella'], ]); }); diff --git a/src/__tests__/web-desktop/acappella-client/client.test.ts b/src/__tests__/web-desktop/acappella-client/client.test.ts new file mode 100644 index 0000000000..f5e84e0786 --- /dev/null +++ b/src/__tests__/web-desktop/acappella-client/client.test.ts @@ -0,0 +1,735 @@ +/** + * The browser reference client, driven against fakes. + * + * These assertions are the conformance items from + * `docs/ios-client/protocol-conformance.md` made executable, which is the only + * reason the client's socket, peer, microphone, clock, and token store are all + * injectable. Each test names the item it pins. + */ + +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; + +import { + RELIABLE_CHANNEL_LABEL, + UNRELIABLE_CHANNEL_LABEL, + encodeDeviceMessage, + type DeviceMessage, +} from '../../../shared/acappella/device-protocol'; +import type { VoiceEvent } from '../../../shared/acappella/protocol'; +import type { SignalingServerMessage } from '../../../shared/acappella/signaling-protocol'; +import { DEFAULT_REMOTE_AUDIO_CONFIG } from '../../../shared/acappella/webrtc-host'; +import { + ACappellaReferenceClient, + type ClientEvent, + type PairingStore, + type SignalingSocket, + type SignalingSocketHandlers, + type StoredPairing, +} from '../../../web-desktop/acappella-client/client'; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +class FakeChannel { + readyState: RTCDataChannelState = 'connecting'; + onopen: (() => void) | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + readonly sent: string[] = []; + closed = false; + + constructor( + readonly label: string, + readonly init: RTCDataChannelInit | undefined + ) {} + + open(): void { + this.readyState = 'open'; + this.onopen?.(); + } + + send(data: string): void { + this.sent.push(data); + } + + close(): void { + this.closed = true; + this.readyState = 'closed'; + } + + /** Every frame this channel sent, decoded. */ + messages(): Array> { + return this.sent.map((raw) => JSON.parse(raw) as Record); + } +} + +const OPUS_OFFER_SDP = [ + 'v=0', + 'm=audio 9 UDP/TLS/RTP/SAVPF 111', + 'a=rtpmap:111 opus/48000/2', + 'a=fmtp:111 minptime=10', + '', +].join('\r\n'); + +class FakePeer { + connectionState: RTCPeerConnectionState = 'new'; + onicecandidate: ((event: { candidate: RTCIceCandidate | null }) => void) | null = null; + ontrack: ((event: { streams: MediaStream[]; track: MediaStreamTrack }) => void) | null = null; + onconnectionstatechange: (() => void) | null = null; + readonly channels: FakeChannel[] = []; + readonly senderTracks: Array = []; + localDescription: { type: string; sdp?: string } | null = null; + closed = false; + /** Order of the negotiation calls, so "channels before the offer" is testable. */ + readonly calls: string[] = []; + + constructor(readonly config: RTCConfiguration) {} + + createDataChannel(label: string, init?: RTCDataChannelInit): FakeChannel { + this.calls.push(`channel:${label}`); + const channel = new FakeChannel(label, init); + this.channels.push(channel); + return channel; + } + + addTransceiver(kind: string, init: { direction: string }): { sender: unknown } { + this.calls.push(`transceiver:${kind}:${init.direction}`); + return { + sender: { + replaceTrack: (track: MediaStreamTrack | null) => { + this.senderTracks.push(track); + return Promise.resolve(); + }, + getParameters: () => ({ encodings: [{}] }), + setParameters: () => Promise.resolve(), + }, + }; + } + + createOffer(): Promise<{ type: string; sdp: string }> { + this.calls.push('createOffer'); + return Promise.resolve({ type: 'offer', sdp: OPUS_OFFER_SDP }); + } + + setLocalDescription(description: { type: string; sdp?: string }): Promise { + this.localDescription = description; + return Promise.resolve(); + } + + setRemoteDescription(): Promise { + return Promise.resolve(); + } + + addIceCandidate(): Promise { + return Promise.resolve(); + } + + getStats(): Promise<{ forEach: (fn: (value: unknown) => void) => void }> { + return Promise.resolve({ forEach: () => {} }); + } + + close(): void { + this.closed = true; + } + + channel(label: string): FakeChannel { + const found = this.channels.find((entry) => entry.label === label); + if (!found) throw new Error(`No channel ${label}`); + return found; + } +} + +class FakeSocket implements SignalingSocket { + readonly sent: Array> = []; + closed = false; + constructor( + readonly url: string, + readonly handlers: SignalingSocketHandlers + ) {} + send(message: Record): void { + this.sent.push(message); + } + close(): void { + this.closed = true; + } + ops(): string[] { + return this.sent.map((message) => String(message.op)); + } +} + +function memoryStore( + initial: StoredPairing | null = null +): PairingStore & { value: StoredPairing | null } { + return { + value: initial, + read(): StoredPairing | null { + return this.value; + }, + write(pairing: StoredPairing): void { + this.value = pairing; + }, + clear(): void { + this.value = null; + }, + }; +} + +function fakeTrack(): MediaStreamTrack { + return { stop: vi.fn(), kind: 'audio' } as unknown as MediaStreamTrack; +} + +function fakeStream(track: MediaStreamTrack): MediaStream { + return { + getTracks: () => [track], + getAudioTracks: () => [track], + } as unknown as MediaStream; +} + +const TARGET = { host: '192.168.1.5', port: 4123, token: 'server-token', code: 'ABC123' }; + +interface Harness { + client: ACappellaReferenceClient; + sockets: FakeSocket[]; + peers: FakePeer[]; + store: PairingStore & { value: StoredPairing | null }; + events: ClientEvent[]; + track: MediaStreamTrack; + socket(): FakeSocket; + peer(): FakePeer; + now: { value: number }; +} + +function harness(options: { stored?: StoredPairing | null } = {}): Harness { + const sockets: FakeSocket[] = []; + const peers: FakePeer[] = []; + const store = memoryStore(options.stored ?? null); + const events: ClientEvent[] = []; + const track = fakeTrack(); + const now = { value: 1_000_000 }; + + const client = new ACappellaReferenceClient({ + identity: { name: 'Reference', platform: 'browser', appVersion: '9.9.9' }, + store, + openSocket: (url, handlers) => { + const socket = new FakeSocket(url, handlers); + sockets.push(socket); + return socket; + }, + createPeerConnection: (config) => { + const peer = new FakePeer(config); + peers.push(peer); + return peer as unknown as RTCPeerConnection; + }, + openMicrophone: () => Promise.resolve(fakeStream(track)), + now: () => now.value, + }); + client.subscribe((event) => events.push(event)); + + return { + client, + sockets, + peers, + store, + events, + track, + now, + socket: () => sockets[sockets.length - 1], + peer: () => peers[peers.length - 1], + }; +} + +/** Let the client's internal promises settle. */ +async function settle(): Promise { + for (let i = 0; i < 8; i += 1) await Promise.resolve(); +} + +const AUTHENTICATED: SignalingServerMessage = { + op: 'authenticated', + deviceId: 'device-1', + // Deliberately not `DEVICE_PROTOCOL_VERSION`: the client must stamp what the + // desktop negotiated, not its own constant. C-35. + protocolVersion: 7, + iceServers: [{ urls: 'stun:stun.example:3478' }], + iceTransportPolicy: 'all', + audio: DEFAULT_REMOTE_AUDIO_CONFIG, +}; + +/** Get to a live peer with both channels open, the way a real session does. */ +async function connected(options: { stored?: StoredPairing | null } = {}): Promise { + const h = harness({ + stored: options.stored ?? { deviceId: 'device-1', token: 'tok', fingerprint: 'fp' }, + }); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage(AUTHENTICATED); + await settle(); + h.peer().channel(RELIABLE_CHANNEL_LABEL).open(); + h.peer().channel(UNRELIABLE_CHANNEL_LABEL).open(); + return h; +} + +function inbound(peer: FakePeer, label: string, message: DeviceMessage, version = 7): void { + peer.channel(label).onmessage?.({ data: encodeDeviceMessage(message, version) }); +} + +function voiceEvent(event: Partial & { type: VoiceEvent['type'] }): VoiceEvent { + return { sessionId: 'voice-1', seq: 1, ts: 0, ...event } as VoiceEvent; +} + +// --------------------------------------------------------------------------- + +describe('ACappellaReferenceClient - pairing', () => { + beforeEach(() => { + vi.useRealTimers(); + }); + + it('claims with a non-empty name and platform (C-02)', () => { + const h = harness(); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + + expect(h.socket().sent[0]).toEqual({ + op: 'pair-claim', + code: 'ABC123', + name: 'Reference', + platform: 'browser', + appVersion: '9.9.9', + }); + }); + + it('polls once a second and keeps the deadline from the FIRST pair-pending (C-03)', () => { + vi.useFakeTimers(); + const h = harness(); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage({ + op: 'pair-pending', + requestId: 'req-1', + expiresAt: 1_000_120_000, + }); + + vi.advanceTimersByTime(3000); + expect( + h + .socket() + .ops() + .filter((op) => op === 'pair-poll') + ).toHaveLength(3); + + // The poll response carries `expiresAt: 0`. Taking it would collapse the + // deadline to 1970 and stop the countdown immediately. + h.socket().handlers.onMessage({ op: 'pair-pending', requestId: 'req-1', expiresAt: 0 }); + vi.advanceTimersByTime(2000); + expect( + h + .socket() + .ops() + .filter((op) => op === 'pair-poll') + ).toHaveLength(5); + vi.useRealTimers(); + }); + + it('writes the token before anything else, then authenticates (C-04, C-06)', () => { + const h = harness(); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage({ op: 'pair-pending', requestId: 'req-1', expiresAt: 9e12 }); + h.socket().handlers.onMessage({ op: 'pair-approved', deviceId: 'device-1', token: 'secret' }); + + expect(h.store.value).toEqual({ + deviceId: 'device-1', + token: 'secret', + fingerprint: 'server-t', + }); + const auth = h.socket().sent.find((message) => message.op === 'auth'); + expect(auth).toMatchObject({ deviceId: 'device-1', token: 'secret' }); + expect(Number.isInteger(auth?.protocolVersion as number)).toBe(true); + expect(auth?.protocolVersion as number).toBeGreaterThanOrEqual(1); + }); + + it('shows a rejection message verbatim and stops (C-05)', () => { + const h = harness(); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage({ + op: 'pair-rejected', + reason: 'expired', + message: 'That pairing code has expired. Start pairing again on the desktop.', + }); + + const state = h.client.snapshot(); + expect(state.message).toBe( + 'That pairing code has expired. Start pairing again on the desktop.' + ); + expect(state.phase).toBe('terminal'); + }); + + it('sends exactly one auth per socket (C-09)', () => { + const h = harness({ stored: { deviceId: 'device-1', token: 'tok', fingerprint: 'fp' } }); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onOpen(); + + expect( + h + .socket() + .ops() + .filter((op) => op === 'auth') + ).toHaveLength(1); + }); + + it('clears the stored pairing on auth-failed (C-13)', () => { + const h = harness({ stored: { deviceId: 'device-1', token: 'stale', fingerprint: 'fp' } }); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage({ + op: 'auth-failed', + reason: 'unauthorized', + message: 'This device is not paired with this computer, or its pairing was revoked.', + }); + + expect(h.store.value).toBeNull(); + expect(h.client.snapshot().phase).toBe('terminal'); + }); +}); + +describe('ACappellaReferenceClient - peer setup', () => { + it('creates both channels with the exact labels and inits, before the offer (C-16, C-17)', async () => { + const h = await connected(); + const peer = h.peer(); + + expect(peer.channel(RELIABLE_CHANNEL_LABEL).init).toEqual({ ordered: true }); + expect(peer.channel(UNRELIABLE_CHANNEL_LABEL).init).toEqual({ + ordered: false, + maxRetransmits: 0, + }); + // Channels and the audio transceiver all exist before the SDP is created, + // so the first offer already carries the SCTP association and the m-line. + expect(peer.calls.indexOf('createOffer')).toBeGreaterThan( + peer.calls.indexOf(`channel:${UNRELIABLE_CHANNEL_LABEL}`) + ); + expect(peer.calls).toContain('transceiver:audio:sendrecv'); + }); + + it('uses the ICE servers as sent and hard-codes none (C-10)', async () => { + const h = await connected(); + expect(h.peer().config.iceServers).toEqual([{ urls: 'stun:stun.example:3478' }]); + expect(h.peer().config.iceTransportPolicy).toBe('all'); + }); + + it('applies the desktop audio config to the offer SDP (C-11)', async () => { + const h = await connected(); + const offer = h.socket().sent.find((message) => message.op === 'offer') as { + sdp: { sdp: string }; + }; + expect(offer.sdp.sdp).toContain('useinbandfec=1'); + expect(offer.sdp.sdp).toContain('usedtx=1'); + expect(offer.sdp.sdp).toContain( + `maxaveragebitrate=${DEFAULT_REMOTE_AUDIO_CONFIG.maxAverageBitrate}` + ); + }); + + it('sends hello first on the state channel, stamped with the NEGOTIATED version (C-18, C-19, C-35)', async () => { + const h = await connected(); + const [first] = h.peer().channel(RELIABLE_CHANNEL_LABEL).messages(); + + expect(first).toMatchObject({ + type: 'hello', + v: 7, + identity: { deviceId: 'device-1', name: 'Reference', platform: 'browser' }, + }); + }); + + it('does not wait for welcome before it is usable (C-31)', async () => { + const h = await connected(); + h.client.pressFloor(); + // No `welcome` has arrived and none ever will from desktop v1. The floor + // request still goes out. + expect(h.peer().channel(UNRELIABLE_CHANNEL_LABEL).messages()).toContainEqual( + expect.objectContaining({ type: 'floor', action: 'press' }) + ); + }); +}); + +describe('ACappellaReferenceClient - the floor', () => { + it('routes each message to the channel the table names (C-20, C-21)', async () => { + const h = await connected(); + h.client.pressFloor({ kind: 'agent', sessionId: 'agent-7' }); + h.client.requestStop(); + + const live = h.peer().channel(UNRELIABLE_CHANNEL_LABEL).messages(); + expect(live).toContainEqual( + expect.objectContaining({ type: 'floor', scope: { kind: 'agent', sessionId: 'agent-7' } }) + ); + expect(live).toContainEqual(expect.objectContaining({ type: 'interrupt', kind: 'stop-word' })); + // Only the five device-originated types are ever sent. + const everySent = [...h.peer().channel(RELIABLE_CHANNEL_LABEL).messages(), ...live].map( + (message) => message.type + ); + for (const type of everySent) { + expect(['hello', 'floor', 'interrupt', 'audio-level', 'link-quality']).toContain(type); + } + }); + + it('opens the microphone only once the desktop says the floor is ours (C-37, C-38)', async () => { + const h = await connected(); + h.client.pressFloor(); + await settle(); + // Pressed, but not yet granted. Nothing is captured. + expect(h.peer().senderTracks).toHaveLength(0); + expect(h.client.microphone).toBeNull(); + + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'floor-state', + holder: 'device-1', + isSelf: true, + }); + await settle(); + expect(h.peer().senderTracks[0]).not.toBeNull(); + expect(h.client.snapshot().sending).toBe(true); + + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'floor-state', + holder: 'local', + isSelf: false, + }); + await settle(); + expect(h.track.stop).toHaveBeenCalled(); + expect(h.peer().senderTracks[h.peer().senderTracks.length - 1]).toBeNull(); + expect(h.client.snapshot().sending).toBe(false); + }); + + it('sends audio-level only while the floor is open, throttled (C-39)', async () => { + const h = await connected(); + h.client.reportAudioLevel(0.4, true); + expect(h.peer().channel(UNRELIABLE_CHANNEL_LABEL).messages()).toHaveLength(0); + + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'floor-state', + holder: 'device-1', + isSelf: true, + }); + await settle(); + + h.client.reportAudioLevel(0.4, true); + h.client.reportAudioLevel(0.5, true); // Same millisecond: throttled away. + h.now.value += 60; + h.client.reportAudioLevel(0.6, false); + + const levels = h + .peer() + .channel(UNRELIABLE_CHANNEL_LABEL) + .messages() + .filter((message) => message.type === 'audio-level'); + expect(levels).toHaveLength(2); + expect(levels[1]).toMatchObject({ level: 0.6, speech: false }); + }); + + it('records a takeover from the desktop rather than the local gesture (C-48)', async () => { + const h = await connected(); + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'floor-state', + holder: 'device-2', + isSelf: false, + takenOverBy: "Pedram's iPhone", + }); + + expect(h.client.snapshot().floor).toEqual({ + holder: 'device-2', + isSelf: false, + takenOverBy: "Pedram's iPhone", + }); + }); +}); + +describe('ACappellaReferenceClient - interrupts', () => { + it('ducks locally BEFORE the interrupt frame goes out (C-44)', async () => { + const h = await connected(); + h.events.length = 0; + h.client.requestBargeIn(); + + const duckIndex = h.events.findIndex((event) => event.type === 'duck' && event.ducked); + expect(duckIndex).toBeGreaterThanOrEqual(0); + // The frame is on the wire only after the duck event was emitted. + expect(h.peer().channel(UNRELIABLE_CHANNEL_LABEL).messages()).toContainEqual( + expect.objectContaining({ type: 'interrupt', kind: 'barge-in' }) + ); + }); + + it('lifts the duck on the authoritative barge-in (C-45, C-46)', async () => { + const h = await connected(); + h.client.requestBargeIn(); + h.events.length = 0; + + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'voice-event', + event: voiceEvent({ type: 'barge-in', source: 'voice' } as Partial & { + type: 'barge-in'; + }), + }); + + expect(h.events).toContainEqual({ type: 'duck', ducked: false }); + // Barge-in keeps the floor. Nothing here releases it. + expect( + h + .peer() + .channel(UNRELIABLE_CHANNEL_LABEL) + .messages() + .filter((message) => message.action === 'release') + ).toHaveLength(0); + }); + + it('lifts the duck by itself when the desktop never answers (C-45)', async () => { + vi.useFakeTimers(); + const h = harness({ stored: { deviceId: 'device-1', token: 'tok', fingerprint: 'fp' } }); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage(AUTHENTICATED); + await vi.advanceTimersByTimeAsync(0); + h.peer().channel(RELIABLE_CHANNEL_LABEL).open(); + h.peer().channel(UNRELIABLE_CHANNEL_LABEL).open(); + + h.client.requestBargeIn(); + h.events.length = 0; + await vi.advanceTimersByTimeAsync(600); + expect(h.events).toContainEqual({ type: 'duck', ducked: false }); + vi.useRealTimers(); + }); +}); + +describe('ACappellaReferenceClient - inbound frames', () => { + it('ignores malformed and unknown frames without closing anything (C-22, C-23)', async () => { + const h = await connected(); + const channel = h.peer().channel(RELIABLE_CHANNEL_LABEL); + + channel.onmessage?.({ data: 'not json' }); + channel.onmessage?.({ data: JSON.stringify({ type: 'floor-state', holder: null }) }); // no `v` + channel.onmessage?.({ data: JSON.stringify({ type: 'made-up', v: 7 }) }); + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'voice-event', + event: voiceEvent({ type: 'invented-event' as VoiceEvent['type'] }), + }); + + expect(channel.closed).toBe(false); + expect(h.peer().closed).toBe(false); + expect(h.client.snapshot().phase).not.toBe('terminal'); + }); + + it('flags a seq gap on the reliable channel and not on the lossy one (C-29)', async () => { + const h = await connected(); + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'voice-event', + event: voiceEvent({ type: 'listen-stop', seq: 1 } as Partial & { + type: 'listen-stop'; + }), + }); + inbound(h.peer(), UNRELIABLE_CHANNEL_LABEL, { + type: 'voice-event', + event: voiceEvent({ type: 'audio-level', seq: 40 } as Partial & { + type: 'audio-level'; + }), + }); + expect(h.client.snapshot().transcriptSuspect).toBe(false); + + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'voice-event', + event: voiceEvent({ type: 'listen-stop', seq: 9 } as Partial & { + type: 'listen-stop'; + }), + }); + expect(h.client.snapshot().transcriptSuspect).toBe(true); + }); + + it('handles welcome if it ever arrives (C-32)', async () => { + const h = await connected(); + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'welcome', + version: 7, + appVersion: '1.2.3', + sessionId: 'voice-9', + }); + + expect(h.client.snapshot().desktopVersion).toBe('1.2.3'); + }); + + it('treats revoked as terminal and forgets the pairing (C-12)', async () => { + vi.useFakeTimers(); + const h = harness({ stored: { deviceId: 'device-1', token: 'tok', fingerprint: 'fp' } }); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage(AUTHENTICATED); + await vi.advanceTimersByTimeAsync(0); + h.peer().channel(RELIABLE_CHANNEL_LABEL).open(); + + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'revoked', + message: 'This computer removed this device.', + }); + + expect(h.client.snapshot().phase).toBe('terminal'); + expect(h.store.value).toBeNull(); + // No reconnect, however long we wait. + await vi.advanceTimersByTimeAsync(60_000); + expect(h.sockets).toHaveLength(1); + vi.useRealTimers(); + }); +}); + +describe('ACappellaReferenceClient - version and teardown', () => { + it('treats a version error as terminal and KEEPS the token (C-33, C-34)', () => { + const h = harness({ stored: { deviceId: 'device-1', token: 'tok', fingerprint: 'fp' } }); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage({ + op: 'error', + code: 'protocol-version', + message: 'This device speaks A Cappella protocol v1; this desktop needs v2 or newer.', + }); + + const state = h.client.snapshot(); + expect(state.phase).toBe('terminal'); + expect(state.canRetry).toBe(false); + expect(state.message).toBe( + 'This device speaks A Cappella protocol v1; this desktop needs v2 or newer.' + ); + // The pairing is still valid; only one end is behind. + expect(h.store.value).not.toBeNull(); + }); + + it('sends bye before a deliberate teardown (C-14)', async () => { + const h = await connected(); + h.client.disconnect(); + + expect(h.socket().ops()).toContain('bye'); + expect(h.peer().closed).toBe(true); + }); + + it('starts the floor closed after a reconnect (C-49)', async () => { + vi.useFakeTimers(); + const h = harness({ stored: { deviceId: 'device-1', token: 'tok', fingerprint: 'fp' } }); + h.client.connect(TARGET); + h.socket().handlers.onOpen(); + h.socket().handlers.onMessage(AUTHENTICATED); + await vi.advanceTimersByTimeAsync(0); + h.peer().channel(RELIABLE_CHANNEL_LABEL).open(); + inbound(h.peer(), RELIABLE_CHANNEL_LABEL, { + type: 'floor-state', + holder: 'device-1', + isSelf: true, + }); + await vi.advanceTimersByTimeAsync(0); + expect(h.client.snapshot().floor.isSelf).toBe(true); + + h.socket().handlers.onClose(); + expect(h.client.snapshot().floor).toEqual({ holder: null, isSelf: false }); + + await vi.advanceTimersByTimeAsync(1200); + expect(h.sockets.length).toBeGreaterThan(1); + expect(h.client.snapshot().floor.isSelf).toBe(false); + vi.useRealTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); +}); diff --git a/src/__tests__/web-desktop/acappella-client/ui.test.tsx b/src/__tests__/web-desktop/acappella-client/ui.test.tsx new file mode 100644 index 0000000000..0a7d9d12c5 --- /dev/null +++ b/src/__tests__/web-desktop/acappella-client/ui.test.tsx @@ -0,0 +1,197 @@ +/** + * The reference client's rendering, which is where three of the easiest + * conformance items to get wrong actually live: the roster is replaced rather + * than merged, a route correction rewrites one row rather than appending a + * second, and a late sentence from a cancelled speech run is dropped. + * + * A `.tsx` extension with no JSX in it, because the jsdom project matches every + * `.tsx` in `src/` and this suite needs a document. + */ + +import { describe, expect, it } from 'vitest'; + +import type { RosterAgent, VoiceEvent, VoiceScope } from '../../../shared/acappella/protocol'; +import { + createTranscript, + micPillText, + renderWheel, + sameScope, +} from '../../../web-desktop/acappella-client/ui'; +import type { ClientState } from '../../../web-desktop/acappella-client/client'; + +function agent(sessionId: string, name: string): RosterAgent { + return { sessionId, name, agentType: 'claude-code', cwd: '/tmp', tabs: [] }; +} + +function event(partial: Partial & { type: VoiceEvent['type'] }): VoiceEvent { + return { sessionId: 'voice-1', seq: 1, ts: 0, ...partial } as VoiceEvent; +} + +function baseState(patch: Partial = {}): ClientState { + return { + phase: 'connected', + message: '', + deviceId: 'device-1', + protocolVersion: 1, + floor: { holder: null, isSelf: false }, + sending: false, + desktopVersion: null, + quality: null, + transcriptSuspect: false, + canRetry: true, + ...patch, + }; +} + +describe('project wheel', () => { + it('replaces the roster wholesale rather than merging it (C-24)', () => { + const container = document.createElement('div'); + const conductor: VoiceScope = { kind: 'conductor' }; + + renderWheel(container, [agent('a', 'Alpha'), agent('b', 'Beta')], conductor, () => {}); + expect(container.textContent).toContain('Alpha'); + + // Beta was closed on the desktop. The next snapshot simply does not have it, + // and a merge here is how a phone offers to talk to something gone. + renderWheel(container, [agent('a', 'Alpha')], conductor, () => {}); + expect(container.textContent).toContain('Alpha'); + expect(container.textContent).not.toContain('Beta'); + }); + + it('always offers the conductor, selected by default', () => { + const container = document.createElement('div'); + renderWheel(container, [], { kind: 'conductor' }, () => {}); + const first = container.querySelector('button'); + expect(first?.textContent).toContain('Conductor'); + expect(first?.getAttribute('aria-pressed')).toBe('true'); + }); + + it('compares agent scopes by session id', () => { + expect(sameScope({ kind: 'agent', sessionId: 'a' }, { kind: 'agent', sessionId: 'a' })).toBe( + true + ); + expect(sameScope({ kind: 'agent', sessionId: 'a' }, { kind: 'agent', sessionId: 'b' })).toBe( + false + ); + expect(sameScope({ kind: 'conductor' }, { kind: 'agent', sessionId: 'a' })).toBe(false); + }); +}); + +describe('microphone pill (C-50)', () => { + it('distinguishes sending from a connected but closed microphone', () => { + expect(micPillText(baseState({ sending: true }))).toBe('Sending'); + expect(micPillText(baseState({ sending: false }))).toBe('Mic off'); + expect(micPillText(baseState({ phase: 'idle' }))).toBe('Not connected'); + }); +}); + +describe('transcript', () => { + it('replaces the in-flight user row on each partial rather than appending', () => { + const container = document.createElement('div'); + const transcript = createTranscript(container); + + transcript.apply(event({ type: 'partial-transcript', text: 'open the', stability: 0.2 })); + transcript.apply( + event({ type: 'partial-transcript', text: 'open the auth tab', stability: 0.8 }) + ); + transcript.apply( + event({ type: 'final-transcript', text: 'open the auth tab', confidence: 0.9 }) + ); + + const rows = container.querySelectorAll('.row-user'); + expect(rows).toHaveLength(1); + expect(rows[0].textContent).toContain('open the auth tab'); + }); + + it('rewrites a caption in place on a route correction (C-25)', () => { + const container = document.createElement('div'); + const transcript = createTranscript(container); + + transcript.apply(event({ type: 'final-transcript', text: 'ship it', confidence: 1 })); + transcript.apply( + event({ + type: 'dispatch', + agentSessionId: 'a', + agentName: 'Alpha', + tabId: 't1', + action: 'focused', + promptSent: true, + }) + ); + transcript.apply( + event({ + type: 'route-correction', + fromAgentSessionId: 'a', + fromTabId: 't1', + agentSessionId: 'b', + agentName: 'Beta', + tabId: 't2', + action: 'focused', + promptSent: true, + source: 'client-button', + }) + ); + + // One row, one caption, and the caption is the corrected one. + expect(container.querySelectorAll('.row-user')).toHaveLength(1); + const captions = container.querySelectorAll('.row-caption'); + expect(captions).toHaveLength(1); + expect(captions[0].textContent).toContain('Beta'); + }); + + it('drops sentences from a run that is no longer current (C-27)', () => { + const container = document.createElement('div'); + const transcript = createTranscript(container); + + transcript.apply( + event({ + type: 'speak-start', + utteranceId: 'u1', + sentenceCount: 2, + ttsProviderId: 'piper', + streaming: true, + }) + ); + transcript.apply( + event({ + type: 'agent-reply', + agentSessionId: 'a', + tabId: 't', + text: 'Done.', + spokenText: 'Done.', + }) + ); + // Index 5 with a sentenceCount of 2 is normal while streaming: the count is a + // lower bound and must never be clamped. C-26. + transcript.apply(event({ type: 'speak-sentence', utteranceId: 'u1', index: 5, text: 'five' })); + expect(container.querySelector('.row-caption')?.textContent).toBe('sentence 6'); + + // A sentence from a cancelled earlier run, arriving late. + transcript.apply(event({ type: 'speak-sentence', utteranceId: 'u0', index: 0, text: 'stale' })); + expect(container.querySelector('.row-caption')?.textContent).toBe('sentence 6'); + }); + + it('shows the egress statement verbatim (C-30)', () => { + const container = document.createElement('div'); + const transcript = createTranscript(container); + transcript.apply( + event({ + type: 'provider-state', + pipeline: 'cascade', + slots: [], + egressStatement: 'Your voice stays on this machine.', + audioLeavesMachine: false, + }) + ); + expect(container.textContent).toContain('Your voice stays on this machine.'); + }); + + it('ignores an event type it does not know (C-23)', () => { + const container = document.createElement('div'); + const transcript = createTranscript(container); + expect(() => + transcript.apply(event({ type: 'something-new' as VoiceEvent['type'] })) + ).not.toThrow(); + expect(container.childElementCount).toBe(0); + }); +}); diff --git a/src/main/__tests__/window-registry.test.ts b/src/main/__tests__/window-registry.test.ts index f6f9cc4278..f564090024 100644 --- a/src/main/__tests__/window-registry.test.ts +++ b/src/main/__tests__/window-registry.test.ts @@ -6,9 +6,21 @@ */ import { describe, it, expect, vi, beforeEach } from 'vitest'; -import type { BrowserWindow } from 'electron'; +import { BrowserWindow } from 'electron'; import { WindowRegistry, type WindowRegistryChange } from '../window-registry'; +// The registry resolves "which window is this" through two Electron statics. +// Mocked as plain spies so the tests can say what the OS would have answered. +vi.mock('electron', () => ({ + BrowserWindow: { + fromWebContents: vi.fn(() => null), + getFocusedWindow: vi.fn(() => null), + }, +})); + +const fromWebContents = vi.mocked(BrowserWindow.fromWebContents); +const getFocusedWindow = vi.mocked(BrowserWindow.getFocusedWindow); + /** A fake BrowserWindow exposing only what the registry touches. */ function makeWindow( bounds: { x: number; y: number; width: number; height: number } = { @@ -30,6 +42,8 @@ describe('WindowRegistry', () => { beforeEach(() => { registry = new WindowRegistry(); + fromWebContents.mockReset().mockReturnValue(null); + getFocusedWindow.mockReset().mockReturnValue(null); }); describe('create / get / remove', () => { @@ -535,4 +549,67 @@ describe('WindowRegistry', () => { expect(listener).not.toHaveBeenCalled(); }); }); + + /** + * Which window a surface belongs to. A Cappella scopes a voice session to the + * window it was opened in, and voice events are broadcast to every window, so + * a wrong answer here draws the same HUD in all of them. + */ + describe('findBySender', () => { + it('resolves the window an IPC message came from', () => { + const bw = makeWindow(); + registry.create({ windowId: 'w1', browserWindow: bw }); + registry.create({ windowId: 'w2', browserWindow: makeWindow() }); + fromWebContents.mockReturnValue(bw); + + expect(registry.findBySender({} as never)?.id).toBe('w1'); + }); + + it('answers "no window" for a sender that is not one', () => { + // The web-desktop bridge invokes handlers with a synthetic event that has + // no sender at all. A web client is not a window, and passing that to + // BrowserWindow.fromWebContents throws. + registry.create({ windowId: 'w1', browserWindow: makeWindow() }); + + expect(registry.findBySender(undefined)).toBeUndefined(); + expect(registry.findBySender(null)).toBeUndefined(); + expect(fromWebContents).not.toHaveBeenCalled(); + }); + + it('answers "no window" when the sender belongs to an unregistered window', () => { + registry.create({ windowId: 'w1', browserWindow: makeWindow() }); + fromWebContents.mockReturnValue(makeWindow()); + + expect(registry.findBySender({} as never)).toBeUndefined(); + }); + }); + + describe('getFocusedAppWindow', () => { + it('prefers the focused window', () => { + registry.create({ windowId: 'w1', browserWindow: makeWindow(), isMain: true }); + const second = makeWindow(); + registry.create({ windowId: 'w2', browserWindow: second }); + getFocusedWindow.mockReturnValue(second); + + expect(registry.getFocusedAppWindow()?.id).toBe('w2'); + }); + + it('falls back to the primary when nothing is focused', () => { + // A global hotkey pressed while another app is in front: the trigger still + // has to land on a window rather than nowhere. + registry.create({ windowId: 'w1', browserWindow: makeWindow(), isMain: true }); + getFocusedWindow.mockReturnValue(null); + + expect(registry.getFocusedAppWindow()?.id).toBe('w1'); + }); + + it('ignores a focused feature window, which owns no agents', () => { + registry.create({ windowId: 'w1', browserWindow: makeWindow(), isMain: true }); + const hud = makeWindow(); + registry.create({ windowId: 'hud', browserWindow: hud, kind: 'cadenza-hud' }); + getFocusedWindow.mockReturnValue(hud); + + expect(registry.getFocusedAppWindow()?.id).toBe('w1'); + }); + }); }); diff --git a/src/main/acappella/audio-host-window.ts b/src/main/acappella/audio-host-window.ts new file mode 100644 index 0000000000..b39abd22f8 --- /dev/null +++ b/src/main/acappella/audio-host-window.ts @@ -0,0 +1,183 @@ +/** + * A Cappella audio host window - the main process's audio I/O device. + * + * Electron's main process has no `AudioContext`, no `getUserMedia`, and no + * `AudioWorklet`: everything that touches a microphone or a speaker has to run + * in a renderer. So A Cappella gets one hidden renderer whose entire job is + * audio. It loads the ordinary Maestro renderer bundle with `?acappellaAudio`, + * which boots into the audio host root instead of the app (see + * `src/renderer/main.tsx`), captures the mic through Chromium's own libwebrtc + * audio processing module (AEC, noise suppression, auto gain), and plays TTS + * back out through the same context so the echo canceller has a real reference + * signal to subtract. No native modules, and the identical capture path will + * terminate the phone's peer connection in Phase 10. + * + * Properties this module is responsible for: + * + * - **Invisible, and invisible everywhere.** `show: false` and never shown, so + * it cannot be raised, cycled to, or moved. It registers as an + * `acappella-audio` kind in the {@link WindowRegistry}, which every + * multi-window consumer (persistence, "Move to Window", empty-window + * auto-close, telemetry) filters out by asking for `getAppWindows()`. + * `skipTaskbar` keeps it off the Windows/Linux taskbar and + * `setExcludedFromShownWindowsMenu` off the macOS Window menu. + * - **Never throttled.** A hidden, never-painted window is exactly what + * Chromium's background throttling targets, and a throttled timer in the + * audio path is a dropout. `backgroundThrottling: false` opts out. + * - **Lazily created, eagerly destroyed.** Built on the first session start + * ({@link ensureAcappellaAudioHostWindow}) rather than at boot, and torn down + * when the Encore Feature is switched off or the app quits, so a user who + * never speaks never pays for a second renderer or an open microphone. + * + * Deliberately NOT offscreen-rendered (`webPreferences.offscreen`): OSR exists + * to get pixels out of a window, and this window has no pixels anyone wants. It + * would add a frame pipeline and lose the GPU compositor for zero benefit. + * `paintWhenInitiallyHidden: false` is the cheaper answer - the window never + * paints at all, while its JS, its `AudioContext`, and its worklet run normally. + */ + +import { BrowserWindow, type WebContents } from 'electron'; + +import { isMacOS } from '../../shared/platformDetection'; +import { logger } from '../utils/logger'; +import type { WindowRegistry } from '../window-registry'; + +const LOG_CONTEXT = 'ACappellaAudio'; + +/** + * Everything the audio host needs to load the renderer bundle. Same shape as + * `CadenzaHudWindowDeps` on purpose: both are host-owned feature windows that + * reuse the main preload plus the main bundle with a boot-mode query. + */ +export interface AudioHostWindowDeps { + isDevelopment: boolean; + preloadPath: string; + /** Custom-protocol URL used to load the production renderer. */ + rendererProductionUrl: string; + /** Development server URL. */ + devServerUrl: string; + /** Registry the window enrolls in as an `acappella-audio` kind. */ + windowRegistry: WindowRegistry; +} + +let audioWindow: BrowserWindow | null = null; +/** Registry id for the audio host, so its `closed` handler can deregister it. */ +let audioWindowId: string | null = null; + +/** The audio host window, or null when A Cappella has never opened one. */ +export function getAcappellaAudioHostWindow(): BrowserWindow | null { + return audioWindow && !audioWindow.isDestroyed() ? audioWindow : null; +} + +/** + * True when `contents` is the audio host's own web contents. + * + * The default session denies every media permission request (see + * `main-window-navigation.ts`), which is the correct posture for the app window + * and for embedded browser tabs. Permission handlers are per-session, not + * per-window, so the microphone grant has to be expressed as "this exact + * webContents" rather than as a looser session-wide allowance. + */ +export function isAcappellaAudioHostContents(contents: WebContents | null | undefined): boolean { + const win = getAcappellaAudioHostWindow(); + if (!win || !contents) return false; + return contents === win.webContents; +} + +/** Append the `acappellaAudio` flag so main.tsx boots into audio-host mode. */ +function withAudioHostFlag(url: string): string { + return url.includes('?') ? `${url}&acappellaAudio` : `${url}?acappellaAudio`; +} + +/** + * Create the audio host window, or return the existing one. Called on session + * start; safe to call repeatedly. + */ +export function ensureAcappellaAudioHostWindow(deps: AudioHostWindowDeps): BrowserWindow { + const existing = getAcappellaAudioHostWindow(); + if (existing) return existing; + + const win = new BrowserWindow({ + // Small rather than zero-sized: Chromium clamps a 0x0 window anyway, and a + // sane size keeps devtools usable when debugging the audio path. + width: 480, + height: 320, + show: false, + // The window is never shown, so nothing here is user-visible; these keep + // the OS from surfacing it in any window-management affordance. + frame: false, + skipTaskbar: true, + focusable: false, + resizable: false, + minimizable: false, + maximizable: false, + fullscreenable: false, + hasShadow: false, + // Never composite a frame for a window nobody sees. `ready-to-show` does + // not fire when this is false, so load completion is observed through + // `did-finish-load` instead. + paintWhenInitiallyHidden: false, + webPreferences: { + preload: deps.preloadPath, + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + // Non-negotiable: Chromium throttles timers in hidden windows, and a + // throttled audio timer is an audible dropout. + backgroundThrottling: false, + }, + }); + + audioWindow = win; + // Tracked as a feature window so teardown and telemetry stay uniform. Every + // multi-window consumer reads `getAppWindows()`, so this kind is invisible to + // "Move to Window", the window switcher, persistence, and auto-close. + audioWindowId = deps.windowRegistry.create({ + browserWindow: win, + kind: 'acappella-audio', + isMain: false, + sessionIds: [], + }); + + // macOS lists every window in the app's Window menu; a hidden audio device is + // not something the user can meaningfully switch to. + if (isMacOS()) win.excludedFromShownWindowsMenu = true; + + const url = deps.isDevelopment + ? withAudioHostFlag(deps.devServerUrl) + : withAudioHostFlag(deps.rendererProductionUrl); + void win.loadURL(url); + + win.on('closed', () => { + if (audioWindow !== win) return; + if (audioWindowId) { + deps.windowRegistry.remove(audioWindowId); + audioWindowId = null; + } + audioWindow = null; + logger.info('A Cappella audio host window closed', LOG_CONTEXT); + }); + + // The audio host only ever runs its own bundle: no popups, no navigation. + win.webContents.setWindowOpenHandler(() => ({ action: 'deny' })); + win.webContents.on('will-navigate', (event, target) => { + if (target !== url) event.preventDefault(); + }); + + logger.info('A Cappella audio host window created', LOG_CONTEXT, { + mode: deps.isDevelopment ? 'development' : 'production', + }); + + return win; +} + +/** + * Destroy the audio host window if it is open. Called when the Encore Feature is + * switched off and on app quit. The `'closed'` handler owns all teardown, so + * nothing is nulled here - doing so would make its identity guard fail and leak + * the registry entry. + */ +export function closeAcappellaAudioHostWindow(): void { + const win = getAcappellaAudioHostWindow(); + if (win) win.close(); +} diff --git a/src/main/acappella/audio/audio-bridge.ts b/src/main/acappella/audio/audio-bridge.ts new file mode 100644 index 0000000000..d642f52bc8 --- /dev/null +++ b/src/main/acappella/audio/audio-bridge.ts @@ -0,0 +1,337 @@ +/** + * A Cappella audio bridge - the composition root for real audio. + * + * Phase 02 built four self-contained pieces and deliberately wired none of them: + * the pipeline (`audio-pipeline.ts`), the detector (`vad.ts`), the meter + * (`level-meter.ts`), and the microphone projection (`mic-state.ts`). This is + * where they meet the session service, and it is the only module that knows all + * of them exist. + * + * What it owns, in one sentence each: + * + * - **Capture follows the floor.** `listen-start` opens the microphone, + * `listen-stop` closes it, and the pipeline decides per frame whether the + * audio reaches the recogniser, the pre-roll, or the drop counter. + * - **A microphone is opened only for a provider that can hear.** The gate is + * `SttProvider.acceptsAudio`, not a list of provider ids: asking a user for a + * microphone permission on behalf of a text-in mock buys a level meter over a + * transcript that is never coming. + * - **Playback closes the duplex loop.** TTS chunks go out as `play` commands + * to the same audio host that captures, which is what gives Chromium's echo + * canceller a reference signal - and is therefore what makes it safe to keep + * the microphone open while the assistant speaks. + * - **Every audio fact becomes a protocol event.** The level meter and the + * microphone tracker publish through the session, so the HUD and the Phase 10 + * phone read one ordered stream rather than two transports. + * + * Free of Electron: frames, statuses, and the command sink are injected. The IPC + * layer owns the channels (see `src/main/ipc/handlers/acappella.ts`), and Phase + * 10's phone will drive the same object with frames that arrived over WebRTC. + */ + +import type { + AudioFrame, + AudioHostCommand, + AudioHostErrorCode, + AudioHostStatus, + PlaybackFormat, +} from '../../../shared/acappella/audio-host'; +import type { InterruptSource, MicState, VoiceEvent } from '../../../shared/acappella/protocol'; +import type { SttProvider, TtsChunk } from '../../../shared/acappella/providers'; +import type { VoiceSessionState } from '../../../shared/acappella/session-state'; +import { logger } from '../../utils/logger'; +import { captureException } from '../../utils/sentry'; +import { AudioPipeline, type AudioPipelineStats } from './audio-pipeline'; +import { AudioLevelMeter, type AudioLevelMeterConfig } from './level-meter'; +import { MicStateTracker } from './mic-state'; +import type { VadConfig } from './vad'; + +const LOG_CONTEXT = 'ACappella'; + +/** + * The slice of `VoiceSessionService` the bridge drives. Narrow on purpose: audio + * publishes facts and performs exactly one action (barge-in). It never routes, + * never speaks, and never starts or stops a session - that is floor control's. + */ +export interface AudioBridgeSession { + getState(): VoiceSessionState; + interrupt(source?: InterruptSource): boolean; + subscribe(listener: (event: VoiceEvent) => void): () => void; + getActiveStt(): SttProvider | null; + publishAudioLevel(level: number, speech: boolean): void; + publishMicState(state: MicState): void; + reportAudioCaptureFailure(code: AudioHostErrorCode, message: string): void; +} + +export interface VoiceAudioBridgeOptions { + session: AudioBridgeSession; + /** Sends one command to the audio host renderer. A closed host is a no-op. */ + sendCommand: (command: AudioHostCommand) => void; + vad?: Partial; + /** Audio retained ahead of the floor opening. Clamped by the pipeline. */ + preRollMs?: number; + meter?: Partial; + /** The user's chosen microphone, read at each capture start. */ + getInputDeviceId?: () => string | undefined; +} + +export class VoiceAudioBridge { + private readonly options: VoiceAudioBridgeOptions; + private readonly pipeline: AudioPipeline; + private readonly meter: AudioLevelMeter; + private readonly mic = new MicStateTracker(); + private readonly unsubscribe: () => void; + + /** + * Whether the host renderer has announced itself. Commands sent before that + * reach a window that is still loading its bundle and are lost, which matters + * for exactly one of them: the `start-capture` that a session start races. + */ + private hostReady = false; + /** Last volume asked for, replayed on `ready`. Null means never set. */ + private playbackVolume: number | null = null; + private disposed = false; + + constructor(options: VoiceAudioBridgeOptions) { + this.options = options; + this.meter = new AudioLevelMeter(options.meter); + this.pipeline = new AudioPipeline({ + session: options.session, + getStt: () => options.session.getActiveStt(), + sendCommand: (command) => this.send(command), + getInputDeviceId: options.getInputDeviceId, + vad: options.vad, + preRollMs: options.preRollMs, + onFrame: ({ result }) => { + const update = this.meter.push(result.rms, result.active); + if (update) options.session.publishAudioLevel(update.level, update.speech); + }, + }); + + this.unsubscribe = options.session.subscribe((event) => this.handleEvent(event)); + } + + /** Counters for the current capture run. Every audio failure is otherwise silent. */ + getStats(): AudioPipelineStats { + return this.pipeline.getStats(); + } + + /** One 20 ms frame from the audio host. */ + handleFrame(frame: AudioFrame): void { + if (this.disposed) return; + this.pipeline.handleFrame(frame); + } + + /** + * One control-plane message from the audio host. + * + * Three consumers, in this order: the readiness latch (so a capture that was + * requested during boot is re-requested), the microphone projection (so a + * client can tell a quiet session from a deaf one), and the pipeline (so a + * device restart does not carry an open speech state across it). + */ + handleStatus(status: AudioHostStatus): void { + if (this.disposed) return; + + if (status.kind === 'ready') this.onHostReady(); + + const micState = this.mic.apply(status); + if (micState) this.options.session.publishMicState(micState); + + if (status.kind === 'mic-error') { + // A dead microphone must never present as a session that is merely quiet. + this.options.session.reportAudioCaptureFailure(status.code, status.message); + } + + if (status.kind === 'capture-stop' || status.kind === 'mic-error') this.meter.reset(); + + this.pipeline.handleStatus(status); + } + + /** + * Force the recogniser to endpoint now. + * + * The seam floor control's hold-to-talk release binds to: a user who let go of + * the key has already said the utterance is finished, so waiting out the VAD's + * endpoint silence would be latency bought with nothing. + */ + endUtterance(): void { + const stt = this.options.session.getActiveStt(); + if (!stt) return; + void stt.flush().catch((error: Error) => { + // Endpointing is a hint. The recogniser still has the audio. + void captureException(error, { + context: 'acappella.audioBridge.endUtterance', + providerId: stt.id, + }); + }); + } + + /** + * Hand one chunk of synthesised speech to the audio host. + * + * Wired as the session's `onSpeechChunk`. Chunks with no samples behind them + * (the mock tier's `format: 'none'`) are dropped here rather than at the + * source: the session's job is to announce the sentence, and what is playable + * is the sink's question. + */ + handleSpeechChunk(chunk: TtsChunk): void { + if (this.disposed) return; + if (!chunk.audio || chunk.audio.byteLength === 0 || chunk.format === 'none') return; + + const format: PlaybackFormat = chunk.format === 'pcm16' ? 'pcm16' : 'encoded'; + if (format === 'pcm16' && !chunk.sampleRate) { + // Raw samples with no rate cannot be played at all, and guessing one is how + // a voice ends up an octave out. Dropping it is the honest failure. + logger.warn( + `Dropping pcm16 speech chunk with no sample rate (utterance ${chunk.utteranceId})`, + LOG_CONTEXT + ); + return; + } + + this.send({ + kind: 'play', + utteranceId: chunk.utteranceId, + format, + sampleRate: chunk.sampleRate, + // Copied because the buffer crosses an IPC boundary and the provider may + // well be reusing it for the next sentence. + data: new Uint8Array(chunk.audio).buffer, + }); + } + + /** + * Drop playback gain, for a barge-in the pipeline did not see coming. + * + * The pipeline already ducks on a CANDIDATE frame, before the detector has + * confirmed anything, which is what makes a spoken interruption feel instant. + * This is the other door: a client button, and the Phase 10 phone, where the + * first the audio path hears of it is that the session cancelled a run. + */ + duckPlayback(gain: number, ms: number): void { + this.send({ kind: 'duck', gain, ms }); + } + + /** Discard audio already queued in the host. Audio the user talked over. */ + flushPlayback(): void { + this.send({ kind: 'flush' }); + } + + /** + * Set the base output volume (0 to 1) the assistant speaks at. + * + * Its own command rather than a duck, because it has to survive every + * barge-in: `flush()` restores gain, and a volume expressed as a duck would be + * silently undone the first time the user interrupted. + */ + setPlaybackVolume(volume: number): void { + this.playbackVolume = volume; + this.send({ kind: 'set-volume', volume }); + } + + /** + * Stop capture, release the host, and drop the subscription. Safe to repeat. + * + * The disposed flag is set LAST on purpose: `pipeline.dispose()` is what sends + * `stop-capture`, and a bridge that marked itself dead first would swallow the + * one message that closes the microphone. + */ + dispose(): void { + if (this.disposed) return; + this.unsubscribe(); + this.pipeline.dispose(); + this.meter.reset(); + this.mic.reset(); + this.disposed = true; + } + + // -- Internals ----------------------------------------------------------- + + /** + * Follow the session's own stream rather than being called at each site. + * + * The floor can open and close from places this module will never see: the + * wake word, a hotkey, the HUD button, a provider failure. Subscribing is what + * keeps "the microphone is open exactly while the session is listening" true no + * matter which of them did it. + */ + private handleEvent(event: VoiceEvent): void { + switch (event.type) { + case 'listen-start': + this.startCapture(); + break; + case 'listen-stop': + this.pipeline.stop(); + this.meter.reset(); + break; + case 'speak-end': + // A completed run drains what is queued; anything else was cut off and + // the queue is stale audio the user has already talked over. + this.send( + event.reason === 'complete' + ? { kind: 'end-utterance', utteranceId: event.utteranceId } + : { kind: 'flush' } + ); + break; + default: + break; + } + } + + /** + * Open the microphone, but only for a recogniser that consumes audio. + * + * The mock tier is text-in by construction. Opening a capture device for it + * would cost the user an OS permission prompt and give back a level meter over + * a transcript that is never coming, which is a worse lie than showing no meter + * at all. + */ + private startCapture(): void { + const stt = this.options.session.getActiveStt(); + if (!stt?.acceptsAudio) { + if (this.pipeline.isRunning) this.pipeline.stop(); + return; + } + this.pipeline.start(); + } + + /** + * The host renderer finished booting. + * + * The window is created on the first session start and the session reaches + * `listening` long before a renderer has loaded its bundle, so the first + * `start-capture` is normally sent into a window that cannot hear it yet. + * Re-requesting on `ready` costs one message and removes the race, which is + * cheaper than a command queue that would have to decide what a stale `duck` or + * `play` means once the host finally arrives. + */ + private onHostReady(): void { + this.hostReady = true; + // Replayed for the same reason capture is: the volume is normally set the + // moment a session starts, which is before the host window has a listener. + // Dropping it would give the user's first sentence the default level and + // silently un-mute a muted session. + if (this.playbackVolume !== null) { + this.send({ kind: 'set-volume', volume: this.playbackVolume }); + } + if (!this.pipeline.isRunning) return; + logger.debug('Audio host became ready mid-capture; re-requesting capture', LOG_CONTEXT); + // Re-sent with the device, not bare: the replay has to reopen the SAME + // microphone the session started with, or the race it exists to fix would + // be traded for a session that quietly moved to the system default. + this.send({ kind: 'start-capture', deviceId: this.options.getInputDeviceId?.() }); + } + + private send(command: AudioHostCommand): void { + if (this.disposed) return; + // Before `ready` the host has no listener attached, so this would vanish. + // `onHostReady` re-requests capture; nothing else is worth replaying. + if (!this.hostReady && command.kind !== 'start-capture') return; + this.options.sendCommand(command); + } +} + +export function createVoiceAudioBridge(options: VoiceAudioBridgeOptions): VoiceAudioBridge { + return new VoiceAudioBridge(options); +} diff --git a/src/main/acappella/audio/audio-pipeline.ts b/src/main/acappella/audio/audio-pipeline.ts new file mode 100644 index 0000000000..bc07b19d5d --- /dev/null +++ b/src/main/acappella/audio/audio-pipeline.ts @@ -0,0 +1,561 @@ +/** + * A Cappella duplex audio pipeline. + * + * The join between the hidden audio host (microphone in, TTS out) and the voice + * session (STT in, speech runs out). Frames arrive here at 50/s and this module + * decides, per frame, one of three things: feed it to the speech recogniser, hold + * it in the pre-roll, or drop it and say so. + * + * **Full duplex, not half.** The microphone stays open while the assistant + * speaks. That is only safe because capture and playback share one `AudioContext` + * in the audio host, so Chromium's echo canceller has our own output as a + * reference and subtracts it: what reaches the VAD during playback is the user, + * not the assistant. Barge-in is therefore just the VAD firing `speech-start` + * while the session is in `speaking`, and the response is immediate - flush the + * queued audio in the host, cancel the TTS run, take the floor back - rather than + * waiting for a sentence boundary. + * + * **Nothing here is buffered without a bound.** The only queue is the pre-roll + * ring, whose capacity is fixed in frames at construction. Audio that arrives + * with nowhere to go is counted and periodically logged, never accumulated: a + * pipeline that quietly grows a queue while the session is busy trades a moment + * of silence for an unbounded memory leak and a transcript minutes out of date. + * + * **The pre-roll is what makes a wake word usable.** Capture runs before the + * floor opens, so the ~500 ms preceding a wake word or a hotkey is already in + * hand and is fed to STT ahead of the live frames. Without it "Maestro, what's + * the status" reaches the recogniser as "...what's the status". The same buffer + * covers barge-in: the syllables spoken over the assistant are in the ring before + * the interrupt lands, so they survive the transition into `listening`. + * + * Deliberately free of Electron and of the concrete session service: frames, + * commands, and the session come in as injected seams. That keeps the whole thing + * unit-testable with generated PCM, and lets Phase 10's phone drive the identical + * pipeline with frames that arrived over WebRTC instead of over IPC. + */ + +import { + ACAPPELLA_AUDIO_FRAME_MS, + type AudioFrame, + type AudioHostCommand, + type AudioHostStatus, +} from '../../../shared/acappella/audio-host'; +import type { InterruptSource } from '../../../shared/acappella/protocol'; +import type { SttProvider } from '../../../shared/acappella/providers'; +import type { VoiceSessionState } from '../../../shared/acappella/session-state'; +import { logger } from '../../utils/logger'; +import { captureException } from '../../utils/sentry'; +import { VoiceActivityDetector } from './vad'; +import type { VadConfig, VadFrameResult } from './vad'; + +const LOG_CONTEXT = 'ACappella'; + +/** + * How much audio preceding the floor opening is kept. 500 ms is about one spoken + * word: enough to cover the gap between a wake word firing and the session + * reaching `listening`, short enough that the ring is 25 frames of 320 samples. + */ +export const DEFAULT_PRE_ROLL_MS = 500; + +/** + * Hard ceiling on the pre-roll, and with it on every byte this module holds. A + * setting is a number a user can get wrong; the cap is what makes "bounded" a + * property of the code rather than of the configuration. + */ +export const MAX_PRE_ROLL_MS = 5000; + +/** Output gain while a possible barge-in is being confirmed. Audible, but the user wins. */ +const DEFAULT_DUCK_GAIN = 0.2; + +/** Ramp for the duck and for its restore. Fast enough to feel instant, slow enough not to click. */ +const DEFAULT_DUCK_RAMP_MS = 60; + +/** Dropped frames between warnings: 500 frames is 10 s of audio going nowhere. */ +const DROP_LOG_INTERVAL = 500; + +// --------------------------------------------------------------------------- +// Seams +// --------------------------------------------------------------------------- + +/** + * The slice of `VoiceSessionService` the pipeline needs. Narrow on purpose: the + * pipeline reads the state machine and performs exactly one action on it, and + * anything wider would invite it to start driving turns. + */ +export interface AudioPipelineSession { + getState(): VoiceSessionState; + /** Barge-in: cancels the speech run and keeps the floor. False when nothing was speaking. */ + interrupt(source?: InterruptSource): boolean; +} + +export interface AudioPipelineOptions { + session: AudioPipelineSession; + /** + * The recogniser for the current session, or null when none is running. Read + * per frame rather than captured, so a provider swap between sessions cannot + * leave the pipeline feeding a stopped recogniser. + */ + getStt: () => SttProvider | null; + /** Sends a command to the audio host window. */ + sendCommand: (command: AudioHostCommand) => void; + /** + * The microphone the user picked, read at each start rather than captured. + * + * A function because the setting can change between sessions, and a value + * frozen at construction would keep opening the device the user has since + * stopped choosing. + */ + getInputDeviceId?: () => string | undefined; + vad?: Partial; + /** Audio retained ahead of the floor opening. Clamped to {@link MAX_PRE_ROLL_MS}. */ + preRollMs?: number; + /** Gain TTS output is ducked to while a barge-in is being confirmed. */ + duckGain?: number; + /** Ramp length for ducking and restoring. */ + duckRampMs?: number; + /** Per frame, after the frame has been classified and routed. The level-meter seam. */ + onFrame?: (info: AudioPipelineFrameInfo) => void; + /** A barge-in was performed: TTS was flushed and cancelled, the floor is back. */ + onBargeIn?: () => void; + /** + * The detector decided the user stopped talking, at the moment it decided. + * + * This is the only place that instant is known: the recogniser learns about it + * as a `flush()` and the session learns about it as a transcript, so a turn + * timed from either would silently exclude the decode. It is the zero point of + * every span in `telemetry/turn-metrics.ts`. + */ + onSpeechEnd?: () => void; +} + +export interface AudioPipelineFrameInfo { + frame: AudioFrame; + result: VadFrameResult; + /** Whether this frame reached the recogniser. False means it went to the pre-roll. */ + delivered: boolean; +} + +/** + * Counters, all monotonic within a capture run. + * + * These exist because every interesting audio failure is silent: a recogniser + * that never hears anything, an IPC channel shedding frames under load, and a + * healthy pipeline all look identical from the outside. + */ +export interface AudioPipelineStats { + framesReceived: number; + /** Frames handed to `SttProvider.feed()`, including replayed pre-roll. */ + framesDelivered: number; + /** Frames that arrived with no recogniser to take them. The pre-roll holds the last few. */ + framesDropped: number; + /** Frames replayed out of the pre-roll when the floor opened. */ + preRollFramesDelivered: number; + /** Frames missing between two `seq` values: the IPC channel shed them, not us. */ + sequenceGaps: number; + bargeIns: number; + /** Throws out of `feed()`. Counted rather than propagated: see {@link AudioPipeline.handleFrame}. */ + feedErrors: number; +} + +// --------------------------------------------------------------------------- +// Pre-roll ring +// --------------------------------------------------------------------------- + +/** + * Fixed-capacity ring of PCM frames, oldest evicted first. + * + * Eviction is the normal case, not an error: this is a sliding window over the + * last N frames, so a frame ageing out has done its job. That is why it keeps no + * drop counter - the pipeline's `framesDropped` counts audio that had nowhere to + * go, which is a different and much more interesting fact. + */ +export class AudioFrameRing { + private readonly frames: Int16Array[] = []; + + constructor(readonly capacity: number) {} + + get size(): number { + return this.frames.length; + } + + push(samples: Int16Array): void { + if (this.capacity <= 0) return; + if (this.frames.length >= this.capacity) this.frames.shift(); + this.frames.push(samples); + } + + /** Take everything, oldest first, and empty the ring. */ + drain(): Int16Array[] { + return this.frames.splice(0, this.frames.length); + } + + clear(): void { + this.frames.length = 0; + } +} + +// --------------------------------------------------------------------------- +// Pipeline +// --------------------------------------------------------------------------- + +export class AudioPipeline { + private readonly options: AudioPipelineOptions; + private readonly vad: VoiceActivityDetector; + private readonly preRoll: AudioFrameRing; + private readonly frameMs: number; + private readonly duckGain: number; + private readonly duckRampMs: number; + + private running = false; + /** Whether the previous frame was delivered, so the pre-roll drains on the edge. */ + private feeding = false; + private ducked = false; + private lastState: VoiceSessionState = 'idle'; + private lastSeq = 0; + private dropsSinceLog = 0; + /** One Sentry report per run: a broken `feed()` breaks on every one of 50 frames a second. */ + private reportedFeedError = false; + private stats = emptyStats(); + + constructor(options: AudioPipelineOptions) { + this.options = options; + this.vad = new VoiceActivityDetector(options.vad); + this.frameMs = this.vad.config.frameMs || ACAPPELLA_AUDIO_FRAME_MS; + this.preRoll = new AudioFrameRing( + Math.max(0, Math.round(resolvePreRollMs(options.preRollMs) / this.frameMs)) + ); + this.duckGain = clamp01(options.duckGain ?? DEFAULT_DUCK_GAIN); + this.duckRampMs = Math.max(0, options.duckRampMs ?? DEFAULT_DUCK_RAMP_MS); + } + + get isRunning(): boolean { + return this.running; + } + + /** Frames of audio the pre-roll can hold. Zero disables it. */ + get preRollCapacity(): number { + return this.preRoll.capacity; + } + + getStats(): AudioPipelineStats { + return { ...this.stats }; + } + + /** Open the microphone. Idempotent: a second start is not a second device. */ + start(): void { + if (this.running) return; + this.running = true; + this.resetRun(); + this.options.sendCommand({ + kind: 'start-capture', + deviceId: this.options.getInputDeviceId?.(), + }); + } + + /** + * Close the microphone and stop any speech. + * + * Both halves, because a session that ends mid-sentence must not keep talking + * into a room whose microphone it just released. + */ + stop(): void { + if (!this.running) return; + this.running = false; + this.options.sendCommand({ kind: 'stop-capture' }); + this.options.sendCommand({ kind: 'flush' }); + this.ducked = false; + this.logDropSummary(); + this.resetRun(); + } + + /** + * Classify one captured frame and route it. + * + * The order matters: the session state is read first so a detector reset lands + * before the frame is classified rather than after it, the VAD then sees every + * frame regardless of state (it is how barge-in is detected in the one state + * where audio is not being fed anywhere), barge-in is handled before delivery + * so the frame the user interrupted with is itself delivered, and only then is + * the frame either fed or shelved. + */ + handleFrame(frame: AudioFrame): void { + if (!this.running) return; + + this.stats.framesReceived += 1; + this.trackSequence(frame.seq); + + this.syncSessionState(); + const samples = new Int16Array(frame.pcm); + const result = this.vad.process(samples); + + this.handleDuplex(result); + + const delivered = this.deliver(samples, result); + this.options.onFrame?.({ frame, result, delivered }); + } + + /** + * React to the audio host's control plane. + * + * Only the events that invalidate what the pipeline is holding: a capture that + * started or stopped resets the detector (carrying an open speech state across + * a device restart would endpoint audio nobody spoke), and a microphone error + * makes the pre-roll a record of a device that is gone. + */ + handleStatus(status: AudioHostStatus): void { + switch (status.kind) { + case 'capture-start': + this.resetRun(); + break; + case 'capture-stop': + case 'mic-error': + this.logDropSummary(); + this.resetRun(); + break; + default: + break; + } + } + + /** Stop and drop everything held. Safe to call more than once. */ + dispose(): void { + this.stop(); + this.preRoll.clear(); + } + + // -- Internals ----------------------------------------------------------- + + /** + * A gap in `seq` means frames were shed between the worklet and here, which is + * the one drop this module cannot prevent and therefore must not hide. + */ + private trackSequence(seq: number): void { + if (this.lastSeq > 0 && seq > this.lastSeq + 1) { + this.stats.sequenceGaps += seq - this.lastSeq - 1; + } + this.lastSeq = seq; + } + + /** + * Reset the detector when the session starts speaking. + * + * Barge-in needs a closed floor to fire `speech-start` from, and the VAD can + * still be open (hangover, or a recogniser that endpointed before the VAD did) + * when playback begins. Leaving it open would swallow the first interruption + * for as long as the hangover lasts. The reset also re-runs the noise-floor + * calibration against whatever the echo canceller leaves behind, which is the + * right floor to measure while our own voice is in the room. + */ + private syncSessionState(): void { + const state = this.options.session.getState(); + if (state === this.lastState) return; + if (state === 'speaking') this.vad.reset(); + this.lastState = state; + } + + /** + * Duck on suspicion, interrupt on confirmation. + * + * The duck fires on the first candidate frame, which is up to `enterFrames` + * ahead of a confirmed `speech-start` - 80 ms of the user hearing themselves + * win the room rather than talking into a wall. If the candidate turns out to + * be a cough or a door, the gain goes straight back up and nothing was + * cancelled. + */ + private handleDuplex(result: VadFrameResult): void { + if (this.lastState !== 'speaking') { + // Playback ended on its own while a duck was in place. Nothing will + // restore the gain if this does not: `flush()` only runs on barge-in. + if (this.ducked) this.setDuck(false); + return; + } + + if (result.event?.type === 'speech-start') { + this.bargeIn(); + return; + } + + if (result.candidate !== this.ducked) this.setDuck(result.candidate); + } + + /** + * Cut the assistant off. + * + * Flush first: it is the only step the user can hear, and it costs one IPC + * message. Cancelling the provider and moving the state machine is bookkeeping + * that can happen while the room is already quiet. + */ + private bargeIn(): void { + this.options.sendCommand({ kind: 'flush' }); + // The host restores gain to 1 as part of a flush, so the local flag has to + // follow or the next duck would be a no-op. + this.ducked = false; + + const interrupted = this.options.session.interrupt('voice'); + if (!interrupted) { + // The session moved on between the frame and here (the speech run ended by + // itself). The flush was still correct, and there is nothing to report. + return; + } + + this.stats.bargeIns += 1; + this.lastState = this.options.session.getState(); + this.options.onBargeIn?.(); + } + + private setDuck(ducked: boolean): void { + this.ducked = ducked; + this.options.sendCommand({ + kind: 'duck', + gain: ducked ? this.duckGain : 1, + ms: this.duckRampMs, + }); + } + + /** + * Feed the frame, or shelve it in the pre-roll. + * + * The pre-roll is filled only while NOT feeding: once frames are reaching the + * recogniser there is nothing to pre-roll, and keeping a second copy would + * replay audio the recogniser already has the next time the floor closes and + * reopens. + */ + private deliver(samples: Int16Array, result: VadFrameResult): boolean { + const stt = this.options.getStt(); + const canFeed = stt !== null && this.lastState === 'listening'; + + if (!canFeed) { + this.feeding = false; + this.preRoll.push(samples); + this.stats.framesDropped += 1; + this.countDropForLogging(); + return false; + } + + if (!this.feeding) { + this.feeding = true; + this.drainPreRoll(stt); + } + + this.feed(stt, samples); + + // The recogniser decides what a final transcript is, but it cannot know the + // room went quiet if it is being fed a continuous stream, so the VAD's + // endpoint is forwarded as an explicit flush. + if (result.event?.type === 'speech-end') this.endpoint(stt); + return true; + } + + private drainPreRoll(stt: SttProvider): void { + const buffered = this.preRoll.drain(); + for (const samples of buffered) this.feed(stt, samples); + this.stats.preRollFramesDelivered += buffered.length; + } + + /** + * One buffer into the recogniser. + * + * A throwing `feed()` is caught rather than propagated: this runs from a frame + * handler 50 times a second, so an unhandled throw would be 50 identical Sentry + * reports a second and would take the whole capture run down with it. One + * report per run, then the frames are counted as errors and dropped. + */ + private feed(stt: SttProvider, samples: Int16Array): void { + try { + stt.feed(samples); + this.stats.framesDelivered += 1; + } catch (error) { + this.stats.feedErrors += 1; + if (this.reportedFeedError) return; + this.reportedFeedError = true; + logger.error( + `Speech provider '${stt.id}' rejected audio: ${(error as Error).message}`, + LOG_CONTEXT + ); + void captureException(error as Error, { + context: 'acappella.audioPipeline.feed', + providerId: stt.id, + }); + } + } + + private endpoint(stt: SttProvider): void { + // Before the flush, not after: the flush is what starts the decode being + // measured, so stamping afterwards would exclude a synchronous recogniser's + // entire first pass. + this.options.onSpeechEnd?.(); + void stt.flush().catch((error: Error) => { + // Endpointing is a hint. A provider that cannot take it still has the + // audio, so this is reported and the run continues. + void captureException(error, { + context: 'acappella.audioPipeline.endpoint', + providerId: stt.id, + }); + }); + } + + /** + * Dropped frames are logged in batches. Per frame it would be 50 lines a second + * of the least useful log in the app; per batch it is one line every 10 s that + * says the microphone is running with nothing listening. + */ + private countDropForLogging(): void { + this.dropsSinceLog += 1; + if (this.dropsSinceLog < DROP_LOG_INTERVAL) return; + logger.debug( + `Dropped ${this.stats.framesDropped} captured frames (state '${this.lastState}')`, + LOG_CONTEXT + ); + this.dropsSinceLog = 0; + } + + private logDropSummary(): void { + const { framesReceived, framesDelivered, framesDropped, sequenceGaps, feedErrors } = this.stats; + if (framesReceived === 0) return; + const message = + `Audio run: ${framesReceived} frames, ${framesDelivered} delivered, ` + + `${framesDropped} dropped, ${sequenceGaps} lost in transit, ${feedErrors} feed errors`; + if (sequenceGaps > 0 || feedErrors > 0) logger.warn(message, LOG_CONTEXT); + else logger.debug(message, LOG_CONTEXT); + } + + private resetRun(): void { + this.vad.reset(); + this.preRoll.clear(); + this.feeding = false; + this.lastSeq = 0; + this.dropsSinceLog = 0; + this.reportedFeedError = false; + this.lastState = this.options.session.getState(); + this.stats = emptyStats(); + } +} + +export function createAudioPipeline(options: AudioPipelineOptions): AudioPipeline { + return new AudioPipeline(options); +} + +// --------------------------------------------------------------------------- + +function emptyStats(): AudioPipelineStats { + return { + framesReceived: 0, + framesDelivered: 0, + framesDropped: 0, + preRollFramesDelivered: 0, + sequenceGaps: 0, + bargeIns: 0, + feedErrors: 0, + }; +} + +/** Clamped, never rejected: this arrives from a user setting, like the VAD's numbers. */ +function resolvePreRollMs(value: number | undefined): number { + if (value === undefined || !Number.isFinite(value)) return DEFAULT_PRE_ROLL_MS; + return Math.min(MAX_PRE_ROLL_MS, Math.max(0, value)); +} + +function clamp01(value: number): number { + if (!Number.isFinite(value)) return DEFAULT_DUCK_GAIN; + return value < 0 ? 0 : value > 1 ? 1 : value; +} diff --git a/src/main/acappella/audio/floor-control.ts b/src/main/acappella/audio/floor-control.ts new file mode 100644 index 0000000000..b6b95217f5 --- /dev/null +++ b/src/main/acappella/audio/floor-control.ts @@ -0,0 +1,505 @@ +/** + * A Cappella floor control - who holds the microphone, and until when. + * + * Every surface that can take or release the floor drives this one object: the + * Phase 06 global hotkey, the HUD button, the Phase 10 phone's push-to-talk + * button, and (later) the wake word. They differ only in the `WakeSource` they + * pass. That matters because floor semantics are where a voice UI feels either + * telepathic or broken, and three independent implementations of "what does a + * second press mean" would drift within a week. + * + * Two modes, and the difference is entirely in what a release means: + * + * - **`tap-to-toggle`**: press opens the floor and the session stays open, + * hands-free, until the stop word, another press, or the idle timeout. The + * release is ignored, so a tap and a two-second hold do the same thing. + * - **`hold-to-talk`**: the floor is open exactly while the control is held. + * Release ends the utterance IMMEDIATELY, bypassing the VAD's endpoint + * silence: the user has told us the sentence is finished, so waiting 700 ms + * to agree with them is 700 ms of latency bought with nothing. + * + * **The idle timeout is the backstop for every way the floor can get stuck + * open**: a VAD that latched on a noisy room, a hotkey release that never + * arrived because the window lost focus mid-chord, a user who walked away. It + * runs only while the session is `listening` - the states that follow are + * progress, not idleness, and a slow agent must never be mistaken for a + * forgotten microphone. + * + * Free of Electron, of the concrete session service, and of any input library: + * the session arrives as an injected seam and the inputs are two methods. The + * whole state machine is therefore testable without a keyboard, a phone, or an + * audio device. + */ + +import type { + InterruptSource, + VoiceEvent, + VoiceOrigin, + VoiceScope, + WakeSource, +} from '../../../shared/acappella/protocol'; +import type { VoiceSessionState } from '../../../shared/acappella/session-state'; +import { + DEFAULT_IDLE_TIMEOUT_MS, + MAX_IDLE_TIMEOUT_MS, + MIN_IDLE_TIMEOUT_MS, +} from '../../../shared/acappella/voice-controls'; +import { logger } from '../../utils/logger'; +import { captureException } from '../../utils/sentry'; + +const LOG_CONTEXT = 'ACappella'; + +export type FloorMode = 'tap-to-toggle' | 'hold-to-talk'; + +export const FLOOR_MODES: readonly FloorMode[] = ['tap-to-toggle', 'hold-to-talk'] as const; + +/** + * Hands-free by default. Holding a key for the length of a spoken request is a + * choice, not something to impose on someone who just wants to talk. + */ +export const DEFAULT_FLOOR_MODE: FloorMode = 'tap-to-toggle'; + +export { + DEFAULT_IDLE_TIMEOUT_MS, + MAX_IDLE_TIMEOUT_MS, + MIN_IDLE_TIMEOUT_MS, +} from '../../../shared/acappella/voice-controls'; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +export interface FloorControlConfig { + mode: FloorMode; + /** + * Listening silence after which the session closes itself. `0` disables the + * timeout entirely, which is a supported choice for a desk setup with a + * hardware mute switch and a terrible one everywhere else. + */ + idleTimeoutMs: number; +} + +export const DEFAULT_FLOOR_CONTROL_CONFIG: FloorControlConfig = { + mode: DEFAULT_FLOOR_MODE, + idleTimeoutMs: DEFAULT_IDLE_TIMEOUT_MS, +}; + +/** + * Fill in and sanitise a partial config. + * + * Clamped rather than rejected, for the same reason as `resolveVadConfig`: these + * numbers arrive from user settings, and a typo in a preference must not be able + * to throw somewhere that leaves the microphone open. + */ +export function resolveFloorControlConfig( + overrides: Partial = {} +): FloorControlConfig { + const merged = { ...DEFAULT_FLOOR_CONTROL_CONFIG, ...overrides }; + return { + mode: FLOOR_MODES.includes(merged.mode) ? merged.mode : DEFAULT_FLOOR_MODE, + idleTimeoutMs: resolveIdleTimeoutMs(merged.idleTimeoutMs), + }; +} + +/** Zero passes through as "disabled"; anything else is pulled into the usable band. */ +function resolveIdleTimeoutMs(value: number): number { + if (!Number.isFinite(value) || value <= 0) return value === 0 ? 0 : DEFAULT_IDLE_TIMEOUT_MS; + return Math.min(MAX_IDLE_TIMEOUT_MS, Math.max(MIN_IDLE_TIMEOUT_MS, Math.round(value))); +} + +// --------------------------------------------------------------------------- +// Seams +// --------------------------------------------------------------------------- + +/** + * The slice of `VoiceSessionService` floor control needs. Narrow on purpose: it + * opens sessions, closes them, and cuts speech off. It never routes, never + * speaks, and never touches a provider. + */ +export interface FloorControlSession { + getState(): VoiceSessionState; + startSession(params: { + scope: VoiceScope; + source?: WakeSource; + origin?: VoiceOrigin; + }): Promise; + stopSession(reason: FloorSessionStopReason): Promise; + /** Barge-in. False when nothing was speaking. */ + interrupt(source?: InterruptSource): boolean; +} + +/** The subset of `VoiceStopReason` floor control can produce. */ +export type FloorSessionStopReason = 'user' | 'timeout' | 'shutdown'; + +/** Why the floor closed. Wider than the session stop reasons: not every close ends a session. */ +export type FloorCloseReason = + /** A second press in `tap-to-toggle`. */ + | 'toggle' + /** The control was released in `hold-to-talk`. */ + | 'release' + /** Nothing was heard for `idleTimeoutMs`. */ + | 'idle-timeout' + /** The session ended elsewhere: the stop word, an error, a replaced session. */ + | 'session-ended' + /** The controller was disposed. */ + | 'shutdown'; + +export interface FloorControlOptions extends Partial { + session: FloorControlSession; + /** What a floor opened by this controller binds to. Defaults to the conductor. */ + getScope?: () => VoiceScope; + /** + * Which microphone the next press opens. Defaults to this machine's. + * + * A seam for the same reason as `getScope`: there is exactly one floor and one + * microphone, so a phone pressing its talk button drives THIS controller + * rather than a second state machine, and the only thing that differs between + * a hotkey press and a remote press is which device the session is credited + * to. See `../transport/remote-session.ts`. + */ + getOrigin?: () => VoiceOrigin | undefined; + /** + * Force the recogniser to endpoint now. The hold-to-talk release path, and the + * only reason this module knows the recogniser exists: a user who let go of + * the key has already told us the utterance is over. + */ + endUtterance?: () => void | Promise; + /** The floor opened or closed. The seam the capture gate binds to. */ + onFloorChange?: (open: boolean, reason: FloorOpenReason | FloorCloseReason) => void; + /** Something the caller could not have awaited went wrong. Already reported to Sentry. */ + onError?: (error: Error) => void; +} + +/** Why the floor opened. Mirrors `FloorCloseReason` for the `onFloorChange` seam. */ +export type FloorOpenReason = 'press' | 'session-started'; + +// --------------------------------------------------------------------------- +// Controller +// --------------------------------------------------------------------------- + +/** + * The floor state machine. + * + * Every mutating entry point returns a promise and is serialised through one + * chain, so a double tap, a key repeat, and a release that lands while the + * session is still starting all resolve in the order they happened rather than + * racing each other into two sessions. + */ +export class FloorController { + private config: FloorControlConfig; + private readonly options: FloorControlOptions; + + private open = false; + /** True between a `press()` and its `release()`. Only meaningful in hold mode. */ + private held = false; + private idleTimer: NodeJS.Timeout | null = null; + private disposed = false; + /** Serialises the async entry points. Never rejects: every link catches. */ + private queue: Promise = Promise.resolve(); + + constructor(options: FloorControlOptions) { + this.options = options; + this.config = resolveFloorControlConfig(options); + } + + get mode(): FloorMode { + return this.config.mode; + } + + get idleTimeoutMs(): number { + return this.config.idleTimeoutMs; + } + + /** Whether audio spoken now belongs to the session. The capture gate reads this. */ + get isFloorOpen(): boolean { + return this.open; + } + + /** Whether the control is currently held down. */ + get isHeld(): boolean { + return this.held; + } + + /** + * Change mode or timeout mid-session. + * + * Switching to `tap-to-toggle` while the control is held keeps the floor open + * and forgets the hold: the alternative is a floor that closes on a release + * the user made under the old rules, which reads as the app dropping the + * sentence they are in the middle of. + */ + configure(overrides: Partial): void { + this.config = resolveFloorControlConfig({ ...this.config, ...overrides }); + if (this.config.mode === 'tap-to-toggle') this.held = false; + // Restarted, not left running: a countdown armed under the old timeout would + // otherwise outlive the setting the user just changed. + this.syncIdleTimer(true); + } + + /** + * The control went down: hotkey, HUD button, or phone button. + * + * Idempotent while held, because a held key repeats on every platform and a + * repeat must not toggle the floor fifty times a second. + */ + press(source: WakeSource = 'client-button'): Promise { + return this.enqueue(async () => { + if (this.disposed) return; + if (this.held) return; + this.held = true; + + // A press over active speech means "stop talking and listen", in both + // modes. Never "end the session": the destructive reading of a gesture + // must not be the one you get for interrupting. + if (this.options.session.getState() === 'speaking') { + this.options.session.interrupt('client-button'); + this.setFloor(true, 'press'); + this.syncIdleTimer(true); + return; + } + + if (this.config.mode === 'tap-to-toggle' && this.open) { + await this.closeFloor('toggle'); + return; + } + + await this.openFloor(source); + }); + } + + /** + * The control came up. + * + * Ignored in `tap-to-toggle` (a tap and a long press are the same gesture) and + * a no-op without a matching press, so a release delivered after a mode change + * or a lost keydown cannot close a floor it never opened. + */ + release(_source: WakeSource = 'client-button'): Promise { + return this.enqueue(async () => { + if (!this.held) return; + this.held = false; + if (this.disposed) return; + if (this.config.mode !== 'hold-to-talk') return; + if (!this.open) return; + + // Endpoint before closing: the recogniser needs the audio it already has + // turned into a final transcript, and the floor closing is what stops + // more arriving. + await this.endUtterance(); + this.setFloor(false, 'release'); + // The session stays alive to answer. The idle timeout is what eventually + // closes it, which is the intended shape of a push-to-talk session: talk, + // listen to the reply, go cold. + this.syncIdleTimer(true); + }); + } + + /** + * Close the floor and end the session. The hotkey's explicit stop, and what + * the idle timeout does on its own. + */ + close(reason: FloorCloseReason = 'toggle'): Promise { + return this.enqueue(() => this.closeFloor(reason)); + } + + /** + * Something was heard. Restarts the idle countdown. + * + * The pipeline calls this on a VAD `speech-start`, which is the earliest + * evidence a human is in the room - a session should not go cold at second 60 + * of a request that started at second 58. + */ + noteActivity(): void { + this.syncIdleTimer(true); + } + + /** + * Follow the session's own event stream. + * + * The floor can change without going through this object: the wake word opens + * a session, the stop word ends one, a provider failure parks it in `error`. + * Subscribing keeps the controller's view honest rather than making every + * other path remember to tell it. + */ + handleEvent(event: VoiceEvent): void { + if (this.disposed) return; + + switch (event.type) { + case 'listen-start': + // Covers a session opened by the wake word or by a client that called + // the service directly: the floor is open whether we opened it or not. + if (!this.open) this.setFloor(true, 'session-started'); + break; + case 'listen-stop': + case 'stop-word': + this.held = false; + if (this.open) this.setFloor(false, 'session-ended'); + break; + case 'session-error': + if (!event.recoverable) { + this.held = false; + if (this.open) this.setFloor(false, 'session-ended'); + } + break; + default: + break; + } + + // A roster push or a tab change is the app talking to itself, not a human in + // the room. Counting it as activity would keep a forgotten microphone alive + // for as long as the user keeps working in another window. + const activity = event.type !== 'agent-roster' && event.type !== 'tab-state'; + this.syncIdleTimer(activity); + } + + /** Close everything and stop accepting input. Safe to call more than once. */ + async dispose(): Promise { + if (this.disposed) return; + this.disposed = true; + this.clearIdleTimer(); + const pending = this.queue; + this.queue = Promise.resolve(); + await pending; + this.held = false; + if (this.open) this.setFloor(false, 'shutdown'); + // Unconditional: a hold-to-talk session whose key was already released has a + // closed floor and a very much open session. + await this.stopSession('shutdown'); + } + + /** Resolves once every queued action has run. Tests and shutdown paths use it. */ + whenSettled(): Promise { + return this.queue; + } + + // -- Internals ----------------------------------------------------------- + + private async openFloor(source: WakeSource): Promise { + const state = this.options.session.getState(); + if (state === 'idle' || state === 'error') { + const scope = this.options.getScope?.() ?? { kind: 'conductor' }; + const origin = this.options.getOrigin?.(); + try { + await this.options.session.startSession({ scope, source, origin }); + } catch (error) { + // The session reports its own classified failures as `session-error` + // events; anything that throws out of `startSession` is unexpected, and + // the floor must not be left claiming to be open. + this.held = false; + this.report(error as Error, 'acappella.floorControl.start'); + return; + } + } + + this.setFloor(true, 'press'); + this.syncIdleTimer(true); + } + + private async closeFloor(reason: FloorCloseReason): Promise { + this.held = false; + this.clearIdleTimer(); + if (this.open) this.setFloor(false, reason); + if (reason === 'session-ended') return; + await this.stopSession(reason === 'idle-timeout' ? 'timeout' : 'user'); + } + + private async stopSession(reason: FloorSessionStopReason): Promise { + if (this.options.session.getState() === 'idle') return; + try { + await this.options.session.stopSession(reason); + } catch (error) { + this.report(error as Error, 'acappella.floorControl.stop'); + } + } + + private async endUtterance(): Promise { + if (!this.options.endUtterance) return; + try { + await this.options.endUtterance(); + } catch (error) { + // Endpointing is a hint, exactly as it is in the audio pipeline: the + // recogniser still has the audio, so a failure here must not stop the + // floor from closing. + this.report(error as Error, 'acappella.floorControl.endUtterance'); + } + } + + private setFloor(open: boolean, reason: FloorOpenReason | FloorCloseReason): void { + if (this.open === open) return; + this.open = open; + if (!open) this.clearIdleTimer(); + try { + this.options.onFloorChange?.(open, reason); + } catch (error) { + // The capture gate is this seam and it sends IPC, so a window destroyed + // between the press and the notify throws right here. A subscriber's + // failure is not the state machine's failure: letting it escape would + // abandon the rest of the action - most importantly the `stopSession` + // that follows a close - and leave a session running with a floor that + // already reports itself shut. + this.report(error as Error, 'acappella.floorControl.onFloorChange'); + } + } + + /** + * The idle countdown runs only while the session is `listening`. + * + * Every other active state is the session making progress on the user's + * behalf, and a slow agent must never be mistaken for an abandoned session. + * State is read rather than tracked, so this cannot drift from the state + * machine no matter which path got us here. + */ + private syncIdleTimer(restart = false): void { + const listening = !this.disposed && this.options.session.getState() === 'listening'; + if (!listening || this.config.idleTimeoutMs <= 0) { + this.clearIdleTimer(); + return; + } + if (!restart && this.idleTimer !== null) return; + + this.clearIdleTimer(); + this.idleTimer = setTimeout(() => { + this.idleTimer = null; + logger.debug( + `Voice floor idle for ${this.config.idleTimeoutMs}ms, closing session`, + LOG_CONTEXT + ); + void this.close('idle-timeout'); + }, this.config.idleTimeoutMs); + // A pending microphone timeout is not a reason to keep the process alive. + this.idleTimer.unref?.(); + } + + private clearIdleTimer(): void { + if (this.idleTimer === null) return; + clearTimeout(this.idleTimer); + this.idleTimer = null; + } + + /** + * One action at a time. + * + * Presses arrive from a hotkey handler that cannot await them, so without a + * queue a double tap would run its two halves against the same observed state + * and open two sessions. Every link swallows its own failure, because a + * rejected chain would silently swallow every later press. + */ + private enqueue(action: () => Promise): Promise { + const next = this.queue.then(action).catch((error: Error) => { + this.report(error, 'acappella.floorControl'); + }); + this.queue = next; + return next; + } + + private report(error: Error, context: string): void { + logger.error(`Floor control failure (${context}): ${error.message}`, LOG_CONTEXT); + void captureException(error, { context, mode: this.config.mode }); + this.options.onError?.(error); + } +} + +/** Sugar for `new FloorController(...)`, matching the rest of A Cappella's factories. */ +export function createFloorController(options: FloorControlOptions): FloorController { + return new FloorController(options); +} diff --git a/src/main/acappella/audio/level-meter.ts b/src/main/acappella/audio/level-meter.ts new file mode 100644 index 0000000000..bfcc46488c --- /dev/null +++ b/src/main/acappella/audio/level-meter.ts @@ -0,0 +1,165 @@ +/** + * A Cappella input level meter. + * + * Turns the 50 frames a second the capture worklet produces into the ~20 a + * second a client can actually draw, so a live meter costs a number rather than + * PCM. Pure and synchronous over frame measurements, like `vad.ts`: no clock, no + * device, no Electron. + * + * **Why downsample at all.** A meter is read by an eye, and an eye cannot see 50 + * updates a second. Every frame that reaches a client past the point of + * perception is a message on the IPC channel, a React render in every open + * window, and - once the phone is a peer client in Phase 10 - a packet on a + * radio. The window is the cheapest place in the whole pipeline to spend that. + * + * **The window is counted in frames, not milliseconds.** Same reasoning as the + * VAD: 3 frames is 60 ms whether the event loop was free or wedged, so a stalled + * main thread makes the meter coarse rather than wrong. With the standard 20 ms + * frame the achievable rates bracket the target (2 frames is 25/s, 3 frames is + * 16.7/s) and this rounds to the nearer one, which is 3. Under-reporting a meter + * is cheaper than over-reporting it: nobody has ever noticed a level bar + * updating 16 times a second instead of 20. + * + * **Silence is not published forever.** Once the meter has visibly fallen to + * rest it stops emitting until something moves again. An open microphone in a + * quiet room is the normal state of a voice session, and 20 identical zeros a + * second is the definition of traffic nobody can use. + */ + +import { ACAPPELLA_AUDIO_FRAME_MS } from '../../../shared/acappella/audio-host'; + +/** Meter updates per second. What a client draws, not what the microphone produces. */ +export const DEFAULT_LEVEL_UPDATE_HZ = 20; + +/** + * Level at or below which the meter counts as at rest. Sits under the VAD's exit + * threshold on purpose: the meter must be able to show room noise the detector + * is ignoring, or a user with a dead microphone and a user in a quiet room would + * see the same thing. + */ +export const DEFAULT_LEVEL_SILENCE = 0.005; + +export interface AudioLevelMeterConfig { + /** Duration of one input frame. Must match the capture frame size. */ + frameMs: number; + /** Target updates per second. Realised to the nearest whole number of frames. */ + updateHz: number; + /** Level at or under which the meter is at rest and stops republishing. */ + silenceLevel: number; +} + +export const DEFAULT_LEVEL_METER_CONFIG: AudioLevelMeterConfig = { + frameMs: ACAPPELLA_AUDIO_FRAME_MS, + updateHz: DEFAULT_LEVEL_UPDATE_HZ, + silenceLevel: DEFAULT_LEVEL_SILENCE, +}; + +/** One meter update, shaped exactly like the body of an `audio-level` event. */ +export interface AudioLevelUpdate { + /** Root mean square across the window, 0 to 1. */ + level: number; + /** Whether the detector held the floor open at any point in the window. */ + speech: boolean; +} + +/** + * Clamped, never thrown. These numbers reach the meter from user settings, and a + * typo in a preference must not be able to throw inside the audio path. + */ +export function resolveLevelMeterConfig( + overrides?: Partial +): AudioLevelMeterConfig { + const merged = { ...DEFAULT_LEVEL_METER_CONFIG, ...(overrides ?? {}) }; + return { + frameMs: positive(merged.frameMs, DEFAULT_LEVEL_METER_CONFIG.frameMs), + updateHz: positive(merged.updateHz, DEFAULT_LEVEL_METER_CONFIG.updateHz), + silenceLevel: clamp01(merged.silenceLevel, DEFAULT_LEVEL_METER_CONFIG.silenceLevel), + }; +} + +/** + * Accumulates frame measurements and yields an update once per window. + * + * Instances are cheap and hold one capture run; the pipeline pushes each frame's + * RMS and the detector's verdict, and publishes whatever comes back. + */ +export class AudioLevelMeter { + readonly config: AudioLevelMeterConfig; + /** Frames per update. At least one, so a pathological config still emits. */ + readonly windowFrames: number; + + private frames = 0; + /** Sum of squares, so the window's level is a true RMS rather than a mean of RMSs. */ + private sumSquares = 0; + private speech = false; + /** Whether the last published update was already at rest. Null before the first. */ + private atRest: boolean | null = null; + + constructor(overrides?: Partial) { + this.config = resolveLevelMeterConfig(overrides); + this.windowFrames = Math.max(1, Math.round(1000 / this.config.updateHz / this.config.frameMs)); + } + + /** Realised update rate, which is the frame quantum's answer rather than the requested one. */ + get updateHz(): number { + return 1000 / (this.windowFrames * this.config.frameMs); + } + + /** + * Add one frame. + * + * @param rms The frame's root mean square, 0 to 1. + * @param speech Whether the detector counted this frame as part of an utterance. + * @returns The update to publish, or null when the window is still filling or + * the meter is already at rest and has not moved. + */ + push(rms: number, speech: boolean): AudioLevelUpdate | null { + const value = clamp01(rms, 0); + this.sumSquares += value * value; + this.speech = this.speech || speech; + this.frames += 1; + if (this.frames < this.windowFrames) return null; + + const level = Math.sqrt(this.sumSquares / this.frames); + const update: AudioLevelUpdate = { level, speech: this.speech }; + this.frames = 0; + this.sumSquares = 0; + this.speech = false; + + const resting = level <= this.config.silenceLevel && !update.speech; + // The first at-rest window is published so the meter visibly falls to zero; + // every one after it says nothing new. + if (resting && this.atRest) return null; + this.atRest = resting; + return update; + } + + /** + * Drop the partial window and forget what was last published. + * + * Called when a capture run ends: the next run must publish its first update + * even if the room is silent, or a client that started listening during the + * gap would show a meter frozen at whatever the last run left behind. + */ + reset(): void { + this.frames = 0; + this.sumSquares = 0; + this.speech = false; + this.atRest = null; + } +} + +export function createAudioLevelMeter(overrides?: Partial): AudioLevelMeter { + return new AudioLevelMeter(overrides); +} + +// --------------------------------------------------------------------------- + +function positive(value: number, fallback: number): number { + return Number.isFinite(value) && value > 0 ? value : fallback; +} + +function clamp01(value: number, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + return value < 0 ? 0 : value > 1 ? 1 : value; +} diff --git a/src/main/acappella/audio/mic-state.ts b/src/main/acappella/audio/mic-state.ts new file mode 100644 index 0000000000..5f4c2fa7ed --- /dev/null +++ b/src/main/acappella/audio/mic-state.ts @@ -0,0 +1,124 @@ +/** + * A Cappella microphone state tracker. + * + * Folds the audio host's control plane (`AudioHostStatus`) into the protocol's + * `mic-state`, which is the one thing a client needs in order to tell a session + * that is quiet from a session that is deaf. Pure and synchronous, with no + * Electron and no device anywhere near it. + * + * **It is a projection, not a translation.** A status message carries one fact; + * the microphone's state is several facts that change at different times. The + * device label arrives with `capture-start` and is still true after + * `capture-stop`; permission is learned once and outlives every capture run; + * `device-change` says the device SET moved without saying anything about ours. + * A stateless per-status mapping would have to blank the fields it does not + * carry, and a HUD would flicker between "MacBook Pro Microphone" and nothing. + * + * **Only real changes come back.** `apply()` returns null when nothing the + * client can see moved, so the redundant statuses a device replug produces do + * not become a burst of identical events. The one deliberate exception is + * `device-change` itself, which always publishes: "something was plugged in" is + * news even when the current capture is unaffected, and it is the only signal a + * client has that the input list is worth re-reading. + */ + +import { + audioHostErrorToMicIssue, + type AudioHostStatus, +} from '../../../shared/acappella/audio-host'; +import type { MicState } from '../../../shared/acappella/protocol'; + +/** Nothing has been attempted yet: not granted, not denied, no device open. */ +export const INITIAL_MIC_STATE: MicState = { + permission: 'unknown', + capturing: false, + deviceId: null, + deviceLabel: null, + issue: null, + deviceChanged: false, +}; + +export class MicStateTracker { + private current: MicState = { ...INITIAL_MIC_STATE }; + + /** The state as of the last applied status. Never mutated by callers. */ + get state(): MicState { + return { ...this.current }; + } + + /** + * Apply one host status. + * + * @returns The state to publish, or null when nothing observable changed. + */ + apply(status: AudioHostStatus): MicState | null { + const next: MicState = { ...this.current, deviceChanged: false }; + + switch (status.kind) { + case 'capture-start': + // The microphone opened, which is proof of permission no query can + // give us: Chromium only hands over a live track once the user agrees. + next.permission = 'granted'; + next.capturing = true; + next.deviceId = status.device.deviceId || null; + next.deviceLabel = status.device.label || null; + next.issue = null; + break; + + case 'capture-stop': + next.capturing = false; + // A device that was taken away is a fault the user has to see; a + // requested stop is the session simply ending and clears the fault. + next.issue = status.reason === 'device-lost' ? 'device-lost' : null; + if (status.reason === 'device-lost') { + next.deviceId = null; + next.deviceLabel = null; + } + break; + + case 'mic-error': { + const issue = audioHostErrorToMicIssue(status.code); + next.capturing = false; + next.issue = issue; + if (issue === 'permission-denied') next.permission = 'denied'; + if (issue === 'no-device' || issue === 'device-lost') { + next.deviceId = null; + next.deviceLabel = null; + } + break; + } + + case 'device-change': + next.deviceChanged = true; + break; + + default: + // `ready` and `playback-state` say nothing about the microphone. + return null; + } + + const changed = next.deviceChanged || !sameMicState(this.current, next); + this.current = { ...next, deviceChanged: false }; + return changed ? next : null; + } + + /** Back to never-attempted. Called when the audio host window goes away. */ + reset(): void { + this.current = { ...INITIAL_MIC_STATE }; + } +} + +export function createMicStateTracker(): MicStateTracker { + return new MicStateTracker(); +} + +/** `deviceChanged` is a one-shot flag on an event, never part of the state. */ +function sameMicState(a: MicState, b: MicState): boolean { + return ( + a.permission === b.permission && + a.capturing === b.capturing && + a.deviceId === b.deviceId && + a.deviceLabel === b.deviceLabel && + a.issue === b.issue + ); +} diff --git a/src/main/acappella/audio/vad.ts b/src/main/acappella/audio/vad.ts new file mode 100644 index 0000000000..e27c07c1ce --- /dev/null +++ b/src/main/acappella/audio/vad.ts @@ -0,0 +1,490 @@ +/** + * A Cappella voice activity detection. + * + * A pure, synchronous classifier over the 20 ms / 16 kHz mono frames the PCM + * worklet produces. Feed it frames, get back a per-frame verdict plus the two + * transitions the pipeline cares about: `speech-start` (the floor has real audio + * on it) and `speech-end` (the utterance endpointed). + * + * **Nothing in here touches a clock, a timer, a device, or Electron.** Time is + * counted in frames, because that is the only clock that stays honest when the + * main thread stalls: 35 frames of silence is 700 ms of silence whether the + * event loop was free or wedged. Callers that need wall time correlate against + * `AudioFrame.capturedAt`, which the worklet derives from the audio clock. The + * practical payoff is that the whole detector is testable from generated tone + * and silence arrays with no audio device anywhere near it. + * + * The classifier is energy plus zero-crossing rate, the classic pairing, with + * three things bolted on that the textbook version leaves out and that decide + * whether it is usable in a real room: + * + * 1. **Hysteresis.** Opening takes `enterRms`, closing takes the lower + * `exitRms`. One threshold makes the detector chatter on every syllable + * whose level happens to sit on the line. + * 2. **A zero-crossing band, applied only on the way in.** Voiced speech + * crosses zero at a moderate rate. A desk thump or a fan rumble is far + * below the band, hiss and broadband transients far above it. Once speech + * is open the band is dropped, because a trailing "sss" is legitimately + * high-ZCR and must not close the floor mid-word. + * 3. **A tracked noise floor.** A fixed absolute threshold is tuned for + * exactly one room and one microphone. The floor follows the quiet parts + * down fast and drifts up slowly, so a laptop fan spinning up raises the + * bar rather than holding the mic open. + * + * The limitation worth stating rather than papering over: no energy/ZCR detector + * can tell a cough from a word. The defences here are the ZCR band (a cough is a + * broadband burst) and `enterFrames`, which demands sustained evidence before + * opening. Anything that survives both is the transcript's problem, and the + * backstop for a floor that opens and will not close is the idle timeout in + * `floor-control.ts`, not a cleverer VAD. + */ + +import { ACAPPELLA_AUDIO_FRAME_MS } from '../../../shared/acappella/audio-host'; +import type { AudioFrame } from '../../../shared/acappella/audio-host'; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +export interface VadConfig { + /** Duration of one frame. Must match the capture frame size, or every ms figure below lies. */ + frameMs: number; + /** RMS (0 to 1) a frame must reach to count toward speech onset. */ + enterRms: number; + /** RMS a frame must fall under to count as silence. Below `enterRms`: this is the hysteresis. */ + exitRms: number; + /** Consecutive qualifying frames required before `speech-start`. The transient defence. */ + enterFrames: number; + /** + * Silent frames tolerated before a frame stops being marked `active`. Stop + * consonants and inter-word gaps are silent; cutting the audio feed at the + * first quiet frame clips the end of every utterance. + */ + hangoverFrames: number; + /** + * Continuous silence that ends an utterance. The endpointing decision, and + * the one knob a user has any business turning: too short truncates anyone + * who thinks mid-sentence, too long makes the assistant feel deaf. + */ + endpointSilenceMs: number; + /** Crossings per sample below which a frame is rumble, not voice. */ + minZeroCrossingRate: number; + /** Crossings per sample above which a frame is hiss or a transient, not voice. */ + maxZeroCrossingRate: number; + /** Track the room's noise floor and lift both thresholds above it. */ + adaptiveNoiseFloor: boolean; + /** Multiple of the noise floor the enter threshold is held above. */ + noiseFloorEnterMargin: number; + /** Multiple of the noise floor the exit threshold is held above. Below the enter margin. */ + noiseFloorExitMargin: number; + /** + * Hard ceiling on the tracked floor. Chosen so the adapted exit threshold + * stays under the level of ordinary conversational speech: a floor that could + * climb past the speaker would endpoint people mid-sentence, which is a far + * worse failure than failing to reject a loud room. + */ + maxNoiseFloor: number; + /** + * Frames at the head of a run during which the floor adapts fast. + * + * Without this the estimator is useless exactly when it matters: a room whose + * noise already clears the absolute enter threshold opens the floor within + * `enterFrames` and then never sees a quiet frame to learn from, so it stays + * latched open. A short fast pass at the start measures the room before that + * can happen. It does NOT suppress onset - a user who talks the instant the + * mic opens is still heard immediately - which is safe only because + * {@link maxNoiseFloor} bounds how wrong a calibration on speech can be. + */ + calibrationFrames: number; +} + +/** + * Tuned for a headset or a laptop mic at arm's length in a normal room. + * + * `endpointSilenceMs: 700` is the number to argue about. It is long enough to + * survive the pause people leave before the second half of a sentence and short + * enough that a finished request does not feel ignored. + */ +export const DEFAULT_VAD_CONFIG: VadConfig = { + frameMs: ACAPPELLA_AUDIO_FRAME_MS, + enterRms: 0.02, + exitRms: 0.01, + enterFrames: 4, + hangoverFrames: 10, + endpointSilenceMs: 700, + minZeroCrossingRate: 0.01, + maxZeroCrossingRate: 0.3, + adaptiveNoiseFloor: true, + noiseFloorEnterMargin: 3, + noiseFloorExitMargin: 1.8, + maxNoiseFloor: 0.03, + calibrationFrames: 10, +}; + +/** How fast the tracked floor follows a quieter room. Fast: quiet is trustworthy. */ +const NOISE_FLOOR_FALL = 0.5; +/** + * How fast it follows a louder one. Deliberately ~4 s: speech is loud too, and a + * floor that chased it would raise the bar above the speaker mid-sentence. + */ +const NOISE_FLOOR_RISE = 0.005; +/** Rise rate during calibration. ~5 frames to the room's level, not four seconds. */ +const NOISE_FLOOR_CALIBRATION_RISE = 0.3; + +/** + * Fill in and sanitise a partial config. + * + * Every out-of-range value is clamped rather than rejected. These numbers reach + * us from user settings, and a typo in a preference must not be able to throw + * inside the audio path - a slightly wrong threshold is recoverable, a dead + * pipeline is not. + */ +export function resolveVadConfig(overrides: Partial = {}): VadConfig { + const merged = { ...DEFAULT_VAD_CONFIG, ...overrides }; + const frameMs = Math.max(1, finite(merged.frameMs, DEFAULT_VAD_CONFIG.frameMs)); + const enterRms = clamp(finite(merged.enterRms, DEFAULT_VAD_CONFIG.enterRms), 0, 1); + const maxNoiseFloor = clamp(finite(merged.maxNoiseFloor, DEFAULT_VAD_CONFIG.maxNoiseFloor), 0, 1); + return { + frameMs, + enterRms, + // An exit threshold at or above the enter threshold would make the detector + // open and close on the same frame, so it is pinned below it. + exitRms: Math.min(clamp(finite(merged.exitRms, DEFAULT_VAD_CONFIG.exitRms), 0, 1), enterRms), + enterFrames: Math.max( + 1, + Math.round(finite(merged.enterFrames, DEFAULT_VAD_CONFIG.enterFrames)) + ), + hangoverFrames: Math.max( + 0, + Math.round(finite(merged.hangoverFrames, DEFAULT_VAD_CONFIG.hangoverFrames)) + ), + // One frame of silence is the shortest endpoint that can exist; anything + // less would fire `speech-end` on the same frame as `speech-start`. + endpointSilenceMs: Math.max( + frameMs, + finite(merged.endpointSilenceMs, DEFAULT_VAD_CONFIG.endpointSilenceMs) + ), + minZeroCrossingRate: clamp( + finite(merged.minZeroCrossingRate, DEFAULT_VAD_CONFIG.minZeroCrossingRate), + 0, + 1 + ), + maxZeroCrossingRate: clamp( + finite(merged.maxZeroCrossingRate, DEFAULT_VAD_CONFIG.maxZeroCrossingRate), + 0, + 1 + ), + adaptiveNoiseFloor: merged.adaptiveNoiseFloor !== false, + noiseFloorEnterMargin: Math.max( + 1, + finite(merged.noiseFloorEnterMargin, DEFAULT_VAD_CONFIG.noiseFloorEnterMargin) + ), + noiseFloorExitMargin: Math.max( + 1, + finite(merged.noiseFloorExitMargin, DEFAULT_VAD_CONFIG.noiseFloorExitMargin) + ), + maxNoiseFloor, + calibrationFrames: Math.max( + 0, + Math.round(finite(merged.calibrationFrames, DEFAULT_VAD_CONFIG.calibrationFrames)) + ), + }; +} + +// --------------------------------------------------------------------------- +// Results +// --------------------------------------------------------------------------- + +export type VadState = 'silence' | 'speech'; + +export type VadEvent = + | { + type: 'speech-start'; + /** Detector time at the START of the first qualifying frame, not at the decision. */ + atMs: number; + } + | { + type: 'speech-end'; + /** Detector time at the END of the last frame that counted as speech. */ + atMs: number; + /** The matching `speech-start.atMs`. */ + startedAtMs: number; + /** `atMs - startedAtMs`. Speech only: the endpoint silence is not in here. */ + durationMs: number; + /** Silence measured before endpointing, rounded up to a whole frame. */ + trailingSilenceMs: number; + }; + +export interface VadFrameResult { + /** State AFTER this frame. */ + state: VadState; + /** + * Whether this frame belongs to the utterance and should be fed to STT. True + * through the hangover, which is the point of the hangover. + */ + active: boolean; + /** + * Whether this frame on its own looks like voice: the full onset test while the + * floor is closed, energy above the exit threshold while it is open. + * + * Deliberately weaker evidence than {@link VadFrameResult.event} - one frame is + * not enough to open the floor - and that is exactly what makes it useful. The + * pipeline ducks TTS output on the first candidate frame, 80 ms before a + * `speech-start` could possibly be confirmed, and restores the gain if the + * candidate does not turn into speech. + */ + candidate: boolean; + /** The transition this frame caused, if any. At most one per frame. */ + event: VadEvent | null; + /** Root mean square of the frame, 0 to 1. */ + rms: number; + /** Zero crossings per sample, 0 to 1. */ + zeroCrossingRate: number; + /** The tracked noise floor after this frame. Always zero when adaptation is off. */ + noiseFloor: number; + /** Detector time at the END of this frame. Frame count times `frameMs`. */ + elapsedMs: number; + /** Continuous silence up to and including this frame. Zero while speech is live. */ + silenceMs: number; +} + +// --------------------------------------------------------------------------- +// Detector +// --------------------------------------------------------------------------- + +/** + * Energy plus zero-crossing voice activity detection over fixed-size frames. + * + * Stateful across frames (that is what hysteresis and hangover mean) but with no + * hidden inputs: the same frame sequence always produces the same event + * sequence. Instances are cheap; the pipeline holds one per capture run and + * calls {@link reset} rather than rebuilding. + */ +export class VoiceActivityDetector { + readonly config: VadConfig; + + private currentState: VadState = 'silence'; + private frameIndex = 0; + /** Consecutive onset candidates seen while in `silence`. */ + private candidateFrames = 0; + /** Consecutive silent frames seen while in `speech`. */ + private silenceFrames = 0; + /** + * Starts at zero, meaning "nothing measured yet". That is deliberately the + * conservative end: with no estimate the configured absolute thresholds + * govern, and the floor can only ever raise them from there. Seeding it at + * `exitRms` instead would mean the absolute settings were never the operative + * numbers in even a silent room, which makes them impossible to reason about. + */ + private noiseFloorValue = 0; + private speechStartedAtMs = 0; + /** Frame index just past the last frame that counted as speech. */ + private lastVoicedFrame = 0; + private readonly endpointFrames: number; + + constructor(overrides: Partial = {}) { + this.config = resolveVadConfig(overrides); + this.endpointFrames = Math.max( + 1, + Math.ceil(this.config.endpointSilenceMs / this.config.frameMs) + ); + } + + get state(): VadState { + return this.currentState; + } + + /** The tracked noise floor. Exposed for the level meter and for tests. */ + get noiseFloor(): number { + return this.config.adaptiveNoiseFloor ? this.noiseFloorValue : 0; + } + + /** Detector time at the end of the last processed frame. */ + get elapsedMs(): number { + return this.frameIndex * this.config.frameMs; + } + + /** + * Back to the state a fresh detector is in, including the frame clock. Call + * between capture runs: carrying an open `speech` state across a stop would + * make the next run's first frame emit a `speech-end` for audio nobody heard. + */ + reset(): void { + this.currentState = 'silence'; + this.frameIndex = 0; + this.candidateFrames = 0; + this.silenceFrames = 0; + this.noiseFloorValue = 0; + this.speechStartedAtMs = 0; + this.lastVoicedFrame = 0; + } + + /** Convenience over the wire type. Uses the samples, not `frame.rms`, so there is one measure. */ + processFrame(frame: AudioFrame): VadFrameResult { + return this.process(new Int16Array(frame.pcm)); + } + + /** Classify one frame of signed 16-bit mono PCM. */ + process(samples: Int16Array): VadFrameResult { + const { rms, zeroCrossingRate } = measure(samples); + return this.processMeasurement(rms, zeroCrossingRate); + } + + /** + * The classifier proper, over an already-measured frame. + * + * Split out so tests can drive threshold and hangover behaviour directly + * rather than by synthesising PCM that happens to land on a level, and so a + * future caller that already has the numbers is not forced to rescan. + */ + processMeasurement(rms: number, zeroCrossingRate: number): VadFrameResult { + const enterThreshold = this.enterThreshold(); + const exitThreshold = this.exitThreshold(); + const quiet = rms < exitThreshold; + + // Measured only while the floor is closed: an open utterance is the one + // stretch we know is not noise, and feeding it into the estimate is how an + // adaptive detector talks itself into cutting the speaker off. Steady state + // also skips a run of onset candidates for the same reason; calibration + // cannot afford to, since a noisy room is nothing but candidates. + const calibrating = this.frameIndex < this.config.calibrationFrames; + if (this.currentState === 'silence' && (calibrating || this.candidateFrames === 0)) { + this.trackNoiseFloor(rms, calibrating); + } + + this.frameIndex += 1; + let event: VadEvent | null = null; + let candidate: boolean; + + if (this.currentState === 'silence') { + candidate = + rms >= enterThreshold && + zeroCrossingRate >= this.config.minZeroCrossingRate && + zeroCrossingRate <= this.config.maxZeroCrossingRate; + this.candidateFrames = candidate ? this.candidateFrames + 1 : 0; + + if (this.candidateFrames >= this.config.enterFrames) { + // Dated to the start of the FIRST qualifying frame, not to the frame + // that tipped the count. The onset is what the pre-roll aligns to, and + // backdating it is the difference between the transcript keeping the + // first syllable and losing it. + this.speechStartedAtMs = (this.frameIndex - this.candidateFrames) * this.config.frameMs; + this.currentState = 'speech'; + this.candidateFrames = 0; + this.silenceFrames = 0; + this.lastVoicedFrame = this.frameIndex; + event = { type: 'speech-start', atMs: this.speechStartedAtMs }; + } + } else { + // Hysteresis: anything not under the exit threshold sustains speech, even + // though it would not have been loud enough to open the floor. + candidate = !quiet; + if (quiet) { + this.silenceFrames += 1; + } else { + this.silenceFrames = 0; + this.lastVoicedFrame = this.frameIndex; + } + + if (this.silenceFrames >= this.endpointFrames) { + const atMs = this.lastVoicedFrame * this.config.frameMs; + event = { + type: 'speech-end', + atMs, + startedAtMs: this.speechStartedAtMs, + durationMs: atMs - this.speechStartedAtMs, + trailingSilenceMs: this.silenceFrames * this.config.frameMs, + }; + this.currentState = 'silence'; + this.candidateFrames = 0; + this.silenceFrames = 0; + } + } + + return { + state: this.currentState, + active: this.currentState === 'speech' && this.silenceFrames <= this.config.hangoverFrames, + candidate, + event, + rms, + zeroCrossingRate, + noiseFloor: this.noiseFloor, + elapsedMs: this.elapsedMs, + silenceMs: this.silenceFrames * this.config.frameMs, + }; + } + + private enterThreshold(): number { + if (!this.config.adaptiveNoiseFloor) return this.config.enterRms; + return Math.max(this.config.enterRms, this.noiseFloorValue * this.config.noiseFloorEnterMargin); + } + + private exitThreshold(): number { + if (!this.config.adaptiveNoiseFloor) return this.config.exitRms; + // Monotone in both arguments and the exit margin is below the enter margin, + // so this can never climb above `enterThreshold()`. + return Math.max(this.config.exitRms, this.noiseFloorValue * this.config.noiseFloorExitMargin); + } + + private trackNoiseFloor(rms: number, calibrating: boolean): void { + if (!this.config.adaptiveNoiseFloor) return; + const rise = calibrating ? NOISE_FLOOR_CALIBRATION_RISE : NOISE_FLOOR_RISE; + const alpha = rms < this.noiseFloorValue ? NOISE_FLOOR_FALL : rise; + this.noiseFloorValue = Math.min( + this.config.maxNoiseFloor, + this.noiseFloorValue + alpha * (rms - this.noiseFloorValue) + ); + } +} + +/** Sugar for `new VoiceActivityDetector(...)`, matching the rest of A Cappella's factories. */ +export function createVoiceActivityDetector( + overrides: Partial = {} +): VoiceActivityDetector { + return new VoiceActivityDetector(overrides); +} + +// --------------------------------------------------------------------------- +// Measurement +// --------------------------------------------------------------------------- + +/** Full-scale magnitude of a signed 16-bit sample. */ +const INT16_SCALE = 0x8000; + +/** + * RMS and zero-crossing rate in one pass over the frame. + * + * The worklet already reports an RMS, but it measures the float samples before + * quantisation while this measures what actually arrived, and the zero-crossing + * scan has to walk the array regardless. One measure of the same bytes beats two + * measures that disagree in the third decimal place. + */ +export function measure(samples: Int16Array): { rms: number; zeroCrossingRate: number } { + const length = samples.length; + if (length === 0) return { rms: 0, zeroCrossingRate: 0 }; + + let sumSquares = 0; + let crossings = 0; + let previousPositive = samples[0] >= 0; + + for (let i = 0; i < length; i++) { + const value = samples[i] / INT16_SCALE; + sumSquares += value * value; + const positive = samples[i] >= 0; + if (i > 0 && positive !== previousPositive) crossings += 1; + previousPositive = positive; + } + + return { + rms: Math.sqrt(sumSquares / length), + zeroCrossingRate: length > 1 ? crossings / (length - 1) : 0, + }; +} + +function clamp(value: number, min: number, max: number): number { + return value < min ? min : value > max ? max : value; +} + +function finite(value: number, fallback: number): number { + return Number.isFinite(value) ? value : fallback; +} diff --git a/src/main/acappella/dispatch/route-executor.ts b/src/main/acappella/dispatch/route-executor.ts new file mode 100644 index 0000000000..58b43aab7d --- /dev/null +++ b/src/main/acappella/dispatch/route-executor.ts @@ -0,0 +1,633 @@ +/** + * A Cappella dispatch executor - where a `RouteDecision` becomes a real agent + * and a real tab. + * + * Main has NO tab authority. Tab state lives in the renderer, and even a web or + * CLI request to open one is forwarded there for execution + * (`src/main/web-server/callbacks/tabCallbacks.ts`). So nothing here creates a + * tab: it resolves a decision into the same `remote:*` messages the web server + * already uses and waits for the renderer's answer. Hand-rolling a parallel tab + * path in main would produce tabs the renderer does not know about. + * + * The roster comes from the persisted sessions store, the same source + * `registerSessionCallbacks` reads, so the Brain routes against what the user + * actually has open. + * + * The renderer round trip is behind `VoiceRendererBridge` for the same reason + * the session service takes its providers injected: the routing rules are worth + * testing without an Electron window, and Phase 08 will hand the phone leg the + * same executor with a different bridge. + */ + +import type { BrowserWindow } from 'electron'; + +import type { RosterAgent, VoiceScope } from '../../../shared/acappella/protocol'; +import type { RouteDecision } from '../../../shared/acappella/route-decision'; +import { isClarification, routeTargetSessionId } from '../../../shared/acappella/route-decision'; +import type { StoredSession } from '../../stores/types'; +import { getSessionsStore } from '../../stores/getters'; +import { isWebContentsAvailable } from '../../utils/safe-send'; +import { logger } from '../../utils/logger'; +import { requestFromRenderer } from '../../web-server/callbacks/remoteRequest'; +import { buildRoutingRoster } from '../router/routing-context'; +import { resolveRecall } from '../router/tab-recall'; +import { VoiceDispatchError } from '../voice-session-service'; +import type { VoiceDispatchResult, VoiceRouteExecutor } from '../voice-session-service'; + +const LOG_CONTEXT = 'ACappella'; + +/** Tab creation is a renderer round trip; 5s matches `tabCallbacks.ts`. */ +const NEW_TAB_TIMEOUT_MS = 5000; + +/** Delivery receipt window, matching `REMOTE_COMMAND_RECEIPT_TIMEOUT_MS`. */ +const COMMAND_RECEIPT_TIMEOUT_MS = 3000; + +// --------------------------------------------------------------------------- +// Roster +// --------------------------------------------------------------------------- + +/** + * Compact the persisted sessions into the roster the Brain routes against and + * the phone's project wheel will later render. + * + * One builder, shared with the router: the Brain and the executor disagreeing + * about which tabs exist is how a decision becomes undispatchable between being + * made and being performed. + */ +export function buildAgentRoster(sessions: StoredSession[]): RosterAgent[] { + return buildRoutingRoster(sessions); +} + +/** The roster as of right now, straight from the store. */ +export function readAgentRoster(): RosterAgent[] { + return buildAgentRoster(readStoredSessions()); +} + +function readStoredSessions(): StoredSession[] { + return getSessionsStore().get('sessions', []); +} + +function readActiveSessionId(): string | null { + return getSessionsStore().get('activeSessionId') ?? null; +} + +// --------------------------------------------------------------------------- +// Renderer bridge +// --------------------------------------------------------------------------- + +/** The renderer's answer to `remote:newAITabWithPrompt`. */ +export interface NewTabWithPromptResult { + success: boolean; + tabId?: string; +} + +/** The renderer's delivery receipt for `remote:executeCommand`. */ +export interface CommandReceipt { + accepted: boolean; + reason?: string; +} + +/** + * What the renderer did to land on a tab. + * + * `focused` is the ordinary case; the other two are the states main cannot + * resolve on its own, because waking a snoozed tab and reopening a closed one + * both need renderer-owned state. Reporting which one happened is what lets the + * `dispatch` event say something true: "back in the auth conversation" is a lie + * if the tab was still snoozed underneath. + */ +export interface FocusTabResult { + ok: boolean; + /** The tab actually landed on. It can differ when a wake found a duplicate. */ + tabId?: string; + action?: 'focused' | 'woke' | 'reopened'; + reason?: string; +} + +/** + * Every renderer operation dispatch needs, and nothing else. Each method maps + * onto one existing `remote:*` channel; adding a method here means adding a + * channel, not inventing a second way to do something the renderer already does. + */ +export interface VoiceRendererBridge { + /** `remote:selectSession` - focus an agent, and a tab within it when given. */ + selectSession(agentSessionId: string, tabId?: string): void; + /** + * `remote:focusAiTab` - land on one AI tab and say what that took. + * + * Distinct from `selectSession` because it is a REQUEST: it waits for the + * renderer, which is the only side that can wake a snoozed tab, reopen a + * closed one, or honour the tiling invariant. Announcing a recall before the + * renderer confirmed it is how a client ends up narrating a tab the user + * cannot see. + */ + focusTab(agentSessionId: string, tabId: string): Promise; + /** `remote:renameTab` - name an existing AI tab. */ + renameTab(agentSessionId: string, tabId: string, name: string): void; + /** `remote:newTab` - open an empty AI tab. Resolves to its id, or null. */ + newTab(agentSessionId: string): Promise; + /** `remote:newAITabWithPrompt` - open a tab and dispatch a prompt atomically. */ + newTabWithPrompt(agentSessionId: string, prompt: string): Promise; + /** `remote:executeCommand` - send a prompt to an existing tab, awaiting its receipt. */ + executeCommand(agentSessionId: string, tabId: string, prompt: string): Promise; +} + +/** + * The real bridge: the window that OWNS the agent, and the existing `remote:*` + * channels. + * + * `getWindowForSession` is what makes dispatch multi-window aware. Agent + * ownership is per window while `activeSessionId` is global, so sending a voice + * dispatch to whichever window happens to be "main" would activate an agent that + * window does not own - the documented way to make a window render "No agents". + * The owning window is also raised, because a spoken instruction that landed + * behind another window has, from the user's side, done nothing. + */ +export function createRendererVoiceBridge( + getWindow: () => BrowserWindow | null, + getWindowForSession?: (agentSessionId: string) => BrowserWindow | null +): VoiceRendererBridge { + const requireWindow = (operation: string, agentSessionId?: string): BrowserWindow => { + const owner = agentSessionId ? getWindowForSession?.(agentSessionId) : null; + const win = isWebContentsAvailable(owner) ? owner : getWindow(); + if (!isWebContentsAvailable(win)) { + throw new VoiceDispatchError(`No renderer is available to ${operation}`); + } + return win; + }; + + /** Raise the window a dispatch is about to land in. Never steals from another app. */ + const revealWindow = (win: BrowserWindow): void => { + if (win.isMinimized()) win.restore(); + win.show(); + }; + + return { + selectSession(agentSessionId, tabId) { + const win = requireWindow('focus an agent', agentSessionId); + revealWindow(win); + win.webContents.send('remote:selectSession', agentSessionId, tabId); + }, + + async focusTab(agentSessionId, tabId) { + const win = requireWindow('focus a tab', agentSessionId); + revealWindow(win); + return requestFromRenderer(win, 'remote:focusAiTab', { + fallback: { ok: false, reason: 'renderer-timeout' }, + timeoutMs: NEW_TAB_TIMEOUT_MS, + parse: parseFocusTabResult, + args: [agentSessionId, tabId], + }); + }, + + renameTab(agentSessionId, tabId, name) { + requireWindow('rename a tab', agentSessionId).webContents.send( + 'remote:renameTab', + agentSessionId, + tabId, + name + ); + }, + + async newTab(agentSessionId) { + const result = await requestFromRenderer<{ tabId?: unknown } | null>( + requireWindow('open a tab', agentSessionId), + 'remote:newTab', + { + fallback: null, + timeoutMs: NEW_TAB_TIMEOUT_MS, + parse: (raw) => + typeof raw === 'object' && raw !== null ? (raw as { tabId?: unknown }) : null, + args: [agentSessionId], + } + ); + return typeof result?.tabId === 'string' ? result.tabId : null; + }, + + async newTabWithPrompt(agentSessionId, prompt) { + // The channel takes `(sessionId, prompt, responseChannel, background?)`, + // so omitting `background` both puts the response channel in the right + // position and gets the focus behaviour voice wants: a tab the user + // asked for out loud should be the tab they are looking at. + return requestFromRenderer( + requireWindow('open a tab', agentSessionId), + 'remote:newAITabWithPrompt', + { + fallback: { success: false }, + timeoutMs: NEW_TAB_TIMEOUT_MS, + parse: parseNewTabWithPromptResult, + args: [agentSessionId, prompt], + } + ); + }, + + async executeCommand(agentSessionId, tabId, prompt) { + return requestFromRenderer( + requireWindow('send a prompt', agentSessionId), + 'remote:executeCommand', + { + fallback: { accepted: false, reason: 'renderer-timeout' }, + timeoutMs: COMMAND_RECEIPT_TIMEOUT_MS, + parse: parseCommandReceipt, + // Positional: (sessionId, command, inputMode, tabId, force, images, + // background) before the receipt channel. `force: false` keeps the + // renderer's busy guard - talking over a working agent must not + // interleave two prompts in one tab. + args: [agentSessionId, prompt, 'ai', tabId, false, undefined, false], + } + ); + }, + }; +} + +function parseNewTabWithPromptResult(raw: unknown): NewTabWithPromptResult { + if (typeof raw === 'object' && raw !== null) { + const result = raw as { success?: unknown; tabId?: unknown }; + return { + success: result.success === true, + tabId: typeof result.tabId === 'string' ? result.tabId : undefined, + }; + } + // Older renderers ack with a bare boolean and no tab id. + return { success: raw === true }; +} + +function parseFocusTabResult(raw: unknown): FocusTabResult { + if (typeof raw !== 'object' || raw === null) return { ok: false, reason: 'malformed-result' }; + const result = raw as { ok?: unknown; tabId?: unknown; action?: unknown; reason?: unknown }; + return { + ok: result.ok === true, + tabId: typeof result.tabId === 'string' ? result.tabId : undefined, + action: + result.action === 'woke' || result.action === 'reopened' || result.action === 'focused' + ? result.action + : undefined, + reason: typeof result.reason === 'string' ? result.reason : undefined, + }; +} + +function parseCommandReceipt(raw: unknown): CommandReceipt { + if (typeof raw === 'object' && raw !== null && 'accepted' in raw) { + const receipt = raw as { accepted?: unknown; reason?: unknown }; + return { + accepted: receipt.accepted === true, + reason: typeof receipt.reason === 'string' ? receipt.reason : undefined, + }; + } + return { accepted: false, reason: 'malformed-receipt' }; +} + +// --------------------------------------------------------------------------- +// Execution +// --------------------------------------------------------------------------- + +export interface VoiceRouteExecutorOptions { + bridge: VoiceRendererBridge; + /** Persisted sessions. Defaults to the main sessions store. */ + getSessions?: () => StoredSession[]; + /** The agent the desktop is on, used when the Brain targets the conductor. */ + getActiveSessionId?: () => string | null; + /** How long an identical decision replays instead of executing again. */ + replayWindowMs?: number; +} + +interface ResolvedExecutorDeps { + bridge: VoiceRendererBridge; + getSessions: () => StoredSession[]; + getActiveSessionId: () => string | null; + recentDispatches?: DispatchReplayCache; +} + +/** + * Remembers what each decision already did, briefly. + * + * A dispatch is retried whenever a renderer round trip times out and the caller + * tries again, and the failure mode that costs the user something is a decision + * that opened a tab, lost the receipt, and opened a second one. Replaying the + * first result is idempotence: the same decision performed twice is the same + * dispatch, not two. + * + * The window is short on purpose. Beyond it, saying the same thing again is a + * person repeating themselves, and they mean it. + */ +export interface DispatchReplayCache { + get(key: string): VoiceDispatchResult | undefined; + set(key: string, value: VoiceDispatchResult): void; +} + +/** Thirty seconds: long enough for a retry, short enough not to swallow intent. */ +export const DEFAULT_REPLAY_WINDOW_MS = 30_000; + +export function createDispatchReplayCache( + ttlMs: number = DEFAULT_REPLAY_WINDOW_MS, + now: () => number = Date.now +): DispatchReplayCache { + const entries = new Map(); + + return { + get(key) { + // A window of zero disables replay outright rather than depending on two + // dispatches landing in different milliseconds. + if (ttlMs <= 0) return undefined; + const entry = entries.get(key); + if (!entry) return undefined; + if (now() - entry.at > ttlMs) { + entries.delete(key); + return undefined; + } + return entry.result; + }, + set(key, result) { + const at = now(); + entries.set(key, { at, result }); + // Swept on write rather than on a timer: the map only grows when someone + // is talking, so the moment it grows is the moment to prune it. + for (const [candidate, entry] of entries) { + if (at - entry.at > ttlMs) entries.delete(candidate); + } + }, + }; +} + +/** Bind an executor for `VoiceSessionServiceOptions.executeRoute`. */ +export function createVoiceRouteExecutor(options: VoiceRouteExecutorOptions): VoiceRouteExecutor { + const deps: ResolvedExecutorDeps = { + bridge: options.bridge, + getSessions: options.getSessions ?? readStoredSessions, + getActiveSessionId: options.getActiveSessionId ?? readActiveSessionId, + recentDispatches: createDispatchReplayCache(options.replayWindowMs), + }; + return (decision, context) => executeRouteDecision(decision, context, deps); +} + +/** + * Perform one decision and report what actually happened. The result becomes the + * `dispatch` event, so every field has to describe the real outcome rather than + * the request: "opened a new tab named Auth Refactor on agent Backend" is only + * true if the renderer says it is. + * + * Every known failure throws `VoiceDispatchError`, which the session service + * turns into a `dispatch-failed` event. Anything else is a bug and reaches + * Sentry unchanged. + */ +export async function executeRouteDecision( + decision: RouteDecision, + context: { roster: RosterAgent[]; scope: VoiceScope }, + deps: ResolvedExecutorDeps +): Promise { + // A clarification is a question, not an instruction. Reaching the executor + // with one means a caller skipped the guard, and dispatching it would send + // the user their own half-finished request. + if (isClarification(decision)) { + throw new VoiceDispatchError('That decision is a question, not a dispatch'); + } + + // Re-read rather than trusting the roster the Brain saw: routing is async and + // the user can close a tab while a decision is in flight. + const sessions = deps.getSessions(); + const roster = buildAgentRoster(sessions); + const agent = resolveAgent(decision, context, roster, deps.getActiveSessionId()); + const prompt = decision.prompt.trim(); + + // Idempotency is checked AFTER the roster read so a retry that is no longer + // performable still fails rather than replaying a stale success, and before + // anything is created so a retried decision cannot open a second tab. + const key = dispatchKey(decision, agent.sessionId); + const replayed = deps.recentDispatches?.get(key); + if (replayed) { + logger.info(`Replaying the dispatch for an identical decision on '${agent.name}'`, LOG_CONTEXT); + return replayed; + } + + const result = await performDispatch(decision, agent, roster, sessions, prompt, deps); + deps.recentDispatches?.set(key, result); + return result; +} + +async function performDispatch( + decision: RouteDecision, + agent: RosterAgent, + roster: RosterAgent[], + sessions: StoredSession[], + prompt: string, + deps: ResolvedExecutorDeps +): Promise { + if (decision.tabAction === 'new') { + return openNewTab(agent, prompt, decision.tabName, deps.bridge); + } + + if (decision.tabAction === 'recall') { + return recallTab(decision, agent, roster, prompt, deps); + } + + // `activeTabId` is deliberately not on `RosterAgent` - the Brain routes by + // name, not by which tab happens to be on screen - so it is read here. + const stored = sessions.find((session) => session.id === agent.sessionId); + const activeTabId = typeof stored?.activeTabId === 'string' ? stored.activeTabId : null; + const tabId = resolveCurrentTab(agent, activeTabId); + + if (!tabId) { + // The agent has no AI tab to talk into. Creating one is the only honest way + // to land the prompt, and the result says `created` so nobody is told a tab + // was focused that never existed. + logger.info(`Agent '${agent.name}' has no open AI tab; creating one`, LOG_CONTEXT); + return openNewTab(agent, prompt, decision.tabName, deps.bridge); + } + + deps.bridge.selectSession(agent.sessionId, tabId); + const promptSent = await sendPrompt(deps.bridge, agent.sessionId, tabId, prompt); + + return { + agentSessionId: agent.sessionId, + agentName: agent.name, + tabId, + tabName: agent.tabs.find((tab) => tab.id === tabId)?.name ?? undefined, + action: 'focused', + promptSent, + }; +} + +/** + * Return to an existing conversation, waking or reopening it if that is what it + * takes. + * + * The focus is a REQUEST rather than a fire-and-forget send, because the three + * states a recalled tab can be in are only distinguishable in the renderer, and + * announcing a recall the renderer did not perform would tell the user they are + * somewhere they are not. + */ +async function recallTab( + decision: RouteDecision, + agent: RosterAgent, + roster: RosterAgent[], + prompt: string, + deps: ResolvedExecutorDeps +): Promise { + const resolution = resolveRecall(decision, roster, { confirmed: true }); + if (resolution.kind === 'missing') { + // Recall is a promise to return somewhere specific. A gone tab is a failure, + // never a silently different tab. + throw new VoiceDispatchError( + decision.tabId + ? `That tab is no longer open on '${agent.name}'` + : `Cannot recall a tab on '${agent.name}' without a tab id` + ); + } + if (resolution.kind === 'offer') { + // The router turns an offer into a spoken question before it ever reaches + // here; arriving with one means the confirmation was skipped. + throw new VoiceDispatchError(`That conversation is closed and was not confirmed for reopening`); + } + + const focus = await deps.bridge.focusTab(resolution.agentSessionId, resolution.tab.id); + if (!focus.ok) { + throw new VoiceDispatchError( + `Could not return to that conversation on '${agent.name}' (${focus.reason ?? 'no reason given'})` + ); + } + + // The renderer may have landed on a different tab: waking a snooze whose + // conversation is already open focuses the copy that exists rather than + // restoring a duplicate. + const tabId = focus.tabId ?? resolution.tab.id; + const promptSent = await sendPrompt(deps.bridge, resolution.agentSessionId, tabId, prompt); + + return { + agentSessionId: resolution.agentSessionId, + agentName: agent.name, + tabId, + tabName: resolution.tab.name ?? undefined, + action: 'recalled', + promptSent, + }; +} + +/** + * Identity of a dispatch, for the replay guard. + * + * The prompt is part of the key because two identical requests to the same tab + * ARE the same dispatch as far as the user is concerned - they said it twice + * because the first one appeared to do nothing - while the same tab with a + * different prompt is a new turn. The agent id rather than the decision's target + * so a conductor-targeted retry resolved to the same agent still matches. + */ +function dispatchKey(decision: RouteDecision, agentSessionId: string): string { + return [ + agentSessionId, + decision.tabAction, + decision.tabId ?? '', + decision.tabName ?? '', + decision.prompt.trim(), + ].join(''); +} + +/** + * A conductor-targeted decision still has to land somewhere. Preference order: + * the session's bound agent, then the agent the desktop is showing, then the + * only agent there is. With several agents and no signal, guessing would put a + * spoken instruction in the wrong repository, so it fails instead. + */ +function resolveAgent( + decision: RouteDecision, + context: { roster: RosterAgent[]; scope: VoiceScope }, + roster: RosterAgent[], + activeSessionId: string | null +): RosterAgent { + const byId = (sessionId: string | null): RosterAgent | undefined => + sessionId ? roster.find((agent) => agent.sessionId === sessionId) : undefined; + + const targetId = routeTargetSessionId(decision.target); + if (targetId) { + const agent = byId(targetId); + if (!agent) { + throw new VoiceDispatchError(`Agent '${targetId}' is no longer running`); + } + return agent; + } + + const scoped = context.scope.kind === 'agent' ? byId(context.scope.sessionId) : undefined; + const fallback = scoped ?? byId(activeSessionId) ?? (roster.length === 1 ? roster[0] : undefined); + if (!fallback) { + throw new VoiceDispatchError( + roster.length === 0 + ? 'No agents are open to dispatch to' + : 'No agent was named and none is active, so the request has no target' + ); + } + return fallback; +} + +/** + * The agent's active tab, or its most recently used one. + * + * Only OPEN tabs are eligible. The roster deliberately lists snoozed and closed + * ones so recall can name them, and "carry on where we were" landing on a tab + * the user put away last week would be the worst possible reading of "current". + */ +function resolveCurrentTab(agent: RosterAgent, activeTabId: string | null): string | null { + const open = agent.tabs.filter((tab) => (tab.state ?? 'open') === 'open'); + if (open.length === 0) return null; + if (activeTabId && open.some((tab) => tab.id === activeTabId)) return activeTabId; + const mostRecent = [...open].sort((a, b) => (b.lastActiveAt ?? 0) - (a.lastActiveAt ?? 0))[0]; + return mostRecent.id; +} + +async function openNewTab( + agent: RosterAgent, + prompt: string, + tabName: string | undefined, + bridge: VoiceRendererBridge +): Promise { + let tabId: string | null; + let promptSent = false; + + if (prompt) { + // One atomic renderer operation: a separate create-then-send would leave an + // orphan tab behind whenever the send is dropped. + const result = await bridge.newTabWithPrompt(agent.sessionId, prompt); + if (!result.success || !result.tabId) { + throw new VoiceDispatchError(`Could not open a new tab on '${agent.name}'`); + } + tabId = result.tabId; + promptSent = true; + } else { + tabId = await bridge.newTab(agent.sessionId); + if (!tabId) { + throw new VoiceDispatchError(`Could not open a new tab on '${agent.name}'`); + } + } + + if (tabName) { + bridge.renameTab(agent.sessionId, tabId, tabName); + } + + return { + agentSessionId: agent.sessionId, + agentName: agent.name, + tabId, + tabName, + action: 'created', + promptSent, + }; +} + +/** + * A rejected receipt is a real failure, not a `promptSent: false` footnote: the + * session holds the floor open waiting for a reply that would never come. + */ +async function sendPrompt( + bridge: VoiceRendererBridge, + agentSessionId: string, + tabId: string, + prompt: string +): Promise { + if (!prompt) return false; + + const receipt = await bridge.executeCommand(agentSessionId, tabId, prompt); + if (!receipt.accepted) { + throw new VoiceDispatchError( + `The prompt was not delivered (${receipt.reason ?? 'no reason given'})` + ); + } + return true; +} diff --git a/src/main/acappella/hotkeys/index.ts b/src/main/acappella/hotkeys/index.ts new file mode 100644 index 0000000000..a663b20651 --- /dev/null +++ b/src/main/acappella/hotkeys/index.ts @@ -0,0 +1,243 @@ +/** + * Wiring for the two A Cappella global hotkeys. + * + * Everything interesting lives in `voice-hotkeys.ts` and `press-hold.ts`; this + * file is the part that knows about Electron, the settings store, and the + * running session, and it exists so those two stay testable without any of them. + * + * The bindings come from the same `shortcuts` settings map the Shortcuts tab + * writes, so rebinding a voice hotkey there rebinds the real system-wide combo + * with no second code path. A settings watcher re-syncs on every change, which + * is also how switching the Encore Feature off releases both combos: a global + * shortcut left registered for a feature nobody has enabled is a combo stolen + * from whatever app the user actually wanted it for. + */ + +import type { BrowserWindow } from 'electron'; + +import type { VoiceOrigin, VoiceScope } from '../../../shared/acappella/protocol'; +import type { Shortcut } from '../../../shared/shortcut-types'; +import { isACappellaEnabled } from '../../../shared/acappella/feature-flag'; +import { getGlobalHotkeyRegistry, summonMainWindow } from '../../global-hotkey-manager'; +import type { GlobalHotkeyStatus } from '../../../shared/global-hotkeys'; +import { logger } from '../../utils/logger'; +import type { FloorControlSession } from '../audio/floor-control'; +import { createFloorController, type FloorController } from '../audio/floor-control'; +import { + VOICE_HOTKEY_IDS, + VoiceHotkeyController, + type VoiceHotkeyId, + type VoiceHotkeyRefusalInfo, +} from './voice-hotkeys'; +import { resolveHoldThresholdMs } from './press-hold'; + +const LOG_CONTEXT = 'ACappella'; + +/** The settings blob key A Cappella keeps everything under. Mirrors the registry's. */ +const ACAPPELLA_SETTINGS_KEY = 'acappella'; + +export interface VoiceHotkeySettingsStore { + get: (key: string, defaultValue?: unknown) => unknown; + onDidChange?: (key: string, callback: (value: unknown) => void) => void; +} + +export interface InstallVoiceHotkeysDeps { + settingsStore: VoiceHotkeySettingsStore; + /** The floor's view of the session service. Supplied by the IPC layer. */ + session: FloorControlSession; + getMainWindow: () => BrowserWindow | null; + /** + * The agent the user is looking at, in the FOCUSED window when there are + * several. Null when nothing is focused, which refuses the agent hotkey rather + * than guessing. + */ + getFocusedAgentSessionId: () => string | null; + /** Force the recogniser to endpoint. The hold-to-talk release path. */ + endUtterance?: () => void | Promise; + /** A press was refused. Surfaced to the user by the caller. */ + onRefused?: (info: VoiceHotkeyRefusalInfo) => void; +} + +export interface VoiceHotkeyInstallation { + controller: VoiceHotkeyController; + /** Re-read settings and rebind. Called by the settings watcher and by tests. */ + sync: () => Record; + statuses: () => GlobalHotkeyStatus[]; + /** + * The one floor controller, aimed at a scope and an origin. + * + * Exposed so a paired device presses the SAME state machine the hotkey does + * (see `../transport/remote-session.ts`). There is one microphone and one + * session, so a second controller would be two state machines racing for the + * same device - which is exactly the failure the hotkey path already avoids by + * keeping one instance behind a mutable scope. + */ + acquireFloor: (scope: VoiceScope, origin?: VoiceOrigin) => FloorController; + dispose: () => void; +} + +/** The A Cappella settings blob's `controls` section, or an empty object. */ +export function readVoiceControlSettings(store: VoiceHotkeySettingsStore): Record { + const blob = (store.get(ACAPPELLA_SETTINGS_KEY, {}) ?? {}) as { controls?: unknown }; + return (blob.controls ?? {}) as Record; +} + +/** + * The user's binding for a hotkey id, or undefined when they have never touched + * it. + * + * Undefined and `[]` mean different things and must not be collapsed: the + * persisted map only holds ids the user has customised, so a missing entry is + * "take the shipped default" while an empty array is "I cleared this on purpose". + */ +function readHotkeyKeys(store: VoiceHotkeySettingsStore, id: VoiceHotkeyId): string[] | undefined { + const shortcuts = (store.get('shortcuts', {}) ?? {}) as Record; + const entry = shortcuts[id]; + return Array.isArray(entry?.keys) ? entry.keys : undefined; +} + +/** + * Register both voice hotkeys and keep them in step with settings. + * + * Safe to call once per process. The returned handle is what the IPC layer uses + * to answer "is this combo actually bound" for the settings rows. + */ +export function installVoiceHotkeys(deps: InstallVoiceHotkeysDeps): VoiceHotkeyInstallation { + const registry = getGlobalHotkeyRegistry(); + + /** + * One floor, whatever the scope. + * + * There is only ever one microphone and one session, so a controller per scope + * would be two state machines racing for the same device. The scope is a + * mutable field the hotkey sets immediately before pressing, read by the + * floor's own `getScope` seam. + */ + let pendingScope: VoiceScope = { kind: 'conductor' }; + /** + * Which microphone the next press opens. Same mutable-field pattern as the + * scope, and for the same reason: one controller, several surfaces, and the + * only thing that differs between them is what the session is credited to. + */ + let pendingOrigin: VoiceOrigin = { kind: 'local' }; + let floor: FloorController | null = null; + + const acquireFloor = (scope: VoiceScope, origin?: VoiceOrigin): FloorController => { + pendingScope = scope; + pendingOrigin = origin ?? { kind: 'local' }; + if (!floor) { + floor = createFloorController({ + session: deps.session, + getScope: () => pendingScope, + getOrigin: () => pendingOrigin, + endUtterance: deps.endUtterance, + idleTimeoutMs: readIdleTimeoutMs(deps.settingsStore), + }); + } + return floor; + }; + + const controller = new VoiceHotkeyController({ + registry, + checkAvailability: () => { + if (!isACappellaEnabled(deps.settingsStore)) { + return { + ok: false, + reason: 'feature-disabled', + message: 'A Cappella is switched off in Encore Features.', + }; + } + // The capability gate itself is async (it stats model files), and a hotkey + // must not wait on disk. The session's own `checkReadiness` refuses the + // start and names the missing slot, so an unsatisfied gate surfaces as a + // `session-error` event rather than as a hotkey that does nothing. + return { ok: true }; + }, + acquireFloor, + resolveFocusedAgent: () => { + const sessionId = deps.getFocusedAgentSessionId(); + return sessionId ? { kind: 'agent', sessionId } : null; + }, + summon: () => { + const win = deps.getMainWindow(); + if (win) summonMainWindow(win); + }, + onRefused: deps.onRefused, + getHoldThresholdMs: () => + resolveHoldThresholdMs(readVoiceControlSettings(deps.settingsStore).holdThresholdMs), + }); + + const sync = (): Record => { + if (!isACappellaEnabled(deps.settingsStore)) { + // Released rather than left bound: see the module header. + for (const id of VOICE_HOTKEY_IDS) registry.clear(id); + return Object.fromEntries( + VOICE_HOTKEY_IDS.map((id) => [ + id, + registry.status(id) ?? { id, keys: [], accelerator: null, registered: false }, + ]) + ) as Record; + } + const keysById: Partial> = {}; + for (const id of VOICE_HOTKEY_IDS) { + const keys = readHotkeyKeys(deps.settingsStore, id); + if (keys) keysById[id] = keys; + } + return controller.sync(keysById); + }; + + sync(); + + deps.settingsStore.onDidChange?.('shortcuts', () => sync()); + deps.settingsStore.onDidChange?.('encoreFeatures', () => sync()); + deps.settingsStore.onDidChange?.(ACAPPELLA_SETTINGS_KEY, () => { + floor?.configure({ idleTimeoutMs: readIdleTimeoutMs(deps.settingsStore) }); + }); + + logger.info(`Voice hotkeys installed (${controller.capability})`, LOG_CONTEXT); + + return { + controller, + sync, + acquireFloor, + statuses: () => + VOICE_HOTKEY_IDS.map( + (id) => registry.status(id) ?? { id, keys: [], accelerator: null, registered: false } + ), + dispose: () => { + controller.dispose(); + void floor?.dispose(); + floor = null; + }, + }; +} + +/** The idle timeout from settings. Clamped by `resolveFloorControlConfig` downstream. */ +function readIdleTimeoutMs(store: VoiceHotkeySettingsStore): number | undefined { + const value = readVoiceControlSettings(store).idleTimeoutMs; + return typeof value === 'number' ? value : undefined; +} + +export { + createVoiceHotkeyController, + defaultVoiceHotkeyKeys, + VOICE_HOTKEY_IDS, + VoiceHotkeyController, +} from './voice-hotkeys'; +export type { + VoiceFloorSurface, + VoiceHotkeyDeps, + VoiceHotkeyId, + VoiceHotkeyRefusal, + VoiceHotkeyRefusalInfo, +} from './voice-hotkeys'; +export { + createPressHoldDetector, + describePressHoldCapability, + PressHoldDetector, + resolveHoldThresholdMs, + resolvePlatformKeyStateProbe, + resolvePressHoldCapability, + setKeyStateProbe, +} from './press-hold'; +export type { KeyStateProbe, PressHoldCapability, PressHoldOptions } from './press-hold'; diff --git a/src/main/acappella/hotkeys/press-hold.ts b/src/main/acappella/hotkeys/press-hold.ts new file mode 100644 index 0000000000..c1bf328363 --- /dev/null +++ b/src/main/acappella/hotkeys/press-hold.ts @@ -0,0 +1,293 @@ +/** + * Tap vs hold classification for a global hotkey. + * + * Electron's `globalShortcut` fires on PRESS and never on release. That is fine + * for "show Maestro" and useless for push-to-talk, where the release is the + * entire gesture: it is what says the sentence is finished. So this module turns + * one press callback into three outcomes - tap, hold-start, hold-end - by + * polling a platform key-state probe until the combo comes back up. + * + * **The probe is a seam, and today every platform returns null.** There is no + * way to read live key state from Electron's own API: macOS would need + * `CGEventSourceKeyState`, Windows `GetAsyncKeyState`, X11 `XQueryKeymap`, and + * all three mean a native module Maestro does not ship. Rather than fake it + * (auto-repeat timing is not a release signal - the OS repeat delay is longer + * than any usable hold threshold) the detector reports `tap-only` and SAYS SO. + * A push-to-talk key that silently behaves like a toggle is the kind of bug a + * user blames themselves for. + * + * The seam is real, not decorative: `setKeyStateProbe()` is what a future native + * module plugs into, and it is how both branches are tested. Surfaces that DO + * have a real release event - the HUD button, the Phase 10 phone button - never + * come through here; they call `FloorController.press()`/`release()` directly, + * which is the same state machine this ends up driving. + */ + +import { resolveHoldThresholdMs } from '../../../shared/acappella/voice-controls'; +import { logger } from '../../utils/logger'; + +const LOG_CONTEXT = 'ACappella'; + +/** What the hotkey can actually do on this machine. */ +export type PressHoldCapability = + /** Both gestures: a quick tap toggles, holding keeps the floor open. */ + | 'hold-and-tap' + /** Press only. Every press is a tap, and hold-to-talk is unavailable. */ + | 'tap-only'; + +export { + DEFAULT_HOLD_THRESHOLD_MS, + MAX_HOLD_THRESHOLD_MS, + MIN_HOLD_THRESHOLD_MS, + /** + * Re-exported so this module stays the one import site for press-hold, even + * though the clamp itself moved to `shared/` when the HUD's talk button + * needed to classify a press against exactly the same number. + */ + resolveHoldThresholdMs, +} from '../../../shared/acappella/voice-controls'; + +/** How often the probe is asked whether the combo is still down. */ +export const DEFAULT_KEY_POLL_MS = 25; + +/** + * A hold this long is a stuck key or a lying probe, not a sentence. The floor's + * idle timeout is the real backstop; this stops the poll timer leaking. + */ +export const MAX_HOLD_MS = 60_000; + +/** + * Reports whether the combo behind an accelerator is still physically down. + * + * Returning `false` on the first poll is a legal answer: it means the key came + * up between the shortcut firing and the first tick, which is exactly what a + * tap is. + */ +export type KeyStateProbe = (accelerator: string) => boolean; + +let installedProbe: KeyStateProbe | null = null; + +/** + * Install the process-wide key-state probe. + * + * Pass `null` to remove it, which puts every detector built afterwards back on + * tap-only. Tests use this; a native input module would too. + */ +export function setKeyStateProbe(probe: KeyStateProbe | null): void { + installedProbe = probe; +} + +/** + * The probe for this platform, or null when there is no reliable release signal. + * + * Null on every platform today - see the module header. It is a function rather + * than a constant so installing a probe at runtime takes effect. + */ +export function resolvePlatformKeyStateProbe(): KeyStateProbe | null { + return installedProbe; +} + +/** What a detector built right now would be able to do. */ +export function resolvePressHoldCapability(probe: KeyStateProbe | null): PressHoldCapability { + return probe ? 'hold-and-tap' : 'tap-only'; +} + +/** The sentence the settings panel and the HUD show. Never silently degrade. */ +export function describePressHoldCapability(capability: PressHoldCapability): string { + return capability === 'hold-and-tap' + ? 'Tap to toggle the microphone, or hold the key and talk.' + : 'Tap to toggle the microphone. Hold-to-talk needs a key-release signal this platform does not provide, so holding the key behaves like a tap.'; +} + +export interface PressHoldOptions { + /** The Electron accelerator the probe is asked about. */ + accelerator: string; + holdThresholdMs?: number; + pollIntervalMs?: number; + /** + * Override the process-wide probe. `undefined` takes the installed one; + * `null` forces tap-only, which is what a caller that wants toggle semantics + * regardless of platform passes. + */ + probe?: KeyStateProbe | null; + /** A short press. Toggles the floor. */ + onTap: () => void; + /** The key has been down past the threshold. Opens the floor. */ + onHoldStart: () => void; + /** The key came up after a hold. Ends the utterance and closes the floor. */ + onHoldEnd: () => void; + /** Injected clock, for tests. */ + now?: () => number; +} + +/** + * One hotkey's press classifier. + * + * Stateless between gestures: a press resolves to exactly one of tap or + * hold-start/hold-end, and nothing is remembered afterwards. + */ +export class PressHoldDetector { + readonly capability: PressHoldCapability; + + private readonly options: PressHoldOptions; + private readonly probe: KeyStateProbe | null; + private readonly thresholdMs: number; + private readonly pollMs: number; + private readonly now: () => number; + + private pressedAt: number | null = null; + private holding = false; + private timer: NodeJS.Timeout | null = null; + private disposed = false; + + constructor(options: PressHoldOptions) { + this.options = options; + this.probe = options.probe === undefined ? resolvePlatformKeyStateProbe() : options.probe; + this.capability = resolvePressHoldCapability(this.probe); + this.thresholdMs = resolveHoldThresholdMs(options.holdThresholdMs); + this.pollMs = Math.max(1, Math.round(options.pollIntervalMs ?? DEFAULT_KEY_POLL_MS)); + this.now = options.now ?? Date.now; + } + + /** True between `onHoldStart` and `onHoldEnd`. */ + get isHolding(): boolean { + return this.holding; + } + + /** True while a press is being classified. */ + get isPressed(): boolean { + return this.pressedAt !== null; + } + + /** + * The global shortcut fired. + * + * Idempotent while a press is in flight, because a held key auto-repeats on + * Windows and Linux and a repeat is not a second gesture. + */ + trigger(): void { + if (this.disposed) return; + + if (!this.probe) { + // Tap-only: there is nothing to wait for, and delaying the toggle to find + // out would add latency to buy an answer that never arrives. + this.options.onTap(); + return; + } + + if (this.pressedAt !== null) return; + this.pressedAt = this.now(); + this.startPolling(); + } + + /** + * Abandon any press in flight, ending a hold cleanly. + * + * Called on rebind and on shutdown: a hold whose detector is torn down must + * still close the floor, or the microphone stays open with nothing watching it. + */ + dispose(): void { + if (this.disposed) return; + this.disposed = true; + this.stopPolling(); + if (this.holding) { + this.holding = false; + this.pressedAt = null; + this.safely(this.options.onHoldEnd, 'onHoldEnd'); + return; + } + this.pressedAt = null; + } + + // -- Internals ----------------------------------------------------------- + + private startPolling(): void { + this.stopPolling(); + this.timer = setInterval(() => this.poll(), this.pollMs); + // A key being watched is not a reason to keep the process alive. + this.timer.unref?.(); + } + + private stopPolling(): void { + if (this.timer === null) return; + clearInterval(this.timer); + this.timer = null; + } + + /** + * One tick. + * + * Release is checked BEFORE the threshold on purpose: a press that came up + * just past the threshold but before the tick resolves as a tap rather than as + * an instantaneous open-and-close of the floor, which is the outcome a user + * would read as a glitch. + */ + private poll(): void { + if (this.pressedAt === null) { + this.stopPolling(); + return; + } + + const elapsed = this.now() - this.pressedAt; + let down: boolean; + try { + down = this.probe!(this.options.accelerator); + } catch (err) { + // A probe that throws is a probe that cannot be trusted to report the + // release either, so end the gesture rather than hold the floor on it. + logger.warn(`Key state probe failed: ${(err as Error).message}`, LOG_CONTEXT); + down = false; + } + + if (!down) { + this.finish(elapsed); + return; + } + + if (elapsed >= MAX_HOLD_MS) { + logger.warn( + `Hold-to-talk key reported down for ${elapsed}ms; releasing the floor`, + LOG_CONTEXT + ); + this.finish(elapsed); + return; + } + + if (!this.holding && elapsed >= this.thresholdMs) { + this.holding = true; + this.safely(this.options.onHoldStart, 'onHoldStart'); + } + } + + private finish(elapsed: number): void { + this.stopPolling(); + this.pressedAt = null; + if (this.holding) { + this.holding = false; + this.safely(this.options.onHoldEnd, 'onHoldEnd'); + return; + } + logger.debug(`Voice hotkey tap (${elapsed}ms)`, LOG_CONTEXT); + this.safely(this.options.onTap, 'onTap'); + } + + /** + * A subscriber's failure is not the classifier's failure. + * + * These callbacks drive the floor, which sends IPC; a window destroyed + * mid-gesture must not leave the poll timer running with a half-finished + * press behind it. + */ + private safely(fn: () => void, name: string): void { + try { + fn(); + } catch (err) { + logger.error(`Press-hold ${name} threw: ${(err as Error).message}`, LOG_CONTEXT); + } + } +} + +/** Sugar, matching the rest of A Cappella's factories. */ +export function createPressHoldDetector(options: PressHoldOptions): PressHoldDetector { + return new PressHoldDetector(options); +} diff --git a/src/main/acappella/hotkeys/voice-hotkeys.ts b/src/main/acappella/hotkeys/voice-hotkeys.ts new file mode 100644 index 0000000000..3da0a2bbe1 --- /dev/null +++ b/src/main/acappella/hotkeys/voice-hotkeys.ts @@ -0,0 +1,312 @@ +/** + * The two A Cappella global hotkeys. + * + * `voiceConductor` opens a Conductor-scoped session and deliberately does NOT + * touch window focus: the point of a voice assistant is talking to it while + * doing something else, and a hotkey that yanks a window to the front every time + * you speak is a hotkey people stop pressing. `voiceCurrentAgent` is the + * opposite gesture and summons Maestro on purpose, because "the current agent" + * is a thing you have to be looking at to mean. + * + * Both route through `audio/floor-control.ts` rather than driving the session + * service themselves. That module already owns what a second press means, what a + * release means, and when an untouched microphone goes cold; a hotkey that + * re-derived any of it would drift from the HUD button and the phone button + * within a week. + * + * Every seam here is SYNCHRONOUS on purpose. A hotkey handler that awaits a + * capability gate before deciding whether to open the floor is a hotkey whose + * behaviour depends on disk latency, and keyboard input that is sometimes + * dropped is worse than a feature that is off. + */ + +import type { VoiceScope, WakeSource } from '../../../shared/acappella/protocol'; +import { + VOICE_AGENT_HOTKEY_ID, + VOICE_CONDUCTOR_HOTKEY_ID, + defaultGlobalHotkeyKeys, + getGlobalHotkeyDefinition, + type GlobalHotkeyStatus, +} from '../../../shared/global-hotkeys'; +import type { GlobalHotkeyRegistry } from '../../global-hotkey-manager'; +import { logger } from '../../utils/logger'; +import type { FloorControlConfig, FloorMode } from '../audio/floor-control'; +import { + PressHoldDetector, + resolvePlatformKeyStateProbe, + resolvePressHoldCapability, + type KeyStateProbe, + type PressHoldCapability, +} from './press-hold'; + +const LOG_CONTEXT = 'ACappella'; + +export const VOICE_HOTKEY_IDS = [VOICE_CONDUCTOR_HOTKEY_ID, VOICE_AGENT_HOTKEY_ID] as const; + +export type VoiceHotkeyId = (typeof VOICE_HOTKEY_IDS)[number]; + +/** The shipped bindings, read from the one shared definition table. */ +export function defaultVoiceHotkeyKeys(): Record { + return { + [VOICE_CONDUCTOR_HOTKEY_ID]: defaultGlobalHotkeyKeys(VOICE_CONDUCTOR_HOTKEY_ID), + [VOICE_AGENT_HOTKEY_ID]: defaultGlobalHotkeyKeys(VOICE_AGENT_HOTKEY_ID), + }; +} + +/** + * The slice of `FloorController` a hotkey drives. + * + * `FloorController` satisfies this structurally, so nothing has to be adapted; + * stating it narrowly is what keeps a hotkey from growing the ability to route a + * turn or cancel speech behind the floor's back. + */ +export interface VoiceFloorSurface { + readonly mode: FloorMode; + configure(overrides: Partial): void; + press(source?: WakeSource): Promise; + release(source?: WakeSource): Promise; +} + +/** Why a press did nothing. Distinct values because the user's next move differs. */ +export type VoiceHotkeyRefusal = + /** The A Cappella Encore Feature is switched off. */ + | 'feature-disabled' + /** A required slot (model, runtime, microphone) is unsatisfied. */ + | 'not-ready' + /** `voiceCurrentAgent` fired with nothing focused. */ + | 'no-focused-agent' + /** Nothing is holding a floor to press. */ + | 'no-floor'; + +export interface VoiceHotkeyRefusalInfo { + id: VoiceHotkeyId; + reason: VoiceHotkeyRefusal; + /** Ready to show. Carries the capability gate's own sentence when it has one. */ + message: string; +} + +export interface VoiceHotkeyDeps { + registry: GlobalHotkeyRegistry; + /** + * Whether A Cappella is usable right now, or the sentence explaining why not. + * + * Synchronous, and therefore a CACHED view of the capability gate rather than + * a fresh resolve. See the module header: a hotkey must not be able to take + * longer to respond because a model file is being stat'ed. + */ + checkAvailability: () => + | { ok: true } + | { ok: false; reason: VoiceHotkeyRefusal; message: string }; + /** The floor for a scope, or null when A Cappella has never been started. */ + acquireFloor: (scope: VoiceScope) => VoiceFloorSurface | null; + /** The agent the user is looking at - in the FOCUSED window, when there are several. */ + resolveFocusedAgent: () => VoiceScope | null; + /** Bring Maestro to the front. Only `voiceCurrentAgent` calls it. */ + summon: () => void; + /** A press was refused. The seam a toast or HUD line binds to. */ + onRefused?: (info: VoiceHotkeyRefusalInfo) => void; + /** Tap-vs-hold threshold, read per press so a settings change takes effect live. */ + getHoldThresholdMs?: () => number; + /** + * Override the key-state probe. `undefined` uses the platform's, which is how + * the capability ends up honest; tests pass one to exercise the hold path. + */ + probe?: KeyStateProbe | null; +} + +/** + * Owns the registration and the press semantics of both voice hotkeys. + * + * One instance for the app. Rebinding is `sync()` with new keys; the registry + * releases the old combo and reports per-id failure on its own. + */ +export class VoiceHotkeyController { + /** What these hotkeys can do on this machine. Shown, never silently assumed. */ + readonly capability: PressHoldCapability; + + private readonly deps: VoiceHotkeyDeps; + private readonly probe: KeyStateProbe | null; + private readonly detectors = new Map(); + /** The floor a hold opened, so the release closes the same one. */ + private readonly heldFloors = new Map(); + /** The mode the floor was in before a hold forced `hold-to-talk`. */ + private readonly restoreModes = new Map(); + private disposed = false; + + constructor(deps: VoiceHotkeyDeps) { + this.deps = deps; + this.probe = deps.probe === undefined ? resolvePlatformKeyStateProbe() : deps.probe; + this.capability = resolvePressHoldCapability(this.probe); + + for (const id of VOICE_HOTKEY_IDS) { + this.deps.registry.define(id, () => this.handleTrigger(id)); + } + } + + /** + * Bind (or rebind) both hotkeys. + * + * A missing entry takes the shipped default rather than unbinding: the + * persisted `shortcuts` blob only contains ids the user has touched, so an + * absent key is "never customised", not "deliberately cleared". An explicitly + * empty array IS a deliberate clear and is honoured. + */ + sync( + keysById: Partial> = {} + ): Record { + const statuses = {} as Record; + for (const id of VOICE_HOTKEY_IDS) { + const keys = keysById[id] ?? defaultGlobalHotkeyKeys(id); + // A rebind mid-hold would leave the floor open with nothing watching it. + this.endHold(id); + statuses[id] = this.deps.registry.setKeys(id, keys); + } + return statuses; + } + + /** Release both combos and end any press in flight. Safe to call twice. */ + dispose(): void { + if (this.disposed) return; + this.disposed = true; + for (const id of VOICE_HOTKEY_IDS) { + this.endHold(id); + this.deps.registry.remove(id); + } + } + + status(id: VoiceHotkeyId): GlobalHotkeyStatus | null { + return this.deps.registry.status(id); + } + + // -- Internals ----------------------------------------------------------- + + /** + * The shortcut fired. + * + * The detector is built lazily per gesture rather than at registration, + * because the accelerator changes on rebind and the threshold changes in + * settings, and a detector captured at startup would answer with both stale. + */ + private handleTrigger(id: VoiceHotkeyId): void { + if (this.disposed) return; + + const existing = this.detectors.get(id); + if (existing?.isPressed) { + // Auto-repeat, or a second fire while a hold is being classified. + existing.trigger(); + return; + } + + const accelerator = this.deps.registry.status(id)?.accelerator ?? id; + const detector = new PressHoldDetector({ + accelerator, + holdThresholdMs: this.deps.getHoldThresholdMs?.(), + probe: this.probe, + onTap: () => this.handleTap(id), + onHoldStart: () => this.handleHoldStart(id), + onHoldEnd: () => this.endHold(id), + }); + this.detectors.set(id, detector); + detector.trigger(); + } + + /** A tap toggles the floor: open it if closed, close it if open. */ + private handleTap(id: VoiceHotkeyId): void { + const floor = this.beginGesture(id); + if (!floor) return; + floor.configure({ mode: 'tap-to-toggle' }); + void floor.press('hotkey'); + } + + /** A hold opens the floor and keeps it open until the key comes up. */ + private handleHoldStart(id: VoiceHotkeyId): void { + const floor = this.beginGesture(id); + if (!floor) return; + this.restoreModes.set(id, floor.mode); + floor.configure({ mode: 'hold-to-talk' }); + this.heldFloors.set(id, floor); + void floor.press('hotkey'); + } + + /** + * End a hold, whatever ended it: the key came up, the binding changed, or the + * app is shutting down. Idempotent, and restores the mode the user chose so a + * push-to-talk gesture does not silently convert their session to push-to-talk + * forever. + */ + private endHold(id: VoiceHotkeyId): void { + const detector = this.detectors.get(id); + const floor = this.heldFloors.get(id); + this.heldFloors.delete(id); + this.detectors.delete(id); + + if (floor) { + void floor.release('hotkey'); + const restore = this.restoreModes.get(id); + if (restore) floor.configure({ mode: restore }); + } + this.restoreModes.delete(id); + // Disposing after the release, so the detector's own `onHoldEnd` finds no + // floor left to release and cannot double-fire. + detector?.dispose(); + } + + /** + * Everything both gestures need before they touch the floor: the feature is + * on, the gate is satisfied, a scope resolves, and a floor exists. + * + * @returns the floor to act on, or null after reporting why not. + */ + private beginGesture(id: VoiceHotkeyId): VoiceFloorSurface | null { + const availability = this.deps.checkAvailability(); + if (!availability.ok) { + this.refuse(id, availability.reason, availability.message); + return null; + } + + let scope: VoiceScope; + if (id === VOICE_AGENT_HOTKEY_ID) { + // Summoned BEFORE the scope is resolved: the window that comes forward is + // the one whose agent the session binds to, and reversing the order would + // let a focus change between the two steps bind the wrong agent. + this.deps.summon(); + const focused = this.deps.resolveFocusedAgent(); + if (!focused) { + // Deliberately not falling back to the Conductor. The user asked to + // talk to a specific agent; guessing one is how a spoken instruction + // lands in the wrong terminal. + this.refuse( + id, + 'no-focused-agent', + 'No agent is focused, so there is nothing to talk to yet.' + ); + return null; + } + scope = focused; + } else { + scope = { kind: 'conductor' }; + } + + const floor = this.deps.acquireFloor(scope); + if (!floor) { + this.refuse(id, 'no-floor', 'A Cappella is not running yet.'); + return null; + } + return floor; + } + + private refuse(id: VoiceHotkeyId, reason: VoiceHotkeyRefusal, message: string): void { + const label = getGlobalHotkeyDefinition(id)?.label ?? id; + logger.info(`Voice hotkey '${label}' refused: ${message}`, LOG_CONTEXT); + try { + this.deps.onRefused?.({ id, reason, message }); + } catch (err) { + logger.warn(`Voice hotkey refusal listener threw: ${err}`, LOG_CONTEXT); + } + } +} + +/** Sugar, matching the rest of A Cappella's factories. */ +export function createVoiceHotkeyController(deps: VoiceHotkeyDeps): VoiceHotkeyController { + return new VoiceHotkeyController(deps); +} diff --git a/src/main/acappella/index.ts b/src/main/acappella/index.ts new file mode 100644 index 0000000000..53feb689c1 --- /dev/null +++ b/src/main/acappella/index.ts @@ -0,0 +1,124 @@ +/** + * A Cappella main-process entry point. + * + * Holds the one voice session service instance and nothing else. The service is + * created lazily by whoever resolves the provider trio (the IPC layer, via the + * provider registry), never here: this module must not import a concrete + * provider, or the "no silent cloud substitution" rule would be decided by an + * import instead of by the registry. + */ + +import { ACappellaTransport } from './transport'; +import type { ACappellaTransportDeps as ACappellaTransportOptions } from './transport'; +import { VoiceSessionService } from './voice-session-service'; +import type { VoiceSessionServiceOptions } from './voice-session-service'; + +export { VoiceSessionService, VoiceDispatchError } from './voice-session-service'; +export { + closeAcappellaAudioHostWindow, + ensureAcappellaAudioHostWindow, + getAcappellaAudioHostWindow, + isAcappellaAudioHostContents, + type AudioHostWindowDeps, +} from './audio-host-window'; +export { + VoiceAudioBridge, + createVoiceAudioBridge, + type AudioBridgeSession, + type VoiceAudioBridgeOptions, +} from './audio/audio-bridge'; +export { + buildAgentRoster, + createRendererVoiceBridge, + createVoiceRouteExecutor, + executeRouteDecision, + readAgentRoster, +} from './dispatch/route-executor'; +export type { + CommandReceipt, + DispatchReplayCache, + FocusTabResult, + NewTabWithPromptResult, + VoiceRendererBridge, + VoiceRouteExecutorOptions, +} from './dispatch/route-executor'; +export * from './router'; +export * from './speech'; +export type { + VoiceDispatchResult, + VoiceEventListener, + VoiceRouteExecutor, + VoiceSessionServiceOptions, + VoiceSessionSnapshot, + VoiceStopReason, +} from './voice-session-service'; + +export { ACappellaTransport } from './transport'; +export type { ACappellaTransportDeps, DeviceStatus, PairingPayload } from './transport'; +export { PairingService } from './pairing/pairing-service'; +export type { PairedDeviceView, PairingOffer, PairingRequest } from './pairing/pairing-service'; + +let instance: VoiceSessionService | null = null; + +/** + * The paired-device transport, or null when nothing has ever paired a device. + * + * Held here alongside the session service, and for the same reason: it owns real + * resources (a Bonjour advert, live signaling sessions, peer connections in the + * audio host) that outlive any one voice session. + */ +let transport: ACappellaTransport | null = null; + +/** Create the transport, replacing any existing one. */ +export function initACappellaTransport(options: ACappellaTransportOptions): ACappellaTransport { + disposeACappellaTransport(); + transport = new ACappellaTransport(options); + return transport; +} + +/** + * The live transport, or null. + * + * Callers must handle null: A Cappella is an Encore Feature that is off by + * default, and the WebSocket route asks for this on every inbound signaling + * message including ones that arrive before anything has been set up. + */ +export function getACappellaTransport(): ACappellaTransport | null { + return transport; +} + +export function disposeACappellaTransport(): void { + if (!transport) return; + const previous = transport; + transport = null; + previous.dispose(); +} + +/** + * Create the singleton, replacing any existing one. Calling this again is how a + * provider change takes effect, so the previous instance is disposed first. + */ +export async function initVoiceSessionService( + options: VoiceSessionServiceOptions +): Promise { + await disposeVoiceSessionService(); + instance = new VoiceSessionService(options); + return instance; +} + +/** + * The live service, or `null` when A Cappella has never been started. Callers + * must handle null: the Encore Feature is off by default, and nothing here runs + * until a session is explicitly started. + */ +export function getVoiceSessionService(): VoiceSessionService | null { + return instance; +} + +/** Tear down the singleton. Safe to call when nothing was ever created. */ +export async function disposeVoiceSessionService(): Promise { + if (!instance) return; + const previous = instance; + instance = null; + await previous.dispose(); +} diff --git a/src/main/acappella/models/capability-gate.ts b/src/main/acappella/models/capability-gate.ts new file mode 100644 index 0000000000..90824191c6 --- /dev/null +++ b/src/main/acappella/models/capability-gate.ts @@ -0,0 +1,351 @@ +/** + * A Cappella capability gate - may voice mode run, and if not, exactly why. + * + * This module answers one question with a structured verdict rather than a + * boolean, because a boolean is what produces the two failure modes A Cappella + * cannot ship with: + * + * 1. **Silent substitution.** "Local Whisper is missing, so use the cloud one" + * spends the user's money and ships their microphone to a service they did + * not choose. There is no code path in this file that can do that: a slot is + * satisfied by the provider it was configured with or it is UNSATISFIED. It + * never resolves to a different provider, and it never returns a provider at + * all - it returns a verdict. Choosing providers is the registry's job, and + * the registry's only fallback is the mock. + * 2. **A disabled button with no explanation.** "Voice mode unavailable" with + * no reason is indistinguishable from a bug. Every unsatisfied slot carries + * a reason code, a sentence naming the missing piece, and a suggested action. + * + * The wake word is reported but does not block a session. Hands-free means + * something is always listening, and that is a real capability with a real + * requirement; click-to-talk is not, and refusing to open a session the user + * explicitly asked for because an optional always-on model is missing would be + * the gate getting in the way rather than doing its job. + */ + +import { OPENWAKEWORD_BASE_ID, getVoiceModel } from '../../../shared/acappella/model-catalog'; +import { + credentialLabel, + voiceProviderRequirement, + type VoiceCredentialService, + type VoiceProviderRequirement, +} from '../../../shared/acappella/provider-catalog'; +import type { NativeRuntimeId } from '../../../shared/acappella/native-runtimes'; +import { hasCredential } from '../providers/credentials'; +import type { MicPermission } from '../../../shared/acappella/protocol'; +import type { + VoiceReadiness, + VoiceSlot, + VoiceSlotReadiness, +} from '../../../shared/acappella/readiness'; +import { getMicPermission } from '../permissions/mic-permission'; +import { + knownNativeRuntimeUnavailability, + type NativeRuntimeUnavailable, +} from '../runtime/native-loader'; +import { DEFAULT_PROVIDER_IDS, type VoiceProviderSettings } from '../providers/provider-registry'; +import { getStatus, type ModelStatus } from './model-store'; + +/** + * Provider ids for the local tier. + * + * Re-exported from the shared catalog rather than re-declared: the requirement + * table, the registry's registrations, and the settings panel all have to spell + * a provider id the same way, and three literals in three files is three chances + * for a slot the gate blocks and the panel says is fine. + */ +export { LOCAL_PROVIDER_IDS } from '../../../shared/acappella/provider-catalog'; + +/** The wake word slot has one implementation and it is always local. */ +export const WAKE_WORD_PROVIDER_ID = 'openwakeword-local'; + +/** + * The microphone slot's "provider" id. + * + * The device is not a provider and there is nothing to choose here, but the slot + * carries an id so the structure stays uniform for every consumer that renders + * `slots` and `blocking` generically. + */ +export const MICROPHONE_PROVIDER_ID = 'system-microphone'; + +/** + * What a provider needs before it can run. + * + * Every entry comes from the shared catalog except the wake word, which has no + * provider slot of its own to be selected in and so is stated here. + * + * A local provider needs BOTH its model and its native runtime, and the runtime + * is checked first: a llama.cpp binary that will not load on this machine is not + * fixed by downloading another gigabyte, so telling the user to download is the + * wrong instruction even though the model may also be missing. + */ +function requirementFor(providerId: string): VoiceProviderRequirement { + if (providerId === WAKE_WORD_PROVIDER_ID) { + return { kind: 'model', modelId: OPENWAKEWORD_BASE_ID, runtimeId: 'onnx' }; + } + return voiceProviderRequirement(providerId); +} + +/** + * Slot order, which is also the order Voice Setup renders and errors list them. + * + * The microphone comes first because it is the one requirement that is true + * regardless of which providers are configured, and because a user who reads + * "microphone access denied" first does not need to read the rest. + */ +const SLOT_ORDER: VoiceSlot[] = ['microphone', 'stt', 'tts', 'brain', 'wake-word']; + +const SLOT_LABELS: Record = { + microphone: 'Microphone', + stt: 'Speech-to-Text', + tts: 'Text-to-Speech', + brain: 'Conductor Brain', + 'wake-word': 'Wake word', +}; + +export interface ResolveVoiceReadinessOptions { + /** The persisted provider selection. Omitted roles take the build default. */ + settings?: VoiceProviderSettings; + /** + * Whether hands-free is switched on. When off, the wake word slot is still + * REPORTED (Voice Setup shows what it would need) but nothing is downloaded + * or required on its account. + */ + handsFreeEnabled?: boolean; + /** + * Whether a service's API key is stored. Defaults to the OS keychain. + * + * A boolean, not the key: the gate has no business holding a credential, and a + * seam that returned one would put keys in every test fixture that configures + * a hosted provider. + */ + hasApiKey?: (service: VoiceCredentialService) => boolean; + /** + * Optional reachability probe for cloud providers, keyed by provider id. + * Absent means "assume reachable": a gate that reported every cloud provider + * unreachable because nobody wired a probe would be worse than one that lets + * the provider's own start() report the truth. + */ + probeProvider?: (providerId: string) => Promise | boolean; + /** Injected for tests. Defaults to the real model store. */ + readModelStatus?: (modelId: string) => Promise; + /** + * The microphone permission. Defaults to the real OS query, which never + * prompts: readiness is resolved on every Settings render, and a gate that + * could raise a TCC dialog would turn drawing a panel into asking for the + * microphone. + */ + readMicPermission?: () => MicPermission; + /** + * Why a native runtime will not load, or null when it will. Defaults to the + * loader's answer: a remembered failure if there is one, otherwise the facts + * knowable from the registry alone (not a dependency of this build, no binary + * for this platform). It deliberately does NOT attempt a load, because + * dlopen'ing an inference engine to draw a settings panel is exactly the + * startup cost the lazy loader exists to avoid. + */ + readRuntimeFailure?: (runtimeId: NativeRuntimeId) => NativeRuntimeUnavailable | null; +} + +/** + * Resolve the readiness of all four slots. + * + * Every branch either satisfies the slot with the provider that was ASKED for or + * marks it unsatisfied. There is deliberately no `else` that reaches for a + * different provider. + */ +export async function resolveVoiceReadiness( + options: ResolveVoiceReadinessOptions = {} +): Promise { + const settings = options.settings ?? {}; + const readModelStatus = options.readModelStatus ?? getStatus; + + const slots: VoiceSlotReadiness[] = []; + for (const slot of SLOT_ORDER) { + if (slot === 'microphone') { + slots.push(resolveMicrophone(options)); + continue; + } + const providerId = providerForSlot(slot, settings); + slots.push(await resolveSlot(slot, providerId, options, readModelStatus)); + } + + const wakeWord = slots.find((slot) => slot.slot === 'wake-word'); + // A session needs a microphone, speech in, speech out, and routing. The wake + // word is a hands-free capability, not a precondition for talking. + const blocking = slots.filter((slot) => slot.slot !== 'wake-word' && !slot.satisfied); + + return { + canStartSession: blocking.length === 0, + canRunHandsFree: blocking.length === 0 && (wakeWord?.satisfied ?? false), + slots, + blocking, + }; +} + +function providerForSlot(slot: VoiceSlot, settings: VoiceProviderSettings): string { + if (slot === 'wake-word') return WAKE_WORD_PROVIDER_ID; + if (slot === 'microphone') return MICROPHONE_PROVIDER_ID; + return settings[slot] ?? DEFAULT_PROVIDER_IDS[slot]; +} + +/** + * The microphone slot. + * + * Only `denied` and `restricted` block. `not-determined` and `unknown` are the + * states of a machine that has never been asked, and blocking on them would mean + * a first-run user is told voice is unavailable BEFORE the app has done the one + * thing that would make it available. The ask happens at session start, in + * `requestMicPermission()`, which is the moment the user asked for a microphone. + */ +function resolveMicrophone(options: ResolveVoiceReadinessOptions): VoiceSlotReadiness { + const readPermission = options.readMicPermission ?? (() => getMicPermission().state); + const permission = readPermission(); + const base = { + slot: 'microphone' as const, + providerId: MICROPHONE_PROVIDER_ID, + micPermission: permission, + }; + + if (permission === 'denied') { + return { + ...base, + satisfied: false, + reason: 'mic-permission-denied', + // Named as a permission, never as "voice unavailable": a user with every + // model on disk and a denied microphone has a one-checkbox problem, and + // this sentence is the difference between fixing it and filing a bug. + detail: 'Microphone: Maestro does not have microphone access.', + suggestedAction: 'Grant microphone access to Maestro in your system privacy settings.', + }; + } + + if (permission === 'restricted') { + return { + ...base, + satisfied: false, + reason: 'mic-permission-restricted', + detail: 'Microphone: access is blocked by a system policy.', + // No privacy-pane link here on purpose. The user cannot change this one, + // so sending them to a checkbox they are not allowed to tick is a dead end. + suggestedAction: 'A device policy controls this. Ask whoever manages this machine.', + }; + } + + return { ...base, satisfied: true }; +} + +/** + * A native runtime that will not load, as a slot verdict. Null when it will. + * + * "Will not load" rather than "has already failed": a runtime that is not part + * of this build, or has no binary for this platform, is unusable before anything + * tries it, and a gate that waited for an attempt would call the slot ready + * right up until the session died in the provider's `start()`. + */ +function runtimeFailureFor( + slot: VoiceSlot, + providerId: string, + runtimeId: NativeRuntimeId, + options: ResolveVoiceReadinessOptions +): VoiceSlotReadiness | null { + const read = options.readRuntimeFailure ?? knownNativeRuntimeUnavailability; + const failure = read(runtimeId); + if (!failure) return null; + + return { + slot, + providerId, + satisfied: false, + reason: 'runtime-unavailable', + detail: `${SLOT_LABELS[slot]}: ${failure.message}`, + suggestedAction: failure.suggestedAction, + }; +} + +async function resolveSlot( + slot: VoiceSlot, + providerId: string, + options: ResolveVoiceReadinessOptions, + readModelStatus: (modelId: string) => Promise +): Promise { + const label = SLOT_LABELS[slot]; + const requirement = requirementFor(providerId); + + if (requirement.kind === 'none') { + return { slot, providerId, satisfied: true }; + } + + if (requirement.kind === 'model') { + // Runtime before model. A binary that will not load on this machine is not + // repaired by a download, and "download 1.1 GB" is the wrong instruction to + // give someone whose real problem is a missing redistributable. + if (requirement.runtimeId) { + const runtimeVerdict = runtimeFailureFor(slot, providerId, requirement.runtimeId, options); + if (runtimeVerdict) return runtimeVerdict; + } + + const model = getVoiceModel(requirement.modelId); + const modelName = model?.displayName ?? requirement.modelId; + const status = await readModelStatus(requirement.modelId); + + if (status.status === 'installed') { + return { slot, providerId, satisfied: true, requiredModelId: requirement.modelId }; + } + if (status.status === 'corrupt') { + return { + slot, + providerId, + satisfied: false, + reason: 'model-corrupt', + requiredModelId: requirement.modelId, + detail: `${label}: ${modelName} is installed but failed verification${ + status.detail ? ` (${status.detail})` : '' + }.`, + suggestedAction: 'Re-verify or re-download it in Settings > Plugins > A Cappella > Models.', + }; + } + return { + slot, + providerId, + satisfied: false, + reason: 'model-not-installed', + requiredModelId: requirement.modelId, + detail: `${label}: ${modelName} is not installed.`, + suggestedAction: 'Download it in Settings > Plugins > A Cappella > Voice Setup.', + }; + } + + const readKey = options.hasApiKey ?? hasCredential; + if (!readKey(requirement.service)) { + return { + slot, + providerId, + satisfied: false, + reason: 'api-key-missing', + detail: `${label}: ${providerId} needs a ${credentialLabel(requirement.service)} API key.`, + // Naming the alternative matters: the honest recovery for "no key" is + // often "use the local model instead", and the gate is the only place + // that knows both options exist. + suggestedAction: `Add the key in Settings, or switch ${label} to a local model.`, + }; + } + + const reachable = (await options.probeProvider?.(providerId)) ?? true; + if (!reachable) { + return { + slot, + providerId, + satisfied: false, + reason: 'provider-unreachable', + detail: `${label}: ${providerId} could not be reached.`, + suggestedAction: `Check your connection, or switch ${label} to a local model.`, + }; + } + + return { slot, providerId, satisfied: true }; +} + +// Re-exported so a caller that already has the gate does not need a second +// import for the one-line formatting of its verdict. +export { readinessErrorMessage } from '../../../shared/acappella/readiness'; diff --git a/src/main/acappella/models/model-downloader.ts b/src/main/acappella/models/model-downloader.ts new file mode 100644 index 0000000000..114e6c1ba6 --- /dev/null +++ b/src/main/acappella/models/model-downloader.ts @@ -0,0 +1,643 @@ +/** + * A Cappella model downloader - resumable, verified, and never optimistic. + * + * The invariant everything else in this subsystem rests on: + * + * **A file only appears at its final path after its bytes have been hashed and + * the hash matched the catalog.** + * + * Until then the bytes live in `.part`. That is not tidiness; it is the + * difference between a killed app resuming a download and a killed app leaving a + * truncated file that passes an existence check and detonates weeks later inside + * a model runtime with no evidence of what happened. + * + * How each requirement is met: + * + * - **Resume.** The `.part` file's length is the resume offset, sent as + * `Range: bytes=N-`. A server that ignores the range (200 instead of 206) + * restarts from zero, and the partial file is truncated to match, because a + * resumed hash over bytes that were not resumed is worse than a slow restart. + * - **Verification.** SHA-256 is computed as the bytes stream past. On resume + * the existing partial is re-hashed first, so the running digest covers the + * whole file rather than only the new tail. + * - **Mismatch.** The `.part` is deleted and BOTH hashes are reported. A + * mismatch means the bytes are not what the catalog promised; keeping them + * around to "resume" would resume a corrupt file forever. + * - **Pause / resume / cancel.** Pause aborts the request and keeps the + * partial. Cancel aborts and deletes it, along with anything else the job + * wrote, so a cancelled download leaves nothing behind. Both leave the + * manifest untouched, because there is no manifest until success. + * - **Retry.** Bounded, with exponential backoff, on transient network errors + * only. An HTTP 404 or a hash mismatch is not transient and fails at once. + * - **Progress.** Emitted on a throttled cadence with bytes, total, rate, and + * ETA. The throttle lives here because the flood originates here; the + * renderer uses `useThrottledCallback` for its own repaints and does not need + * a second throttle helper. + * - **Concurrency.** At most {@link MAX_ACTIVE_DOWNLOADS} models transfer at + * once. A 1.4 GB set downloaded four-wide saturates a domestic connection and + * makes every individual file slower, which reads to the user as a hang. + */ + +import { createHash, type Hash } from 'crypto'; +import * as fs from 'fs/promises'; +import { createReadStream, createWriteStream } from 'fs'; +import { Readable } from 'stream'; +import { pipeline } from 'stream/promises'; + +import { + getVoiceModel, + type VoiceModelEntry, + type VoiceModelFile, +} from '../../../shared/acappella/model-catalog'; +import { logger } from '../../utils/logger'; +import { + ensureModelDir, + markInstalled, + modelFilePath, + PARTIAL_SUFFIX, + remove, + type ModelManifest, +} from './model-store'; + +const LOG_CONTEXT = 'ACappella'; + +/** + * Two at a time. One is too conservative for a set of four small files; three or + * more starves each stream on a domestic uplink and makes the whole set slower. + */ +export const MAX_ACTIVE_DOWNLOADS = 2; + +/** Progress push interval. ~4 Hz: fast enough to look live, slow enough to be free. */ +export const PROGRESS_INTERVAL_MS = 250; + +/** Attempts per file before a transient failure becomes a real one. */ +export const MAX_RETRIES = 4; + +const BASE_RETRY_DELAY_MS = 500; + +export type DownloadPhase = + | 'queued' + | 'downloading' + | 'verifying' + | 'paused' + | 'complete' + | 'cancelled' + | 'error'; + +export interface DownloadProgress { + modelId: string; + phase: DownloadPhase; + /** Bytes of the whole model transferred so far, resumed bytes included. */ + bytesReceived: number; + /** Total bytes of every file in the model. */ + bytesTotal: number; + /** Bytes per second over the recent window. Zero before the first sample. */ + bytesPerSecond: number; + /** Seconds remaining at the current rate, or null when it cannot be estimated. */ + etaSeconds: number | null; + /** The file currently transferring, for a per-file line in the UI. */ + currentFile?: string; + /** Set on `error`. */ + error?: string; + /** Set when the failure was a hash mismatch, so the UI can show both sides. */ + mismatch?: { path: string; expected: string; actual: string }; +} + +export type DownloadProgressListener = (progress: DownloadProgress) => void; + +export interface DownloadResult { + modelId: string; + status: 'complete' | 'cancelled' | 'paused' | 'error'; + manifest?: ModelManifest; + error?: string; + mismatch?: { path: string; expected: string; actual: string }; +} + +/** Injectable fetch, so tests drive the transport without a network. */ +export type FetchLike = (url: string, init?: RequestInit) => Promise; + +export interface ModelDownloaderOptions { + fetchImpl?: FetchLike; + /** Overridable so tests do not wait real backoff. */ + retryDelayMs?: (attempt: number) => number; + maxActiveDownloads?: number; + progressIntervalMs?: number; + /** Injectable clock, so rate and ETA are testable. */ + now?: () => number; +} + +/** Errors we retry. Everything else is a real failure and fails immediately. */ +function isTransient(error: unknown): boolean { + if (error instanceof TransientHttpError) return true; + const code = (error as NodeJS.ErrnoException | undefined)?.code; + if ( + code === 'ECONNRESET' || + code === 'ETIMEDOUT' || + code === 'ECONNREFUSED' || + code === 'EAI_AGAIN' || + code === 'ENOTFOUND' || + code === 'EPIPE' + ) { + return true; + } + // Undici surfaces most network faults as a generic TypeError with a cause. + return error instanceof TypeError && (error as { cause?: unknown }).cause !== undefined; +} + +class TransientHttpError extends Error {} + +/** A hash that did not match. Never retried: more attempts cannot change bytes. */ +export class HashMismatchError extends Error { + constructor( + readonly filePath: string, + readonly expected: string, + readonly actual: string + ) { + super(`Hash mismatch for ${filePath}: expected ${expected}, got ${actual}`); + this.name = 'HashMismatchError'; + } +} + +/** Raised when a job is paused or cancelled mid-transfer. */ +class AbortedError extends Error { + constructor(readonly kind: 'paused' | 'cancelled') { + super(`Download ${kind}`); + this.name = 'AbortedError'; + } +} + +interface Job { + entry: VoiceModelEntry; + controller: AbortController; + /** Set when the abort was deliberate, so a fetch abort is not read as a fault. */ + intent: 'paused' | 'cancelled' | null; + phase: DownloadPhase; + bytesReceived: number; + currentFile?: string; + /** Promise of the in-flight run, awaited by `cancel()` so teardown is ordered. */ + running: Promise; + /** + * Releases this job from the concurrency queue. Called when a slot frees up + * AND when the job is cancelled while still waiting - without the second path + * a `cancel()` on a queued download would await a promise nothing will ever + * settle. + */ + releaseSlot: (() => void) | null; + lastProgressAt: number; + lastSampleBytes: number; + lastSampleAt: number; + bytesPerSecond: number; +} + +export class ModelDownloader { + private readonly fetchImpl: FetchLike; + private readonly retryDelayMs: (attempt: number) => number; + private readonly maxActive: number; + private readonly progressIntervalMs: number; + private readonly now: () => number; + + private readonly listeners = new Set(); + private readonly jobs = new Map(); + /** Ids waiting for a slot, oldest first. */ + private readonly queue: string[] = []; + private readonly slotWaiters: Array<{ modelId: string; resolve: () => void }> = []; + private activeCount = 0; + + constructor(options: ModelDownloaderOptions = {}) { + this.fetchImpl = options.fetchImpl ?? ((url, init) => fetch(url, init)); + this.retryDelayMs = + options.retryDelayMs ?? ((attempt) => BASE_RETRY_DELAY_MS * Math.pow(2, attempt)); + this.maxActive = options.maxActiveDownloads ?? MAX_ACTIVE_DOWNLOADS; + this.progressIntervalMs = options.progressIntervalMs ?? PROGRESS_INTERVAL_MS; + this.now = options.now ?? (() => Date.now()); + } + + /** Subscribe to progress. Returns the unsubscribe function. */ + onProgress(listener: DownloadProgressListener): () => void { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + } + + /** Ids with a job in flight (running, queued, or paused-in-place). */ + activeIds(): string[] { + return [...this.jobs.keys()]; + } + + /** + * Start (or resume) a model download. + * + * Idempotent while a job is live: calling it twice for the same model returns + * the same promise rather than opening a second set of requests at the same + * `.part` file, which is how two writers end up interleaving into one file. + */ + download(modelId: string): Promise { + const existing = this.jobs.get(modelId); + if (existing) return existing.running; + + const entry = getVoiceModel(modelId); + if (!entry) return Promise.resolve({ modelId, status: 'error', error: 'Unknown model' }); + + const job: Job = { + entry, + controller: new AbortController(), + intent: null, + phase: 'queued', + bytesReceived: 0, + running: Promise.resolve({ modelId, status: 'error' }), + releaseSlot: null, + lastProgressAt: 0, + lastSampleBytes: 0, + lastSampleAt: this.now(), + bytesPerSecond: 0, + }; + this.jobs.set(modelId, job); + job.running = this.runWhenSlotFree(job); + this.emit(job, true); + return job.running; + } + + /** + * Pause: abort the transfer and KEEP the `.part` file. The next `download()` + * resumes from where this stopped. + */ + pause(modelId: string): boolean { + const job = this.jobs.get(modelId); + if (!job || job.phase === 'complete') return false; + job.intent = 'paused'; + job.controller.abort(); + job.releaseSlot?.(); + return true; + } + + /** Resume a paused model. Same call as starting one; the `.part` does the rest. */ + resume(modelId: string): Promise { + return this.download(modelId); + } + + /** + * Cancel: abort, then delete everything the job wrote. Awaits the running job + * so the deletion cannot race the writer and leave the file it just recreated. + */ + async cancel(modelId: string): Promise { + const job = this.jobs.get(modelId); + if (!job) { + // Nothing running, but a partial from a previous boot may still be on + // disk, and Cancel has to mean "leave nothing behind" either way. + await this.cleanupPartials(modelId); + return false; + } + job.intent = 'cancelled'; + job.controller.abort(); + job.releaseSlot?.(); + await job.running.catch(() => undefined); + return true; + } + + /** Cancel every in-flight job. Used on teardown. */ + async cancelAll(): Promise { + await Promise.all([...this.jobs.keys()].map((id) => this.cancel(id))); + } + + // -- Internals ----------------------------------------------------------- + + private async runWhenSlotFree(job: Job): Promise { + this.queue.push(job.entry.id); + await this.waitForSlot(job); + this.activeCount++; + try { + // A job cancelled while it sat in the queue never opened a connection, + // but Cancel still has to mean "leave nothing behind". + if (job.intent) return await this.handleFailure(job, new AbortedError(job.intent)); + return await this.run(job); + } finally { + this.activeCount--; + this.jobs.delete(job.entry.id); + this.drainQueue(); + } + } + + private waitForSlot(job: Job): Promise { + const modelId = job.entry.id; + if (this.canStart(modelId)) { + this.dequeue(modelId); + return Promise.resolve(); + } + return new Promise((resolve) => { + const waiter = { modelId, resolve }; + this.slotWaiters.push(waiter); + job.releaseSlot = () => { + const index = this.slotWaiters.indexOf(waiter); + if (index < 0) return; + this.slotWaiters.splice(index, 1); + this.dequeue(modelId); + job.releaseSlot = null; + resolve(); + }; + }); + } + + private canStart(modelId: string): boolean { + return this.activeCount < this.maxActive && this.queue[0] === modelId; + } + + private dequeue(modelId: string): void { + const index = this.queue.indexOf(modelId); + if (index >= 0) this.queue.splice(index, 1); + } + + private drainQueue(): void { + while (this.slotWaiters.length > 0 && this.activeCount < this.maxActive) { + const next = this.slotWaiters.find((waiter) => this.queue[0] === waiter.modelId); + if (!next) break; + this.slotWaiters.splice(this.slotWaiters.indexOf(next), 1); + this.dequeue(next.modelId); + const waiting = this.jobs.get(next.modelId); + if (waiting) waiting.releaseSlot = null; + next.resolve(); + } + } + + private async run(job: Job): Promise { + const { entry } = job; + job.phase = 'downloading'; + this.emit(job, true); + + try { + await ensureModelDir(entry); + + for (const file of entry.files) { + job.currentFile = file.path; + await this.downloadFile(job, file); + } + + job.phase = 'verifying'; + this.emit(job, true); + const manifest = await markInstalled(entry); + + job.phase = 'complete'; + job.bytesReceived = entry.bytes; + this.emit(job, true); + return { modelId: entry.id, status: 'complete', manifest }; + } catch (error) { + return await this.handleFailure(job, error); + } + } + + private async handleFailure(job: Job, error: unknown): Promise { + const { entry } = job; + + if (error instanceof AbortedError || job.intent) { + const kind = error instanceof AbortedError ? error.kind : job.intent; + if (kind === 'cancelled') { + // Cancel means nothing left behind: partials AND any file that already + // completed and was renamed into place during this run. + await remove(entry.id).catch(() => undefined); + job.phase = 'cancelled'; + this.emit(job, true); + return { modelId: entry.id, status: 'cancelled' }; + } + job.phase = 'paused'; + this.emit(job, true); + return { modelId: entry.id, status: 'paused' }; + } + + if (error instanceof HashMismatchError) { + job.phase = 'error'; + this.emit(job, true, { + error: error.message, + mismatch: { path: error.filePath, expected: error.expected, actual: error.actual }, + }); + logger.warn(`Model ${entry.id} failed verification: ${error.message}`, LOG_CONTEXT); + return { + modelId: entry.id, + status: 'error', + error: error.message, + mismatch: { path: error.filePath, expected: error.expected, actual: error.actual }, + }; + } + + const message = error instanceof Error ? error.message : String(error); + job.phase = 'error'; + this.emit(job, true, { error: message }); + logger.warn(`Model ${entry.id} download failed: ${message}`, LOG_CONTEXT); + return { modelId: entry.id, status: 'error', error: message }; + } + + /** + * Fetch one file with retry, verify it, and rename it into place. + * + * A retry re-enters `transferFile`, which re-reads the partial's length, so a + * connection dropped at 80% resumes at 80% rather than starting over. + */ + private async downloadFile(job: Job, file: VoiceModelFile): Promise { + const finalPath = modelFilePath(job.entry.id, file.path); + const partPath = `${finalPath}${PARTIAL_SUFFIX}`; + + // A file already at its final path with the right length is already + // verified: it only got there by passing verification. + const existing = await statSize(finalPath); + if (existing === file.bytes) { + job.bytesReceived += file.bytes; + this.emit(job, true); + return; + } + if (existing !== null) await fs.rm(finalPath, { force: true }); + + let lastError: unknown; + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + const digest = await this.transferFile(job, file, partPath); + if (digest !== file.sha256) { + // Delete the partial. Keeping it would mean every future resume + // continues a file whose bytes are already known to be wrong. + await fs.rm(partPath, { force: true }); + throw new HashMismatchError(file.path, file.sha256, digest); + } + await fs.rename(partPath, finalPath); + return; + } catch (error) { + if (error instanceof AbortedError || error instanceof HashMismatchError) throw error; + if (job.intent) throw new AbortedError(job.intent); + if (!isTransient(error) || attempt === MAX_RETRIES) throw error; + lastError = error; + await delay(this.retryDelayMs(attempt)); + } + } + + throw lastError instanceof Error ? lastError : new Error(String(lastError)); + } + + /** + * Stream one file into `.part`, resuming if bytes are already there. + * + * @returns the SHA-256 of the complete file. + */ + private async transferFile(job: Job, file: VoiceModelFile, partPath: string): Promise { + let resumeFrom = (await statSize(partPath)) ?? 0; + if (resumeFrom > file.bytes) { + // Longer than the catalog says the file is. Something else wrote here; + // resuming past the end would produce a file that can never hash right. + await fs.rm(partPath, { force: true }); + resumeFrom = 0; + } + + const headers: Record = {}; + if (resumeFrom > 0) headers.Range = `bytes=${resumeFrom}-`; + + const response = await this.fetchImpl(file.sourceUrl, { + headers, + signal: job.controller.signal, + }).catch((error) => { + if (job.intent) throw new AbortedError(job.intent); + throw error; + }); + + if (response.status === 416) { + // The server says our offset is past the end. The partial is not what we + // think it is; drop it and let the retry start clean. + await fs.rm(partPath, { force: true }); + throw new TransientHttpError('Partial file rejected by server (416)'); + } + if (!response.ok) { + const message = `HTTP ${response.status} for ${file.sourceUrl}`; + if (response.status >= 500 || response.status === 429) throw new TransientHttpError(message); + throw new Error(message); + } + + // A 200 to a ranged request means the server ignored the range and is + // sending the whole file. Appending would splice the file into itself. + const serverResumed = response.status === 206; + if (resumeFrom > 0 && !serverResumed) { + await fs.rm(partPath, { force: true }); + resumeFrom = 0; + } + + const hash = createHash('sha256'); + // The digest has to cover the whole file, not just this leg, so the bytes + // already on disk are folded in before the new ones arrive. + if (resumeFrom > 0) await hashInto(hash, partPath); + + job.bytesReceived = this.completedBytesBefore(job, file) + resumeFrom; + job.lastSampleBytes = job.bytesReceived; + job.lastSampleAt = this.now(); + this.emit(job, true); + + if (!response.body) throw new TransientHttpError(`Empty body for ${file.sourceUrl}`); + + const sink = createWriteStream(partPath, { flags: resumeFrom > 0 ? 'a' : 'w' }); + const source = Readable.fromWeb(response.body as Parameters[0]); + + source.on('data', (chunk: Buffer) => { + hash.update(chunk); + job.bytesReceived += chunk.length; + this.emit(job, false); + }); + + try { + await pipeline(source, sink); + } catch (error) { + if (job.intent) throw new AbortedError(job.intent); + throw error; + } + + if (job.intent) throw new AbortedError(job.intent); + + return hash.digest('hex'); + } + + /** Bytes of the model's earlier files, so per-model progress stays monotonic. */ + private completedBytesBefore(job: Job, file: VoiceModelFile): number { + let total = 0; + for (const candidate of job.entry.files) { + if (candidate.path === file.path) break; + total += candidate.bytes; + } + return total; + } + + private async cleanupPartials(modelId: string): Promise { + const entry = getVoiceModel(modelId); + if (!entry) return; + for (const file of entry.files) { + await fs + .rm(`${modelFilePath(modelId, file.path)}${PARTIAL_SUFFIX}`, { force: true }) + .catch(() => undefined); + } + } + + /** + * Push a progress event, throttled unless `force`. + * + * Throttling here rather than in the renderer is deliberate: a 1 GB file at + * 20 MB/s produces hundreds of chunk events a second, and every one of them + * would otherwise cross the IPC boundary and wake React. Phase transitions and + * terminal states always go through. + */ + private emit(job: Job, force: boolean, extra?: Partial): void { + const now = this.now(); + if (!force && now - job.lastProgressAt < this.progressIntervalMs) return; + job.lastProgressAt = now; + + const elapsed = (now - job.lastSampleAt) / 1000; + if (elapsed >= 0.2) { + const delta = job.bytesReceived - job.lastSampleBytes; + job.bytesPerSecond = delta > 0 ? delta / elapsed : 0; + job.lastSampleBytes = job.bytesReceived; + job.lastSampleAt = now; + } + + const remaining = Math.max(0, job.entry.bytes - job.bytesReceived); + const progress: DownloadProgress = { + modelId: job.entry.id, + phase: job.phase, + bytesReceived: job.bytesReceived, + bytesTotal: job.entry.bytes, + bytesPerSecond: job.bytesPerSecond, + etaSeconds: + job.bytesPerSecond > 0 && job.phase === 'downloading' + ? Math.round(remaining / job.bytesPerSecond) + : null, + currentFile: job.currentFile, + ...extra, + }; + + for (const listener of this.listeners) listener(progress); + } +} + +// --------------------------------------------------------------------------- +// Singleton +// --------------------------------------------------------------------------- + +let instance: ModelDownloader | null = null; + +/** The app-wide downloader. Built on first use; nothing runs until then. */ +export function getModelDownloader(): ModelDownloader { + if (!instance) instance = new ModelDownloader(); + return instance; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function statSize(filePath: string): Promise { + try { + const stat = await fs.stat(filePath); + return stat.isFile() ? stat.size : null; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +async function hashInto(hash: Hash, filePath: string): Promise { + const stream = createReadStream(filePath); + for await (const chunk of stream) hash.update(chunk as Buffer); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/main/acappella/models/model-store.ts b/src/main/acappella/models/model-store.ts new file mode 100644 index 0000000000..2366321982 --- /dev/null +++ b/src/main/acappella/models/model-store.ts @@ -0,0 +1,488 @@ +/** + * A Cappella model store - what is on disk, and whether it can be trusted. + * + * Install layout, one directory per model: + * + * userData/models/acappella// + * manifest.json what was installed, from where, and when it was last verified + * + * + * The single most important rule in this file: **`isInstalled` never answers + * from `existsSync`.** A killed download leaves a real file at the real path + * with the wrong length, and an existence check would call that installed. The + * failure then surfaces hours later as an inference crash inside a model + * runtime, which is about the worst possible place to learn that a download was + * interrupted. So an install is only an install when a manifest exists, its + * recorded hashes match the catalog's, and every file's byte length on disk + * matches the recorded length exactly. + * + * `verify()` is the stronger, slower check: it re-hashes the bytes. A mismatch + * is recorded as corrupt and REPORTED - never silently re-downloaded. Silently + * repairing would spend a gigabyte of someone's connection without asking, and + * would hide the fact that something on this machine is modifying model files. + * + * Manifests are written through `atomicWriteJson` plus a per-model write queue, + * matching `src/main/utils/atomic-json-store.ts`. Concurrent non-atomic writes + * have already corrupted JSON state in this codebase once (history files); this + * store does not get to relearn that lesson. + */ + +import { app } from 'electron'; +import { createHash } from 'crypto'; +import * as fs from 'fs/promises'; +import { createReadStream, type Dirent } from 'fs'; +import * as path from 'path'; + +import { + getVoiceModel, + VOICE_MODEL_CATALOG, + type VoiceModelEntry, + type VoiceModelFile, +} from '../../../shared/acappella/model-catalog'; +import { atomicWriteJson, createKeyedWriteQueue } from '../../utils/atomic-json-store'; + +/** Directory name under userData. Also the thing the reclaim-disk flow deletes. */ +export const ACAPPELLA_MODELS_DIRNAME = path.join('models', 'acappella'); + +export const MODEL_MANIFEST_FILENAME = 'manifest.json'; + +/** Suffix an in-flight download writes to. Never treated as installed. */ +export const PARTIAL_SUFFIX = '.part'; + +/** Per-file record inside a manifest. */ +export interface ModelManifestFile { + path: string; + sha256: string; + bytes: number; +} + +/** + * What a completed install recorded about itself. Deliberately self-contained: + * a manifest has to be readable against a catalog whose revision has since moved + * on, so it repeats the revision and the source rather than pointing at them. + */ +export interface ModelManifest { + id: string; + revision: string; + /** Hash over the model's files, in catalog order. See {@link modelDigest}. */ + sha256: string; + bytes: number; + sourceUrl: string; + license: string; + files: ModelManifestFile[]; + /** Epoch ms the install completed. */ + installedAt: number; + /** Epoch ms of the last successful `verify()`. Equals `installedAt` on install. */ + verifiedAt: number; +} + +/** Why a model is not usable. `ok` is the only state voice mode may start in. */ +export type ModelStatusKind = 'installed' | 'not-installed' | 'corrupt'; + +export interface ModelStatus { + id: string; + status: ModelStatusKind; + /** Present for `installed` and for a `corrupt` install whose manifest still parses. */ + manifest: ModelManifest | null; + /** Human-readable reason, present when the status is not `installed`. */ + detail?: string; + /** Bytes actually occupied on disk by this model's directory. */ + bytesOnDisk: number; +} + +export interface ModelFootprint { + /** Sum of `bytesOnDisk` over every model directory, including stray ones. */ + bytes: number; + models: Array<{ id: string; bytes: number }>; +} + +// --------------------------------------------------------------------------- +// Paths +// --------------------------------------------------------------------------- + +/** + * Resolve the Maestro data dir, matching the pianola / plugin store semantics. + * A new path constant is deliberately NOT introduced: `MAESTRO_USER_DATA` has to + * keep working, and it only does if every store asks the same question. + */ +function dataDir(): string { + if (process.env.MAESTRO_USER_DATA) return path.resolve(process.env.MAESTRO_USER_DATA); + return app.getPath('userData'); +} + +/** Root every A Cappella model lives under. */ +export function modelsRoot(): string { + return path.join(dataDir(), ACAPPELLA_MODELS_DIRNAME); +} + +/** + * Install directory for one model. + * + * Guarded rather than trusting the caller: ids reach this function from IPC, and + * a `../` in one would let a caller point the installer (and, worse, `remove()`) + * at an arbitrary directory. Only ids that are in the catalog are accepted, which + * makes the guard a whitelist rather than a sanitiser. + */ +export function modelDir(id: string): string { + if (!getVoiceModel(id)) throw new Error(`UnknownVoiceModel: ${id}`); + return path.join(modelsRoot(), id); +} + +/** Absolute path of one file within a model's install directory. */ +export function modelFilePath(id: string, filePath: string): string { + const dir = modelDir(id); + const resolved = path.resolve(dir, filePath); + // Belt and braces: catalog paths are authored in this repo, but a relative + // path that escapes its root is the one mistake that turns a delete into a + // disaster, so it is checked rather than assumed. + if (resolved !== dir && !resolved.startsWith(dir + path.sep)) { + throw new Error(`UnsafeModelFilePath: ${filePath}`); + } + return resolved; +} + +function manifestPath(id: string): string { + return path.join(modelDir(id), MODEL_MANIFEST_FILENAME); +} + +// --------------------------------------------------------------------------- +// Hashing +// --------------------------------------------------------------------------- + +/** + * Stable digest identifying a whole model: the per-file SHA-256s joined in + * catalog order and hashed again. + * + * A single-file model would let the manifest just carry the file hash, but two of + * the catalog's models are multi-file, and "the model's hash" has to mean the + * same thing for both. Deriving it from the catalog rather than storing it means + * a manifest written by an older build still compares correctly. + */ +export function modelDigest(files: readonly { path: string; sha256: string }[]): string { + const hash = createHash('sha256'); + for (const file of files) hash.update(`${file.path}:${file.sha256}\n`); + return hash.digest('hex'); +} + +/** SHA-256 of a file on disk, streamed so a 1 GB model does not land in memory. */ +export async function hashFile(filePath: string): Promise { + const hash = createHash('sha256'); + const stream = createReadStream(filePath); + for await (const chunk of stream) hash.update(chunk as Buffer); + return hash.digest('hex'); +} + +// --------------------------------------------------------------------------- +// Manifest I/O +// --------------------------------------------------------------------------- + +/** + * Serialises every mutation of a given model's manifest. Two callers doing a + * read-modify-write on the same file (install finishing while a verify stamps + * `verifiedAt`) is precisely the lost-update case this queue exists for. + */ +const manifestWrites = createKeyedWriteQueue(); + +/** Read a manifest. Null when absent or unparseable - both mean "not installed". */ +export async function readManifest(id: string): Promise { + let raw: string; + try { + raw = await fs.readFile(manifestPath(id), 'utf-8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } + + try { + const parsed = JSON.parse(raw) as ModelManifest; + if (!parsed || typeof parsed !== 'object' || parsed.id !== id) return null; + if (!Array.isArray(parsed.files)) return null; + return parsed; + } catch { + // A manifest we cannot read is not an install. Reporting it as corrupt + // rather than throwing keeps a hand-edited file from bricking the panel. + return null; + } +} + +/** Atomically replace a model's manifest, serialised against other writers. */ +export async function writeManifest(manifest: ModelManifest): Promise { + await manifestWrites.enqueue(manifest.id, async () => { + await fs.mkdir(modelDir(manifest.id), { recursive: true }); + await atomicWriteJson(manifestPath(manifest.id), manifest); + }); +} + +/** Build the manifest a freshly completed install should record. */ +export function buildManifest(entry: VoiceModelEntry, installedAt: number): ModelManifest { + return { + id: entry.id, + revision: entry.revision, + sha256: modelDigest(entry.files), + bytes: entry.bytes, + sourceUrl: entry.files[0]?.sourceUrl ?? '', + license: entry.license, + files: entry.files.map((file) => ({ + path: file.path, + sha256: file.sha256, + bytes: file.bytes, + })), + installedAt, + verifiedAt: installedAt, + }; +} + +// --------------------------------------------------------------------------- +// Status +// --------------------------------------------------------------------------- + +async function fileSize(filePath: string): Promise { + try { + const stat = await fs.stat(filePath); + return stat.isFile() ? stat.size : null; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +/** Recursive byte total for a directory. Missing directory reads as zero. */ +async function dirBytes(dir: string): Promise { + let total = 0; + let entries: Dirent[]; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 0; + throw error; + } + + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + total += await dirBytes(full); + } else if (entry.isFile()) { + total += (await fileSize(full)) ?? 0; + } + } + return total; +} + +/** + * Full status of one model: manifest present, hashes matching the catalog, and + * every file the right length. + * + * The length check is the whole point. A `.part` renamed too early, a disk that + * filled, or an app killed mid-write all produce a file that exists and is + * short, and every one of them reads as installed to an `existsSync`. + */ +export async function getStatus(id: string): Promise { + const entry = getVoiceModel(id); + if (!entry) { + return { id, status: 'not-installed', manifest: null, detail: 'Unknown model', bytesOnDisk: 0 }; + } + + const bytesOnDisk = await dirBytes(modelDir(id)); + const manifest = await readManifest(id); + + if (!manifest) { + return { + id, + status: 'not-installed', + manifest: null, + detail: bytesOnDisk > 0 ? 'Files present with no manifest' : 'Not installed', + bytesOnDisk, + }; + } + + // An install from an older catalog revision is not corrupt, it is stale: the + // bytes are exactly what they claimed to be, they are just no longer what we + // ship. Reporting it as not-installed points the user at Download, which is + // the correct recovery. + if (manifest.sha256 !== modelDigest(entry.files)) { + return { + id, + status: 'not-installed', + manifest, + detail: `Installed revision ${manifest.revision} no longer matches the catalog`, + bytesOnDisk, + }; + } + + for (const file of entry.files) { + const size = await fileSize(modelFilePath(id, file.path)); + if (size === null) { + return { + id, + status: 'not-installed', + manifest, + detail: `Missing file ${file.path}`, + bytesOnDisk, + }; + } + if (size !== file.bytes) { + return { + id, + status: 'corrupt', + manifest, + detail: `${file.path} is ${size} bytes, expected ${file.bytes}`, + bytesOnDisk, + }; + } + } + + return { id, status: 'installed', manifest, bytesOnDisk }; +} + +/** + * Cheap installed check: manifest plus byte lengths, never a bare `existsSync`. + * This is what the capability gate calls on every readiness query, so it must not + * re-hash a gigabyte. + */ +export async function isInstalled(id: string): Promise { + return (await getStatus(id)).status === 'installed'; +} + +/** Status of every catalog model, in catalog order. */ +export async function listStatuses(): Promise { + const statuses: ModelStatus[] = []; + for (const entry of VOICE_MODEL_CATALOG) statuses.push(await getStatus(entry.id)); + return statuses; +} + +// --------------------------------------------------------------------------- +// Verification +// --------------------------------------------------------------------------- + +export interface VerifyResult { + id: string; + ok: boolean; + status: ModelStatusKind; + detail?: string; + /** Populated on a hash mismatch so the UI can show both sides. */ + mismatch?: { path: string; expected: string; actual: string }; + verifiedAt?: number; +} + +/** + * Re-hash a model's files and compare against the catalog. + * + * On success the manifest's `verifiedAt` is stamped. On a mismatch the model is + * reported CORRUPT and left exactly as it is: no delete, no silent re-download. + * The user decides whether to spend the bandwidth again, and gets told which file + * disagreed and by what hash, because "your model is corrupt" with no evidence is + * indistinguishable from a bug in this function. + */ +export async function verify(id: string): Promise { + const entry = getVoiceModel(id); + if (!entry) return { id, ok: false, status: 'not-installed', detail: 'Unknown model' }; + + const status = await getStatus(id); + if (status.status === 'not-installed') { + return { id, ok: false, status: 'not-installed', detail: status.detail }; + } + + for (const file of entry.files) { + const full = modelFilePath(id, file.path); + const actual = await hashFile(full); + if (actual !== file.sha256) { + return { + id, + ok: false, + status: 'corrupt', + detail: `${file.path} failed verification`, + mismatch: { path: file.path, expected: file.sha256, actual }, + }; + } + } + + // A model whose lengths were wrong but whose hashes match is not a thing that + // can happen; if getStatus said corrupt on length, hashing said otherwise, and + // we got here, the lengths are right by construction. + const verifiedAt = Date.now(); + const manifest = status.manifest ?? buildManifest(entry, verifiedAt); + await writeManifest({ ...manifest, verifiedAt }); + + return { id, ok: true, status: 'installed', verifiedAt }; +} + +// --------------------------------------------------------------------------- +// Install completion and removal +// --------------------------------------------------------------------------- + +/** + * Record a completed install. Called by the downloader AFTER every file has been + * hashed and renamed into place, never before: a manifest is the store's promise + * that the bytes are good, so writing one over an incomplete install would break + * the one guarantee `isInstalled` rests on. + */ +export async function markInstalled(entry: VoiceModelEntry): Promise { + const manifest = buildManifest(entry, Date.now()); + await writeManifest(manifest); + return manifest; +} + +/** + * Delete a model's entire directory, manifest and stray `.part` files included. + * + * @returns bytes reclaimed. + */ +export async function remove(id: string): Promise { + const dir = modelDir(id); + const bytes = await dirBytes(dir); + await fs.rm(dir, { recursive: true, force: true }); + return bytes; +} + +/** + * Disk used by A Cappella models. + * + * Walks the models root rather than the catalog, so a directory left behind by a + * model that has since been dropped from the catalog is still counted and still + * reclaimable. Disk the user cannot see is disk they cannot get back. + */ +export async function totalFootprint(): Promise { + const root = modelsRoot(); + let entries: Dirent[]; + try { + entries = await fs.readdir(root, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { bytes: 0, models: [] }; + throw error; + } + + const models: Array<{ id: string; bytes: number }> = []; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + models.push({ id: entry.name, bytes: await dirBytes(path.join(root, entry.name)) }); + } + + return { bytes: models.reduce((total, model) => total + model.bytes, 0), models }; +} + +/** + * Delete every A Cappella model directory. The reclaim-disk action behind the + * Encore Feature being switched off; scoped to the A Cappella root so it can + * never reach another feature's models. + * + * @returns bytes reclaimed. + */ +export async function removeAll(): Promise { + const footprint = await totalFootprint(); + await fs.rm(modelsRoot(), { recursive: true, force: true }); + return footprint.bytes; +} + +/** Ensure a model's install directory (and any nested file directories) exist. */ +export async function ensureModelDir(entry: VoiceModelEntry): Promise { + await fs.mkdir(modelDir(entry.id), { recursive: true }); + for (const file of entry.files) { + const dir = path.dirname(modelFilePath(entry.id, file.path)); + await fs.mkdir(dir, { recursive: true }); + } +} + +/** Absolute on-disk path of a catalog file. Exported for the bill of materials. */ +export function installPathFor(entry: VoiceModelEntry, file: VoiceModelFile): string { + return modelFilePath(entry.id, file.path); +} diff --git a/src/main/acappella/pairing/discovery.ts b/src/main/acappella/pairing/discovery.ts new file mode 100644 index 0000000000..38e216fc8d --- /dev/null +++ b/src/main/acappella/pairing/discovery.ts @@ -0,0 +1,239 @@ +/** + * Zero-config LAN discovery: a Bonjour `_maestro._tcp` advert so a phone on the + * same network finds this desktop without anybody typing an IP address. + * + * Three things this module is careful about. + * + * **It is optional in both directions.** The advert is a convenience, not the + * connection. Every path it shortcuts is reachable by scanning the QR code, + * which carries the host candidates and the port directly, or by typing a host + * name. So an environment with no mDNS responder available degrades to "manual + * entry", which is a slightly worse first-run experience rather than a broken + * feature. `{@link DiscoveryService.status}` says which of those you are in, out + * loud, because a user who cannot see their Mac in a list needs to know whether + * to fix their network or just type an address. + * + * **It is off-switchable, and the switch is real.** Broadcasting the machine + * name and a port to every device on a network is a disclosure. Some people are + * on networks where they do not want to make it. Turning it off stops the advert + * entirely rather than hiding it from a UI. + * + * **It advertises no secret.** The TXT record carries the app version, the + * protocol version, and the pairing fingerprint. It does NOT carry the server + * token or a pairing code: an advert is readable by everything on the network, + * so anything in it is public by construction. + * + * The mDNS responder itself arrives through {@link MdnsResponderFactory}. The + * default loader tries the optional `bonjour-service` package and reports its + * absence rather than failing: a pure-JavaScript multicast responder is not a + * dependency worth forcing on every Maestro install for a feature most users + * will never turn on. + */ + +import { DEVICE_PROTOCOL_VERSION } from '../../../shared/acappella/device-protocol'; +import { logger } from '../../utils/logger'; + +const LOG_CONTEXT = 'ACappella'; + +/** The service type a Maestro desktop advertises itself under. */ +export const MAESTRO_SERVICE_TYPE = 'maestro'; +export const MAESTRO_SERVICE_PROTOCOL = 'tcp'; + +/** The full name, as it appears on the wire and in every discovery tool. */ +export const MAESTRO_SERVICE_FQDN = `_${MAESTRO_SERVICE_TYPE}._${MAESTRO_SERVICE_PROTOCOL}`; + +/** What goes in the TXT record. Public by construction: an advert is readable by all. */ +export interface DiscoveryTxtRecord { + /** Maestro's version, so a device can say what it found. */ + version: string; + /** The A Cappella device protocol version, so an old client fails early. */ + proto: string; + /** The pairing fingerprint, so the user can confirm they found the right Mac. */ + fingerprint: string; + /** Human name of this desktop. */ + host: string; +} + +/** The one verb an mDNS library has to provide for this to work. */ +export interface MdnsAdvertisement { + stop(): void | Promise; +} + +export interface MdnsResponder { + publish(options: { + name: string; + type: string; + protocol: 'tcp' | 'udp'; + port: number; + txt: Record; + }): MdnsAdvertisement; + destroy?(): void | Promise; +} + +export type MdnsResponderFactory = () => Promise; + +export type DiscoveryStatus = + /** Off by user choice. */ + | { state: 'disabled' } + /** Advertising right now. */ + | { state: 'advertising'; name: string; port: number } + /** + * No responder is available on this machine, so nothing is being advertised. + * The QR code and manual host entry still work; this is the sentence that says + * so instead of leaving a user staring at an empty device list. + */ + | { state: 'unavailable'; reason: string } + /** The responder was there and publishing failed anyway. */ + | { state: 'error'; message: string }; + +export interface DiscoveryServiceOptions { + /** The port the signaling WebSocket is served on. */ + getPort: () => number | null; + /** Display name of this desktop. */ + getName: () => string; + getAppVersion: () => string; + /** The pairing fingerprint, so a discovered host can be verified. */ + getFingerprint: () => string; + /** Injectable for tests, and for anyone who wants a different responder. */ + createResponder?: MdnsResponderFactory; +} + +/** + * The default loader. + * + * A non-literal specifier so the bundler leaves the import alone and so a build + * without the optional package still type-checks. Its absence is a reported + * status, never a throw: this runs during app startup, and a missing optional + * discovery library must not be able to stop Maestro from booting. + */ +export const loadOptionalBonjour: MdnsResponderFactory = async () => { + const specifier = 'bonjour-service'; + try { + const mod = (await import(/* @vite-ignore */ specifier)) as { + Bonjour?: new () => MdnsResponder; + default?: new () => MdnsResponder; + }; + const Ctor = mod.Bonjour ?? mod.default; + if (!Ctor) return null; + return new Ctor(); + } catch { + return null; + } +}; + +export class DiscoveryService { + private readonly options: DiscoveryServiceOptions; + private responder: MdnsResponder | null = null; + private advertisement: MdnsAdvertisement | null = null; + private state: DiscoveryStatus = { state: 'disabled' }; + /** Serialises start/stop so a fast toggle cannot leave two adverts running. */ + private queue: Promise = Promise.resolve(); + + constructor(options: DiscoveryServiceOptions) { + this.options = options; + } + + get status(): DiscoveryStatus { + return this.state; + } + + /** + * Publish the advert. Idempotent: a second call with the advert already up + * republishes it, which is what a port change needs. + */ + start(): Promise { + return this.enqueue(async () => { + await this.stopInternal(); + + const port = this.options.getPort(); + if (!port) { + this.state = { + state: 'unavailable', + reason: 'The Maestro web server is not running, so there is no port to advertise.', + }; + return; + } + + const factory = this.options.createResponder ?? loadOptionalBonjour; + this.responder = await factory(); + if (!this.responder) { + this.state = { + state: 'unavailable', + reason: + 'No mDNS responder is available in this build, so Maestro is not advertising itself. ' + + 'Scan the pairing QR code, or enter the address of this computer on the device instead.', + }; + return; + } + + const name = this.options.getName(); + const txt: DiscoveryTxtRecord = { + version: this.options.getAppVersion(), + proto: String(DEVICE_PROTOCOL_VERSION), + fingerprint: this.options.getFingerprint(), + host: name, + }; + + try { + this.advertisement = this.responder.publish({ + name, + type: MAESTRO_SERVICE_TYPE, + protocol: MAESTRO_SERVICE_PROTOCOL, + port, + txt: { ...txt }, + }); + this.state = { state: 'advertising', name, port }; + logger.info( + `Advertising ${MAESTRO_SERVICE_FQDN} as '${name}' on port ${port}`, + LOG_CONTEXT + ); + } catch (error) { + this.state = { state: 'error', message: (error as Error).message }; + logger.warn(`Bonjour advert failed: ${(error as Error).message}`, LOG_CONTEXT); + } + }); + } + + /** Take the advert down. Safe when nothing is up. */ + stop(): Promise { + return this.enqueue(async () => { + await this.stopInternal(); + this.state = { state: 'disabled' }; + }); + } + + private async stopInternal(): Promise { + try { + await this.advertisement?.stop(); + await this.responder?.destroy?.(); + } catch (error) { + // A responder that will not shut down cleanly is not a reason to keep the + // caller waiting or to fail a settings toggle. + logger.warn(`Bonjour teardown failed: ${(error as Error).message}`, LOG_CONTEXT); + } + this.advertisement = null; + this.responder = null; + } + + private enqueue(action: () => Promise): Promise { + const next = this.queue.then(action).catch((error: Error) => { + this.state = { state: 'error', message: error.message }; + }); + this.queue = next; + return next; + } +} + +/** + * The address a user types when discovery is unavailable or switched off. + * + * Returned as a list because a machine on WiFi and an overlay network has more + * than one right answer, and the phone knows which network it is on better than + * the desktop does. + */ +export function manualEntryHint(hosts: string[], port: number | null): string { + if (!port) return 'Start the Maestro web server to pair a device manually.'; + if (hosts.length === 0) + return `Enter the address of this computer and port ${port} on the device.`; + return `Enter ${hosts.map((host) => `${host}:${port}`).join(' or ')} on the device.`; +} diff --git a/src/main/acappella/pairing/pairing-service.ts b/src/main/acappella/pairing/pairing-service.ts new file mode 100644 index 0000000000..879b1612c3 --- /dev/null +++ b/src/main/acappella/pairing/pairing-service.ts @@ -0,0 +1,600 @@ +/** + * Device pairing: how a phone earns the right to hold this desktop's microphone. + * + * The security model, stated before the code because the code only makes sense + * against it: + * + * - **Knowing the code is not enough.** A pairing code is a short string that + * is shown on a screen, photographed, and typed. Anything that short is + * guessable given enough attempts and shoulder-surfable given one glance, so + * it is treated as a POINTER to a request, not as an authorisation. Pairing + * completes only when a human clicks Approve on the desktop, looking at the + * name and platform of the thing asking. + * - **The pairing window is short.** A code is valid for + * {@link DEFAULT_PAIRING_TTL_MS} and is consumed by the first claim. A code + * left on screen while its owner goes to lunch is a code that has already + * expired. + * - **The long-lived token is never stored in plain text.** What is persisted + * is a salted SHA-256 of it, so the device file is not a credential. A stolen + * `devices.json` lets an attacker enumerate device NAMES, which is the + * smallest disclosure this design could arrive at while still being able to + * authenticate a returning device without a server. + * - **Revocation is immediate.** `revoke()` marks the device and fires + * {@link PairingService.onRevoke}, which the signaling service turns into a + * torn-down peer connection and a closed voice session. A revocation that + * only took effect at the next connect would be useless in the one situation + * anybody ever uses it: a device that is connected right now. + * + * Free of Electron: the file path and the clock arrive as options, so the whole + * lifecycle is testable without an app object or a real sleep. + */ + +import { createHash, randomBytes, timingSafeEqual } from 'crypto'; +import * as fs from 'fs/promises'; +import * as path from 'path'; + +import type { DeviceCandidateType } from '../../../shared/acappella/device-protocol'; +import { atomicWriteJson, createKeyedWriteQueue } from '../../utils/atomic-json-store'; +import { logger } from '../../utils/logger'; + +const LOG_CONTEXT = 'ACappella'; + +/** + * How long a pairing code lives. Two minutes is long enough to walk to the phone + * and short enough that a code left on a screen is not a standing invitation. + */ +export const DEFAULT_PAIRING_TTL_MS = 120_000; + +/** + * How long an approved request can go unredeemed. + * + * The token exists from the moment of approval, so this is the window in which + * an approved-but-uncollected credential is sitting in memory. Short. + */ +export const DEFAULT_APPROVAL_TTL_MS = 60_000; + +/** Characters a pairing code is drawn from: no 0/O, no 1/I/L, no vowels. */ +const CODE_ALPHABET = '23456789BCDFGHJKMNPQRSTVWXYZ'; + +/** Length of a pairing code. Six is what fits on a phone keypad without hating it. */ +const CODE_LENGTH = 6; + +// --------------------------------------------------------------------------- +// Shapes +// --------------------------------------------------------------------------- + +/** A device as it is persisted. The hash never leaves the main process. */ +export interface PairedDeviceRecord { + id: string; + name: string; + platform: string; + appVersion?: string; + createdAt: number; + lastConnectedAt: number | null; + /** How the last connection actually reached us. Displayed in the device list. */ + lastCandidateType: DeviceCandidateType; + revokedAt: number | null; + /** Salted SHA-256 of the device token, hex. */ + tokenHash: string; + tokenSalt: string; +} + +/** A device as anything outside main sees it. No credential material at all. */ +export type PairedDeviceView = Omit; + +/** The code on screen, plus what a device needs to confirm it is the right desktop. */ +export interface PairingOffer { + code: string; + expiresAt: number; + /** + * Short digest of this desktop's server token, shown on both ends. + * + * It is what turns "I scanned a QR code" into "I scanned THIS Mac's QR code": + * a device that renders the fingerprint it derived from the connection lets + * the user compare four characters and notice a man in the middle. + */ + fingerprint: string; +} + +/** A device asking to pair, waiting for a human on the desktop. */ +export interface PairingRequest { + requestId: string; + name: string; + platform: string; + appVersion?: string; + requestedAt: number; + expiresAt: number; + /** Where the request came from, so an approval is not made blind. */ + remoteAddress?: string; +} + +export type PairingClaimResult = + | { status: 'pending'; requestId: string; expiresAt: number } + | { status: 'rejected'; reason: 'unknown-code' | 'expired' | 'already-used' | 'busy' }; + +export type PairingRedeemResult = + | { status: 'pending' } + | { status: 'approved'; deviceId: string; token: string } + | { status: 'denied' } + | { status: 'expired' }; + +export interface PairingServiceOptions { + /** Where `devices.json` lives. */ + filePath: string; + /** + * The desktop's server token, hashed into the pairing fingerprint. Never + * stored and never transmitted; only its digest is. + */ + hostSecret?: string; + /** Injectable clock. Tests drive expiry without sleeping. */ + now?: () => number; + pairingTtlMs?: number; + approvalTtlMs?: number; +} + +// --------------------------------------------------------------------------- +// Service +// --------------------------------------------------------------------------- + +export class PairingService { + private readonly options: Required> & + Pick; + private readonly writes = createKeyedWriteQueue(); + + private devices = new Map(); + private loaded = false; + /** The most recently queued write, for {@link whenPersisted}. */ + private lastWrite: Promise = Promise.resolve(); + /** Set by {@link close}; a persist after this point is dropped. */ + private closed = false; + + /** The one open pairing window. A second `startPairing` replaces it. */ + private offer: (PairingOffer & { claimed: boolean }) | null = null; + + /** Claims waiting for, or just given, a decision. */ + private requests = new Map< + string, + { + request: PairingRequest; + decision: 'pending' | 'approved' | 'denied'; + /** Present only between approval and redemption, then dropped. */ + token?: string; + deviceId?: string; + } + >(); + + private readonly requestListeners = new Set<(request: PairingRequest | null) => void>(); + private readonly revokeListeners = new Set<(deviceId: string, reason: string) => void>(); + private readonly changeListeners = new Set<() => void>(); + + constructor(options: PairingServiceOptions) { + this.options = { + filePath: options.filePath, + hostSecret: options.hostSecret, + now: options.now ?? (() => Date.now()), + pairingTtlMs: options.pairingTtlMs ?? DEFAULT_PAIRING_TTL_MS, + approvalTtlMs: options.approvalTtlMs ?? DEFAULT_APPROVAL_TTL_MS, + }; + } + + // -- Subscriptions ------------------------------------------------------- + + /** A device is asking to pair, or the pending request went away (`null`). */ + onPairingRequest(listener: (request: PairingRequest | null) => void): () => void { + this.requestListeners.add(listener); + return () => this.requestListeners.delete(listener); + } + + /** + * A device lost its pairing. The signaling service turns this into an + * immediate teardown, which is the entire reason revocation is an event + * rather than a flag somebody remembers to check. + */ + onRevoke(listener: (deviceId: string, reason: string) => void): () => void { + this.revokeListeners.add(listener); + return () => this.revokeListeners.delete(listener); + } + + /** The device list changed in any way. Repaints the settings panel. */ + onChange(listener: () => void): () => void { + this.changeListeners.add(listener); + return () => this.changeListeners.delete(listener); + } + + // -- Pairing window ------------------------------------------------------ + + /** + * Open a pairing window and return the code to put on screen. + * + * Replaces any open window: two live codes would mean a user cannot tell + * which QR the phone in their hand actually scanned. + */ + startPairing(): PairingOffer { + const now = this.options.now(); + this.offer = { + code: generatePairingCode(), + expiresAt: now + this.options.pairingTtlMs, + fingerprint: this.fingerprint(), + claimed: false, + }; + this.clearPendingRequest(); + return this.currentOffer() as PairingOffer; + } + + /** The open window, or null when there is none (or it has expired). */ + currentOffer(): PairingOffer | null { + if (!this.offer) return null; + if (this.offer.expiresAt <= this.options.now()) { + this.offer = null; + this.clearPendingRequest(); + return null; + } + const { code, expiresAt, fingerprint } = this.offer; + return { code, expiresAt, fingerprint }; + } + + /** Close the window without pairing anything. */ + cancelPairing(): void { + this.offer = null; + this.clearPendingRequest(); + } + + /** + * A device presented a code. Creates a request for a human to approve. + * + * Deliberately does NOT hand back anything usable. The code buys exactly one + * thing: the right to appear in a dialog on the desktop. + */ + claim(input: { + code: string; + name: string; + platform: string; + appVersion?: string; + remoteAddress?: string; + }): PairingClaimResult { + const offer = this.currentOffer(); + if (!offer) return { status: 'rejected', reason: 'expired' }; + if (!constantTimeEquals(input.code.trim().toUpperCase(), offer.code)) { + return { status: 'rejected', reason: 'unknown-code' }; + } + if (this.offer?.claimed) return { status: 'rejected', reason: 'already-used' }; + if (this.pendingRequest()) return { status: 'rejected', reason: 'busy' }; + + // One-time use: the code is spent the moment it is presented, whether or + // not the human approves. A denied request must not leave a live code + // behind for whoever was watching over a shoulder. + if (this.offer) this.offer.claimed = true; + + const now = this.options.now(); + const request: PairingRequest = { + requestId: randomBytes(9).toString('base64url'), + name: input.name.trim() || 'Unnamed device', + platform: input.platform.trim() || 'unknown', + appVersion: input.appVersion, + requestedAt: now, + expiresAt: now + this.options.pairingTtlMs, + remoteAddress: input.remoteAddress, + }; + this.requests.set(request.requestId, { request, decision: 'pending' }); + this.emitRequest(request); + return { status: 'pending', requestId: request.requestId, expiresAt: request.expiresAt }; + } + + /** The request a human is being asked about, or null. */ + pendingRequest(): PairingRequest | null { + const now = this.options.now(); + for (const entry of this.requests.values()) { + if (entry.decision !== 'pending') continue; + if (entry.request.expiresAt <= now) continue; + return entry.request; + } + return null; + } + + /** + * The affirmative action. Mints the device token, persists its hash, and + * leaves the plain token in memory for exactly one redemption. + */ + async approve(requestId: string, nameOverride?: string): Promise { + const entry = this.requests.get(requestId); + if (!entry || entry.decision !== 'pending') return null; + if (entry.request.expiresAt <= this.options.now()) return null; + + await this.load(); + const now = this.options.now(); + const token = randomBytes(32).toString('base64url'); + const tokenSalt = randomBytes(16).toString('hex'); + const record: PairedDeviceRecord = { + id: randomBytes(12).toString('hex'), + name: (nameOverride ?? entry.request.name).trim() || 'Unnamed device', + platform: entry.request.platform, + appVersion: entry.request.appVersion, + createdAt: now, + lastConnectedAt: null, + lastCandidateType: 'unknown', + revokedAt: null, + tokenHash: hashToken(token, tokenSalt), + tokenSalt, + }; + this.devices.set(record.id, record); + entry.decision = 'approved'; + entry.token = token; + entry.deviceId = record.id; + entry.request.expiresAt = now + this.options.approvalTtlMs; + + // The window closes with the approval: it has done its job, and a code + // that outlived the pairing it authorised would pair a second device. + this.offer = null; + this.emitRequest(null); + await this.persist(); + logger.info(`Paired device '${record.name}' (${record.platform})`, LOG_CONTEXT); + return toView(record); + } + + /** The other affirmative action. The code is already spent either way. */ + deny(requestId: string): void { + const entry = this.requests.get(requestId); + if (!entry || entry.decision !== 'pending') return; + entry.decision = 'denied'; + this.offer = null; + this.emitRequest(null); + } + + /** + * The device collects its credential. Exactly once: the token is deleted from + * memory as it is handed over, so a replayed redemption gets nothing. + */ + redeem(requestId: string): PairingRedeemResult { + const entry = this.requests.get(requestId); + if (!entry) return { status: 'expired' }; + if (entry.decision === 'denied') { + this.requests.delete(requestId); + return { status: 'denied' }; + } + if (entry.request.expiresAt <= this.options.now()) { + this.requests.delete(requestId); + return { status: 'expired' }; + } + if (entry.decision === 'pending') return { status: 'pending' }; + + const { token, deviceId } = entry; + this.requests.delete(requestId); + if (!token || !deviceId) return { status: 'expired' }; + return { status: 'approved', deviceId, token }; + } + + // -- Authentication ------------------------------------------------------ + + /** + * Check a returning device's credential. + * + * Returns null for unknown, revoked, and wrong-token alike. The caller gets no + * more detail than that on the wire: distinguishing "no such device" from + * "wrong token" hands an attacker an enumeration oracle for free. + */ + async authenticate(deviceId: string, token: string): Promise { + await this.load(); + const record = this.devices.get(deviceId); + if (!record || record.revokedAt !== null) return null; + if (!constantTimeEquals(hashToken(token, record.tokenSalt), record.tokenHash)) return null; + return toView(record); + } + + // -- Device list --------------------------------------------------------- + + /** Every device, revoked ones included, newest first. */ + async list(): Promise { + await this.load(); + return [...this.devices.values()].sort((a, b) => b.createdAt - a.createdAt).map(toView); + } + + /** Synchronous read of what is already loaded. For hot paths that cannot await. */ + listLoaded(): PairedDeviceView[] { + return [...this.devices.values()].sort((a, b) => b.createdAt - a.createdAt).map(toView); + } + + async rename(deviceId: string, name: string): Promise { + await this.load(); + const record = this.devices.get(deviceId); + if (!record) return false; + record.name = name.trim() || record.name; + await this.persist(); + return true; + } + + /** + * End a pairing, now. + * + * The listeners fire BEFORE the write completes on purpose: tearing down a + * live connection is the urgent half and it must not wait on a disk flush. + */ + async revoke( + deviceId: string, + reason = 'This device was revoked on the desktop.' + ): Promise { + await this.load(); + const record = this.devices.get(deviceId); + if (!record || record.revokedAt !== null) return false; + record.revokedAt = this.options.now(); + this.emitRevoke(deviceId, reason); + await this.persist(); + logger.info(`Revoked device '${record.name}'`, LOG_CONTEXT); + return true; + } + + /** Revoke every device. The panic button. */ + async revokeAll(reason = 'All devices were revoked on the desktop.'): Promise { + await this.load(); + let count = 0; + for (const record of this.devices.values()) { + if (record.revokedAt !== null) continue; + record.revokedAt = this.options.now(); + this.emitRevoke(record.id, reason); + count += 1; + } + if (count > 0) await this.persist(); + return count; + } + + /** Forget a revoked device entirely, so the list stops showing it. */ + async forget(deviceId: string): Promise { + await this.load(); + if (!this.devices.delete(deviceId)) return false; + await this.persist(); + return true; + } + + /** Record a successful connection and how it got here. */ + async noteConnected(deviceId: string, candidateType: DeviceCandidateType): Promise { + await this.load(); + const record = this.devices.get(deviceId); + if (!record) return; + record.lastConnectedAt = this.options.now(); + record.lastCandidateType = candidateType; + await this.persist(); + } + + // -- Storage ------------------------------------------------------------- + + /** Read `devices.json` once. A missing or corrupt file is an empty list. */ + async load(): Promise { + if (this.loaded) return; + this.loaded = true; + try { + const raw = await fs.readFile(this.options.filePath, 'utf-8'); + const parsed = JSON.parse(raw) as { devices?: PairedDeviceRecord[] }; + for (const record of parsed.devices ?? []) { + if (!record || typeof record.id !== 'string' || typeof record.tokenHash !== 'string') { + continue; + } + this.devices.set(record.id, { + ...record, + lastConnectedAt: record.lastConnectedAt ?? null, + lastCandidateType: record.lastCandidateType ?? 'unknown', + revokedAt: record.revokedAt ?? null, + }); + } + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + // A file that has never existed is the first run, not a failure. Anything + // else is worth a line in the log, but never worth refusing to run: a + // device list that cannot be read is a user who cannot pair, which is a + // worse outcome than a user who has to pair again. + if (code !== 'ENOENT') { + logger.warn(`Could not read paired devices: ${(error as Error).message}`, LOG_CONTEXT); + } + } + } + + private async persist(): Promise { + if (this.closed) return; + const devices = [...this.devices.values()]; + this.emitChange(); + this.lastWrite = this.writes.enqueue(this.options.filePath, async () => { + await fs.mkdir(path.dirname(this.options.filePath), { recursive: true }); + await atomicWriteJson(this.options.filePath, { version: 1, devices }); + }); + await this.lastWrite; + } + + /** + * Resolves once every queued write has hit disk. + * + * Several callers persist as a side effect and do not await it - + * `noteConnected()` runs off a peer connecting - so this is how a shutdown or + * a test knows the device file is settled rather than half written. + */ + whenPersisted(): Promise { + return this.lastWrite.then( + () => {}, + () => {} + ); + } + + /** + * Drain the queued writes and stop accepting new ones. + * + * {@link whenPersisted} alone is not enough to settle this file, because + * `noteConnected()` is fire-and-forget off a peer reaching `connected` and it + * awaits {@link load} before it persists. A caller that only drains can have a + * write enqueued a microtask after it looked, which is how tearing down a + * conformance world renamed into a directory that had just been removed. + * Closing sets the flag synchronously first, so anything still in flight + * becomes a no-op rather than a late write. + */ + async close(): Promise { + this.closed = true; + await this.whenPersisted(); + } + + // -- Internals ----------------------------------------------------------- + + /** + * The short digest a user compares between the two screens. + * + * Public because the Bonjour advert carries it too: a device that found this + * desktop by discovery has to be able to show the same four-plus-four + * characters the pairing sheet shows, or the check is not a check. + */ + fingerprint(): string { + const secret = this.options.hostSecret ?? ''; + const digest = createHash('sha256').update(`acappella-pairing:${secret}`).digest('hex'); + return `${digest.slice(0, 4)}-${digest.slice(4, 8)}`.toUpperCase(); + } + + private clearPendingRequest(): void { + for (const [id, entry] of this.requests) { + if (entry.decision === 'pending') this.requests.delete(id); + } + this.emitRequest(null); + } + + private emitRequest(request: PairingRequest | null): void { + for (const listener of this.requestListeners) listener(request); + } + + private emitRevoke(deviceId: string, reason: string): void { + for (const listener of this.revokeListeners) listener(deviceId, reason); + } + + private emitChange(): void { + for (const listener of this.changeListeners) listener(); + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function toView(record: PairedDeviceRecord): PairedDeviceView { + const { tokenHash: _hash, tokenSalt: _salt, ...view } = record; + return view; +} + +/** Salted SHA-256, hex. The salt is per device so two devices never collide. */ +export function hashToken(token: string, salt: string): string { + return createHash('sha256').update(`${salt}:${token}`).digest('hex'); +} + +/** + * Compare without leaking length or position through timing. + * + * `timingSafeEqual` throws on a length mismatch, so both sides are hashed to a + * fixed width first - which also makes the comparison safe for the pairing code, + * where the lengths genuinely differ between a good guess and a bad one. + */ +export function constantTimeEquals(a: string, b: string): boolean { + const left = createHash('sha256').update(a).digest(); + const right = createHash('sha256').update(b).digest(); + return timingSafeEqual(left, right); +} + +/** A six-character code from an alphabet with no lookalike glyphs. */ +export function generatePairingCode(): string { + const bytes = randomBytes(CODE_LENGTH); + let code = ''; + for (let index = 0; index < CODE_LENGTH; index += 1) { + code += CODE_ALPHABET[bytes[index] % CODE_ALPHABET.length]; + } + return code; +} diff --git a/src/main/acappella/permissions/mic-permission.ts b/src/main/acappella/permissions/mic-permission.ts new file mode 100644 index 0000000000..8845b36552 --- /dev/null +++ b/src/main/acappella/permissions/mic-permission.ts @@ -0,0 +1,220 @@ +/** + * The microphone permission, kept strictly separate from everything else that + * can make voice mode unavailable. + * + * A denied microphone and a missing model are not the same failure and must + * never be reported as one. "Voice unavailable" in front of a user who has + * already downloaded 1.4 GB of models, when the real problem is a TCC checkbox, + * is a support ticket the app could have answered itself. So this module answers + * exactly one question, the capability gate turns it into its own slot with its + * own reason code, and the two can never be collapsed. + * + * **When the prompt happens.** Never at app launch, and never when the Encore + * Feature is switched on. An app that asks for the microphone on first run, for + * a feature the user has not turned on, has spent trust it did not earn. The ask + * happens at the first real session start, which is the moment the user has + * asked for something that genuinely needs a microphone. {@link getMicPermission} + * is a pure query and NEVER prompts, which is what makes it safe to call from + * the capability gate on every Settings render. + * + * **Per platform.** macOS has a real TCC state and a real prompt, so it gets + * both. Windows has a queryable state but no in-app prompt: the OS setting is + * the only recovery. Linux has neither, so the state stays `unknown` until a + * `getUserMedia` call fails and {@link noteGetUserMediaFailure} records what it + * failed with. That failure path is the only permission signal Linux has, which + * is why it is a first-class input here rather than a special case buried in the + * audio host. + */ + +import { shell, systemPreferences } from 'electron'; + +import type { AudioHostErrorCode } from '../../../shared/acappella/audio-host'; +import { micSettingsLabel, micSettingsUrl } from '../../../shared/acappella/mic-settings'; +import type { MicPermission } from '../../../shared/acappella/protocol'; +import { isLinux, isMacOS } from '../../../shared/platformDetection'; + +/** The permission, plus everything a caller needs to decide what to render. */ +export interface MicPermissionInfo { + readonly state: MicPermission; + /** + * Whether asking would actually show the OS prompt. False once the user has + * answered, and false everywhere except macOS: a "Grant access" button that + * silently does nothing is worse than no button. + */ + readonly canPrompt: boolean; + /** Deep link to the OS privacy pane, or null where none exists (Linux). */ + readonly settingsUrl: string | null; + /** Button text naming the place the user is being sent. */ + readonly settingsLabel: string; + readonly platform: string; +} + +/** + * What a capture attempt taught us, on a platform that cannot be asked. + * + * Remembered because Chromium reports the denial once, at the moment of the + * failed call, and the Linux query will keep saying `unknown` forever after. + * Forgetting it immediately would mean the gate says "microphone: fine" one + * second after the session died because the microphone was not fine. + * + * It is deliberately NOT permanent, and deliberately NOT authoritative over the + * OS. Both would deadlock: a denial that outranks a `granted` query, or that + * survives the user fixing the setting, blocks every future session through the + * capability gate, and the only thing that could clear it is the successful + * capture the gate is now preventing. So the OS wins wherever it has an answer, + * and a fresh session start clears the observation (see + * {@link requestMicPermission}) because a retry is the user telling us the old + * evidence is stale. + */ +let observedDenial: MicPermission | null = null; + +/** Clear the remembered `getUserMedia` denial. Tests, and after a granted capture. */ +export function resetMicPermissionObservation(): void { + observedDenial = null; +} + +/** + * Record what a capture attempt reported. + * + * This IS the `getUserMedia` failure path, one step downstream: the audio host + * classifies the DOMException with `classifyCaptureError()` and main receives + * the code. Feeding the classified code rather than re-parsing the exception + * here keeps one mapping from browser error to meaning, in the process that + * actually saw the exception. + * + * On Windows and Linux this is the ONLY permission signal there is, which is why + * it is a first-class input to this module rather than something the audio + * bridge quietly knows on its own. + * + * @returns The permission state after folding the failure in. + */ +export function noteCaptureFailure(code: AudioHostErrorCode): MicPermission { + // `no-device` and `device-lost` mean there is no microphone, which is a + // completely different problem with a completely different recovery. Recording + // them as a denial would send a user to a privacy pane that has nothing wrong + // with it. + if (code === 'permission-denied') observedDenial = 'denied'; + return observedDenial ?? 'unknown'; +} + +/** Record that capture actually started, which is the only proof of a grant. */ +export function noteCaptureStarted(): void { + observedDenial = null; +} + +/** + * The current permission. Pure query: this never prompts and never opens a + * device, so it is safe on any render path. + */ +export function getMicPermission(): MicPermissionInfo { + return buildInfo(readPlatformState()); +} + +/** + * Ask the OS for microphone access. + * + * Only macOS has an in-app prompt. Everywhere else this resolves to the current + * state without side effects rather than pretending to ask, because a caller + * that believes it prompted will show the wrong recovery when it did not. + * + * Safe to call repeatedly: macOS shows the prompt once and returns the recorded + * answer thereafter, so "ask at first session start" does not become "nag on + * every session start". + */ +export async function requestMicPermission(): Promise { + if (!isMacOS()) { + // A new session start voids what a previous failed capture taught us. On + // Linux that observation is the only permission signal there is, so leaving + // it in place would mean one denial blocks every future session forever, + // including after the user granted access, with no way back: the capture + // that would clear it is the one the gate is refusing to allow. + resetMicPermissionObservation(); + return getMicPermission(); + } + + const current = readPlatformState(); + // Asking again after an answer is a no-op at the OS level, but skipping it + // keeps a denied state from looking like an attempted re-prompt in logs. + if (current !== 'not-determined') return buildInfo(current); + + try { + const granted = await systemPreferences.askForMediaAccess('microphone'); + if (granted) observedDenial = null; + return buildInfo(granted ? 'granted' : 'denied'); + } catch { + // A throw here means the API is unavailable, not that the user said no. + // Reporting a denial would send them to a settings pane to fix a checkbox + // that is already correct. + return buildInfo(readPlatformState()); + } +} + +/** + * Open the OS microphone privacy settings. + * + * @returns false on a platform with no such link, so the caller can offer words + * instead of a button that does nothing. + */ +export async function openMicSystemSettings(): Promise { + const url = micSettingsUrl(process.platform); + if (!url) return false; + await shell.openExternal(url); + return true; +} + +/** + * The OS's answer, falling back to what a failed capture told us. + * + * The OS wins wherever it has one. It is the thing that will actually decide + * whether the next capture works, and it updates the moment the user changes the + * setting, whereas the observation is a memory of one past attempt. The + * observation fills the gap where there is no query at all: Linux, and any + * platform where the API is missing or throws. + */ +function readPlatformState(): MicPermission { + // Electron only implements this on macOS and Windows. On Linux it is absent + // entirely, and calling it would throw rather than return a state. + if (!isLinux()) { + try { + const status = normalize(systemPreferences.getMediaAccessStatus('microphone')); + if (status !== 'unknown') return status; + } catch { + // Fall through to the observation: an API that is not there tells us + // nothing, and a failed capture tells us something. + } + } + + return observedDenial ?? 'unknown'; +} + +/** + * Electron's four states, kept as four states. + * + * `not-determined` is deliberately NOT folded into `denied`: one means "we have + * not asked yet", which is the normal state of a first run and blocks nothing, + * and the other means the user said no and only they can undo it. + */ +function normalize(status: string): MicPermission { + switch (status) { + case 'granted': + return 'granted'; + case 'denied': + return 'denied'; + case 'restricted': + return 'restricted'; + case 'not-determined': + return 'not-determined'; + default: + return 'unknown'; + } +} + +function buildInfo(state: MicPermission): MicPermissionInfo { + return { + state, + canPrompt: isMacOS() && state === 'not-determined', + settingsUrl: micSettingsUrl(process.platform), + settingsLabel: micSettingsLabel(process.platform), + platform: process.platform, + }; +} diff --git a/src/main/acappella/providers/brain-prompt.ts b/src/main/acappella/providers/brain-prompt.ts new file mode 100644 index 0000000000..83292f1d72 --- /dev/null +++ b/src/main/acappella/providers/brain-prompt.ts @@ -0,0 +1,371 @@ +/** + * The Brain's prompts and its output validator, shared by every Brain backend. + * + * Three implementations (a local Qwen3, OpenAI, Anthropic) have to produce the + * SAME `RouteDecision` for the same utterance, or switching Brain providers would + * quietly change where a spoken instruction lands. So the prompt is written once + * here and the parser is the only thing that turns model output into a decision. + * + * The parser is deliberately paranoid. A model asked for JSON will eventually + * return a fenced block, a preamble, a `sessionId` for an agent that closed while + * it was thinking, or a confidence of 7. None of those may become a dispatch: + * sending someone's spoken instruction to the wrong agent is the single worst + * thing this feature can do, and it is worse than doing nothing. Every field is + * therefore validated against the roster that was actually passed in, and + * anything unrecognised collapses to the conductor rather than to a guess. + */ + +import type { RosterAgent, RosterTab } from '../../../shared/acappella/protocol'; +import type { VoiceConverseContext, VoiceRouteContext } from '../../../shared/acappella/providers'; +import type { RouteDecision, RouteTabAction } from '../../../shared/acappella/route-decision'; +import { ROUTE_TAB_ACTIONS } from '../../../shared/acappella/route-decision'; +import { splitIntoSpokenSentences } from '../../../shared/acappella/sentences'; +import { PROMPT_IDS } from '../../../shared/promptDefinitions'; +import { stripMarkdown } from '../../../shared/markdown'; +import { getPrompt } from '../../prompt-manager'; + +/** Spoken replies stay short unless the caller asks for more. */ +const DEFAULT_SPOKEN_SENTENCES = 2; + +/** Cap on the roster handed to a model. A hundred agents is a prompt, not context. */ +const MAX_ROSTER_AGENTS = 40; +const MAX_TABS_PER_AGENT = 12; + +/** + * The routing instructions, as the user may have edited them. + * + * `src/prompts/acappella-router.md` is a registered core prompt, so it shows up + * in Settings > Maestro Prompts and someone whose agents are all called "api" + * can teach the Conductor how to tell them apart. The built-in constant below is + * the fallback for the two cases where the prompt store cannot answer: before + * `initializePrompts()` has run, and in a unit test that never boots one. A + * routing turn that threw because a settings subsystem was not up yet would be a + * worse failure than routing on the default text. + */ +export function routeSystemPrompt(): string { + try { + const edited = getPrompt(PROMPT_IDS.ACAPPELLA_ROUTER).trim(); + if (edited) return edited; + } catch { + /* prompts not initialised, or the id was removed: use the built-in text */ + } + return ROUTE_SYSTEM_PROMPT; +} + +export const ROUTE_SYSTEM_PROMPT = [ + 'You route spoken instructions inside Maestro, a desktop app that runs several AI coding agents at once.', + 'Given one utterance and the list of running agents, decide which agent it is for, what to do with tabs, and what prompt to actually send.', + '', + 'Rules:', + '- Answer with ONE JSON object and nothing else. No prose, no code fence.', + '- "target" is either the string "conductor" or {"sessionId": ""}. Never invent an id.', + '- Use "conductor" when the utterance is about Maestro itself, or when no agent is clearly meant.', + '- "tabAction" is "current" (use the active tab), "new" (open a fresh tab), or "recall" (go back to an existing tab, and then "tabId" is required and must come from that agent\'s tabs).', + '- "prompt" is what the agent should receive: the request itself, with the routing words removed. Keep the user\'s own wording.', + '- "confidence" is 0 to 1. Be honest: a guess is 0.4, hearing an agent name by name is 0.9.', + '', + 'Talking versus sending (only when the conversation section below says you may reply):', + '- "reply" is one short spoken line back to the user. Setting it means you are TALKING: no agent is contacted, and the floor stays with the user.', + '- Reply while the user is still thinking out loud, describing a problem, or has said something that is not yet a doable task.', + '- Do NOT reply once one concrete, doable thing has been stated. Send it instead. An agent can work out the details; your job is to notice that there is a job.', + '- When you send after a conversation, "prompt" is the distilled request - a sentence or two in the user\'s own words, not a transcript of the discussion.', + '- Keep a reply to one or two sentences. It is spoken aloud, not read.', +].join('\n'); + +/** + * The translator instructions, as the user may have edited them. + * + * Same arrangement as {@link routeSystemPrompt}: `src/prompts/acappella-translator.md` + * is the editable core prompt and the constant below is the fallback for a + * process that has not initialised the prompt store. It is read by every Brain's + * `converse()` and by `speech/conversational-translator.ts`, so the voice a user + * tuned is the voice they get on all three backends rather than on whichever one + * happened to import the file. + */ +export function converseSystemPrompt(): string { + try { + const edited = getPrompt(PROMPT_IDS.ACAPPELLA_TRANSLATOR).trim(); + if (edited) return edited; + } catch { + /* prompts not initialised, or the id was removed: use the built-in text */ + } + return CONVERSE_SYSTEM_PROMPT; +} + +export const CONVERSE_SYSTEM_PROMPT = [ + "You turn an AI coding agent's written answer into something worth hearing out loud.", + '', + 'Rules:', + '- Speak the outcome, not the transcript. The listener has no screen.', + '- Never read code, diffs, file paths, URLs, or command output aloud. Say what changed instead.', + '- Plain sentences. No markdown, no bullet points, no headings.', + '- If the answer is a question for the user, ask it directly.', + '- Answer with the spoken text only. No preamble, no quotes around it.', +].join('\n'); + +/** + * The roster block, as every Brain and the routing-context assembler render it. + * + * One renderer, deliberately: the assembler measures its size cap against this + * exact text, so a second copy that formatted a tab differently would cap the + * wrong string and the prompt would quietly overrun. + */ +export function serializeRoster(agents: readonly RosterAgent[]): string[] { + const lines: string[] = ['Running agents:']; + if (agents.length === 0) lines.push(' (none)'); + + for (const agent of agents.slice(0, MAX_ROSTER_AGENTS)) { + const status = agent.status ? ` ${agent.status}` : ''; + lines.push( + `- ${agent.name} [${agent.sessionId}] (${agent.agentType}${status}) in ${agent.cwd}` + ); + if (agent.recentWork) lines.push(` recently: ${agent.recentWork}`); + for (const tab of agent.tabs.slice(0, MAX_TABS_PER_AGENT)) { + lines.push(` tab ${tab.id}: ${describeTab(tab)}`); + } + } + + return lines; +} + +/** The user-side message for a routing call. */ +export function buildRouteUserPrompt(input: string, context: VoiceRouteContext): string { + const lines: string[] = serializeRoster(context.roster); + + if (context.scope.kind === 'agent') { + lines.push('', `The user is currently bound to agent ${context.scope.sessionId}.`); + } else if (context.activeAgentSessionId) { + lines.push('', `The user is looking at agent ${context.activeAgentSessionId}.`); + } + + const recent = context.recentUtterances ?? []; + if (recent.length > 0) { + lines.push('', 'Earlier in this conversation:'); + for (const utterance of recent.slice(-5)) lines.push(`- ${utterance}`); + } + + const conversation = context.conversation ?? []; + if (conversation.length > 0) { + lines.push('', 'The conversation so far:'); + for (const turn of conversation) { + lines.push(`${turn.role === 'user' ? 'User' : 'You'}: ${turn.text}`); + } + } + + if (context.conversational) { + lines.push( + '', + 'You may answer with "reply" instead of dispatching. Use it while the user is still working out what they want.', + 'Send the request the moment one concrete, doable thing has been stated - do not keep asking for detail an agent could work out for itself.' + ); + } + + if (context.clarification) { + // The answer alone is a fragment ("the API one"). Routed on its own it + // becomes a prompt, and the request it was answering is lost. + lines.push( + '', + `The user asked: ${context.clarification.utterance}`, + `You asked back: ${context.clarification.question}`, + 'Their answer follows. Route the ORIGINAL request, using the answer only to pick the target.' + ); + } + + if (context.retryNotes && context.retryNotes.length > 0) { + lines.push('', 'Your previous answer was rejected:'); + for (const note of context.retryNotes) lines.push(`- ${note}`); + lines.push('Answer again, fixing exactly those problems.'); + } + + lines.push('', `Utterance: ${input}`); + return lines.join('\n'); +} + +/** + * One tab, as a line in the roster. + * + * The topic is what makes recall possible at all: a tab called "Tab 3" tells a + * model nothing, and "auth middleware rewrite" is the phrase the user will say + * six hours later. The state is what stops a recall from being a lie - a snoozed + * or closed tab is a legitimate target, but only if whoever picks it knows it has + * to be woken first. + */ +function describeTab(tab: RosterTab): string { + const label = tab.name ?? 'untitled'; + const parts = [label]; + if (tab.topic && tab.topic !== label) parts.push(`- ${tab.topic}`); + if (tab.state && tab.state !== 'open') parts.push(`(${tab.state})`); + return parts.join(' '); +} + +/** How many earlier spoken lines a rewrite is shown. Enough to avoid repeating itself. */ +const MAX_SPOKEN_MEMORY_LINES = 4; + +export function buildConverseUserPrompt(agentText: string, context: VoiceConverseContext): string { + const limit = context.maxSentences ?? DEFAULT_SPOKEN_SENTENCES; + const lines: string[] = []; + + const spoken = context.recentSpoken ?? []; + if (spoken.length > 0) { + // What the listener HEARD, so the rewrite can refer back instead of + // re-explaining, and so a second chunk of one long answer does not repeat + // the headline the first chunk already delivered. + lines.push('You already said, out loud:'); + for (const line of spoken.slice(-MAX_SPOKEN_MEMORY_LINES)) lines.push(`- ${line}`); + lines.push(''); + } + + lines.push(`Say this in at most ${limit} sentence${limit === 1 ? '' : 's'}:`, '', agentText); + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// Parsing +// --------------------------------------------------------------------------- + +/** + * Turn raw model output into a decision that is safe to dispatch. + * + * `fallbackPrompt` is what the agent receives when the model gave no usable + * prompt: the user's own words. An utterance that reaches an agent verbatim is a + * worse prompt than a cleaned one and an infinitely better outcome than a turn + * that silently did nothing. + */ +export function parseRouteDecision( + raw: string, + context: VoiceRouteContext, + fallbackPrompt: string +): RouteDecision { + const parsed = extractJsonObject(raw); + const agent = resolveAgent(parsed?.target, context.roster); + + let tabAction = asTabAction(parsed?.tabAction); + let tabId = typeof parsed?.tabId === 'string' ? parsed.tabId : undefined; + + if (tabAction === 'recall') { + // A recall the executor cannot perform is worse than no recall: it fails the + // turn instead of using the tab the user is already looking at. + const known = agent?.tabs.some((tab) => tab.id === tabId); + if (!known) { + tabAction = 'current'; + tabId = undefined; + } + } else { + tabId = undefined; + } + + const prompt = asNonEmptyString(parsed?.prompt) ?? fallbackPrompt.trim(); + const tabName = tabAction === 'new' ? asNonEmptyString(parsed?.tabName) : undefined; + + return { + target: agent ? { sessionId: agent.sessionId } : 'conductor', + tabAction, + tabId, + tabName, + prompt, + confidence: clampConfidence(parsed?.confidence), + // A model that asked a question instead of guessing did the right thing, so + // the question survives parsing. Everything else on the decision is still + // filled in: if the user answers, the answer routes; if the turn is + // abandoned, nothing was dispatched. + clarify: asSpokenLine(parsed?.clarify), + reply: asSpokenLine(parsed?.reply), + }; +} + +/** + * Pull the first JSON object out of a model response. + * + * Brace matching rather than a regex: a `prompt` field can legally contain + * braces, and the non-greedy regex that "worked" would truncate the object at the + * first one. + */ +export function extractJsonObject(raw: string): Record | null { + const text = raw.trim(); + const start = text.indexOf('{'); + if (start === -1) return null; + + let depth = 0; + let inString = false; + let escaped = false; + + for (let i = start; i < text.length; i++) { + const char = text[i]; + + if (inString) { + if (escaped) escaped = false; + else if (char === '\\') escaped = true; + else if (char === '"') inString = false; + continue; + } + + if (char === '"') inString = true; + else if (char === '{') depth++; + else if (char === '}' && --depth === 0) { + try { + const parsed: unknown = JSON.parse(text.slice(start, i + 1)); + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : null; + } catch { + return null; + } + } + } + + return null; +} + +/** The roster agent a target names, or null for the conductor and for junk. */ +function resolveAgent(target: unknown, roster: RosterAgent[]): RosterAgent | null { + const sessionId = + typeof target === 'string' + ? null + : ((target as { sessionId?: unknown })?.sessionId as string | undefined); + if (typeof sessionId !== 'string' || !sessionId) return null; + // The id must be one that is RUNNING. A hallucinated id would otherwise reach + // the session service, which would correctly refuse the turn - but this is the + // layer that knows the honest recovery is "the conductor takes it". + return roster.find((agent) => agent.sessionId === sessionId) ?? null; +} + +function asTabAction(value: unknown): RouteTabAction { + return typeof value === 'string' && (ROUTE_TAB_ACTIONS as readonly string[]).includes(value) + ? (value as RouteTabAction) + : 'current'; +} + +function asNonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + +/** A clarification is spoken, so it is one line: newlines would be read aloud. */ +function asSpokenLine(value: unknown): string | undefined { + const text = asNonEmptyString(value); + return text ? text.replace(/\s+/g, ' ') : undefined; +} + +function clampConfidence(value: unknown): number { + const num = typeof value === 'number' ? value : Number(value); + if (!Number.isFinite(num)) return 0.5; + return Math.min(1, Math.max(0, Math.round(num * 100) / 100)); +} + +// --------------------------------------------------------------------------- +// Spoken form +// --------------------------------------------------------------------------- + +/** + * Trim a model's spoken rewrite to the sentence budget. + * + * Applied to every Brain's `converse()` output, including a hosted one that was + * asked for two sentences and returned five. The session service already + * announced `sentenceCount` from the same splitter, so a reply that overruns + * would leave a client's "3 of 2" progress permanently wrong. + */ +export function limitSpokenReply(text: string, maxSentences?: number): string { + const plain = stripMarkdown(text).replace(/\s+/g, ' ').trim(); + if (!plain) return ''; + const limit = Math.max(1, maxSentences ?? DEFAULT_SPOKEN_SENTENCES); + return splitIntoSpokenSentences(plain).slice(0, limit).join(' '); +} diff --git a/src/main/acappella/providers/cascade-pipeline.ts b/src/main/acappella/providers/cascade-pipeline.ts new file mode 100644 index 0000000000..75ebf519dd --- /dev/null +++ b/src/main/acappella/providers/cascade-pipeline.ts @@ -0,0 +1,62 @@ +/** + * The cascade pipeline: three independent engines, speech to text to speech. + * + * One of exactly two `VoicePipeline` implementations, and the DEFAULT one, + * because it is the only shape that a fully local install or an ElevenLabs voice + * can take. Realtime is a latency optimisation available to one provider; this is + * the pipeline the feature is actually built on. + * + * It is a thin wrapper by design. All the interesting decisions - which provider + * fills which slot, and what happens when one is unavailable - belong to the + * registry; all this adds is the lifetime. That lifetime is the reason the class + * exists at all: a Whisper model, an ONNX session, and a llama.cpp context are + * each hundreds of megabytes, and a hot-swap that dropped the reference without + * calling `dispose()` would leak every one of them for the life of the process. + */ + +import type { VoicePipeline, VoiceProviderTrio } from '../../../shared/acappella/providers'; + +/** A provider that holds something worth releasing. Duck-typed on purpose. */ +interface Disposable { + dispose?: () => Promise | void; + /** The local Brain's name for the same thing. */ + unload?: () => Promise | void; + /** Every STT provider has this, and it is the right teardown for one. */ + stop?: () => Promise | void; + /** Every TTS provider has this: stop speaking before going away. */ + cancel?: () => void; +} + +export class CascadePipeline implements VoicePipeline { + readonly shape = 'cascade' as const; + + constructor(readonly providers: VoiceProviderTrio) {} + + /** + * Release all three slots. + * + * Every teardown is attempted even when an earlier one throws: a provider that + * fails to close is not a reason to leak the other two, and the swap that + * called this has already decided the pipeline is going away. + */ + async dispose(): Promise { + // TTS first: it is the one that might still be making noise, and the user + // should stop hearing the old voice before anything else is torn down. + await release(this.providers.tts); + await release(this.providers.stt); + await release(this.providers.brain); + } +} + +async function release(provider: unknown): Promise { + const target = provider as Disposable; + try { + target.cancel?.(); + await target.stop?.(); + await target.unload?.(); + await target.dispose?.(); + } catch { + // Best effort. Teardown runs from a swap and from app quit, and neither has + // anywhere useful to send a failure to close a model file. + } +} diff --git a/src/main/acappella/providers/credentials.ts b/src/main/acappella/providers/credentials.ts new file mode 100644 index 0000000000..1af9aedb32 --- /dev/null +++ b/src/main/acappella/providers/credentials.ts @@ -0,0 +1,388 @@ +/** + * A Cappella credentials - API keys in the OS keychain and nowhere else. + * + * The rule, in full, because every part of it has been broken by a well-meaning + * change in some codebase somewhere: + * + * 1. **The key never touches `settings.json`.** Settings are plain JSON in + * userData, they end up in screenshots and support bundles, and they sync. + * A key lives in one place: a per-service entry in the OS credential store. + * 2. **The key is never logged.** Not at debug level, not "just the prefix" at + * a call site that will later be widened, not inside an error message built + * from a request URL. {@link redactSecrets} exists so the debug package and + * any log line can be scrubbed by a function rather than by discipline. + * 3. **The key never reaches Sentry.** `beforeSend` cannot know which string is + * a key, so nothing that holds one may be attached to a captured exception. + * The classified errors this module raises carry a service name, never the + * credential. + * 4. **"Is it valid" is answered by the service, not by a regex.** A key that + * is well-formed and revoked looks fine to a pattern and fails at the worst + * possible moment - mid-utterance, with no screen to read. So validation is + * a real authenticated request against the cheapest endpoint each provider + * offers, and its three outcomes are told apart: valid, rejected, and + * rate-limited. Rate-limited is NOT invalid; telling a user their key is + * wrong because they are briefly over quota would have them paste a new one + * to fix a problem that fixes itself. + * + * A machine with no usable keyring (headless Linux, a locked login keychain) can + * still run: {@link setCredential} reports the failure and the hosted providers + * stay unavailable through the capability gate. There is deliberately no + * "remember it in a file instead" path. + */ + +import { + VOICE_CREDENTIALS, + VOICE_CREDENTIAL_SERVICES, + credentialLabel, + type VoiceCredentialService, +} from '../../../shared/acappella/provider-catalog'; +import { logger } from '../../utils/logger'; +import { createKeyringEntry, type KeyringEntry } from '../../utils/keyring'; + +const LOG_CONTEXT = 'ACappella'; + +/** The keychain service every A Cappella key is filed under. */ +export const CREDENTIAL_KEYRING_SERVICE = 'com.maestro.acappella'; + +/** Per-request ceiling for a validation call. A key check must not hang a panel. */ +const VALIDATE_TIMEOUT_MS = 10_000; + +/** What a validation attempt found out. */ +export type CredentialValidationStatus = + | 'valid' + | 'invalid' + | 'rate-limited' + | 'network-error' + | 'missing'; + +export interface CredentialValidation { + service: VoiceCredentialService; + status: CredentialValidationStatus; + /** One sentence for the user, naming the next action where there is one. */ + message: string; + /** HTTP status, when the failure came back from the service. */ + httpStatus?: number; +} + +export interface CredentialState { + service: VoiceCredentialService; + label: string; + /** Whether a key is stored. The key itself is never returned to a caller. */ + configured: boolean; + /** False when this machine has no usable credential store at all. */ + keyringAvailable: boolean; +} + +/** Injectable transport, so the validation paths are testable without a network. */ +export type CredentialFetch = (url: string, init?: RequestInit) => Promise; + +// --------------------------------------------------------------------------- +// Storage +// --------------------------------------------------------------------------- + +/** + * Entries are cached per service because constructing one is a native call and + * `has()` is on the capability gate's path, which runs on every Settings render. + * `undefined` means "not tried yet"; `null` means "this machine has no keyring", + * which is a real, cacheable answer. + */ +const entries = new Map(); + +/** Test seam. Replaces the entry factory; pass null to restore the real keyring. */ +let entryFactory: ((service: VoiceCredentialService) => KeyringEntry | null) | null = null; + +export function __setCredentialEntryFactory( + factory: ((service: VoiceCredentialService) => KeyringEntry | null) | null +): void { + entryFactory = factory; + entries.clear(); +} + +function entryFor(service: VoiceCredentialService): KeyringEntry | null { + const cached = entries.get(service); + if (cached !== undefined) return cached; + + const entry = entryFactory + ? entryFactory(service) + : createKeyringEntry(CREDENTIAL_KEYRING_SERVICE, service); + entries.set(service, entry); + return entry; +} + +/** True when this machine has a credential store A Cappella can write to. */ +export function isKeyringAvailable(service: VoiceCredentialService): boolean { + return entryFor(service) !== null; +} + +/** + * The stored key, or null. + * + * Main-process only, and deliberately not exposed over IPC: nothing in the + * renderer needs to read a key back, and a channel that returned one would put + * it in a renderer heap, a devtools frame, and any crash dump taken afterwards. + */ +export function getCredential(service: VoiceCredentialService): string | null { + const entry = entryFor(service); + if (!entry) return null; + try { + const value = entry.getPassword(); + return value && value.trim() ? value.trim() : null; + } catch (error) { + // A locked keychain throws here. Report the shape of the failure, never the + // entry contents. + logger.warn( + `Could not read the ${credentialLabel(service)} key: ${describe(error)}`, + LOG_CONTEXT + ); + return null; + } +} + +/** Whether a key is stored. The cheap question the capability gate asks. */ +export function hasCredential(service: VoiceCredentialService): boolean { + return getCredential(service) !== null; +} + +export interface SetCredentialResult { + ok: boolean; + /** Present when the write failed. Never contains the key. */ + error?: string; +} + +/** + * Store a key. An empty value clears the entry, which is how the settings panel + * removes one without a second channel. + */ +export function setCredential(service: VoiceCredentialService, key: string): SetCredentialResult { + const trimmed = key.trim(); + if (!trimmed) return clearCredential(service); + + const entry = entryFor(service); + if (!entry) { + return { + ok: false, + error: `This machine has no credential store Maestro can use, so the ${credentialLabel(service)} key cannot be saved. Keys are never written to disk in plain text.`, + }; + } + + try { + entry.setPassword(trimmed); + return { ok: true }; + } catch (error) { + return { ok: false, error: `Could not save the key: ${describe(error)}` }; + } +} + +/** Remove a stored key. Succeeds when there was nothing to remove. */ +export function clearCredential(service: VoiceCredentialService): SetCredentialResult { + const entry = entryFor(service); + if (!entry) return { ok: true }; + try { + entry.deletePassword(); + return { ok: true }; + } catch (error) { + return { ok: false, error: `Could not remove the key: ${describe(error)}` }; + } +} + +/** Configured state for every service, for the settings panel. No keys. */ +export function listCredentialStates(): CredentialState[] { + return VOICE_CREDENTIAL_SERVICES.map((service) => ({ + service, + label: credentialLabel(service), + configured: hasCredential(service), + keyringAvailable: isKeyringAvailable(service), + })); +} + +// --------------------------------------------------------------------------- +// Validation +// --------------------------------------------------------------------------- + +/** + * The cheapest authenticated endpoint each service offers. + * + * A list call rather than an inference call on purpose: it costs nothing, it + * cannot be mistaken for usage on someone's bill, and it distinguishes a bad key + * (401/403) from a throttled account (429) without spending a token. + */ +const VALIDATION_ENDPOINTS: Record [string, RequestInit]> = + { + openai: (key) => [ + 'https://api.openai.com/v1/models', + { headers: { Authorization: `Bearer ${key}` } }, + ], + elevenlabs: (key) => ['https://api.elevenlabs.io/v1/user', { headers: { 'xi-api-key': key } }], + anthropic: (key) => [ + 'https://api.anthropic.com/v1/models?limit=1', + { headers: { 'x-api-key': key, 'anthropic-version': '2023-06-01' } }, + ], + }; + +/** + * Verify a key with the service. + * + * @param key Optional. Given, the key is checked WITHOUT being stored, which is + * what the Test button does before a Save. Omitted, the stored key is + * used. + */ +export async function validateCredential( + service: VoiceCredentialService, + key?: string, + fetchImpl: CredentialFetch = globalThis.fetch +): Promise { + const label = credentialLabel(service); + const secret = (key ?? getCredential(service) ?? '').trim(); + + if (!secret) { + return { service, status: 'missing', message: `No ${label} key is stored.` }; + } + + const prefix = VOICE_CREDENTIALS[service].keyPrefix; + if (prefix && !secret.startsWith(prefix)) { + // Caught locally because it is almost always a paste error, and telling + // someone their key is invalid after a round trip they did not need is + // slower and less specific than saying "that is not the right kind of key". + return { + service, + status: 'invalid', + message: `That does not look like a ${label} key: they start with "${prefix}".`, + }; + } + + const [url, init] = VALIDATION_ENDPOINTS[service](secret); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), VALIDATE_TIMEOUT_MS); + + try { + const response = await fetchImpl(url, { ...init, method: 'GET', signal: controller.signal }); + return classifyValidation(service, response.status); + } catch (error) { + return { + service, + status: 'network-error', + // The URL is safe to omit and the key must not be in here, so the message + // says what happened and nothing about what was sent. + message: `Could not reach ${label}: ${describe(error)}`, + }; + } finally { + clearTimeout(timer); + } +} + +function classifyValidation( + service: VoiceCredentialService, + httpStatus: number +): CredentialValidation { + const label = credentialLabel(service); + + if (httpStatus >= 200 && httpStatus < 300) { + return { service, status: 'valid', message: `${label} key works.`, httpStatus }; + } + if (httpStatus === 429) { + // Explicitly NOT invalid. The key is fine; the account is busy. + return { + service, + status: 'rate-limited', + message: `${label} is rate limiting this key right now. The key itself looks fine - try again in a moment.`, + httpStatus, + }; + } + if (httpStatus === 401 || httpStatus === 403) { + return { + service, + status: 'invalid', + message: `${label} rejected this key. Check that it is current and has not been revoked.`, + httpStatus, + }; + } + if (httpStatus >= 500) { + return { + service, + status: 'network-error', + message: `${label} returned a server error (${httpStatus}). That is on their side; try again shortly.`, + httpStatus, + }; + } + return { + service, + status: 'network-error', + message: `${label} answered with an unexpected status (${httpStatus}).`, + httpStatus, + }; +} + +// --------------------------------------------------------------------------- +// Redaction +// --------------------------------------------------------------------------- + +/** + * Key shapes, for scrubbing text that may quote one. + * + * Pattern-based rather than "replace the keys we know we stored", because the + * text being scrubbed is usually a support bundle written on a machine whose + * keys this process may not be able to read (locked keychain), and a key pasted + * into a log by a third-party library was never in our map anyway. + */ +const SECRET_PATTERNS: readonly RegExp[] = [ + // OpenAI and Anthropic: sk-, sk-ant-, sk-proj-, plus the long opaque tail. + /\bsk-[A-Za-z0-9_-]{8,}/g, + // ElevenLabs: a bare 32-hex key, usually behind its own header name. + /\b(xi-api-key["'\s:=]+)([A-Za-z0-9]{16,})/gi, + // Anything that named itself. Catches `"api_key": "..."` in a copied payload. + /\b(api[_-]?key["'\s:=]+)([A-Za-z0-9_-]{12,})/gi, + /\b(authorization["'\s:=]+bearer\s+)([A-Za-z0-9._-]{12,})/gi, +]; + +/** The stand-in every scrubbed secret becomes. */ +export const REDACTED = '[redacted]'; + +/** + * Replace anything that looks like a credential. + * + * Called by the debug package before a bundle is written and available to any + * log site that is about to print something a key could have landed in. It is + * intentionally eager: a false positive costs a support engineer one unreadable + * token, and a false negative ships someone's key to a bug tracker. + */ +export function redactSecrets(text: string): string { + let out = text; + for (const pattern of SECRET_PATTERNS) { + pattern.lastIndex = 0; + out = out.replace(pattern, (_match, prefix?: string) => + prefix ? `${prefix}${REDACTED}` : REDACTED + ); + } + return out; +} + +/** + * Deep-scrub a JSON-shaped value. Strings are run through + * {@link redactSecrets}; a key whose NAME says it holds a secret is redacted + * whole, because a value that does not match a pattern is not thereby safe. + */ +export function redactSecretsDeep(value: T): T { + return scrub(value) as T; +} + +const SECRET_KEY_NAMES = /(api[_-]?key|secret|token|password|authorization)/i; + +function scrub(value: unknown): unknown { + if (typeof value === 'string') return redactSecrets(value); + if (Array.isArray(value)) return value.map(scrub); + if (value && typeof value === 'object') { + const out: Record = {}; + for (const [key, inner] of Object.entries(value as Record)) { + out[key] = SECRET_KEY_NAMES.test(key) && typeof inner === 'string' ? REDACTED : scrub(inner); + } + return out; + } + return value; +} + +function describe(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + // Belt and braces: an error thrown by a transport can quote the request it + // failed on, and that request carried the key. + return redactSecrets(message); +} diff --git a/src/main/acappella/providers/echo-stt.ts b/src/main/acappella/providers/echo-stt.ts new file mode 100644 index 0000000000..12ddc914a5 --- /dev/null +++ b/src/main/acappella/providers/echo-stt.ts @@ -0,0 +1,265 @@ +/** + * Echo speech-to-text: a development provider that hears audio and says how much + * of it was speech. + * + * It transcribes nothing. What it does is close the loop that no other provider + * can close until Phase 05 lands Whisper and OpenAI: PCM goes in, speech segments + * come out, and every downstream stage - partial transcripts, routing, dispatch, + * a spoken reply, barge-in - runs against a real microphone rather than against + * typed text. Without it the whole audio path from Phase 02 is unexercised code + * until a model download lands. + * + * **Why it runs its own detector.** The pipeline already has a VAD, and it + * already forwards its endpoint as a `flush()`. Echo could lean on that, and then + * it would only work when fed by that one caller. A recogniser owns its own + * segmentation - that is most of what a streaming recogniser IS - so this one + * does too, and a test can drive it with generated tone frames and no pipeline at + * all. The cost is one extra pass over 320 samples per frame. + * + * **Placeholder text, not fake words.** The transcript says `Echo utterance 2: + * 1.4s of speech` rather than an invented sentence. Inventing plausible words + * would make a demo that reads as a real recogniser failing, and the first person + * to see one would file the bug that the model is terrible. The measured duration + * is genuinely useful, too: it is the fastest way to see that a room's noise + * floor is opening the mic on the fan. + * + * Registered as a development-only provider (see `provider-registry.ts`), so a + * packaged build cannot resolve it even if the setting names it. + */ + +import { ACAPPELLA_AUDIO_SAMPLE_RATE } from '../../../shared/acappella/audio-host'; +import type { SttCallbacks, SttProvider } from '../../../shared/acappella/providers'; +import { estimateSpokenDurationMs } from '../../../shared/acappella/sentences'; +import { VoiceActivityDetector, type VadConfig } from '../audio/vad'; + +/** The id the registry and the settings key use. Shared so the two cannot drift. */ +export const ECHO_STT_PROVIDER_ID = 'echo-stt'; + +/** + * Audio time between partials inside one segment. A real streaming recogniser + * revises its hypothesis a few times a second; less often than that and the HUD + * looks frozen mid-utterance, more often and it is jitter nobody reads. + */ +const DEFAULT_PARTIAL_INTERVAL_MS = 400; + +/** + * Simulated decoder latency between the endpoint and the final transcript. Real + * recognisers all have some; a pipeline demonstrated with none would hide every + * ordering bug that only shows up when the final lands late. + */ +const DEFAULT_FINAL_DELAY_MS = 250; + +/** Rising across the partials of one segment, the way a real hypothesis firms up. */ +const FIRST_PARTIAL_STABILITY = 0.3; +const PARTIAL_STABILITY_STEP = 0.15; +const MAX_PARTIAL_STABILITY = 0.9; + +/** + * What the placeholder claims. Not 1: this provider is a stand-in for a + * recogniser, and a client that dims low-confidence transcripts should have + * something to dim. + */ +const ECHO_FINAL_CONFIDENCE = 0.9; + +/** Typed text is not a guess, so the text-in seam reports no doubt. */ +const INJECTED_FINAL_CONFIDENCE = 1; + +export interface EchoSttOptions { + /** Detector overrides. The defaults are the pipeline's. */ + vad?: Partial; + /** Audio time between partials within a segment. */ + partialIntervalMs?: number; + /** Simulated decoder latency before the final. Tests pass 0 to stay synchronous. */ + finalDelayMs?: number; +} + +export class EchoSttProvider implements SttProvider { + readonly id = ECHO_STT_PROVIDER_ID; + readonly label = 'Echo (development)'; + /** Mock tier: no model, no network, nothing to install. */ + readonly tier = 'mock' as const; + readonly sampleRate = ACAPPELLA_AUDIO_SAMPLE_RATE; + readonly acceptsAudio = true; + /** + * It hears, but it does not transcribe. The session service reads this and + * stops the turn after the transcript rather than routing a measurement into + * somebody's agent. + */ + readonly transcribesSpeech = false; + + private readonly vad: VoiceActivityDetector; + private readonly partialIntervalMs: number; + private readonly finalDelayMs: number; + private readonly timers = new Set>(); + + private callbacks: SttCallbacks | null = null; + /** Detector time the open segment started, or null when the floor is closed. */ + private segmentStartedAtMs: number | null = null; + /** Segments completed this run, so a transcript can be told from the one before it. */ + private segmentIndex = 0; + private partialsInSegment = 0; + private lastPartialAtMs = 0; + + constructor(options: EchoSttOptions = {}) { + this.vad = new VoiceActivityDetector(options.vad); + this.partialIntervalMs = Math.max(0, options.partialIntervalMs ?? DEFAULT_PARTIAL_INTERVAL_MS); + this.finalDelayMs = Math.max(0, options.finalDelayMs ?? DEFAULT_FINAL_DELAY_MS); + } + + async start(callbacks: SttCallbacks): Promise { + this.callbacks = callbacks; + this.resetRun(); + } + + /** + * One 20 ms frame. Segmentation, partials, and the final all come out of here: + * synchronous, allocation-free beyond the detector's own measurement, and safe + * to call 50 times a second. + */ + feed(pcm: Int16Array): void { + if (!this.callbacks) return; + + const result = this.vad.process(pcm); + + if (result.event?.type === 'speech-start') { + this.segmentStartedAtMs = result.event.atMs; + this.partialsInSegment = 0; + this.lastPartialAtMs = result.event.atMs; + return; + } + + if (result.event?.type === 'speech-end') { + // The detector's own duration excludes the endpoint silence, which is what + // a transcript should report: the user spoke for 1.4 s, not 2.1 s. + this.completeSegment(result.event.durationMs); + return; + } + + if (this.segmentStartedAtMs === null) return; + if (this.partialIntervalMs <= 0) return; + if (result.elapsedMs - this.lastPartialAtMs < this.partialIntervalMs) return; + + this.lastPartialAtMs = result.elapsedMs; + this.partialsInSegment += 1; + const elapsed = result.elapsedMs - this.segmentStartedAtMs; + this.emit((callbacks) => + callbacks.onPartial(partialText(this.segmentIndex + 1, elapsed), this.partialStability()) + ); + } + + /** + * Endpoint now. + * + * Two callers, and both mean the same thing: the pipeline forwarding its own + * VAD's `speech-end`, and floor control releasing a push-to-talk key. Silence + * produces nothing - a flush with no open segment is not an empty transcript, + * it is no transcript. + */ + async flush(): Promise { + if (this.segmentStartedAtMs === null) return; + const durationMs = this.vad.elapsedMs - this.segmentStartedAtMs; + // The detector is still holding the floor open, and its own `speech-end` is + // coming. Reset so that endpoint cannot report the same audio a second time, + // and so whatever is said next opens a segment of its own rather than + // waiting out the pause that the user already declared over. + this.vad.reset(); + this.completeSegment(durationMs); + } + + async stop(): Promise { + this.clearTimers(); + this.callbacks = null; + this.resetRun(); + } + + /** + * The text-in seam: the Phase 01 dev harness, and any client that typed + * instead of spoke. + * + * It lands as a synthetic FINAL transcript with no partials in front of it, + * because there is no hypothesis to revise - the text was already settled when + * it arrived. Anything pending from the microphone is dropped first: a typed + * utterance supersedes whatever was being spoken over it. + */ + injectUtterance(text: string): void { + this.clearTimers(); + this.segmentStartedAtMs = null; + this.vad.reset(); + + const utterance = text.trim(); + this.emit((callbacks) => + callbacks.onFinal( + utterance, + INJECTED_FINAL_CONFIDENCE, + utterance ? estimateSpokenDurationMs(utterance) : 0 + ) + ); + } + + // -- Internals ----------------------------------------------------------- + + /** Close the open segment and schedule its final transcript. */ + private completeSegment(durationMs: number): void { + this.segmentStartedAtMs = null; + this.partialsInSegment = 0; + const index = ++this.segmentIndex; + const text = segmentText(index, Math.max(0, durationMs)); + + if (this.finalDelayMs <= 0) { + this.emit((callbacks) => callbacks.onFinal(text, ECHO_FINAL_CONFIDENCE, durationMs)); + return; + } + + const timer = setTimeout(() => { + this.timers.delete(timer); + this.emit((callbacks) => callbacks.onFinal(text, ECHO_FINAL_CONFIDENCE, durationMs)); + }, this.finalDelayMs); + this.timers.add(timer); + } + + private partialStability(): number { + return Math.min( + MAX_PARTIAL_STABILITY, + FIRST_PARTIAL_STABILITY + PARTIAL_STABILITY_STEP * (this.partialsInSegment - 1) + ); + } + + /** A scheduled final can outlive `stop()`, so every emission re-checks the run. */ + private emit(emit: (callbacks: SttCallbacks) => void): void { + if (!this.callbacks) return; + emit(this.callbacks); + } + + private resetRun(): void { + this.vad.reset(); + this.segmentStartedAtMs = null; + this.segmentIndex = 0; + this.partialsInSegment = 0; + this.lastPartialAtMs = 0; + } + + private clearTimers(): void { + for (const timer of this.timers) clearTimeout(timer); + this.timers.clear(); + } +} + +/** Sugar matching the rest of A Cappella's factories. */ +export function createEchoSttProvider(options: EchoSttOptions = {}): EchoSttProvider { + return new EchoSttProvider(options); +} + +// --------------------------------------------------------------------------- + +function seconds(durationMs: number): string { + return (durationMs / 1000).toFixed(1); +} + +/** Growing, and visibly unfinished, so nobody mistakes a partial for a result. */ +function partialText(index: number, elapsedMs: number): string { + return `Echo utterance ${index}: ${seconds(elapsedMs)}s...`; +} + +function segmentText(index: number, durationMs: number): string { + return `Echo utterance ${index}: ${seconds(durationMs)}s of speech.`; +} diff --git a/src/main/acappella/providers/hosted/anthropic-brain.ts b/src/main/acappella/providers/hosted/anthropic-brain.ts new file mode 100644 index 0000000000..934474aa57 --- /dev/null +++ b/src/main/acappella/providers/hosted/anthropic-brain.ts @@ -0,0 +1,154 @@ +/** + * Anthropic Conductor Brain. + * + * Here so that a user with a Claude key is not made to open an OpenAI account to + * use voice mode. Most Maestro users already have one; asking for a second + * vendor relationship to route a sentence would be a tax on the feature. + * + * Same prompts, same parser, same validation as every other Brain (see + * `../brain-prompt.ts`), so switching the Brain slot changes the vendor and not + * the behaviour. Claude has no `response_format`, so the JSON discipline comes + * from the system prompt plus an assistant prefill of `{`, which is the reliable + * way to stop a model prefacing its object with "Sure! Here is the JSON:". + */ + +import { ANTHROPIC_BRAIN_PROVIDER_ID } from '../../../../shared/acappella/provider-catalog'; +import type { + BrainProvider, + VoiceConverseContext, + VoiceRouteContext, +} from '../../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; +import { + buildConverseUserPrompt, + buildRouteUserPrompt, + converseSystemPrompt, + limitSpokenReply, + parseRouteDecision, + routeSystemPrompt, +} from '../brain-prompt'; +import { getCredential } from '../credentials'; +import { hostedJson, requireCredential, type HostedFetch } from './http'; + +const MESSAGES_URL = 'https://api.anthropic.com/v1/messages'; + +/** The version header the Messages API requires. Not a model version. */ +const API_VERSION = '2023-06-01'; + +/** The fastest model in the family. Routing is a latency problem, not a hard one. */ +const DEFAULT_MODEL = 'claude-haiku-4-5-20251001'; + +const ROUTE_TIMEOUT_MS = 8_000; +const CONVERSE_TIMEOUT_MS = 12_000; + +const ROUTE_MAX_TOKENS = 400; +const CONVERSE_MAX_TOKENS = 300; + +/** + * Prefill. The model continues from this rather than starting a sentence, so the + * response is an object body and `parseRouteDecision` gets the brace back below. + */ +const JSON_PREFILL = '{'; + +export interface AnthropicBrainOptions { + model?: string; + fetchImpl?: HostedFetch; + readCredential?: typeof getCredential; + routeTimeoutMs?: number; + converseTimeoutMs?: number; +} + +export class AnthropicBrainProvider implements BrainProvider { + readonly id = ANTHROPIC_BRAIN_PROVIDER_ID; + readonly label = 'Anthropic (hosted)'; + readonly tier = 'cloud' as const; + + private readonly model: string; + private readonly fetchImpl?: HostedFetch; + private readonly readCredential: typeof getCredential; + private readonly routeTimeoutMs: number; + private readonly converseTimeoutMs: number; + + constructor(options: AnthropicBrainOptions = {}) { + this.model = options.model ?? DEFAULT_MODEL; + this.fetchImpl = options.fetchImpl; + this.readCredential = options.readCredential ?? getCredential; + this.routeTimeoutMs = options.routeTimeoutMs ?? ROUTE_TIMEOUT_MS; + this.converseTimeoutMs = options.converseTimeoutMs ?? CONVERSE_TIMEOUT_MS; + } + + async route(input: string, context: VoiceRouteContext): Promise { + const content = await this.complete({ + system: routeSystemPrompt(), + user: buildRouteUserPrompt(input, context), + prefill: JSON_PREFILL, + timeoutMs: this.routeTimeoutMs, + maxTokens: ROUTE_MAX_TOKENS, + }); + + // The prefill is not echoed back, so it is restored before parsing. + return parseRouteDecision(`${JSON_PREFILL}${content}`, context, input); + } + + async converse(agentText: string, context: VoiceConverseContext): Promise { + const content = await this.complete({ + system: converseSystemPrompt(), + user: buildConverseUserPrompt(agentText, context), + timeoutMs: this.converseTimeoutMs, + maxTokens: CONVERSE_MAX_TOKENS, + }); + + return limitSpokenReply(content, context.maxSentences); + } + + // -- Internals ----------------------------------------------------------- + + private async complete(params: { + system: string; + user: string; + prefill?: string; + timeoutMs: number; + maxTokens: number; + }): Promise { + const key = requireCredential(this.id, 'anthropic', this.readCredential); + + const messages: Array<{ role: string; content: string }> = [ + { role: 'user', content: params.user }, + ]; + if (params.prefill) messages.push({ role: 'assistant', content: params.prefill }); + + const payload = await hostedJson({ + providerId: this.id, + service: 'anthropic', + url: MESSAGES_URL, + init: { + method: 'POST', + headers: { + 'x-api-key': key, + 'anthropic-version': API_VERSION, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model: this.model, + // Same reasoning as the OpenAI Brain: routing has to be reproducible + // or a misroute cannot be investigated. + temperature: 0, + max_tokens: params.maxTokens, + system: params.system, + messages, + }), + }, + timeoutMs: params.timeoutMs, + fetchImpl: this.fetchImpl, + }); + + return (payload.content ?? []) + .filter((block) => block?.type === 'text' && typeof block.text === 'string') + .map((block) => block.text as string) + .join(''); + } +} + +interface MessagesResponse { + content?: Array<{ type?: string; text?: string }>; +} diff --git a/src/main/acappella/providers/hosted/elevenlabs-tts.ts b/src/main/acappella/providers/hosted/elevenlabs-tts.ts new file mode 100644 index 0000000000..d6e07f6b35 --- /dev/null +++ b/src/main/acappella/providers/hosted/elevenlabs-tts.ts @@ -0,0 +1,230 @@ +/** + * ElevenLabs text-to-speech. + * + * **One request per sentence, not one per reply.** The session service announces + * a sentence count and emits one `speak-sentence` per chunk, and barge-in has to + * cut speech off mid-reply without waiting for a whole paragraph to synthesise. + * Synthesising sentence by sentence means the first words are audible while the + * rest are still being made, and `cancel()` has something to abort that is at + * most one sentence long. + * + * **`cancel()` aborts the socket, it does not set a flag.** A cancelled run that + * merely stops yielding leaves the request running and the account paying for + * audio nobody will hear. Barge-in is the most common interaction in a voice UI; + * it has to be free. + * + * Audio comes back as raw 16 kHz PCM rather than MP3 so it can go straight to the + * audio host's `pcm16` playback path with no decoder in the main process. + */ + +import { ACAPPELLA_AUDIO_SAMPLE_RATE } from '../../../../shared/acappella/audio-host'; +import { ELEVENLABS_TTS_PROVIDER_ID } from '../../../../shared/acappella/provider-catalog'; +import type { + TtsChunk, + TtsProvider, + TtsSpeakOptions, +} from '../../../../shared/acappella/providers'; +import { splitIntoSpokenSentences } from '../../../../shared/acappella/sentences'; +import { getCredential } from '../credentials'; +import { hostedJson, hostedRequest, requireCredential, type HostedFetch } from './http'; + +const API_ROOT = 'https://api.elevenlabs.io/v1'; + +/** Their low-latency model. A voice assistant is the case it exists for. */ +const DEFAULT_MODEL = 'eleven_flash_v2_5'; + +/** "Rachel". Overridden the moment the user picks a voice. */ +const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM'; + +const DEFAULT_TIMEOUT_MS = 15_000; + +/** + * Rate is sent as ElevenLabs' `speed`, which they bound to 0.7 - 1.2. Anything + * outside that is rejected for the whole request, so a user's slider is clamped + * here rather than becoming a failed reply. + */ +const MIN_SPEED = 0.7; +const MAX_SPEED = 1.2; + +/** One selectable voice, as the voice picker renders it. */ +export interface TtsVoice { + id: string; + name: string; + /** Free-text descriptor from the service ("american, calm"). Optional. */ + description?: string; + /** A sample the picker can play without spending a synthesis call. */ + previewUrl?: string; +} + +/** Per-voice knobs. Sent verbatim; the service owns the ranges. */ +export interface ElevenLabsVoiceSettings { + stability?: number; + similarityBoost?: number; + style?: number; +} + +export interface ElevenLabsTtsOptions { + model?: string; + voiceId?: string; + voiceSettings?: ElevenLabsVoiceSettings; + timeoutMs?: number; + fetchImpl?: HostedFetch; + readCredential?: typeof getCredential; +} + +export class ElevenLabsTtsProvider implements TtsProvider { + readonly id = ELEVENLABS_TTS_PROVIDER_ID; + readonly label = 'ElevenLabs (hosted)'; + readonly tier = 'cloud' as const; + + private readonly model: string; + private readonly defaultVoiceId: string; + private readonly voiceSettings?: ElevenLabsVoiceSettings; + private readonly timeoutMs: number; + private readonly fetchImpl?: HostedFetch; + private readonly readCredential: typeof getCredential; + + /** Bumped by `cancel()` and by every new run, so a stale iterator returns. */ + private run = 0; + private inFlight: AbortController | null = null; + + constructor(options: ElevenLabsTtsOptions = {}) { + this.model = options.model ?? DEFAULT_MODEL; + this.defaultVoiceId = options.voiceId ?? DEFAULT_VOICE_ID; + this.voiceSettings = options.voiceSettings; + this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + this.fetchImpl = options.fetchImpl; + this.readCredential = options.readCredential ?? getCredential; + } + + speak(text: string, options: TtsSpeakOptions): AsyncIterable { + // The run is claimed here, not in the generator body: a generator does not + // start until its first `next()`, and a second `speak()` must supersede the + // first immediately. + return this.stream(splitIntoSpokenSentences(text), ++this.run, options); + } + + cancel(): void { + this.run += 1; + this.inFlight?.abort(); + this.inFlight = null; + } + + /** Every voice on the account, for the picker. */ + async listVoices(): Promise { + const key = requireCredential(this.id, 'elevenlabs', this.readCredential); + const payload = await hostedJson<{ voices?: RawVoice[] }>({ + providerId: this.id, + service: 'elevenlabs', + url: `${API_ROOT}/voices`, + init: { method: 'GET', headers: { 'xi-api-key': key } }, + timeoutMs: this.timeoutMs, + fetchImpl: this.fetchImpl, + }); + + return (payload.voices ?? []) + .filter((voice): voice is RawVoice & { voice_id: string } => Boolean(voice?.voice_id)) + .map((voice) => ({ + id: voice.voice_id, + name: voice.name ?? voice.voice_id, + description: voice.labels ? Object.values(voice.labels).join(', ') : undefined, + previewUrl: voice.preview_url, + })); + } + + // -- Internals ----------------------------------------------------------- + + private async *stream( + sentences: string[], + run: number, + options: TtsSpeakOptions + ): AsyncGenerator { + for (let index = 0; index < sentences.length; index++) { + if (this.run !== run) return; + + let audio: Uint8Array; + try { + audio = await this.synthesize(sentences[index], options); + } catch (error) { + // A barge-in aborts the request, and an abort is not a failure: the run + // it belonged to is already over, so it ends quietly. Everything else + // travels, classified or not - the session announces the classified ones + // and Sentry gets the rest. + if (this.run !== run) return; + throw error; + } + + // Re-checked after the await: a barge-in during synthesis must not deliver + // the sentence it interrupted. + if (this.run !== run) return; + + yield { + utteranceId: options.utteranceId, + index, + text: sentences[index], + format: 'pcm16', + audio, + sampleRate: ACAPPELLA_AUDIO_SAMPLE_RATE, + }; + } + } + + private async synthesize(sentence: string, options: TtsSpeakOptions): Promise { + const key = requireCredential(this.id, 'elevenlabs', this.readCredential); + const controller = new AbortController(); + this.inFlight = controller; + + const voiceId = options.voiceId ?? this.defaultVoiceId; + const body: Record = { + text: sentence, + model_id: this.model, + }; + const settings = this.buildVoiceSettings(options.rate); + if (settings) body.voice_settings = settings; + + try { + const response = await hostedRequest({ + providerId: this.id, + service: 'elevenlabs', + url: `${API_ROOT}/text-to-speech/${encodeURIComponent(voiceId)}/stream?output_format=pcm_16000`, + init: { + method: 'POST', + headers: { 'xi-api-key': key, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }, + timeoutMs: this.timeoutMs, + signal: controller.signal, + // Retrying a sentence the user may already have heard the start of + // would repeat words. One attempt; a failure ends the run honestly. + retry: false, + fetchImpl: this.fetchImpl, + }); + + return new Uint8Array(await response.arrayBuffer()); + } finally { + if (this.inFlight === controller) this.inFlight = null; + } + } + + private buildVoiceSettings(rate?: number): Record | null { + const settings: Record = {}; + if (this.voiceSettings?.stability !== undefined) { + settings.stability = this.voiceSettings.stability; + } + if (this.voiceSettings?.similarityBoost !== undefined) { + settings.similarity_boost = this.voiceSettings.similarityBoost; + } + if (this.voiceSettings?.style !== undefined) settings.style = this.voiceSettings.style; + if (rate !== undefined && rate > 0) { + settings.speed = Math.min(MAX_SPEED, Math.max(MIN_SPEED, rate)); + } + return Object.keys(settings).length > 0 ? settings : null; + } +} + +interface RawVoice { + voice_id?: string; + name?: string; + preview_url?: string; + labels?: Record; +} diff --git a/src/main/acappella/providers/hosted/http.ts b/src/main/acappella/providers/hosted/http.ts new file mode 100644 index 0000000000..2efa556b38 --- /dev/null +++ b/src/main/acappella/providers/hosted/http.ts @@ -0,0 +1,322 @@ +/** + * The transport every hosted A Cappella provider goes through. + * + * Three providers, one set of rules, because the rules are the hard part and + * three copies of them would drift on the first bug fix: + * + * - **Every request has a deadline.** A voice turn that is still waiting on a + * transcript after ten seconds has already failed; the user has been staring + * at a listening indicator with nothing coming back. The timeout is enforced + * with an `AbortController` so the socket actually closes rather than being + * abandoned. + * - **Cancellation is real.** A caller's signal is chained into the same + * controller, so barge-in aborts the in-flight HTTP request instead of + * letting a superseded turn finish paying for itself. + * - **Retry is bounded and only for the failures retrying can fix.** 429 and + * 5xx, with exponential backoff, at most {@link MAX_ATTEMPTS} attempts. A 401 + * is never retried: the key will not become valid, and hammering an auth + * endpoint is how an account gets locked. + * - **Failures are classified, never generic.** Auth, quota, network, timeout, + * and server come back as distinct {@link VoiceProviderError} kinds so the + * session can tell a user which one it is. Anything unexpected is left to + * throw as itself and reach Sentry. + * + * Nothing in this file logs a request URL with its headers, and no error message + * built here quotes a request body. See `../credentials.ts` for why. + */ + +import { + VoiceProviderError, + type VoiceProviderFailureKind, +} from '../../../../shared/acappella/provider-errors'; +import { + credentialLabel, + type VoiceCredentialService, +} from '../../../../shared/acappella/provider-catalog'; + +/** Attempts in total, not retries after the first. */ +export const MAX_ATTEMPTS = 3; + +/** First backoff step. Doubles per attempt. */ +const BASE_BACKOFF_MS = 400; + +/** Ceiling, so a Retry-After of an hour does not become a wait of an hour. */ +const MAX_BACKOFF_MS = 4_000; + +/** Injectable transport. Tests pass a stub; production uses global `fetch`. */ +export type HostedFetch = (url: string, init?: RequestInit) => Promise; + +export interface HostedRequestOptions { + /** Provider id, so a failure can name the engine the user configured. */ + providerId: string; + /** Whose key this is, for the message text. */ + service: VoiceCredentialService; + url: string; + init?: RequestInit; + /** Per-request deadline. */ + timeoutMs: number; + /** Caller's cancellation, chained into the request's own controller. */ + signal?: AbortSignal; + fetchImpl?: HostedFetch; + /** Sleep, injectable so tests do not wait out real backoff. */ + delayMs?: (ms: number) => Promise; + /** + * False for a streaming request that must not be replayed. A retried stream + * would re-synthesise audio the user already heard the start of. + */ + retry?: boolean; +} + +/** + * Perform one hosted request with timeout, cancellation, bounded retry, and + * classified failures. + * + * @returns the `Response`, which the caller owns and must consume or cancel. + */ +export async function hostedRequest(options: HostedRequestOptions): Promise { + const fetchImpl = options.fetchImpl ?? globalThis.fetch; + const sleep = options.delayMs ?? defaultDelay; + const attempts = options.retry === false ? 1 : MAX_ATTEMPTS; + + let lastError: VoiceProviderError | null = null; + + for (let attempt = 0; attempt < attempts; attempt++) { + throwIfAborted(options); + + let response: Response; + try { + response = await withDeadline(options, fetchImpl); + } catch (error) { + const classified = classifyTransportError(error, options); + // A caller-cancelled request is not a fault and must not be retried: the + // turn it belonged to is already over. + if (options.signal?.aborted) throw classified; + lastError = classified; + if (attempt === attempts - 1) throw classified; + await sleep(backoffMs(attempt)); + continue; + } + + if (response.ok) return response; + + const failure = await classifyHttpStatus(response, options); + // Retryable statuses only. An auth failure retried three times is three + // chances to trip a lockout for no gain. + if (!isRetryableStatus(response.status) || attempt === attempts - 1) throw failure; + + lastError = failure; + await sleep(retryAfterMs(response) ?? backoffMs(attempt)); + } + + // Unreachable: the loop either returns or throws on its final attempt. Kept so + // a future edit to the loop cannot silently return undefined. + throw lastError ?? providerError('unavailable', 'The request could not be completed.', options); +} + +/** + * The response body as text, or a classified failure. Used for the small JSON + * responses; streaming callers read `response.body` themselves. + */ +export async function hostedJson(options: HostedRequestOptions): Promise { + const response = await hostedRequest(options); + try { + return (await response.json()) as T; + } catch (error) { + throw providerError( + 'server', + `${credentialLabel(options.service)} returned a response Maestro could not read.`, + options, + error + ); + } +} + +// --------------------------------------------------------------------------- +// Internals +// --------------------------------------------------------------------------- + +/** + * Run one attempt under a deadline, with the caller's cancellation chained in. + * + * `AbortSignal.any` would be the tidy way to do this and is too new to rely on + * across the Node versions Electron has shipped, so the two signals are wired by + * hand and the listener is always removed - an abandoned listener on a long-lived + * caller signal is a leak that only shows up after a few hundred turns. + */ +async function withDeadline( + options: HostedRequestOptions, + fetchImpl: HostedFetch +): Promise { + const controller = new AbortController(); + const onAbort = () => controller.abort(); + options.signal?.addEventListener('abort', onAbort, { once: true }); + + const timer = setTimeout(() => controller.abort(new DeadlineExceeded()), options.timeoutMs); + + try { + return await fetchImpl(options.url, { ...options.init, signal: controller.signal }); + } finally { + clearTimeout(timer); + options.signal?.removeEventListener('abort', onAbort); + } +} + +/** Marker for our own deadline, so it can be told from a caller's cancellation. */ +class DeadlineExceeded extends Error { + constructor() { + super('Deadline exceeded'); + this.name = 'DeadlineExceeded'; + } +} + +function throwIfAborted(options: HostedRequestOptions): void { + if (!options.signal?.aborted) return; + throw providerError('network', 'The request was cancelled.', options); +} + +function isRetryableStatus(status: number): boolean { + return status === 429 || status >= 500; +} + +function backoffMs(attempt: number): number { + return Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** attempt); +} + +/** `Retry-After` in ms, when the service gave one we can honour. */ +function retryAfterMs(response: Response): number | null { + const header = response.headers?.get?.('retry-after'); + if (!header) return null; + const seconds = Number(header); + if (!Number.isFinite(seconds) || seconds < 0) return null; + return Math.min(MAX_BACKOFF_MS, seconds * 1000); +} + +async function classifyHttpStatus( + response: Response, + options: HostedRequestOptions +): Promise { + const label = credentialLabel(options.service); + const status = response.status; + + // The body is read and discarded rather than quoted: it can echo the request, + // and the request carried the key. + await response.text().catch(() => ''); + + if (status === 401 || status === 403) { + return providerError( + 'auth', + `${label} rejected the API key. Add a current key in Settings > Plugins > A Cappella > Voice Providers.`, + options, + undefined, + status + ); + } + if (status === 429) { + return providerError( + 'quota', + `${label} is rate limiting this key. Wait a moment, or switch this slot to a local model.`, + options, + undefined, + status + ); + } + if (status === 402) { + return providerError( + 'quota', + `The ${label} account is out of credit. Top it up, or switch this slot to a local model.`, + options, + undefined, + status + ); + } + if (status >= 500) { + return providerError( + 'server', + `${label} is having a problem on their side (${status}). Try again shortly.`, + options, + undefined, + status + ); + } + return providerError( + 'request', + `${label} refused the request (${status}). This is a Maestro bug rather than something you can fix.`, + options, + undefined, + status + ); +} + +function classifyTransportError(error: unknown, options: HostedRequestOptions): VoiceProviderError { + const label = credentialLabel(options.service); + + if (error instanceof DeadlineExceeded || isAbortForDeadline(error)) { + return providerError( + 'timeout', + `${label} did not answer within ${Math.round(options.timeoutMs / 1000)}s.`, + options, + error + ); + } + if (options.signal?.aborted) { + return providerError('network', 'The request was cancelled.', options, error); + } + return providerError( + 'network', + `Could not reach ${label}. Check your connection, or switch this slot to a local model.`, + options, + error + ); +} + +/** + * Whether an abort came from our deadline. + * + * Undici reports the abort REASON on modern runtimes and a bare `AbortError` on + * older ones, so both shapes are checked. Getting this wrong only mislabels a + * timeout as a network failure, but those two have different recoveries and the + * user reads the difference. + */ +function isAbortForDeadline(error: unknown): boolean { + if ((error as { cause?: unknown })?.cause instanceof DeadlineExceeded) return true; + return (error as { name?: string })?.name === 'TimeoutError'; +} + +function providerError( + kind: VoiceProviderFailureKind, + message: string, + options: HostedRequestOptions, + cause?: unknown, + httpStatus?: number +): VoiceProviderError { + return new VoiceProviderError(message, { + kind, + providerId: options.providerId, + httpStatus, + cause, + }); +} + +function defaultDelay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * The key for a hosted provider, or a classified `unavailable` failure. + * + * Every hosted provider starts with this call, which is what makes "no key + * configured" a stated refusal at the top of a turn rather than a 401 three + * hundred milliseconds later. + */ +export function requireCredential( + providerId: string, + service: VoiceCredentialService, + read: (service: VoiceCredentialService) => string | null +): string { + const key = read(service); + if (key) return key; + throw new VoiceProviderError( + `No ${credentialLabel(service)} API key is configured. Add one in Settings > Plugins > A Cappella > Voice Providers, or switch this slot to a local model.`, + { kind: 'unavailable', providerId } + ); +} diff --git a/src/main/acappella/providers/hosted/openai-brain.ts b/src/main/acappella/providers/hosted/openai-brain.ts new file mode 100644 index 0000000000..c9e06c0a05 --- /dev/null +++ b/src/main/acappella/providers/hosted/openai-brain.ts @@ -0,0 +1,151 @@ +/** + * OpenAI Conductor Brain. + * + * Two calls, both short and both on a cheap fast model, because routing latency + * is felt directly: it sits between the user finishing a sentence and anything + * happening at all. A large model would route marginally better and make the + * feature feel broken. + * + * Routing uses structured outputs against the shared `RouteDecision` schema, so + * the model cannot emit a shape the executor would have to reject. It is still + * run through `parseRouteDecision`, which validates the ids against the roster + * that was actually passed in - a schema guarantees a well-formed `sessionId`, + * not a real one, and dispatching an utterance to a hallucinated agent is the + * failure this whole subsystem is built to avoid. + */ + +import { OPENAI_BRAIN_PROVIDER_ID } from '../../../../shared/acappella/provider-catalog'; +import type { + BrainProvider, + VoiceConverseContext, + VoiceRouteContext, +} from '../../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; +import { ROUTE_DECISION_JSON_SCHEMA } from '../../../../shared/acappella/route-decision'; +import { + buildConverseUserPrompt, + buildRouteUserPrompt, + converseSystemPrompt, + limitSpokenReply, + parseRouteDecision, + routeSystemPrompt, +} from '../brain-prompt'; +import { getCredential } from '../credentials'; +import { hostedJson, requireCredential, type HostedFetch } from './http'; + +const CHAT_URL = 'https://api.openai.com/v1/chat/completions'; + +/** Cheap, fast, and good enough to pick an agent out of a list. */ +const DEFAULT_MODEL = 'gpt-4o-mini'; + +/** + * Routing is on the critical path between a finished sentence and any visible + * response, so it gets a tighter deadline than the rewrite that follows it. + */ +const ROUTE_TIMEOUT_MS = 8_000; +const CONVERSE_TIMEOUT_MS = 12_000; + +/** A route decision is a small object; a spoken reply is two sentences. */ +const ROUTE_MAX_TOKENS = 400; +const CONVERSE_MAX_TOKENS = 300; + +export interface OpenAiBrainOptions { + model?: string; + fetchImpl?: HostedFetch; + readCredential?: typeof getCredential; + routeTimeoutMs?: number; + converseTimeoutMs?: number; +} + +export class OpenAiBrainProvider implements BrainProvider { + readonly id = OPENAI_BRAIN_PROVIDER_ID; + readonly label = 'OpenAI (hosted)'; + readonly tier = 'cloud' as const; + + private readonly model: string; + private readonly fetchImpl?: HostedFetch; + private readonly readCredential: typeof getCredential; + private readonly routeTimeoutMs: number; + private readonly converseTimeoutMs: number; + + constructor(options: OpenAiBrainOptions = {}) { + this.model = options.model ?? DEFAULT_MODEL; + this.fetchImpl = options.fetchImpl; + this.readCredential = options.readCredential ?? getCredential; + this.routeTimeoutMs = options.routeTimeoutMs ?? ROUTE_TIMEOUT_MS; + this.converseTimeoutMs = options.converseTimeoutMs ?? CONVERSE_TIMEOUT_MS; + } + + async route(input: string, context: VoiceRouteContext): Promise { + const content = await this.complete({ + system: routeSystemPrompt(), + user: buildRouteUserPrompt(input, context), + timeoutMs: this.routeTimeoutMs, + maxTokens: ROUTE_MAX_TOKENS, + responseFormat: { + type: 'json_schema', + json_schema: { + name: 'route_decision', + strict: false, + schema: ROUTE_DECISION_JSON_SCHEMA, + }, + }, + }); + + return parseRouteDecision(content, context, input); + } + + async converse(agentText: string, context: VoiceConverseContext): Promise { + const content = await this.complete({ + system: converseSystemPrompt(), + user: buildConverseUserPrompt(agentText, context), + timeoutMs: this.converseTimeoutMs, + maxTokens: CONVERSE_MAX_TOKENS, + }); + + return limitSpokenReply(content, context.maxSentences); + } + + // -- Internals ----------------------------------------------------------- + + private async complete(params: { + system: string; + user: string; + timeoutMs: number; + maxTokens: number; + responseFormat?: unknown; + }): Promise { + const key = requireCredential(this.id, 'openai', this.readCredential); + + const payload = await hostedJson({ + providerId: this.id, + service: 'openai', + url: CHAT_URL, + init: { + method: 'POST', + headers: { Authorization: `Bearer ${key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + model: this.model, + // Deterministic on purpose: the same utterance against the same + // roster should route the same way twice, or "it went to the wrong + // agent" becomes unreproducible. + temperature: 0, + max_tokens: params.maxTokens, + messages: [ + { role: 'system', content: params.system }, + { role: 'user', content: params.user }, + ], + ...(params.responseFormat ? { response_format: params.responseFormat } : {}), + }), + }, + timeoutMs: params.timeoutMs, + fetchImpl: this.fetchImpl, + }); + + return payload.choices?.[0]?.message?.content ?? ''; + } +} + +interface ChatCompletion { + choices?: Array<{ message?: { content?: string } }>; +} diff --git a/src/main/acappella/providers/hosted/openai-stt.ts b/src/main/acappella/providers/hosted/openai-stt.ts new file mode 100644 index 0000000000..a8f7ad6451 --- /dev/null +++ b/src/main/acappella/providers/hosted/openai-stt.ts @@ -0,0 +1,292 @@ +/** + * OpenAI speech-to-text. + * + * **The rule that shapes this file: audio is only sent after the floor opens.** + * `start()` is called by the session service at the moment a wake word, a hotkey, + * or a client button has already opened a session, and this provider buffers + * nothing before that call and drops everything on `stop()`. There is no + * always-on connection, no pre-roll, and no reconnect that outlives a session. A + * hosted recogniser that held a socket open between turns would be a microphone + * pointed at someone's room with a network cable attached to it. + * + * **Why the utterance is uploaded rather than streamed frame by frame.** The + * transcription endpoint streams its OUTPUT (server-sent `delta` events, which + * become partials) but takes its input as one request. The alternative is the + * realtime WebSocket API, and that is a different pipeline shape with a different + * privacy story, which is exactly what `providers/realtime/` is. Keeping the + * cascade's hosted STT on the plain endpoint means one hop, one deadline, and a + * request that is provably scoped to a single utterance. + */ + +import { ACAPPELLA_AUDIO_SAMPLE_RATE } from '../../../../shared/acappella/audio-host'; +import { OPENAI_STT_PROVIDER_ID } from '../../../../shared/acappella/provider-catalog'; +import { VoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import type { SttCallbacks, SttProvider } from '../../../../shared/acappella/providers'; +import { estimateSpokenDurationMs } from '../../../../shared/acappella/sentences'; +import { getCredential } from '../credentials'; +import { PcmBuffer } from '../pcm'; +import { hostedRequest, requireCredential, type HostedFetch } from './http'; + +const TRANSCRIBE_URL = 'https://api.openai.com/v1/audio/transcriptions'; + +/** Fast and cheap. Routing does not need the large model's last percent. */ +const DEFAULT_MODEL = 'gpt-4o-mini-transcribe'; + +/** + * A spoken turn that has not produced a transcript in this long has already + * failed as a conversation, whatever the network eventually says. + */ +const DEFAULT_TIMEOUT_MS = 20_000; + +/** Below this, the "utterance" is a cough. Uploading it would cost a request. */ +const MIN_UTTERANCE_SAMPLES = ACAPPELLA_AUDIO_SAMPLE_RATE / 5; + +export interface OpenAiSttOptions { + model?: string; + timeoutMs?: number; + fetchImpl?: HostedFetch; + /** Injected in tests. Production reads the OS keychain. */ + readCredential?: typeof getCredential; + /** Optional BCP-47 hint. Given, it measurably improves both speed and accuracy. */ + language?: string; +} + +export class OpenAiSttProvider implements SttProvider { + readonly id = OPENAI_STT_PROVIDER_ID; + readonly label = 'OpenAI (hosted)'; + readonly tier = 'cloud' as const; + readonly sampleRate = ACAPPELLA_AUDIO_SAMPLE_RATE; + readonly acceptsAudio = true; + + private readonly model: string; + private readonly timeoutMs: number; + private readonly fetchImpl?: HostedFetch; + private readonly readCredential: typeof getCredential; + private readonly language?: string; + + private callbacks: SttCallbacks | null = null; + private buffer = new PcmBuffer(); + /** Aborts the in-flight upload. Replaced per utterance, cleared on stop. */ + private inFlight: AbortController | null = null; + + constructor(options: OpenAiSttOptions = {}) { + this.model = options.model ?? DEFAULT_MODEL; + this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + this.fetchImpl = options.fetchImpl; + this.readCredential = options.readCredential ?? getCredential; + this.language = options.language; + } + + /** + * Open the floor. + * + * The key is checked HERE rather than at the first upload, so a missing key is + * a refusal before the microphone is ever opened rather than a failure after + * the user has finished speaking. + */ + async start(callbacks: SttCallbacks): Promise { + requireCredential(this.id, 'openai', this.readCredential); + this.callbacks = callbacks; + this.buffer = new PcmBuffer(); + } + + feed(pcm: Int16Array): void { + // No callbacks means no session. Buffering here would be audio held for a + // floor that is not open, which is the one thing this provider must not do. + if (!this.callbacks) return; + this.buffer.push(pcm); + } + + /** Endpoint: upload what was said and stream the transcript back. */ + async flush(): Promise { + if (!this.callbacks) return; + const pcm = this.buffer.toInt16(); + const durationMs = this.buffer.durationMs; + this.buffer.clear(); + + if (pcm.length < MIN_UTTERANCE_SAMPLES) return; + await this.transcribe(pcm, durationMs); + } + + async stop(): Promise { + this.inFlight?.abort(); + this.inFlight = null; + this.callbacks = null; + this.buffer.clear(); + } + + /** + * The text-in seam, for a client that did its own transcription. It lands on + * the same callbacks and, importantly, sends nothing: on-device dictation must + * not become an upload just because the configured provider is hosted. + */ + injectUtterance(text: string): void { + this.buffer.clear(); + const utterance = text.trim(); + this.callbacks?.onFinal(utterance, 1, utterance ? estimateSpokenDurationMs(utterance) : 0); + } + + // -- Internals ----------------------------------------------------------- + + private async transcribe(pcm: Int16Array, durationMs: number): Promise { + // A new utterance supersedes an upload still in flight: the user has moved + // on, and two transcripts racing for one turn is worse than losing the old. + this.inFlight?.abort(); + const controller = new AbortController(); + this.inFlight = controller; + + const form = new FormData(); + form.append('model', this.model); + form.append('response_format', 'json'); + form.append('stream', 'true'); + if (this.language) form.append('language', this.language); + // The cast is safe by construction: `encodeUtterance` allocates its own + // ArrayBuffer, so the widened `ArrayBufferLike` can never be a SharedArrayBuffer. + const wav = encodeUtterance(pcm); + form.append( + 'file', + new Blob([wav.buffer as ArrayBuffer], { type: 'audio/wav' }), + 'utterance.wav' + ); + + try { + const response = await hostedRequest({ + providerId: this.id, + service: 'openai', + url: TRANSCRIBE_URL, + init: { + method: 'POST', + headers: { + Authorization: `Bearer ${requireCredential(this.id, 'openai', this.readCredential)}`, + }, + body: form, + }, + timeoutMs: this.timeoutMs, + signal: controller.signal, + // One utterance, one upload. A retry would re-send audio the user has + // already moved past and could deliver a second transcript for a turn + // that is over. + retry: false, + fetchImpl: this.fetchImpl, + }); + + await this.consume(response, controller, durationMs); + } catch (error) { + if (controller.signal.aborted) return; + // Classified failures are the provider's own report; anything else is a + // bug and belongs in Sentry rather than in a spoken apology. + if (!(error instanceof VoiceProviderError)) throw error; + this.callbacks?.onError(error); + } finally { + if (this.inFlight === controller) this.inFlight = null; + } + } + + /** + * Read the server-sent stream, turning `delta` events into partials and the + * final `done` (or a plain JSON body) into the transcript. + */ + private async consume( + response: Response, + controller: AbortController, + durationMs: number + ): Promise { + const body = response.body; + if (!body) { + // Not every deployment honours `stream=true`. A plain JSON body is a + // complete transcript with no partials, which is a worse experience and a + // perfectly correct turn. + const payload = (await response.json().catch(() => null)) as { text?: string } | null; + this.emitFinal(payload?.text ?? '', durationMs); + return; + } + + let text = ''; + for await (const event of readServerSentEvents(body)) { + if (controller.signal.aborted) return; + const delta = typeof event.delta === 'string' ? event.delta : ''; + if (event.type === 'transcript.text.delta' && delta) { + text += delta; + // Stability rises with length: a hypothesis that has been building for a + // while is less likely to be rewritten than its first word. + this.callbacks?.onPartial(text, partialStability(text)); + continue; + } + if (event.type === 'transcript.text.done' && typeof event.text === 'string') { + text = event.text; + } + } + + this.emitFinal(text, durationMs); + } + + private emitFinal(text: string, durationMs: number): void { + const utterance = text.trim(); + if (!utterance) return; + // The endpoint reports no per-utterance confidence, and inventing one would + // give a client something to dim that means nothing. 1 is the honest value + // for "this provider does not tell us". + this.callbacks?.onFinal(utterance, 1, durationMs); + } +} + +// --------------------------------------------------------------------------- + +function encodeUtterance(pcm: Int16Array): Uint8Array { + const buffer = new PcmBuffer(); + buffer.push(pcm); + return buffer.toWav(); +} + +/** Longer hypotheses are firmer, capped short of certainty. */ +function partialStability(text: string): number { + return Math.min(0.9, 0.3 + text.length / 400); +} + +interface SseEvent { + type?: string; + delta?: string; + text?: string; +} + +/** + * Parse a `text/event-stream` body into its JSON payloads. + * + * Written out rather than pulled from a library because the whole grammar we + * need is "lines starting with `data: `, blank line ends an event", and a + * dependency for that would be a dependency in the audio path. + */ +export async function* readServerSentEvents( + body: ReadableStream +): AsyncGenerator { + const decoder = new TextDecoder(); + const reader = body.getReader(); + let pending = ''; + + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + pending += decoder.decode(value, { stream: true }); + + let newline: number; + while ((newline = pending.indexOf('\n')) !== -1) { + const line = pending.slice(0, newline).trim(); + pending = pending.slice(newline + 1); + if (!line.startsWith('data:')) continue; + + const payload = line.slice(5).trim(); + if (!payload || payload === '[DONE]') continue; + try { + yield JSON.parse(payload) as SseEvent; + } catch { + // A partial frame that split mid-JSON. Dropping one delta costs a + // partial nobody was going to read twice; throwing would lose the + // whole transcript. + } + } + } + } finally { + reader.releaseLock(); + } +} diff --git a/src/main/acappella/providers/local/kokoro-tts.ts b/src/main/acappella/providers/local/kokoro-tts.ts new file mode 100644 index 0000000000..fd7dd8f1bb --- /dev/null +++ b/src/main/acappella/providers/local/kokoro-tts.ts @@ -0,0 +1,289 @@ +/** + * Local text-to-speech on Kokoro, through ONNX Runtime. + * + * **Per sentence, and cancellable between sentences.** One inference run per + * sentence means the first words are audible while the rest of the reply is still + * being made, and `cancel()` has something small to interrupt. Barge-in that has + * to wait out a whole synthesised paragraph does not feel like barge-in. + * + * **`cancel()` cuts the run, it does not wait for it.** The generator checks the + * run token before every yield and after every await, so a cancelled run stops + * delivering audio immediately even though the ONNX call it was inside has to + * finish - ONNX Runtime has no mid-inference abort. The user hears silence at the + * moment they interrupted, which is the property that matters; one orphaned + * tensor is cheaper than a voice that talks over its interruption. + * + * ## The phoneme front end + * + * Kokoro takes PHONEME ids, not characters. Turning English text into phonemes is + * a grapheme-to-phoneme step (espeak-ng, or the misaki front end Kokoro ships + * with) and it is a real dependency, not a lookup table. This build does not have + * one yet, so `phonemize` is an injected seam with no default: without it the + * provider reports itself unavailable through the same classified path as a + * missing model, and Voice Setup says so. It does NOT approximate. A character + * level fallback would synthesise confident nonsense, and a voice reading nonsense + * aloud is a worse failure than a voice that says nothing and explains why. + */ + +import { KOKORO_82M_ID } from '../../../../shared/acappella/model-catalog'; +import { LOCAL_TTS_PROVIDER_ID } from '../../../../shared/acappella/provider-catalog'; +import { VoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import type { + TtsChunk, + TtsProvider, + TtsSpeakOptions, +} from '../../../../shared/acappella/providers'; +import { splitIntoSpokenSentences } from '../../../../shared/acappella/sentences'; +import { modelFilePath } from '../../models/model-store'; +import { float32ToInt16 } from '../pcm'; +import { loadLocalRuntime } from './runtime'; + +/** Catalog files this provider loads. */ +const MODEL_FILE = 'onnx/model.onnx'; + +/** + * The bundled voice. Kokoro voice packs are one file each, and only this one is + * in the catalog, so it is the only id that can resolve without another download. + */ +const BUNDLED_VOICE = { + id: 'af_heart', + name: 'Heart (American English)', + file: 'voices/af_heart.bin', +}; + +/** Kokoro outputs 24 kHz audio, whatever the capture path runs at. */ +const KOKORO_SAMPLE_RATE = 24_000; + +/** + * A voice pack is a table of style vectors, one row per token count, each 256 + * floats wide. The row is picked by the length of the phoneme sequence. + */ +const STYLE_VECTOR_WIDTH = 256; + +/** Kokoro's own bounds. Outside these the model produces artefacts, not speech. */ +const MIN_SPEED = 0.5; +const MAX_SPEED = 2; + +/** Turns text into the phoneme ids Kokoro's vocabulary uses. */ +export type Phonemizer = (text: string) => Promise | number[]; + +/** The ONNX Runtime surface this provider uses, structurally. */ +interface OnnxTensor { + data: Float32Array | BigInt64Array; + dims: readonly number[]; +} + +interface OnnxSession { + run(feeds: Record): Promise>; + release?(): Promise; +} + +interface OnnxModule { + InferenceSession: { + create(path: string, options?: Record): Promise; + }; + Tensor: new ( + type: string, + data: Float32Array | BigInt64Array, + dims: readonly number[] + ) => OnnxTensor; +} + +export interface KokoroTtsOptions { + /** Required for real synthesis. See the module comment. */ + phonemize?: Phonemizer; + modelPath?: string; + voicePackPath?: string; + /** Injected in tests; production goes through `native-loader.ts`. */ + loadRuntime?: typeof loadLocalRuntime; + /** Injected in tests. Production reads the installed voice pack from disk. */ + readVoicePack?: (path: string) => Promise; +} + +export class KokoroTtsProvider implements TtsProvider { + readonly id = LOCAL_TTS_PROVIDER_ID; + readonly label = 'Kokoro (local)'; + readonly tier = 'local' as const; + + private readonly phonemize?: Phonemizer; + private readonly modelPathOverride?: string; + private readonly voicePackPathOverride?: string; + private readonly loadRuntime: typeof loadLocalRuntime; + private readonly readVoicePack: (path: string) => Promise; + + /** Bumped by `cancel()` and by every new run, so a stale iterator returns. */ + private run = 0; + private session: OnnxSession | null = null; + private tensorFactory: OnnxModule['Tensor'] | null = null; + private voicePack: Float32Array | null = null; + /** In-flight load, so two sentences racing do not each open the model. */ + private loading: Promise | null = null; + + constructor(options: KokoroTtsOptions = {}) { + this.phonemize = options.phonemize; + this.modelPathOverride = options.modelPath; + this.voicePackPathOverride = options.voicePackPath; + this.loadRuntime = options.loadRuntime ?? loadLocalRuntime; + this.readVoicePack = options.readVoicePack ?? readVoicePackFromDisk; + } + + /** The voices this install can actually speak with, for the picker. */ + listVoices(): Array<{ id: string; name: string }> { + return [{ id: BUNDLED_VOICE.id, name: BUNDLED_VOICE.name }]; + } + + speak(text: string, options: TtsSpeakOptions): AsyncIterable { + // The run is claimed here, not in the generator body: a generator does not + // start until its first `next()`, and a second `speak()` must supersede the + // first immediately. + return this.stream(splitIntoSpokenSentences(text), ++this.run, options); + } + + cancel(): void { + this.run += 1; + } + + /** Release the session. Called when the pipeline is torn down or swapped. */ + async dispose(): Promise { + this.cancel(); + const session = this.session; + this.session = null; + this.voicePack = null; + this.loading = null; + try { + await session?.release?.(); + } catch { + // A session that will not close must not wedge a provider swap. + } + } + + // -- Internals ----------------------------------------------------------- + + private async *stream( + sentences: string[], + run: number, + options: TtsSpeakOptions + ): AsyncGenerator { + if (sentences.length === 0) return; + await this.ensureLoaded(); + if (this.run !== run) return; + + for (let index = 0; index < sentences.length; index++) { + if (this.run !== run) return; + + const audio = await this.synthesize(sentences[index], options.rate); + // Re-checked after the inference: a barge-in during synthesis must not + // deliver the sentence it interrupted. + if (this.run !== run) return; + + yield { + utteranceId: options.utteranceId, + index, + text: sentences[index], + format: 'pcm16', + audio, + sampleRate: KOKORO_SAMPLE_RATE, + }; + } + } + + private async ensureLoaded(): Promise { + if (this.session) return; + this.loading ??= this.load(); + try { + await this.loading; + } finally { + this.loading = null; + } + } + + private async load(): Promise { + if (!this.phonemize) { + throw new VoiceProviderError( + 'Local speech synthesis needs a phoneme front end, which is not part of this build yet. Switch Text-to-Speech to a hosted voice, or keep it on the mock until the front end ships.', + { kind: 'unavailable', providerId: this.id } + ); + } + + const module = await this.loadRuntime('onnx', this.id); + const modelPath = this.modelPathOverride ?? modelFilePath(KOKORO_82M_ID, MODEL_FILE); + const voicePath = + this.voicePackPathOverride ?? modelFilePath(KOKORO_82M_ID, BUNDLED_VOICE.file); + + try { + this.session = await module.InferenceSession.create(modelPath); + this.tensorFactory = module.Tensor; + this.voicePack = await this.readVoicePack(voicePath); + } catch (error) { + this.session = null; + throw new VoiceProviderError( + 'The Kokoro voice could not be opened. Re-verify it in Settings > Plugins > A Cappella > Models.', + { kind: 'unavailable', providerId: this.id, cause: error } + ); + } + } + + private async synthesize(sentence: string, rate?: number): Promise { + const session = this.session; + const Tensor = this.tensorFactory; + const voicePack = this.voicePack; + if (!session || !Tensor || !voicePack || !this.phonemize) { + throw new VoiceProviderError('The local voice is not loaded.', { + kind: 'unavailable', + providerId: this.id, + }); + } + + const tokens = await this.phonemize(sentence); + const style = styleVectorFor(voicePack, tokens.length); + const speed = Math.min(MAX_SPEED, Math.max(MIN_SPEED, rate && rate > 0 ? rate : 1)); + + const outputs = await session.run({ + // The leading and trailing zero are Kokoro's sequence boundary tokens. + // Without them the first phoneme is clipped and the last is held. + input_ids: new Tensor('int64', BigInt64Array.from([0n, ...tokens.map(BigInt), 0n]), [ + 1, + tokens.length + 2, + ]), + style: new Tensor('float32', style, [1, STYLE_VECTOR_WIDTH]), + speed: new Tensor('float32', Float32Array.from([speed]), [1]), + }); + + const waveform = Object.values(outputs)[0]?.data; + if (!(waveform instanceof Float32Array)) { + throw new VoiceProviderError('The local voice produced no audio for that sentence.', { + kind: 'unavailable', + providerId: this.id, + }); + } + + const pcm = float32ToInt16(waveform); + return new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength); + } +} + +// --------------------------------------------------------------------------- + +/** + * The style row for a phoneme sequence of this length. + * + * Clamped rather than trusted: a sentence longer than the pack has rows for would + * read past the end of the buffer and produce a vector of whatever followed it in + * memory, which comes out as a burst of noise at full volume in someone's + * headphones. + */ +export function styleVectorFor(pack: Float32Array, tokenCount: number): Float32Array { + const rows = Math.max(1, Math.floor(pack.length / STYLE_VECTOR_WIDTH)); + const row = Math.min(rows - 1, Math.max(0, tokenCount)); + const offset = row * STYLE_VECTOR_WIDTH; + return pack.slice(offset, offset + STYLE_VECTOR_WIDTH); +} + +/** Read a Kokoro voice pack: a flat little-endian float32 table. */ +async function readVoicePackFromDisk(path: string): Promise { + const { readFile } = await import('fs/promises'); + const buffer = await readFile(path); + return new Float32Array( + buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) + ); +} diff --git a/src/main/acappella/providers/local/llama-brain.ts b/src/main/acappella/providers/local/llama-brain.ts new file mode 100644 index 0000000000..72799a7456 --- /dev/null +++ b/src/main/acappella/providers/local/llama-brain.ts @@ -0,0 +1,278 @@ +/** + * Local Conductor Brain on Qwen3 1.7B, through node-llama-cpp. + * + * **The context stays loaded between turns.** Loading a 1.1 GB GGUF and building + * a context takes seconds; a routing decision takes a few hundred milliseconds. + * If the model were loaded per turn, every utterance would pay the load, and the + * feature would be unusable. So the model, the context, and the grammar are built + * once and held. + * + * **And it unloads when nobody is talking.** A gigabyte of resident RAM for a + * feature the user finished using twenty minutes ago is not acceptable either, so + * an idle timer frees everything after {@link DEFAULT_IDLE_UNLOAD_MS}. The next + * utterance pays the load again, once. Both halves of that trade are deliberate: + * warm within a conversation, cold between them. + * + * **Routing is grammar-constrained.** llama.cpp can be handed a GBNF grammar + * built from the shared `RouteDecision` JSON Schema, which makes the model + * structurally incapable of emitting a malformed decision. It is still run + * through `parseRouteDecision`, because a grammar guarantees a well-formed + * `sessionId` and not a real one, and the roster is the only thing that knows the + * difference. + */ + +import { QWEN3_1_7B_ID } from '../../../../shared/acappella/model-catalog'; +import { LOCAL_BRAIN_PROVIDER_ID } from '../../../../shared/acappella/provider-catalog'; +import { VoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import type { + BrainProvider, + VoiceConverseContext, + VoiceRouteContext, +} from '../../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; +import { ROUTE_DECISION_JSON_SCHEMA } from '../../../../shared/acappella/route-decision'; +import { modelFilePath } from '../../models/model-store'; +import { + buildConverseUserPrompt, + buildRouteUserPrompt, + converseSystemPrompt, + limitSpokenReply, + parseRouteDecision, + routeSystemPrompt, +} from '../brain-prompt'; +import { loadLocalRuntime } from './runtime'; + +/** The catalog file this provider loads. */ +const MODEL_FILE = 'Qwen3-1.7B-Q4_K_M.gguf'; + +/** + * How long the model stays resident after the last turn. + * + * Five minutes is longer than a pause in a conversation and shorter than a + * coffee break, which is the line this timer is trying to draw. + */ +export const DEFAULT_IDLE_UNLOAD_MS = 5 * 60_000; + +/** Small context: a roster and one utterance, never a codebase. */ +const CONTEXT_SIZE = 4096; + +const ROUTE_MAX_TOKENS = 400; +const CONVERSE_MAX_TOKENS = 300; + +/** The node-llama-cpp surface this provider uses, structurally. */ +interface LlamaGrammar { + readonly _grammar?: unknown; +} + +interface LlamaChatSessionInstance { + prompt(text: string, options?: Record): Promise; + dispose?(): Promise | void; +} + +interface LlamaContextInstance { + getSequence(): unknown; + dispose?(): Promise | void; +} + +interface LlamaModelInstance { + createContext(options?: Record): Promise; + dispose?(): Promise | void; +} + +interface LlamaInstance { + loadModel(options: { modelPath: string }): Promise; + createGrammarForJsonSchema(schema: unknown): Promise; +} + +interface LlamaModule { + getLlama(options?: Record): Promise; + LlamaChatSession: new (options: Record) => LlamaChatSessionInstance; +} + +/** Everything the loaded model holds, so unloading is one object to drop. */ +interface LoadedBrain { + llama: LlamaInstance; + model: LlamaModelInstance; + context: LlamaContextInstance; + ChatSession: LlamaModule['LlamaChatSession']; + routeGrammar: LlamaGrammar | null; +} + +export interface LlamaBrainOptions { + modelPath?: string; + /** Idle time before the model is freed. 0 keeps it loaded forever. */ + idleUnloadMs?: number; + /** Injected in tests; production goes through `native-loader.ts`. */ + loadRuntime?: typeof loadLocalRuntime; +} + +export class LlamaBrainProvider implements BrainProvider { + readonly id = LOCAL_BRAIN_PROVIDER_ID; + readonly label = 'Qwen3 1.7B (local)'; + readonly tier = 'local' as const; + + private readonly modelPathOverride?: string; + private readonly idleUnloadMs: number; + private readonly loadRuntime: typeof loadLocalRuntime; + + private loaded: LoadedBrain | null = null; + /** In-flight load, so two turns racing do not each open the model. */ + private loading: Promise | null = null; + private idleTimer: ReturnType | null = null; + + constructor(options: LlamaBrainOptions = {}) { + this.modelPathOverride = options.modelPath; + this.idleUnloadMs = options.idleUnloadMs ?? DEFAULT_IDLE_UNLOAD_MS; + this.loadRuntime = options.loadRuntime ?? loadLocalRuntime; + } + + async route(input: string, context: VoiceRouteContext): Promise { + const brain = await this.ensureLoaded(); + const raw = await this.prompt( + brain, + routeSystemPrompt(), + buildRouteUserPrompt(input, context), + ROUTE_MAX_TOKENS, + brain.routeGrammar + ); + return parseRouteDecision(raw, context, input); + } + + async converse(agentText: string, context: VoiceConverseContext): Promise { + const brain = await this.ensureLoaded(); + const raw = await this.prompt( + brain, + converseSystemPrompt(), + buildConverseUserPrompt(agentText, context), + CONVERSE_MAX_TOKENS, + null + ); + return limitSpokenReply(raw, context.maxSentences); + } + + /** Free the model now. Called on pipeline teardown and by the idle timer. */ + async unload(): Promise { + this.clearIdleTimer(); + const brain = this.loaded; + this.loaded = null; + if (!brain) return; + + // Innermost first: disposing a model out from under a live context is how + // llama.cpp gets a use-after-free instead of a clean shutdown. + await safeDispose(() => brain.context.dispose?.()); + await safeDispose(() => brain.model.dispose?.()); + } + + /** True while the model is resident. Read by the metrics panel and by tests. */ + get isLoaded(): boolean { + return this.loaded !== null; + } + + // -- Internals ----------------------------------------------------------- + + private async ensureLoaded(): Promise { + this.clearIdleTimer(); + if (this.loaded) return this.loaded; + + this.loading ??= this.load(); + try { + this.loaded = await this.loading; + return this.loaded; + } finally { + this.loading = null; + } + } + + private async load(): Promise { + const module = await this.loadRuntime('llama', this.id); + const modelPath = this.modelPathOverride ?? modelFilePath(QWEN3_1_7B_ID, MODEL_FILE); + + try { + const llama = await module.getLlama(); + const model = await llama.loadModel({ modelPath }); + const context = await model.createContext({ contextSize: CONTEXT_SIZE }); + // Built once, reused per turn: compiling a grammar is not free and the + // schema never changes. + const routeGrammar = await llama + .createGrammarForJsonSchema(ROUTE_DECISION_JSON_SCHEMA) + // A build of llama.cpp without JSON-schema grammars still routes; it + // just relies on the prompt and the parser instead of being unable to + // emit bad JSON. Losing the guarantee is worth more than losing the slot. + .catch(() => null); + + return { llama, model, context, ChatSession: module.LlamaChatSession, routeGrammar }; + } catch (error) { + throw new VoiceProviderError( + 'The local Conductor Brain model could not be opened. Re-verify it in Settings > Plugins > A Cappella > Models.', + { kind: 'unavailable', providerId: this.id, cause: error } + ); + } + } + + /** + * One turn against the loaded context. + * + * A fresh chat session per call, on the same context: routing has no memory + * between utterances (the roster and the recent utterances are in the prompt), + * and a session that accumulated history would grow until it evicted the + * system prompt. + */ + private async prompt( + brain: LoadedBrain, + systemPrompt: string, + userPrompt: string, + maxTokens: number, + grammar: LlamaGrammar | null + ): Promise { + const session = new brain.ChatSession({ + contextSequence: brain.context.getSequence(), + systemPrompt, + }); + + try { + return await session.prompt(userPrompt, { + // Deterministic, for the same reason the hosted Brains are: a misroute + // that cannot be reproduced cannot be fixed. + temperature: 0, + maxTokens, + ...(grammar ? { grammar } : {}), + }); + } catch (error) { + throw new VoiceProviderError( + `The local Conductor Brain failed on this turn: ${(error as Error).message}`, + { kind: 'unavailable', providerId: this.id, cause: error } + ); + } finally { + await safeDispose(() => session.dispose?.()); + // Restarted after every turn, not before: the clock should run from the + // last thing the user said, not from the start of a slow inference. + this.scheduleIdleUnload(); + } + } + + private scheduleIdleUnload(): void { + this.clearIdleTimer(); + if (this.idleUnloadMs <= 0) return; + this.idleTimer = setTimeout(() => { + this.idleTimer = null; + void this.unload(); + }, this.idleUnloadMs); + // The timer must never be the reason the app stays alive. + this.idleTimer.unref?.(); + } + + private clearIdleTimer(): void { + if (!this.idleTimer) return; + clearTimeout(this.idleTimer); + this.idleTimer = null; + } +} + +/** Teardown must not throw: it runs from `finally` blocks and from disposal. */ +async function safeDispose(dispose: () => Promise | void | undefined): Promise { + try { + await dispose(); + } catch { + /* best-effort */ + } +} diff --git a/src/main/acappella/providers/local/runtime.ts b/src/main/acappella/providers/local/runtime.ts new file mode 100644 index 0000000000..cc27b56d30 --- /dev/null +++ b/src/main/acappella/providers/local/runtime.ts @@ -0,0 +1,42 @@ +/** + * The one way a local provider reaches a native runtime. + * + * `native-loader.ts` already owns the dynamic imports, the classification, and + * the memory of what has failed. What it does NOT do is speak the providers' + * language: it returns a `NativeRuntimeUnavailable`, and a provider needs a + * `VoiceProviderError` so the session can announce the failure with the right + * code and the right recovery. + * + * That translation is three lines and it was about to be written three times, in + * three providers, each with slightly different wording for the same event. Doing + * it once also keeps one property true by construction: **a local provider that + * cannot load its runtime fails as itself and never reaches for another + * provider.** There is no branch in here that could. + * + * The failure stays remembered inside the loader, which is what the capability + * gate reads to explain the blocked slot afterwards. + */ + +import type { NativeRuntimeId } from '../../../../shared/acappella/native-runtimes'; +import { VoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import { tryLoadNativeRuntime } from '../../runtime/native-loader'; + +/** + * Load a native runtime for a provider, or throw a classified provider failure. + * + * @param runtimeId The runtime, as named in `shared/acappella/native-runtimes.ts`. + * @param providerId The provider asking, so the error can name the engine. + */ +export async function loadLocalRuntime( + runtimeId: NativeRuntimeId, + providerId: string +): Promise { + const result = await tryLoadNativeRuntime(runtimeId); + if (result.ok) return result.module; + + throw new VoiceProviderError(result.error.message, { + kind: 'unavailable', + providerId, + cause: result.error, + }); +} diff --git a/src/main/acappella/providers/local/whisper-stt.ts b/src/main/acappella/providers/local/whisper-stt.ts new file mode 100644 index 0000000000..d6f0d76caf --- /dev/null +++ b/src/main/acappella/providers/local/whisper-stt.ts @@ -0,0 +1,247 @@ +/** + * Local speech-to-text on whisper.cpp. + * + * **Chunked, not truly streaming.** whisper.cpp transcribes a buffer, not a + * stream: there is no incremental decoder to feed. The standard way to get live + * text out of it, and the one used here, is to re-transcribe the utterance so far + * on a cadence and publish the result as a partial. Words near the start stop + * changing between passes (that is what "stabilise" means here) while the tail + * keeps being revised, which is exactly what a partial transcript is supposed to + * look like. The final pass runs on endpointing and is the only one whose text is + * dispatched. + * + * **One decode at a time.** A pass takes longer than the interval on a slow + * machine, so a second pass starting while the first is running would queue + * decodes until the process fell over. Partials are SKIPPED while busy rather + * than queued: a partial that arrives late is worthless, and the next pass will + * cover the same audio anyway. + * + * **Nothing loads until a session starts.** The model is opened on `start()` + * through `native-loader.ts` and freed on `stop()`. A 148 MB model resident for + * the life of an app whose voice feature is off is exactly the cost the lazy + * loader exists to avoid, and the failure to load reaches the user through the + * capability gate rather than as a dlopen string. + */ + +import { ACAPPELLA_AUDIO_SAMPLE_RATE } from '../../../../shared/acappella/audio-host'; +import { WHISPER_BASE_EN_ID } from '../../../../shared/acappella/model-catalog'; +import { LOCAL_STT_PROVIDER_ID } from '../../../../shared/acappella/provider-catalog'; +import { VoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import type { SttCallbacks, SttProvider } from '../../../../shared/acappella/providers'; +import { estimateSpokenDurationMs } from '../../../../shared/acappella/sentences'; +import { modelFilePath } from '../../models/model-store'; +import { loadLocalRuntime } from './runtime'; +import { PcmBuffer } from '../pcm'; + +/** The catalog file this provider loads. */ +const MODEL_FILE = 'ggml-base.en.bin'; + +/** + * Audio accumulated between partial passes. Under half a second the re-decode + * costs more than the extra word is worth; over about a second and a half the + * transcript visibly lags the speaker. + */ +const DEFAULT_PARTIAL_INTERVAL_MS = 900; + +/** Rising across the passes of one utterance, the way a hypothesis firms up. */ +const FIRST_PARTIAL_STABILITY = 0.3; +const PARTIAL_STABILITY_STEP = 0.15; +const MAX_PARTIAL_STABILITY = 0.9; + +/** + * whisper.cpp is a local decode with no confidence to report. 0.95 rather than 1 + * says "a recogniser produced this" without claiming certainty the model never + * expressed. + */ +const LOCAL_FINAL_CONFIDENCE = 0.95; + +/** The `smart-whisper` surface this provider uses, structurally. */ +interface WhisperSegment { + text: string; +} + +interface WhisperTask { + result: Promise; +} + +interface WhisperInstance { + transcribe( + pcm: Float32Array, + params?: Record + ): Promise | WhisperTask; + free(): Promise | void; +} + +interface WhisperModule { + Whisper: new (modelPath: string, options?: Record) => WhisperInstance; +} + +export interface WhisperSttOptions { + partialIntervalMs?: number; + /** Absolute path override. Defaults to the installed catalog model. */ + modelPath?: string; + /** Whether to ask whisper.cpp for GPU offload. */ + gpu?: boolean; + /** Injected in tests; production goes through `native-loader.ts`. */ + loadRuntime?: typeof loadLocalRuntime; +} + +export class WhisperSttProvider implements SttProvider { + readonly id = LOCAL_STT_PROVIDER_ID; + readonly label = 'Whisper (local)'; + readonly tier = 'local' as const; + readonly sampleRate = ACAPPELLA_AUDIO_SAMPLE_RATE; + readonly acceptsAudio = true; + + private readonly partialIntervalMs: number; + private readonly modelPathOverride?: string; + private readonly gpu: boolean; + private readonly loadRuntime: typeof loadLocalRuntime; + + private callbacks: SttCallbacks | null = null; + private whisper: WhisperInstance | null = null; + private buffer = new PcmBuffer(); + + /** Audio duration at the last partial pass, so the cadence is in AUDIO time. */ + private lastPartialAtMs = 0; + private partialsInUtterance = 0; + private decoding = false; + + constructor(options: WhisperSttOptions = {}) { + this.partialIntervalMs = Math.max(0, options.partialIntervalMs ?? DEFAULT_PARTIAL_INTERVAL_MS); + this.modelPathOverride = options.modelPath; + this.gpu = options.gpu ?? true; + this.loadRuntime = options.loadRuntime ?? loadLocalRuntime; + } + + async start(callbacks: SttCallbacks): Promise { + const module = await this.loadRuntime('whisper', this.id); + const modelPath = this.modelPathOverride ?? modelFilePath(WHISPER_BASE_EN_ID, MODEL_FILE); + + try { + this.whisper = new module.Whisper(modelPath, { gpu: this.gpu }); + } catch (error) { + // The runtime loaded and the model did not. Distinct from a runtime + // failure, and its recovery is the model page rather than a bug report. + throw new VoiceProviderError( + 'The Whisper model could not be opened. Re-verify it in Settings > Plugins > A Cappella > Models.', + { kind: 'unavailable', providerId: this.id, cause: error } + ); + } + + this.callbacks = callbacks; + this.resetUtterance(); + } + + feed(pcm: Int16Array): void { + if (!this.callbacks) return; + this.buffer.push(pcm); + + if (this.partialIntervalMs <= 0 || this.decoding) return; + if (this.buffer.durationMs - this.lastPartialAtMs < this.partialIntervalMs) return; + + this.lastPartialAtMs = this.buffer.durationMs; + // Not awaited: `feed` runs 50 times a second on the frame path and must stay + // synchronous. A rejected pass is reported through the callbacks. + void this.decode('partial'); + } + + /** Endpoint: decode everything buffered and publish it as the transcript. */ + async flush(): Promise { + if (!this.callbacks) return; + if (this.buffer.length === 0) return; + await this.decode('final'); + } + + async stop(): Promise { + this.callbacks = null; + this.buffer.clear(); + + const whisper = this.whisper; + this.whisper = null; + try { + await whisper?.free(); + } catch { + // A model that will not close cleanly must not wedge session teardown. + // The process is about to drop the handle either way. + } + } + + /** + * The text-in seam, so the dev harness and a client that did its own + * transcription land on the same callbacks with no decode at all. + */ + injectUtterance(text: string): void { + this.buffer.clear(); + this.resetUtterance(); + const utterance = text.trim(); + this.callbacks?.onFinal(utterance, 1, utterance ? estimateSpokenDurationMs(utterance) : 0); + } + + // -- Internals ----------------------------------------------------------- + + private async decode(kind: 'partial' | 'final'): Promise { + const whisper = this.whisper; + const callbacks = this.callbacks; + if (!whisper || !callbacks) return; + + const samples = this.buffer.toFloat32(); + const durationMs = this.buffer.durationMs; + if (samples.length === 0) return; + + this.decoding = true; + try { + const task = await whisper.transcribe(samples, { + language: 'en', + // Suppresses whisper.cpp's "[BLANK_AUDIO]" style annotations, which are + // not words and would be routed as if they were. + suppress_non_speech_tokens: true, + }); + const segments = await task.result; + // The session may have ended, or the utterance been superseded, while the + // decode ran. Publishing now would put an old transcript on a new turn. + if (this.callbacks !== callbacks) return; + + const text = segments + .map((segment) => segment.text) + .join(' ') + .replace(/\s+/g, ' ') + .trim(); + + if (kind === 'final') { + this.buffer.clear(); + this.resetUtterance(); + if (text) callbacks.onFinal(text, LOCAL_FINAL_CONFIDENCE, durationMs); + return; + } + + if (!text) return; + this.partialsInUtterance += 1; + callbacks.onPartial(text, this.partialStability()); + } catch (error) { + // A decode failure is classified rather than thrown: it arrives from a + // frame callback with no caller, and the session has a path for a named + // provider failure but not for a rejected promise from nowhere. + callbacks.onError( + new VoiceProviderError( + `Whisper could not transcribe this utterance: ${(error as Error).message}`, + { kind: 'unavailable', providerId: this.id, cause: error } + ) + ); + } finally { + this.decoding = false; + } + } + + private partialStability(): number { + return Math.min( + MAX_PARTIAL_STABILITY, + FIRST_PARTIAL_STABILITY + PARTIAL_STABILITY_STEP * (this.partialsInUtterance - 1) + ); + } + + private resetUtterance(): void { + this.lastPartialAtMs = 0; + this.partialsInUtterance = 0; + } +} diff --git a/src/main/acappella/providers/mock/index.ts b/src/main/acappella/providers/mock/index.ts new file mode 100644 index 0000000000..6e13250bab --- /dev/null +++ b/src/main/acappella/providers/mock/index.ts @@ -0,0 +1,32 @@ +/** + * The mock provider tier: the trio that makes A Cappella runnable with no + * model, no device, and no network. It is also the fallback the registry falls + * back TO, never away from, which is why it has to stay dependency-free. + */ + +import type { VoiceProviderTrio } from '../../../../shared/acappella/providers'; +import { MockBrainProvider } from './mock-brain'; +import { MockSttProvider } from './mock-stt'; +import { MockTtsProvider } from './mock-tts'; +import type { MockSttOptions } from './mock-stt'; +import type { MockTtsOptions } from './mock-tts'; + +export { MockBrainProvider } from './mock-brain'; +export { MockSttProvider } from './mock-stt'; +export { MockTtsProvider } from './mock-tts'; +export type { MockSttOptions } from './mock-stt'; +export type { MockTtsOptions } from './mock-tts'; + +export interface MockProviderOptions { + stt?: MockSttOptions; + tts?: MockTtsOptions; +} + +/** A fresh mock trio. Providers hold per-session state, so never share one. */ +export function createMockProviderTrio(options: MockProviderOptions = {}): VoiceProviderTrio { + return { + stt: new MockSttProvider(options.stt), + tts: new MockTtsProvider(options.tts), + brain: new MockBrainProvider(), + }; +} diff --git a/src/main/acappella/providers/mock/mock-brain.ts b/src/main/acappella/providers/mock/mock-brain.ts new file mode 100644 index 0000000000..1ae6380eab --- /dev/null +++ b/src/main/acappella/providers/mock/mock-brain.ts @@ -0,0 +1,310 @@ +/** + * Mock Brain: deterministic keyword routing over the real agent roster. + * + * Every decision here comes from string matching, never from a model, so the + * whole pipeline runs offline and a test can assert an exact `RouteDecision`. + * The rules are deliberately dumb and deliberately few: Phase 07 replaces this + * with a grammar-constrained local model that emits the same shape, and a + * cleverer mock would only make that swap harder to trust. + */ + +import type { RosterAgent, RosterTab } from '../../../../shared/acappella/protocol'; +import type { + BrainProvider, + VoiceConverseContext, + VoiceRouteContext, +} from '../../../../shared/acappella/providers'; +import type { RouteDecision, RouteTabAction } from '../../../../shared/acappella/route-decision'; +import { splitIntoSpokenSentences } from '../../../../shared/acappella/sentences'; +import { stripMarkdown } from '../../../../shared/markdown'; +import { escapeRegExp } from '../../../../shared/stringUtils'; + +/** Cue words for "put this somewhere I already had open". Checked before `new`. */ +const RECALL_CUES = [ + 'back to', + 'go back', + 'going back', + 'return to', + 'earlier', + 'that tab', + 'the one about', + 'resume', + 'previously', +]; + +/** Cue words for "give me a clean slate". */ +const NEW_CUES = ['new', 'start over', 'fresh', 'from scratch', 'another tab']; + +/** An agent named "a" or "go" would match every utterance, so short names are ignored. */ +const MIN_AGENT_NAME_LENGTH = 3; + +/** Words too generic to prove a tab is the one being recalled. */ +const RECALL_STOPWORDS = new Set(['the', 'and', 'for', 'with', 'tab', 'new', 'about', 'that']); + +const BASE_CONFIDENCE = 0.5; +const AGENT_MATCH_BONUS = 0.25; +/** Falling back to the bound agent is weaker evidence than hearing its name. */ +const SCOPE_MATCH_BONUS = 0.1; +const CUE_MATCH_BONUS = 0.15; +/** Keyword matching is never certain, so the mock never claims to be. */ +const MAX_CONFIDENCE = 0.95; + +/** Leading chatter stripped off a prompt, applied repeatedly until it stops shrinking. */ +const PROMPT_PREAMBLE_PATTERNS: RegExp[] = [ + /^(?:hey|ok|okay|yo|maestro)\b[\s,]*/i, + /^(?:please|can you|could you|i want you to|i'd like you to|go ahead and)\b[\s,]*/i, + /^(?:open|start|create|make|spin up|fire up|kick off)\b\s*(?:a|an|the)?\s*(?:new|fresh|another)?\s*(?:ai\s+)?(?:tab|session|chat|conversation|thread)\b[\s,]*/i, + /^(?:switch|go|jump|head|take me)\s+(?:back\s+)?(?:to|over to)\b[\s,]*/i, + /^(?:back to|return to|resume|start over)\b[\s,]*/i, + /^(?:the\s+)?(?:tab|session|chat|conversation|thread)\b[\s,]*/i, + /^(?:on|in|with|for|about|and|then|to)\b[\s,]*/i, +]; + +/** How many preamble passes before giving up. Bounded so a pattern cycle cannot spin. */ +const MAX_PREAMBLE_PASSES = 8; + +/** Words a tab name never opens with. */ +const TAB_NAME_STOPWORDS = new Set([ + 'the', + 'a', + 'an', + 'about', + 'my', + 'our', + 'this', + 'that', + 'some', + 'please', + 'and', +]); + +const TAB_NAME_MAX_WORDS = 4; +const TAB_NAME_MAX_LENGTH = 32; + +/** A spoken sentence longer than this is cut at a word boundary. */ +const SPOKEN_SENTENCE_MAX_LENGTH = 140; + +/** Spoken replies stay short unless the caller asks for more. */ +const DEFAULT_SPOKEN_SENTENCES = 2; + +export class MockBrainProvider implements BrainProvider { + readonly id = 'mock-brain'; + readonly label = 'Mock (keyword routing)'; + readonly tier = 'mock' as const; + + async route(input: string, context: VoiceRouteContext): Promise { + const normalized = normalize(input); + const named = matchAgentByName(normalized, context.roster); + const scope = context.scope; + const scoped = + scope.kind === 'agent' + ? (context.roster.find((agent) => agent.sessionId === scope.sessionId) ?? null) + : null; + + // A name in the utterance beats the binding: "ask backend about X" said to + // an agent-scoped session means backend, not the agent on screen. + const agent = named ?? scoped; + + const cued = detectTabAction(normalized); + let tabAction: RouteTabAction = cued ?? 'current'; + let tabId: string | undefined; + + if (tabAction === 'recall') { + const tab = agent ? pickRecallTab(agent, normalized) : null; + // Nothing to go back to. Downgrading beats emitting a `recall` with no + // `tabId`, which the executor could only fail on. + if (tab) tabId = tab.id; + else tabAction = 'current'; + } + + const prompt = cleanPrompt(input, named?.name) || input.trim(); + const tabName = tabAction === 'new' ? deriveTabName(prompt) : undefined; + + let confidence = BASE_CONFIDENCE; + if (named) confidence += AGENT_MATCH_BONUS; + else if (scoped) confidence += SCOPE_MATCH_BONUS; + if (cued) confidence += CUE_MATCH_BONUS; + + return { + target: agent ? { sessionId: agent.sessionId } : 'conductor', + tabAction, + tabId, + tabName, + prompt, + confidence: Math.min(MAX_CONFIDENCE, Math.round(confidence * 100) / 100), + }; + } + + /** + * Reshape an agent's terminal-shaped answer for the ear: markdown out, first + * couple of sentences only, each short enough to be interrupted. + */ + async converse(agentText: string, context: VoiceConverseContext): Promise { + const plain = stripMarkdown(agentText).replace(/\s+/g, ' ').trim(); + if (!plain) return ''; + + const limit = context.maxSentences ?? DEFAULT_SPOKEN_SENTENCES; + return splitIntoSpokenSentences(plain) + .slice(0, Math.max(1, limit)) + .map(shortenSentence) + .join(' '); + } +} + +// --------------------------------------------------------------------------- +// Matching helpers +// --------------------------------------------------------------------------- + +/** Lowercase, punctuation to spaces, single-spaced. Both sides of every match. */ +function normalize(text: string): string { + return text + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, ' ') + .trim(); +} + +/** + * Whole-word containment on two normalized strings. Padding with spaces is + * enough because normalization already removed everything a word boundary + * would have had to guard against. + */ +function containsPhrase(haystack: string, phrase: string): boolean { + return ` ${haystack} `.includes(` ${phrase} `); +} + +/** + * Drop the conductor address off the front of an utterance. + * + * "hey maestro" is how you address the conductor, not how you name an agent, + * but an agent called "Maestro" is extremely common - it is what you call the + * agent working on Maestro itself. Without this, "hey maestro, ask Scratch to + * check the disk usage" matched the word `maestro` and every routed sentence + * landed on that one agent. Only the LEADING address is removed, so "ask + * Maestro about the release" still means that agent. + */ +function stripConductorAddress(normalizedInput: string): string { + return normalizedInput.replace(/^(?:hey |ok |okay |yo )?maestro\b\s*/, '').trim(); +} + +/** The longest agent name mentioned, so "backend api" wins over "backend". */ +function matchAgentByName(input: string, roster: RosterAgent[]): RosterAgent | null { + const normalizedInput = stripConductorAddress(input); + let best: RosterAgent | null = null; + let bestLength = 0; + + for (const agent of roster) { + const name = normalize(agent.name); + if (name.length < MIN_AGENT_NAME_LENGTH) continue; + if (!containsPhrase(normalizedInput, name)) continue; + if (name.length > bestLength) { + best = agent; + bestLength = name.length; + } + } + + return best; +} + +/** The cued tab action, or null when the utterance said nothing about tabs. */ +function detectTabAction(normalizedInput: string): RouteTabAction | null { + if (RECALL_CUES.some((cue) => containsPhrase(normalizedInput, cue))) return 'recall'; + if (NEW_CUES.some((cue) => containsPhrase(normalizedInput, cue))) return 'new'; + return null; +} + +/** Best tab-name overlap with the utterance, falling back to the most recent tab. */ +function pickRecallTab(agent: RosterAgent, normalizedInput: string): RosterTab | null { + let best: RosterTab | null = null; + let bestScore = -1; + + for (const tab of agent.tabs) { + const score = tabNameOverlap(tab, normalizedInput); + if (score > bestScore) { + best = tab; + bestScore = score; + continue; + } + if (score === bestScore && (tab.lastActiveAt ?? 0) > (best?.lastActiveAt ?? 0)) { + best = tab; + } + } + + return best; +} + +/** How many distinctive words of a tab's name the utterance repeated. */ +function tabNameOverlap(tab: RosterTab, normalizedInput: string): number { + if (!tab.name) return 0; + const words = normalize(tab.name) + .split(' ') + .filter((word) => word.length >= MIN_AGENT_NAME_LENGTH && !RECALL_STOPWORDS.has(word)); + return words.filter((word) => containsPhrase(normalizedInput, word)).length; +} + +// --------------------------------------------------------------------------- +// Prompt and tab name +// --------------------------------------------------------------------------- + +/** + * Strip the routing chatter so the agent receives the request rather than the + * sentence that steered it. The agent name is dropped wherever it appears, + * which optimizes for the common "on the backend agent" phrasing and can nick a + * word out of an unusual one. That is the accepted cost of a keyword mock. + */ +function cleanPrompt(input: string, agentName?: string): string { + let text = input.trim(); + + if (agentName) { + const pattern = new RegExp( + `(?:^|\\s)(?:(?:on|with|for|to|in|at|from)\\s+)?(?:the\\s+)?${escapeRegExp(agentName)}(?:\\s+agent)?(?=$|[\\s,.!?])`, + 'i' + ); + text = text.replace(pattern, ' ').trim(); + } + + for (let pass = 0; pass < MAX_PREAMBLE_PASSES; pass++) { + const next = PROMPT_PREAMBLE_PATTERNS.reduce( + (current, pattern) => current.replace(pattern, ''), + text + ).trim(); + if (next === text) break; + text = next; + } + + return text.replace(/\s+/g, ' ').trim(); +} + +/** A short title-cased name for a new tab, or undefined when nothing survives. */ +function deriveTabName(prompt: string): string | undefined { + const words = prompt + .replace(/[^\p{L}\p{N}\s-]/gu, ' ') + .split(/\s+/) + .filter(Boolean); + while (words.length > 0 && TAB_NAME_STOPWORDS.has(words[0].toLowerCase())) words.shift(); + if (words.length === 0) return undefined; + + const name = words + .slice(0, TAB_NAME_MAX_WORDS) + // Only the first letter is touched: "OAuth" must not become "Oauth". + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + + return name.length > TAB_NAME_MAX_LENGTH ? name.slice(0, TAB_NAME_MAX_LENGTH).trimEnd() : name; +} + +/** + * Cut a long sentence at a word boundary. The result keeps terminal punctuation + * so re-splitting it yields the same sentence count the session already + * announced in `speak-start`. + */ +function shortenSentence(sentence: string): string { + const trimmed = sentence.trim(); + if (trimmed.length <= SPOKEN_SENTENCE_MAX_LENGTH) { + return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`; + } + + const window = trimmed.slice(0, SPOKEN_SENTENCE_MAX_LENGTH); + const cut = window.lastIndexOf(' '); + const head = (cut > 0 ? window.slice(0, cut) : window).replace(/[\s,;:]+$/, ''); + return `${head}.`; +} diff --git a/src/main/acappella/providers/mock/mock-stt.ts b/src/main/acappella/providers/mock/mock-stt.ts new file mode 100644 index 0000000000..81c2b80a6a --- /dev/null +++ b/src/main/acappella/providers/mock/mock-stt.ts @@ -0,0 +1,137 @@ +/** + * Mock speech-to-text: text in, partials and a final out. + * + * There is no audio path here on purpose. The mock tier exists so the whole + * voice pipeline runs with no model download, no device, and no network, which + * makes it the tier every test and the dev harness drive. Because it lands on + * the same `SttCallbacks` a real recognizer does, nothing downstream can tell a + * typed utterance from a spoken one. + */ + +import type { SttCallbacks, SttProvider } from '../../../../shared/acappella/providers'; +import { estimateSpokenDurationMs } from '../../../../shared/acappella/sentences'; + +/** Gap between the two synthetic partials and the final. 0 emits synchronously. */ +const DEFAULT_PARTIAL_DELAY_MS = 90; + +/** Rising stability across the two partials, matching what a real recognizer reports. */ +const FIRST_PARTIAL_STABILITY = 0.35; +const SECOND_PARTIAL_STABILITY = 0.7; + +/** Typed text is not a guess, so the mock reports no transcription doubt. */ +const MOCK_FINAL_CONFIDENCE = 1; + +export interface MockSttOptions { + /** + * Delay between the injected partials and the final. The default makes the + * live-transcript UI visibly stream; tests pass 0 to stay synchronous. + */ + partialDelayMs?: number; +} + +export class MockSttProvider implements SttProvider { + readonly id = 'mock-stt'; + readonly label = 'Mock (typed input)'; + readonly tier = 'mock' as const; + /** What a real recognizer would want. Nothing here consumes audio. */ + readonly sampleRate = 16_000; + /** + * Text in, transcript out. Declaring this keeps the audio path from opening a + * microphone for a provider that would drop every frame: the mock tier's whole + * promise is no model, no network, and no device. + */ + readonly acceptsAudio = false; + + private callbacks: SttCallbacks | null = null; + private readonly partialDelayMs: number; + private readonly timers = new Set>(); + + constructor(options: MockSttOptions = {}) { + this.partialDelayMs = options.partialDelayMs ?? DEFAULT_PARTIAL_DELAY_MS; + } + + async start(callbacks: SttCallbacks): Promise { + this.callbacks = callbacks; + } + + feed(_pcm: Int16Array): void { + // Dropped deliberately. Inventing a transcript from audio this provider + // cannot hear would emit words nobody said. + } + + async flush(): Promise { + // Endpointing is meaningless without an audio stream: `injectUtterance()` + // already delivers a complete utterance. + } + + async stop(): Promise { + this.clearTimers(); + this.callbacks = null; + } + + /** + * Treat `text` as an already-final transcript, preceded by two synthetic + * partials so the live-transcript UI has something to render. + * + * A second call supersedes the first: pending emissions from the previous + * utterance are dropped rather than interleaved with the new one. + */ + injectUtterance(text: string): void { + this.clearTimers(); + if (!this.callbacks) return; + + const utterance = text.trim(); + if (!utterance) { + // The session service has its own empty-utterance path (it hands the + // floor straight back); partials for nothing would be noise. + this.emit((callbacks) => callbacks.onFinal('', MOCK_FINAL_CONFIDENCE, 0)); + return; + } + + const [first, second] = partialPrefixes(utterance); + this.schedule(1, (callbacks) => callbacks.onPartial(first, FIRST_PARTIAL_STABILITY)); + this.schedule(2, (callbacks) => callbacks.onPartial(second, SECOND_PARTIAL_STABILITY)); + this.schedule(3, (callbacks) => + callbacks.onFinal(utterance, MOCK_FINAL_CONFIDENCE, estimateSpokenDurationMs(utterance)) + ); + } + + // -- Internals ----------------------------------------------------------- + + /** Run `emit` after `step` delay slots, or immediately when there is no delay. */ + private schedule(step: number, emit: (callbacks: SttCallbacks) => void): void { + if (this.partialDelayMs <= 0) { + this.emit(emit); + return; + } + + const timer = setTimeout(() => { + this.timers.delete(timer); + this.emit(emit); + }, this.partialDelayMs * step); + this.timers.add(timer); + } + + /** Callbacks can outlive `stop()`, so every emission re-checks the session. */ + private emit(emit: (callbacks: SttCallbacks) => void): void { + if (!this.callbacks) return; + emit(this.callbacks); + } + + private clearTimers(): void { + for (const timer of this.timers) clearTimeout(timer); + this.timers.clear(); + } +} + +/** + * Growing prefixes at word boundaries, roughly a third and two thirds in. Short + * utterances still get two partials: a client counting them must not have to + * special-case a one-word sentence. + */ +function partialPrefixes(utterance: string): [string, string] { + const words = utterance.split(/\s+/); + const firstCount = Math.max(1, Math.ceil(words.length / 3)); + const secondCount = Math.max(firstCount, Math.ceil((words.length * 2) / 3)); + return [words.slice(0, firstCount).join(' '), words.slice(0, secondCount).join(' ')]; +} diff --git a/src/main/acappella/providers/mock/mock-tts.ts b/src/main/acappella/providers/mock/mock-tts.ts new file mode 100644 index 0000000000..8415baa463 --- /dev/null +++ b/src/main/acappella/providers/mock/mock-tts.ts @@ -0,0 +1,125 @@ +/** + * Mock text-to-speech: sentence events on a timer, no audio. + * + * `format: 'none'` with `audio: null` is a legal `TtsChunk`, which is what lets + * the mock tier drive the real speaking path: the HUD renders one sentence at a + * time, `speak-end` fires, and barge-in cuts the run off, all without a voice + * model. The sentence boundaries come from the shared splitter and nowhere else, + * because the session already announced `sentenceCount` from that same splitter + * and a second opinion would leave a client's "3 of 5" stuck forever. + */ + +import type { + TtsChunk, + TtsProvider, + TtsSpeakOptions, +} from '../../../../shared/acappella/providers'; +import { splitIntoSpokenSentences } from '../../../../shared/acappella/sentences'; + +/** Roughly a natural reading pace, so the HUD looks like something is speaking. */ +const DEFAULT_MS_PER_CHARACTER = 32; +const DEFAULT_MIN_SENTENCE_MS = 220; +/** Ceiling so one long sentence cannot stall a demo. */ +const DEFAULT_MAX_SENTENCE_MS = 2_500; + +export interface MockTtsOptions { + /** Simulated speech rate. Tests pass 0 to emit every sentence immediately. */ + msPerCharacter?: number; + minSentenceMs?: number; + maxSentenceMs?: number; +} + +/** A sleep that `cancel()` can cut short, so barge-in does not wait out a sentence. */ +interface PendingSleep { + wake: () => void; +} + +export class MockTtsProvider implements TtsProvider { + readonly id = 'mock-tts'; + readonly label = 'Mock (silent)'; + readonly tier = 'mock' as const; + + private readonly msPerCharacter: number; + private readonly minSentenceMs: number; + private readonly maxSentenceMs: number; + + /** Bumped by `cancel()` and by every new run, so a stale iterator returns. */ + private run = 0; + private pending: PendingSleep | null = null; + + constructor(options: MockTtsOptions = {}) { + this.msPerCharacter = options.msPerCharacter ?? DEFAULT_MS_PER_CHARACTER; + this.minSentenceMs = options.minSentenceMs ?? DEFAULT_MIN_SENTENCE_MS; + this.maxSentenceMs = options.maxSentenceMs ?? DEFAULT_MAX_SENTENCE_MS; + } + + speak(text: string, options: TtsSpeakOptions): AsyncIterable { + // The run is claimed here, not inside the generator: a generator body does + // not run until the first `next()`, and a second `speak()` has to supersede + // the first one immediately. + return this.stream(splitIntoSpokenSentences(text), ++this.run, options); + } + + /** Barge-in. Wakes the in-flight sentence delay so the iterator ends now. */ + cancel(): void { + this.run += 1; + const pending = this.pending; + this.pending = null; + pending?.wake(); + } + + // -- Internals ----------------------------------------------------------- + + private async *stream( + sentences: string[], + run: number, + options: TtsSpeakOptions + ): AsyncGenerator { + for (let index = 0; index < sentences.length; index++) { + // Checked before every sentence: a run cancelled mid-sentence must not + // deliver the one that follows it. + if (this.run !== run) return; + + const sentence = sentences[index]; + yield { + utteranceId: options.utteranceId, + index, + text: sentence, + format: 'none', + audio: null, + }; + + // The delay follows the sentence because a real voice starts speaking as + // soon as the sentence begins: the wait IS the speech. + await this.sleep(this.sentenceDurationMs(sentence, options.rate)); + } + } + + private sentenceDurationMs(sentence: string, rate?: number): number { + // A zero rate per character means "no simulated speech time at all", which + // is what tests want; the floor below must not put the delay back. + if (this.msPerCharacter <= 0) return 0; + + const speed = rate && rate > 0 ? rate : 1; + const raw = (sentence.length * this.msPerCharacter) / speed; + return Math.min(this.maxSentenceMs, Math.max(this.minSentenceMs, Math.round(raw))); + } + + private sleep(ms: number): Promise { + if (ms <= 0) return Promise.resolve(); + + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.pending = null; + resolve(); + }, ms); + + this.pending = { + wake: () => { + clearTimeout(timer); + resolve(); + }, + }; + }); + } +} diff --git a/src/main/acappella/providers/pcm.ts b/src/main/acappella/providers/pcm.ts new file mode 100644 index 0000000000..fba2ae1e92 --- /dev/null +++ b/src/main/acappella/providers/pcm.ts @@ -0,0 +1,175 @@ +/** + * PCM plumbing shared by the providers. + * + * The capture path produces 16 kHz mono `Int16Array` frames (see + * `src/shared/acappella/audio-host.ts`), and two providers need that same audio + * in a different wrapper: a hosted STT wants an uploadable container, and a local + * recogniser wants one contiguous float buffer. Both conversions are three lines + * of arithmetic that are wrong in an interesting way if you get the endianness or + * the divisor off by one, so they live here once with the reason attached. + */ + +import { ACAPPELLA_AUDIO_SAMPLE_RATE } from '../../../shared/acappella/audio-host'; + +/** Bytes of a canonical 44-byte PCM WAV header. */ +const WAV_HEADER_BYTES = 44; + +/** + * Accumulates capture frames for a provider that transcribes an utterance rather + * than a stream. + * + * Bounded on purpose: a microphone left open by a forgotten session must not grow + * a buffer until the process dies. Past the cap the OLDEST audio is dropped, + * because for speech recognition the end of an utterance is the part that matters + * and the alternative (dropping the newest) would silently truncate what the user + * just said. + */ +export class PcmBuffer { + private readonly chunks: Int16Array[] = []; + private samples = 0; + + constructor( + private readonly maxSamples: number = ACAPPELLA_AUDIO_SAMPLE_RATE * 60, + readonly sampleRate: number = ACAPPELLA_AUDIO_SAMPLE_RATE + ) {} + + get length(): number { + return this.samples; + } + + get durationMs(): number { + return Math.round((this.samples / this.sampleRate) * 1000); + } + + push(pcm: Int16Array): void { + if (pcm.length === 0) return; + // Copied, not retained: the capture path reuses its frame buffers, so + // keeping the reference would hand the transcriber whatever audio happened + // to be in that slot later. + this.chunks.push(Int16Array.from(pcm)); + this.samples += pcm.length; + this.trim(); + } + + clear(): void { + this.chunks.length = 0; + this.samples = 0; + } + + /** Everything buffered, as one contiguous buffer. Does not clear. */ + toInt16(): Int16Array { + const out = new Int16Array(this.samples); + let offset = 0; + for (const chunk of this.chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; + } + + /** Everything buffered as normalised floats, which is what whisper.cpp takes. */ + toFloat32(): Float32Array { + return int16ToFloat32(this.toInt16()); + } + + /** Everything buffered as an uploadable WAV. Does not clear. */ + toWav(): Uint8Array { + return encodeWav(this.toInt16(), this.sampleRate); + } + + private trim(): void { + while (this.samples > this.maxSamples && this.chunks.length > 1) { + const dropped = this.chunks.shift(); + this.samples -= dropped?.length ?? 0; + } + } +} + +/** + * Wrap 16-bit mono samples in a WAV container. + * + * A container rather than raw PCM because every hosted transcription endpoint + * takes a file and infers the format from it; posting bare samples means also + * posting a sample rate in a side channel that half of them ignore. + */ +export function encodeWav(pcm: Int16Array, sampleRate = ACAPPELLA_AUDIO_SAMPLE_RATE): Uint8Array { + const dataBytes = pcm.length * 2; + const buffer = new ArrayBuffer(WAV_HEADER_BYTES + dataBytes); + const view = new DataView(buffer); + + writeAscii(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataBytes, true); + writeAscii(view, 8, 'WAVE'); + writeAscii(view, 12, 'fmt '); + view.setUint32(16, 16, true); // PCM header length + view.setUint16(20, 1, true); // format: PCM + view.setUint16(22, 1, true); // channels: mono + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); // byte rate: rate * blockAlign + view.setUint16(32, 2, true); // block align: 1 channel * 16 bit + view.setUint16(34, 16, true); // bits per sample + writeAscii(view, 36, 'data'); + view.setUint32(40, dataBytes, true); + + // Little-endian explicitly. `new Uint8Array(pcm.buffer)` would inherit the + // host's endianness, which is right on every machine Maestro ships for and + // wrong in a way nobody would find until it was. + for (let i = 0; i < pcm.length; i++) { + view.setInt16(WAV_HEADER_BYTES + i * 2, pcm[i], true); + } + + return new Uint8Array(buffer); +} + +/** 16-bit samples to the -1..1 floats every inference runtime expects. */ +export function int16ToFloat32(pcm: Int16Array): Float32Array { + const out = new Float32Array(pcm.length); + for (let i = 0; i < pcm.length; i++) { + // 32768 for negatives and 32767 for positives is the pedantically correct + // pair; using one divisor for both is standard and keeps the waveform + // symmetric, which matters more than the half-LSB. + out[i] = pcm[i] / 32768; + } + return out; +} + +/** + * Linear resample between two rates. + * + * Linear interpolation, not a windowed filter, and that is a deliberate ceiling: + * this exists for the 16 kHz capture path feeding a service that insists on + * 24 kHz, where the content is speech that was band-limited at 8 kHz before it + * ever got here. There is nothing above the old Nyquist for a better kernel to + * preserve, and a proper resampler in the per-frame hot path would cost more than + * it could possibly recover. + */ +export function resampleLinear(pcm: Int16Array, fromRate: number, toRate: number): Int16Array { + if (fromRate === toRate || pcm.length === 0) return pcm; + + const ratio = toRate / fromRate; + const out = new Int16Array(Math.max(1, Math.round(pcm.length * ratio))); + + for (let i = 0; i < out.length; i++) { + const position = i / ratio; + const left = Math.floor(position); + const right = Math.min(pcm.length - 1, left + 1); + const fraction = position - left; + out[i] = Math.round(pcm[left] * (1 - fraction) + pcm[right] * fraction); + } + + return out; +} + +/** Floats back to 16-bit samples, clamped. The TTS side of the same conversion. */ +export function float32ToInt16(samples: Float32Array): Int16Array { + const out = new Int16Array(samples.length); + for (let i = 0; i < samples.length; i++) { + const clamped = samples[i] < -1 ? -1 : samples[i] > 1 ? 1 : samples[i]; + out[i] = Math.round(clamped * 32767); + } + return out; +} + +function writeAscii(view: DataView, offset: number, text: string): void { + for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i)); +} diff --git a/src/main/acappella/providers/provider-registry.ts b/src/main/acappella/providers/provider-registry.ts new file mode 100644 index 0000000000..1429f45c7f --- /dev/null +++ b/src/main/acappella/providers/provider-registry.ts @@ -0,0 +1,695 @@ +/** + * A Cappella provider registry - the one place that decides what a session runs + * on. + * + * This is the ONLY module allowed to import a concrete provider. Everything + * downstream takes a `VoicePipeline` at construction, so if resolution lived + * anywhere else the rules below would be decided by an import rather than by a + * policy. + * + * The rules, in full: + * + * 1. **A slot resolves to what was asked for, or to nothing.** There is no + * fallback engine. Not to the cloud, which would spend the user's money and + * send their microphone somewhere they did not choose; and not to the mock + * either, because a session that transcribes nothing while looking healthy + * hides the reason. An unbuildable slot becomes an `Unresolved*` provider + * that refuses BY NAME the first time it is used (see `./unresolved.ts`). + * 2. **The mock tier is selected, never substituted.** It exists for tests and + * the dev harness, and it is what an unconfigured install runs on purpose. + * Nothing ever lands on it because something else was missing. + * 3. **Exactly two pipeline shapes.** `CascadePipeline` (three independent + * engines) and `RealtimePipeline` (one fused speech-to-speech adapter). The + * choice is made here, once, and nothing downstream branches on it. + * 4. **A swap is refused mid-utterance.** Changing providers while a turn is in + * flight would splice two engines into one exchange: a sentence transcribed + * by Whisper, routed by a model that never heard it, spoken in a different + * voice. The swap waits for the floor. + * + * The STT slot defaults to the microphone check in EVERY build. It consumes real + * PCM and reports the speech it heard without transcribing it. That default used + * to be the mock in a packaged app, which opened no device at all, so a user with + * no configuration had a session that said "Listening" and could not possibly + * hear them - and no way to tell that apart from a broken microphone. + */ + +import type { BackgroundAnnouncementSetting } from '../../../shared/acappella/announcements'; +import type { ProviderSlotState } from '../../../shared/acappella/protocol'; +import { + clampSendHoldMs, + clampTtsVolume, + clampTurnSettleMs, + parseSendPhrases, +} from '../../../shared/acappella/voice-controls'; +import { + summariseVoiceEgress, + OPENAI_REALTIME_PROVIDER_ID, +} from '../../../shared/acappella/provider-catalog'; +import type { + BrainProvider, + SttProvider, + TtsProvider, + VoicePipeline, + VoicePipelineShape, + VoiceProviderRole, + VoiceProviderSubstitution, + VoiceProviderSubstitutionReason, + VoiceProviderTier, + VoiceProviderTrio, +} from '../../../shared/acappella/providers'; +import { logger } from '../../utils/logger'; +import { CascadePipeline } from './cascade-pipeline'; +import { ECHO_STT_PROVIDER_ID, EchoSttProvider } from './echo-stt'; +import { AnthropicBrainProvider } from './hosted/anthropic-brain'; +import { ElevenLabsTtsProvider } from './hosted/elevenlabs-tts'; +import { OpenAiBrainProvider } from './hosted/openai-brain'; +import { OpenAiSttProvider } from './hosted/openai-stt'; +import { KokoroTtsProvider } from './local/kokoro-tts'; +import { LlamaBrainProvider } from './local/llama-brain'; +import { WhisperSttProvider } from './local/whisper-stt'; +import { MockBrainProvider, MockSttProvider, MockTtsProvider } from './mock'; +import type { MockProviderOptions } from './mock'; +import { createRealtimePipeline } from './realtime/realtime-session'; +import { + UnresolvedBrainProvider, + UnresolvedSttProvider, + UnresolvedTtsProvider, + unresolvedMessage, + type UnresolvedReason, +} from './unresolved'; + +const LOG_CONTEXT = 'ACappella'; + +/** Settings key holding everything A Cappella persists. */ +export const ACAPPELLA_SETTINGS_KEY = 'acappella'; + +// Re-exported so existing importers keep their one import site. The shapes +// themselves live in shared/ because they travel to the renderer and, later, to +// the phone. +export type { VoiceProviderRole, VoiceProviderSubstitution, VoiceProviderSubstitutionReason }; + +/** The provider type each role resolves to. */ +interface VoiceProviderByRole { + stt: SttProvider; + tts: TtsProvider; + brain: BrainProvider; +} + +/** What the user picked in Voice Setup. Absent means "whatever is default". */ +export interface VoiceProviderSettings { + stt?: string; + tts?: string; + brain?: string; + /** Cascade unless the user opted into the realtime tier. */ + pipeline?: VoicePipelineShape; + /** Which realtime provider, when the shape is `realtime`. */ + realtime?: string; + /** Voice id for the TTS slot, when its provider offers a choice. */ + voiceId?: string; + /** Speech rate. 1 is the provider's natural pace. */ + rate?: number; + /** Output volume for the assistant's voice, 0 to 1. */ + volume?: number; + /** + * Whether an agent finishing in the background is spoken about. + * + * Defaults to `auto`, which is on for the Conductor scope and off inside a + * focused agent session. See `src/shared/acappella/announcements.ts`. + */ + speakBackgroundCompletions?: BackgroundAnnouncementSetting; + /** + * Silence after a sentence before it counts as a finished thought, in ms. + * + * On top of the recogniser's own endpointing, not instead of it. Zero + * dispatches every fragment the moment it endpoints, which is what the session + * did before `speech/utterance-composer.ts` existed. + */ + turnSettleMs?: number; + /** + * Whether the Conductor may talk with the user instead of dispatching every + * utterance. See `router/conversation-buffer.ts`. + */ + conversationalMode?: boolean; + /** Whether a request waits for a spoken send phrase (or the long pause). */ + holdUntilSend?: boolean; + /** The pause that sends a held request when no phrase was said. */ + sendHoldMs?: number; + /** Phrases that mean "send it". Undefined takes the built-in set. */ + sendPhrases?: string[]; + /** + * Which microphone to open. Undefined follows the system default. + * + * Deliberately NOT part of {@link pipelineKey}: changing the input device is + * not a provider change, and treating it as one would tear down and rebuild + * a loaded Whisper model to swap a headset. + */ + inputDeviceId?: string; +} + +export interface VoiceProviderRegistration { + role: R; + /** Stable id, the same one that travels in `listen-start` / `speak-start`. */ + id: string; + label: string; + tier: VoiceProviderTier; + /** + * False when the provider is registered but cannot run in this build at all + * (a development-only provider in a packaged app). It is deliberately NOT the + * place to check for a downloaded model or a stored key: that is the capability + * gate's job, and answering it here would mean two subsystems deciding + * readiness with two different answers. + */ + isAvailable?: () => boolean; + create: (options: VoiceProviderCreateOptions) => VoiceProviderByRole[R]; +} + +/** What a factory is told about the configuration it is being built for. */ +export interface VoiceProviderCreateOptions { + settings: VoiceProviderSettings; + /** Timing overrides for the mock tier, so tests can run without timers. */ + mock?: MockProviderOptions; +} + +export interface VoiceProviderResolution { + pipeline: VoicePipeline; + shape: VoicePipelineShape; + /** The trio the session service is handed. Same object for a realtime shape. */ + providers: VoiceProviderTrio; + /** Empty on the happy path. Anything here belongs in front of the user. */ + substitutions: VoiceProviderSubstitution[]; + /** What each role actually resolved to, for the HUD and for `get-state`. */ + resolvedIds: Record; +} + +/** The mock tier's ids. Selected explicitly; never a fallback. */ +export const MOCK_PROVIDER_IDS: Record = { + stt: 'mock-stt', + tts: 'mock-tts', + brain: 'mock-brain', +}; + +/** + * What a role resolves to when the user has picked nothing. + * + * Identical to {@link MOCK_PROVIDER_IDS} except for STT, which defaults to the + * microphone check. That default is deliberate: out of the box the one thing a + * user needs to establish is that their microphone reaches the app, and a + * default that cannot hear makes that impossible to tell from a broken device. + */ +export const DEFAULT_PROVIDER_IDS: Record = { + stt: ECHO_STT_PROVIDER_ID, + tts: MOCK_PROVIDER_IDS.tts, + brain: MOCK_PROVIDER_IDS.brain, +}; + +// --------------------------------------------------------------------------- +// Catalog +// --------------------------------------------------------------------------- + +const catalog: { [R in VoiceProviderRole]: Map> } = { + stt: new Map(), + tts: new Map(), + brain: new Map(), +}; + +/** + * Add a provider to the catalog. Exported so a later phase can register one + * without this file learning about it. + */ +export function registerVoiceProvider( + registration: VoiceProviderRegistration +): void { + catalog[registration.role].set(registration.id, registration); +} + +/** Everything selectable for a role, for the settings panel to list. */ +export function listVoiceProviders( + role: VoiceProviderRole +): Array & { available: boolean }> { + return [...catalog[role].values()].map((entry) => ({ + id: entry.id, + label: entry.label, + tier: entry.tier, + available: entry.isAvailable?.() ?? true, + })); +} + +// -- The mock tier ---------------------------------------------------------- + +registerVoiceProvider({ + role: 'stt', + id: MOCK_PROVIDER_IDS.stt, + label: 'Mock (typed input)', + tier: 'mock', + create: ({ mock }) => new MockSttProvider(mock?.stt), +}); + +registerVoiceProvider({ + role: 'stt', + id: ECHO_STT_PROVIDER_ID, + // Named for what it is FOR, not for the build it came from. It answers one + // question - "is my microphone reaching Maestro at all?" - and the label has + // to say it transcribes nothing, because a row called "Echo" that produces no + // words reads as a broken recogniser rather than a working meter. + label: 'Microphone check (no transcription)', + tier: 'mock', + // Available in EVERY build, deliberately. It used to be development-only, and + // the result was a packaged app with no provider that consumes audio at all: + // the microphone was never opened, the HUD said "Listening", and there was no + // way to tell a dead device from a missing model from a wrong input. This is + // the one provider that can answer that, so it ships. + create: () => new EchoSttProvider(), +}); + +registerVoiceProvider({ + role: 'tts', + id: MOCK_PROVIDER_IDS.tts, + label: 'Mock (silent)', + tier: 'mock', + create: ({ mock }) => new MockTtsProvider(mock?.tts), +}); + +registerVoiceProvider({ + role: 'brain', + id: MOCK_PROVIDER_IDS.brain, + label: 'Mock (keyword routing)', + tier: 'mock', + create: () => new MockBrainProvider(), +}); + +// -- The local tier --------------------------------------------------------- + +registerVoiceProvider({ + role: 'stt', + id: 'whisper-local', + label: 'Whisper (local)', + tier: 'local', + create: () => new WhisperSttProvider(), +}); + +registerVoiceProvider({ + role: 'tts', + id: 'kokoro-local', + label: 'Kokoro (local)', + tier: 'local', + // No voice id is threaded through: Kokoro ships exactly one voice pack in the + // model catalog, and handing it an id chosen for ElevenLabs would point it at + // a pack that was never downloaded. + create: () => new KokoroTtsProvider(), +}); + +registerVoiceProvider({ + role: 'brain', + id: 'qwen3-local', + label: 'Qwen3 1.7B (local)', + tier: 'local', + create: () => new LlamaBrainProvider(), +}); + +// -- The hosted tier -------------------------------------------------------- + +registerVoiceProvider({ + role: 'stt', + id: 'openai-stt', + label: 'OpenAI (hosted)', + tier: 'cloud', + create: () => new OpenAiSttProvider(), +}); + +registerVoiceProvider({ + role: 'tts', + id: 'elevenlabs-tts', + label: 'ElevenLabs (hosted)', + tier: 'cloud', + create: ({ settings }) => new ElevenLabsTtsProvider({ voiceId: settings.voiceId }), +}); + +registerVoiceProvider({ + role: 'brain', + id: 'openai-brain', + label: 'OpenAI (hosted)', + tier: 'cloud', + create: () => new OpenAiBrainProvider(), +}); + +registerVoiceProvider({ + role: 'brain', + id: 'anthropic-brain', + label: 'Anthropic (hosted)', + tier: 'cloud', + create: () => new AnthropicBrainProvider(), +}); + +// --------------------------------------------------------------------------- +// Resolution +// --------------------------------------------------------------------------- + +export interface ResolveVoiceProvidersOptions { + /** Provider ids from settings. Omitted roles take {@link DEFAULT_PROVIDER_IDS}. */ + settings?: VoiceProviderSettings; + /** Timing overrides for the mock tier, so tests can run without timers. */ + mock?: MockProviderOptions; +} + +/** + * Build the pipeline a session will run on, plus anything the user has to be + * told about it. + */ +export function resolveVoicePipeline( + options: ResolveVoiceProvidersOptions = {} +): VoiceProviderResolution { + const settings = options.settings ?? {}; + + if (settings.pipeline === 'realtime') return resolveRealtime(settings); + return resolveCascade(settings, options.mock); +} + +/** + * Backwards-compatible entry point: the trio only. + * + * Kept because the audio bridge and the tests care about the providers and not + * about the pipeline that owns them, and a second call site rebuilding the whole + * pipeline to read `.providers` would construct two of everything. + */ +export function resolveVoiceProviders(options: ResolveVoiceProvidersOptions = {}): { + providers: VoiceProviderTrio; + substitutions: VoiceProviderSubstitution[]; + resolvedIds: Record; +} { + const { providers, substitutions, resolvedIds } = resolveVoicePipeline(options); + return { providers, substitutions, resolvedIds }; +} + +function resolveCascade( + settings: VoiceProviderSettings, + mock: MockProviderOptions | undefined +): VoiceProviderResolution { + const substitutions: VoiceProviderSubstitution[] = []; + const createOptions: VoiceProviderCreateOptions = { settings, mock }; + + const stt = resolveRole('stt', settings.stt, substitutions, createOptions); + const tts = resolveRole('tts', settings.tts, substitutions, createOptions); + const brain = resolveRole('brain', settings.brain, substitutions, createOptions); + + const providers: VoiceProviderTrio = { + stt: stt.provider, + tts: tts.provider, + brain: brain.provider, + }; + + return { + pipeline: new CascadePipeline(providers), + shape: 'cascade', + providers, + substitutions, + resolvedIds: { stt: stt.id, tts: tts.id, brain: brain.id }, + }; +} + +/** + * The realtime shape. One adapter fills all three slots, so there is nothing to + * resolve per role and nothing that could be substituted per role either. + */ +function resolveRealtime(settings: VoiceProviderSettings): VoiceProviderResolution { + const requestedId = settings.realtime ?? OPENAI_REALTIME_PROVIDER_ID; + + if (requestedId !== OPENAI_REALTIME_PROVIDER_ID) { + // Exactly one realtime provider exists. An unknown one becomes three + // refusing slots rather than quietly becoming the one that does exist. + const substitutions: VoiceProviderSubstitution[] = []; + const providers: VoiceProviderTrio = { + stt: unresolved('stt', requestedId, 'unknown-provider', substitutions).provider, + tts: unresolved('tts', requestedId, 'unknown-provider', substitutions).provider, + brain: unresolved('brain', requestedId, 'unknown-provider', substitutions).provider, + }; + return { + pipeline: new CascadePipeline(providers), + shape: 'cascade', + providers, + substitutions, + resolvedIds: { stt: providers.stt.id, tts: providers.tts.id, brain: providers.brain.id }, + }; + } + + const pipeline = createRealtimePipeline({ voice: settings.voiceId }); + return { + pipeline, + shape: 'realtime', + providers: pipeline.providers, + substitutions: [], + resolvedIds: { + stt: OPENAI_REALTIME_PROVIDER_ID, + tts: OPENAI_REALTIME_PROVIDER_ID, + brain: OPENAI_REALTIME_PROVIDER_ID, + }, + }; +} + +function resolveRole( + role: R, + requestedId: string | undefined, + substitutions: VoiceProviderSubstitution[], + createOptions: VoiceProviderCreateOptions +): { id: string; provider: VoiceProviderByRole[R] } { + // No selection is the documented default, not a substitution: A Cappella ships + // on the mock tier until the user picks something, with the one per-build + // exception in DEFAULT_PROVIDER_IDS. + const selectedId = requestedId ?? DEFAULT_PROVIDER_IDS[role]; + const registration = catalog[role].get(selectedId) as VoiceProviderRegistration | undefined; + + if (!registration) { + return unresolved(role, selectedId, 'unknown-provider', substitutions); + } + + if (registration.isAvailable && !registration.isAvailable()) { + // A DEFAULT this build cannot run falls back to the mock rather than + // refusing something nobody chose - but it is REPORTED, which it did not + // used to be. "Nobody asked for it, so say nothing" is how a packaged build + // ended up on a text-only recogniser that opens no device: the session read + // "Listening", the microphone was never touched, and the one fact that + // explained it existed only in this function. Rule 2 above says the mock is + // selected and never substituted; landing here IS a substitution, so it + // travels like one. + if (!requestedId) { + const fallback = catalog[role].get(MOCK_PROVIDER_IDS[role]) as VoiceProviderRegistration; + const message = `${role.toUpperCase()}: '${selectedId}' cannot run in this build, so this slot fell back to '${fallback.id}'.`; + logger.warn(message, LOG_CONTEXT); + substitutions.push({ + role, + requestedId: selectedId, + resolvedId: fallback.id, + reason: 'unavailable', + message, + }); + return { id: fallback.id, provider: fallback.create(createOptions) }; + } + return unresolved(role, selectedId, 'unavailable', substitutions); + } + + return { id: registration.id, provider: registration.create(createOptions) }; +} + +/** + * Build the refusing provider for a slot and record why. + * + * The only path that does not construct what was asked for, and it deliberately + * constructs nothing that WORKS: there is no lookup here that could land on + * another engine. + */ +function unresolved( + role: R, + requestedId: string, + reason: UnresolvedReason, + substitutions: VoiceProviderSubstitution[] +): { id: string; provider: VoiceProviderByRole[R] } { + const message = unresolvedMessage(role, requestedId, reason); + logger.warn(message, LOG_CONTEXT); + + const provider = (role === 'stt' + ? new UnresolvedSttProvider(requestedId, reason) + : role === 'tts' + ? new UnresolvedTtsProvider(requestedId, reason) + : new UnresolvedBrainProvider(requestedId, reason)) as unknown as VoiceProviderByRole[R]; // cannot narrow a generic role parameter through it. // The ternary provably picks the right class for each `role`, but TypeScript + + substitutions.push({ + role, + requestedId, + resolvedId: provider.id, + reason, + message, + }); + + return { id: provider.id, provider }; +} + +// --------------------------------------------------------------------------- +// Settings +// --------------------------------------------------------------------------- + +/** Read the persisted provider selection. Anything malformed reads as unset. */ +export function readVoiceProviderSettings(store: { + get: (key: string, defaultValue: unknown) => unknown; +}): VoiceProviderSettings { + const stored = store.get(ACAPPELLA_SETTINGS_KEY, {}) as + | { + providers?: unknown; + pipeline?: unknown; + voice?: unknown; + speech?: unknown; + audio?: unknown; + controls?: unknown; + } + | undefined; + const providers = (stored?.providers ?? {}) as Record; + const voice = (stored?.voice ?? {}) as Record; + const speech = (stored?.speech ?? {}) as Record; + const audio = (stored?.audio ?? {}) as Record; + // The conversation's timing lives with the other floor controls (hold + // threshold, idle timeout), which is the blob the Voice Controls panel writes. + const controls = (stored?.controls ?? {}) as Record; + + return { + speakBackgroundCompletions: asAnnouncementSetting(speech.speakBackgroundCompletions), + // Clamped rather than passed through: this becomes a timer in front of every + // dispatch, and a hand-edited settings file holding a NaN or a 600000 would + // be a voice assistant that never answers. + turnSettleMs: clampTurnSettleMs(controls.turnSettleMs), + conversationalMode: controls.conversationalMode === true, + holdUntilSend: controls.holdUntilSend === true, + sendHoldMs: clampSendHoldMs(controls.sendHoldMs), + sendPhrases: parseSendPhrases(controls.sendPhrases), + inputDeviceId: asProviderId(audio.inputDeviceId), + stt: asProviderId(providers.stt), + tts: asProviderId(providers.tts), + brain: asProviderId(providers.brain), + pipeline: stored?.pipeline === 'realtime' ? 'realtime' : 'cascade', + realtime: asProviderId(providers.realtime), + voiceId: asProviderId(voice.voiceId), + rate: typeof voice.rate === 'number' && voice.rate > 0 ? voice.rate : undefined, + // Clamped rather than passed through: this number becomes a gain on a live + // output node, and a stored NaN or a 40 from a hand-edited settings file + // would be a burst of distortion in the user's headphones. + volume: clampTtsVolume(voice.volume), + }; +} + +/** + * A stable identity for a selection, so a caller can tell whether the live + * pipeline still matches settings without rebuilding it to find out. + */ +export function pipelineKey(settings: VoiceProviderSettings): string { + return [ + settings.pipeline ?? 'cascade', + settings.realtime ?? '', + settings.stt ?? '', + settings.tts ?? '', + settings.brain ?? '', + settings.voiceId ?? '', + settings.rate ?? '', + ].join('|'); +} + +function asProviderId(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + +/** Anything unrecognised reads as unset, which resolves to the scope default. */ +function asAnnouncementSetting(value: unknown): BackgroundAnnouncementSetting | undefined { + return value === 'on' || value === 'off' || value === 'auto' ? value : undefined; +} + +// --------------------------------------------------------------------------- +// Provider state +// --------------------------------------------------------------------------- + +/** The `provider-state` event body for a resolution. */ +export function buildProviderState(resolution: VoiceProviderResolution): { + pipeline: VoicePipelineShape; + slots: ProviderSlotState[]; + egressStatement: string; + audioLeavesMachine: boolean; +} { + const roles: VoiceProviderRole[] = ['stt', 'tts', 'brain']; + const substitutionByRole = new Map(resolution.substitutions.map((entry) => [entry.role, entry])); + + const slots: ProviderSlotState[] = roles.map((role) => { + const provider = resolution.providers[role]; + return { + role, + providerId: provider.id, + label: provider.label, + tier: provider.tier, + substitutedFor: substitutionByRole.get(role)?.requestedId, + // Read off the resolved provider rather than inferred from its id or + // tier: whether a recogniser consumes PCM is its own declaration, and a + // list of "ids that hear" here would drift the first time one is added. + hearsAudio: role === 'stt' ? resolution.providers.stt.acceptsAudio : undefined, + }; + }); + + // Computed from what RESOLVED, not from what was configured: a slot that fell + // through to a refusing provider sends nothing anywhere, and saying otherwise + // would be the one sentence in this feature that must never be wrong. + const egress = summariseVoiceEgress(slots.map((slot) => slot.providerId)); + + return { + pipeline: resolution.shape, + slots, + egressStatement: egress.statement, + audioLeavesMachine: egress.audioLeaves, + }; +} + +// --------------------------------------------------------------------------- +// Hot swap +// --------------------------------------------------------------------------- + +export type PipelineSwapStatus = 'swapped' | 'unchanged' | 'refused'; + +export interface PipelineSwapResult { + status: PipelineSwapStatus; + /** Present when the swap happened. */ + resolution?: VoiceProviderResolution; + /** Present when refused, written for the user. */ + reason?: string; +} + +export interface PipelineSwapRequest { + settings: VoiceProviderSettings; + /** The live pipeline and the key it was built from, or null on first build. */ + current: { pipeline: VoicePipeline; key: string } | null; + /** + * True while a turn is in flight. A swap is refused rather than queued: the + * user is mid-sentence, and the honest answer is "not now", not a silent change + * of voice halfway through a reply. + */ + isBusy: boolean; + mock?: MockProviderOptions; +} + +/** + * Apply a settings change to the live pipeline. + * + * Tears the old one down BEFORE returning the new one, so two llama contexts are + * never resident at the same time; a swap on a machine that could only just fit + * one model would otherwise fail by running out of memory rather than by saying + * no. + */ +export async function swapVoicePipeline(request: PipelineSwapRequest): Promise { + const key = pipelineKey(request.settings); + if (request.current && request.current.key === key) return { status: 'unchanged' }; + + if (request.isBusy) { + return { + status: 'refused', + reason: + 'Voice providers cannot change in the middle of a turn. Finish speaking, then try again.', + }; + } + + await request.current?.pipeline.dispose(); + return { + status: 'swapped', + resolution: resolveVoicePipeline({ settings: request.settings, mock: request.mock }), + }; +} diff --git a/src/main/acappella/providers/realtime/realtime-session.ts b/src/main/acappella/providers/realtime/realtime-session.ts new file mode 100644 index 0000000000..71a2a0a24b --- /dev/null +++ b/src/main/acappella/providers/realtime/realtime-session.ts @@ -0,0 +1,690 @@ +/** + * The realtime speech-to-speech tier. + * + * The cascade runs three engines in series: speech to text, text to a decision + * and a rewrite, text back to speech. Each hop has its own round trip, and the + * sum is what makes a hands-free assistant feel like a form submission. A + * provider's realtime API collapses all three into one bidirectional socket with + * the model's own endpointing and interruption, which is worth roughly a second + * of turn latency. + * + * What it costs, stated here because it is stated in the settings copy too: the + * assistant speaks in that provider's voice, and the microphone's samples go to + * their servers. Neither is true of the local cascade, which is why realtime is + * an opt-in and never a fallback. + * + * ## How it satisfies three interfaces at once + * + * `RealtimeVoiceAdapter` is ONE object registered as the STT, TTS, and Brain of a + * {@link RealtimePipeline}. That is not a trick to fit an interface: it is the + * accurate model of what a realtime session is - a single conversation that + * happens to be sampled at three points. Because it satisfies the same three + * seams the cascade does, the session service, the protocol, the router, and + * every client are byte-for-byte unaware of which shape is running. + * + * ## Routing stays Maestro's + * + * The realtime model is NOT allowed to decide where a prompt goes by talking + * about it. A `route_utterance` tool is declared on the session, and the model's + * function call carries a `RouteDecision` that Maestro validates against the live + * roster and executes itself. Tab and agent dispatch therefore work identically + * in both pipeline shapes, and a model that invents an agent id gets the same + * treatment it gets in the cascade: the conductor takes the turn. + */ + +import { ACAPPELLA_AUDIO_SAMPLE_RATE } from '../../../../shared/acappella/audio-host'; +import { OPENAI_REALTIME_PROVIDER_ID } from '../../../../shared/acappella/provider-catalog'; +import { VoiceProviderError } from '../../../../shared/acappella/provider-errors'; +import type { + BrainProvider, + SttCallbacks, + SttProvider, + TtsChunk, + TtsProvider, + TtsSpeakOptions, + VoiceConverseContext, + VoicePipeline, + VoiceProviderTrio, + VoiceRouteContext, +} from '../../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../../shared/acappella/route-decision'; +import { ROUTE_DECISION_JSON_SCHEMA } from '../../../../shared/acappella/route-decision'; +import { splitIntoSpokenSentences } from '../../../../shared/acappella/sentences'; +import { logger } from '../../../utils/logger'; +import { buildRouteUserPrompt, parseRouteDecision, routeSystemPrompt } from '../brain-prompt'; +import { getCredential } from '../credentials'; +import { requireCredential } from '../hosted/http'; +import { resampleLinear } from '../pcm'; + +const LOG_CONTEXT = 'ACappella'; + +const DEFAULT_MODEL = 'gpt-4o-realtime-preview'; +const REALTIME_URL = 'wss://api.openai.com/v1/realtime'; + +/** The realtime API speaks 24 kHz PCM in both directions. */ +const REALTIME_SAMPLE_RATE = 24_000; + +/** The tool the model must call to route. Its schema is the shared one. */ +export const ROUTE_TOOL_NAME = 'route_utterance'; + +/** + * How long a turn waits for the model's routing tool call. + * + * Past this the conductor takes the turn rather than the session hanging: a + * spoken instruction that produces nothing at all is the worst outcome, and the + * conductor can always be asked to hand it on. + */ +const ROUTE_TIMEOUT_MS = 6_000; + +/** How long a spoken rewrite may take before the turn gives up on audio. */ +const RESPONSE_TIMEOUT_MS = 20_000; + +// --------------------------------------------------------------------------- +// Socket seam +// --------------------------------------------------------------------------- + +/** + * The slice of a WebSocket this file uses. + * + * Injected rather than imported so the whole protocol - tool calls, barge-in, + * transcript deltas - is testable without a network or an API key. The default + * factory below is the only place `ws` is touched. + */ +export interface RealtimeSocket { + send(data: string): void; + close(): void; + on(event: 'open', handler: () => void): void; + on(event: 'message', handler: (data: string) => void): void; + on(event: 'close', handler: () => void): void; + on(event: 'error', handler: (error: Error) => void): void; +} + +export type RealtimeSocketFactory = ( + url: string, + headers: Record +) => RealtimeSocket; + +/** The production factory. `ws` is already a Maestro dependency. */ +export const defaultRealtimeSocketFactory: RealtimeSocketFactory = (url, headers) => { + // Required lazily: a user who never turns on the realtime tier should not pay + // for the module, and the import must not run in a test that stubs the socket. + const { WebSocket } = require('ws') as typeof import('ws'); + const socket = new WebSocket(url, { headers }); + + return { + send: (data) => socket.send(data), + close: () => socket.close(), + on: (event: string, handler: (...args: never[]) => void) => { + if (event === 'message') { + socket.on('message', (data: Buffer | string) => + (handler as unknown as (text: string) => void)(data.toString()) + ); + return; + } + socket.on(event as 'open', handler as () => void); + }, + } as RealtimeSocket; +}; + +// --------------------------------------------------------------------------- +// Adapter +// --------------------------------------------------------------------------- + +export interface RealtimeSessionOptions { + model?: string; + voice?: string; + socketFactory?: RealtimeSocketFactory; + readCredential?: typeof getCredential; + routeTimeoutMs?: number; + responseTimeoutMs?: number; +} + +/** One realtime conversation, wearing all three provider hats. */ +export class RealtimeVoiceAdapter implements SttProvider, TtsProvider, BrainProvider { + readonly id = OPENAI_REALTIME_PROVIDER_ID; + readonly label = 'OpenAI Realtime'; + readonly tier = 'cloud' as const; + /** + * What `feed()` takes, which is the capture path's rate. The API wants 24 kHz, + * so the conversion happens inside `feed()` rather than being pushed onto the + * audio pipeline: the capture rate is a property of the microphone and of the + * VAD tuned against it, not of whichever provider happens to be selected. + */ + readonly sampleRate = ACAPPELLA_AUDIO_SAMPLE_RATE; + readonly acceptsAudio = true; + + private readonly model: string; + private readonly voice: string; + private readonly socketFactory: RealtimeSocketFactory; + private readonly readCredential: typeof getCredential; + private readonly routeTimeoutMs: number; + private readonly responseTimeoutMs: number; + + private socket: RealtimeSocket | null = null; + private callbacks: SttCallbacks | null = null; + private opened: Promise | null = null; + + /** Resolved by the model's routing tool call, or by its timeout. */ + private pendingRoute: Deferred> | null = null; + /** Resolved when the model finishes a spoken response. */ + private pendingResponse: Deferred | null = null; + + /** Sentence-aligned audio for the reply currently being generated. */ + private building: ResponseBuilder = newResponseBuilder(); + /** The last completed response, waiting for `speak()` to hand it out. */ + private spoken: SpokenResponse | null = null; + + /** Bumped by `cancel()` and by every speech run, so a stale iterator returns. */ + private run = 0; + + constructor(options: RealtimeSessionOptions = {}) { + this.model = options.model ?? DEFAULT_MODEL; + this.voice = options.voice ?? 'alloy'; + this.socketFactory = options.socketFactory ?? defaultRealtimeSocketFactory; + this.readCredential = options.readCredential ?? getCredential; + this.routeTimeoutMs = options.routeTimeoutMs ?? ROUTE_TIMEOUT_MS; + this.responseTimeoutMs = options.responseTimeoutMs ?? RESPONSE_TIMEOUT_MS; + } + + // -- SttProvider --------------------------------------------------------- + + async start(callbacks: SttCallbacks): Promise { + const key = requireCredential(this.id, 'openai', this.readCredential); + this.callbacks = callbacks; + + const socket = this.socketFactory(`${REALTIME_URL}?model=${encodeURIComponent(this.model)}`, { + Authorization: `Bearer ${key}`, + 'OpenAI-Beta': 'realtime=v1', + }); + this.socket = socket; + + this.opened = new Promise((resolve, reject) => { + socket.on('open', () => { + this.configureSession(); + resolve(); + }); + socket.on('error', (error) => + reject( + new VoiceProviderError( + `The realtime connection failed: ${error.message}. Check your connection, or switch to the cascade pipeline.`, + { kind: 'network', providerId: this.id, cause: error } + ) + ) + ); + }); + + socket.on('message', (data) => this.handleMessage(data)); + socket.on('close', () => this.handleClose()); + + await this.opened; + } + + /** + * Push audio to the model. + * + * Base64 over the socket, which is what the API takes. No local endpointing: + * the model's own semantic VAD decides when a turn ended, and that judgement is + * most of what the realtime tier is buying. + */ + feed(pcm: Int16Array): void { + if (!this.socket) return; + const upsampled = resampleLinear(pcm, ACAPPELLA_AUDIO_SAMPLE_RATE, REALTIME_SAMPLE_RATE); + this.send({ + type: 'input_audio_buffer.append', + audio: Buffer.from(upsampled.buffer, upsampled.byteOffset, upsampled.byteLength).toString( + 'base64' + ), + }); + } + + /** Force endpointing. Used by push-to-talk, where the key release IS the end. */ + async flush(): Promise { + if (!this.socket) return; + this.send({ type: 'input_audio_buffer.commit' }); + } + + async stop(): Promise { + this.callbacks = null; + this.pendingRoute?.reject(sessionClosed(this.id)); + this.pendingRoute = null; + this.pendingResponse?.reject(sessionClosed(this.id)); + this.pendingResponse = null; + this.spoken = null; + + const socket = this.socket; + this.socket = null; + try { + socket?.close(); + } catch { + // A socket that will not close cleanly must not wedge session teardown. + } + } + + // -- BrainProvider ------------------------------------------------------- + + /** + * The routing decision for an utterance. + * + * The model has already heard the audio, so this does not re-send the words: it + * publishes the roster the decision has to be made against and waits for the + * tool call. Maestro validates and executes it, exactly as in the cascade. + */ + async route(input: string, context: VoiceRouteContext): Promise { + if (!this.socket) throw sessionClosed(this.id); + + const pending = deferred>(); + this.pendingRoute = pending; + + // The roster changes between turns, so it is pushed per turn rather than + // baked into the session instructions at connect time. + this.send({ + type: 'conversation.item.create', + item: { + type: 'message', + role: 'system', + content: [{ type: 'input_text', text: buildRouteUserPrompt(input, context) }], + }, + }); + this.send({ + type: 'response.create', + response: { modalities: ['text'], tool_choice: { type: 'function', name: ROUTE_TOOL_NAME } }, + }); + + const args = await withTimeout(pending.promise, this.routeTimeoutMs, null); + if (this.pendingRoute === pending) this.pendingRoute = null; + + if (!args) { + logger.warn('Realtime session did not route in time; the conductor takes it', LOG_CONTEXT); + return { target: 'conductor', tabAction: 'current', prompt: input.trim(), confidence: 0.3 }; + } + + // Same parser as every other Brain: a tool call is a well-formed shape, not + // a true one, and only the roster knows which agent ids exist. + return parseRouteDecision(JSON.stringify(args), context, input); + } + + /** + * Turn an agent's written answer into the spoken reply. + * + * The realtime model produces the words AND the audio in one response, so this + * generates both and keeps the audio for the `speak()` that follows. Returning + * the text first is what lets the session service count sentences and announce + * the run before a sample is played, which is the ordering every client's + * transcript depends on. + */ + async converse(agentText: string, context: VoiceConverseContext): Promise { + if (!this.socket) throw sessionClosed(this.id); + + const pending = deferred(); + this.pendingResponse = pending; + this.building = newResponseBuilder(); + + const limit = context.maxSentences ?? 2; + this.send({ + type: 'conversation.item.create', + item: { + type: 'message', + role: 'system', + content: [ + { + type: 'input_text', + text: `An agent answered. Say this out loud in at most ${limit} sentence${ + limit === 1 ? '' : 's' + }, plainly, with no code or file paths:\n\n${agentText}`, + }, + ], + }, + }); + this.send({ type: 'response.create', response: { modalities: ['audio', 'text'] } }); + + const response = await withTimeout(pending.promise, this.responseTimeoutMs, null); + if (this.pendingResponse === pending) this.pendingResponse = null; + + if (!response) { + throw new VoiceProviderError('The realtime model did not answer in time.', { + kind: 'timeout', + providerId: this.id, + }); + } + + this.spoken = response; + return response.text; + } + + // -- TtsProvider --------------------------------------------------------- + + /** + * Hand out the audio the model already generated in `converse()`. + * + * `text` is ignored on purpose: re-synthesising it would be a second + * generation of something the user is about to hear, in a tier whose entire + * point is doing this once. The sentences were cut from the model's own + * transcript, so the chunk boundaries line up with what it actually said. + */ + speak(text: string, options: TtsSpeakOptions): AsyncIterable { + const response = this.spoken; + this.spoken = null; + return this.stream(response ?? { text, sentences: [] }, ++this.run, options); + } + + /** Barge-in. Cancels generation AND drops audio already queued on the server. */ + cancel(): void { + this.run += 1; + this.spoken = null; + if (!this.socket) return; + this.send({ type: 'response.cancel' }); + // Without this the server keeps streaming the audio it had already made, + // and the user hears the assistant talk over the interruption. + this.send({ type: 'output_audio_buffer.clear' }); + } + + // -- Internals ----------------------------------------------------------- + + private async *stream( + response: SpokenResponse, + run: number, + options: TtsSpeakOptions + ): AsyncGenerator { + const sentences = response.sentences.length + ? response.sentences + : splitIntoSpokenSentences(response.text).map((sentence) => ({ + text: sentence, + audio: null, + })); + + for (let index = 0; index < sentences.length; index++) { + if (this.run !== run) return; + yield { + utteranceId: options.utteranceId, + index, + text: sentences[index].text, + format: sentences[index].audio ? 'pcm16' : 'none', + audio: sentences[index].audio, + sampleRate: REALTIME_SAMPLE_RATE, + }; + } + } + + /** + * Declare the session: audio format, transcription, the routing tool, and + * server-side turn detection. + */ + private configureSession(): void { + this.send({ + type: 'session.update', + session: { + modalities: ['text', 'audio'], + voice: this.voice, + instructions: routeSystemPrompt(), + input_audio_format: 'pcm16', + output_audio_format: 'pcm16', + input_audio_transcription: { model: 'whisper-1' }, + // Server VAD with interruption: the model stops speaking when the user + // starts, which is the behaviour the cascade has to emulate with its own + // detector and its own barge-in path. + turn_detection: { type: 'server_vad', create_response: false, interrupt_response: true }, + tools: [ + { + type: 'function', + name: ROUTE_TOOL_NAME, + description: + 'Route the user utterance to an agent and a tab. Always call this instead of describing where it should go.', + parameters: ROUTE_DECISION_JSON_SCHEMA, + }, + ], + }, + }); + } + + private handleMessage(raw: string): void { + let event: RealtimeEvent; + try { + event = JSON.parse(raw) as RealtimeEvent; + } catch { + // A frame we cannot parse is one lost delta, not a dead session. + return; + } + + switch (event.type) { + case 'conversation.item.input_audio_transcription.delta': + if (event.delta) this.callbacks?.onPartial(event.delta, 0.5); + return; + + case 'conversation.item.input_audio_transcription.completed': + if (event.transcript?.trim()) this.callbacks?.onFinal(event.transcript.trim(), 1); + return; + + case 'response.audio_transcript.delta': + if (event.delta) appendTranscript(this.building, event.delta); + return; + + case 'response.audio.delta': + if (event.delta) appendAudio(this.building, Buffer.from(event.delta, 'base64')); + return; + + case 'response.function_call_arguments.done': + this.resolveRoute(event.arguments); + return; + + case 'response.done': + this.pendingResponse?.resolve(finishResponse(this.building)); + this.pendingResponse = null; + this.building = newResponseBuilder(); + return; + + case 'error': + this.reportError(event.error?.message ?? 'The realtime session reported an error.'); + return; + + default: + return; + } + } + + private resolveRoute(rawArguments?: string): void { + if (!this.pendingRoute || !rawArguments) return; + try { + this.pendingRoute.resolve(JSON.parse(rawArguments) as Record); + } catch { + // Unparseable arguments are the same as no decision: the timeout path + // hands the turn to the conductor rather than guessing. + this.pendingRoute.resolve({}); + } + this.pendingRoute = null; + } + + private handleClose(): void { + this.pendingRoute?.resolve({}); + this.pendingRoute = null; + this.pendingResponse?.reject(sessionClosed(this.id)); + this.pendingResponse = null; + if (!this.callbacks) return; + this.reportError('The realtime connection closed. Start voice mode again to reconnect.'); + } + + private reportError(message: string): void { + this.callbacks?.onError( + new VoiceProviderError(message, { kind: 'network', providerId: this.id }) + ); + } + + private send(payload: Record): void { + this.socket?.send(JSON.stringify(payload)); + } +} + +// --------------------------------------------------------------------------- +// Pipeline +// --------------------------------------------------------------------------- + +/** + * The realtime pipeline: one adapter in all three slots. + * + * One of exactly two `VoicePipeline` implementations. Everything downstream is + * handed the same `VoiceProviderTrio` it would get from the cascade. + */ +export class RealtimePipeline implements VoicePipeline { + readonly shape = 'realtime' as const; + readonly providers: VoiceProviderTrio; + + constructor(private readonly adapter: RealtimeVoiceAdapter) { + this.providers = { stt: adapter, tts: adapter, brain: adapter }; + } + + async dispose(): Promise { + await this.adapter.stop(); + } +} + +export function createRealtimePipeline(options: RealtimeSessionOptions = {}): RealtimePipeline { + return new RealtimePipeline(new RealtimeVoiceAdapter(options)); +} + +// --------------------------------------------------------------------------- +// Response assembly +// --------------------------------------------------------------------------- + +/** One sentence of a spoken reply, with the audio that was generated for it. */ +export interface SpokenSentence { + text: string; + audio: Uint8Array | null; +} + +export interface SpokenResponse { + text: string; + sentences: SpokenSentence[]; +} + +interface ResponseBuilder { + transcript: string; + audio: Buffer[]; + /** Sentences already cut, with the audio that had arrived when they were. */ + sentences: SpokenSentence[]; + /** Transcript length already accounted for by a cut sentence. */ + cutAt: number; +} + +function newResponseBuilder(): ResponseBuilder { + return { transcript: '', audio: [], sentences: [], cutAt: 0 }; +} + +/** + * Fold a transcript delta in, cutting a sentence whenever one completes. + * + * The audio and transcript deltas of a realtime response are interleaved in + * generation order, so the audio that has arrived by the time a sentence closes + * IS that sentence's audio, near enough to a frame. This is what lets a realtime + * reply emit one `speak-sentence` per sentence like every other provider, instead + * of one giant chunk that no client could show progress through. + */ +function appendTranscript(builder: ResponseBuilder, delta: string): void { + builder.transcript += delta; + + const pending = builder.transcript.slice(builder.cutAt); + const sentences = splitIntoSpokenSentences(pending); + // The last fragment may still be growing, so only completed ones are cut. + if (sentences.length < 2) return; + + for (const sentence of sentences.slice(0, -1)) { + builder.sentences.push({ text: sentence, audio: drainAudio(builder) }); + builder.cutAt += pending.indexOf(sentence) + sentence.length; + } +} + +function appendAudio(builder: ResponseBuilder, chunk: Buffer): void { + builder.audio.push(chunk); +} + +/** Everything buffered since the last cut, as one buffer. */ +function drainAudio(builder: ResponseBuilder): Uint8Array | null { + if (builder.audio.length === 0) return null; + const joined = Buffer.concat(builder.audio); + builder.audio = []; + return new Uint8Array(joined.buffer, joined.byteOffset, joined.byteLength); +} + +function finishResponse(builder: ResponseBuilder): SpokenResponse { + const tail = builder.transcript.slice(builder.cutAt).trim(); + if (tail) builder.sentences.push({ text: tail, audio: drainAudio(builder) }); + else if (builder.audio.length && builder.sentences.length) { + // Trailing audio with no trailing text: append it to the last sentence + // rather than dropping it, or the reply is cut off mid-word. + const last = builder.sentences[builder.sentences.length - 1]; + last.audio = concatAudio(last.audio, drainAudio(builder)); + } + + return { + text: builder.sentences + .map((sentence) => sentence.text) + .join(' ') + .trim(), + sentences: builder.sentences, + }; +} + +function concatAudio(a: Uint8Array | null, b: Uint8Array | null): Uint8Array | null { + if (!a) return b; + if (!b) return a; + const out = new Uint8Array(a.length + b.length); + out.set(a, 0); + out.set(b, a.length); + return out; +} + +// --------------------------------------------------------------------------- + +interface RealtimeEvent { + type: string; + delta?: string; + transcript?: string; + arguments?: string; + error?: { message?: string }; +} + +interface Deferred { + promise: Promise; + resolve: (value: T) => void; + reject: (error: Error) => void; +} + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + // Attached so a rejection that nobody is awaiting yet (a socket that closed + // between turns) cannot become an unhandled rejection and kill the process. + promise.catch(() => {}); + return { promise, resolve, reject }; +} + +/** Resolve with `fallback` when `promise` has not settled in time. */ +async function withTimeout( + promise: Promise, + ms: number, + fallback: T | null +): Promise { + let timer: ReturnType | null = null; + try { + return await Promise.race([ + promise, + new Promise((resolve) => { + timer = setTimeout(() => resolve(fallback), ms); + timer.unref?.(); + }), + ]); + } catch { + return fallback; + } finally { + if (timer) clearTimeout(timer); + } +} + +function sessionClosed(providerId: string): VoiceProviderError { + return new VoiceProviderError( + 'The realtime session is not connected. Start voice mode again to reconnect.', + { kind: 'network', providerId } + ); +} diff --git a/src/main/acappella/providers/unresolved.ts b/src/main/acappella/providers/unresolved.ts new file mode 100644 index 0000000000..249082416d --- /dev/null +++ b/src/main/acappella/providers/unresolved.ts @@ -0,0 +1,139 @@ +/** + * The provider a slot gets when the one it was configured with cannot be built. + * + * There is no fallback engine in A Cappella. Not to the cloud (that spends the + * user's money and ships their microphone somewhere they did not choose), and not + * to the mock either (a "working" session that transcribes nothing and speaks + * nothing is indistinguishable from a broken feature, and it hides the reason). + * A slot whose provider is unknown or unbuildable therefore resolves to one of + * these: an object that satisfies the interface and refuses, by name, the first + * time anything asks it to work. + * + * The refusal is a classified `VoiceProviderError`, so the session service turns + * it into a `session-error` with the provider id attached rather than a crash, + * and the HUD can say which slot is broken and what was asked for. + */ + +import { ACAPPELLA_AUDIO_SAMPLE_RATE } from '../../../shared/acappella/audio-host'; +import { VoiceProviderError } from '../../../shared/acappella/provider-errors'; +import type { + BrainProvider, + SttCallbacks, + SttProvider, + TtsChunk, + TtsProvider, + TtsSpeakOptions, + VoiceProviderRole, + VoiceProviderTier, +} from '../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../shared/acappella/route-decision'; + +/** Why a slot could not be built. */ +export type UnresolvedReason = 'unknown-provider' | 'unavailable'; + +const ROLE_LABELS: Record = { + stt: 'Speech-to-Text', + tts: 'Text-to-Speech', + brain: 'Conductor Brain', +}; + +/** The id an unresolved slot reports. Distinct per role so a log names the slot. */ +export function unresolvedProviderId(role: VoiceProviderRole): string { + return `unresolved-${role}`; +} + +export function unresolvedMessage( + role: VoiceProviderRole, + requestedId: string, + reason: UnresolvedReason +): string { + const label = ROLE_LABELS[role]; + return reason === 'unknown-provider' + ? `${label}: '${requestedId}' is not a provider Maestro knows about. Pick one in Settings > Plugins > A Cappella > Voice Providers.` + : `${label}: '${requestedId}' cannot run in this build. Pick another in Settings > Plugins > A Cappella > Voice Providers.`; +} + +/** Shared identity for the three refusing providers. */ +abstract class UnresolvedProvider { + readonly id: string; + readonly label: string; + readonly tier: VoiceProviderTier; + + constructor( + protected readonly role: VoiceProviderRole, + protected readonly requestedId: string, + protected readonly reason: UnresolvedReason + ) { + this.id = unresolvedProviderId(role); + this.label = `Unavailable (${requestedId})`; + // Its own tier, never the tier of what was asked for and never `mock`: this + // slot does not run anything, and a client that renders tiers has to be able + // to say "nothing is filling this" rather than "the mock is". + this.tier = 'unresolved'; + } + + protected refuse(): VoiceProviderError { + return new VoiceProviderError(unresolvedMessage(this.role, this.requestedId, this.reason), { + kind: 'unavailable', + providerId: this.requestedId, + }); + } +} + +export class UnresolvedSttProvider extends UnresolvedProvider implements SttProvider { + readonly sampleRate = ACAPPELLA_AUDIO_SAMPLE_RATE; + /** False, so no microphone is opened for a recogniser that will never run. */ + readonly acceptsAudio = false; + + constructor(requestedId: string, reason: UnresolvedReason) { + super('stt', requestedId, reason); + } + + async start(_callbacks: SttCallbacks): Promise { + // Thrown from `start()` rather than reported through the callbacks: the + // session service refuses to open the floor at all, which is the correct + // outcome for a recogniser that cannot exist. + throw this.refuse(); + } + + feed(_pcm: Int16Array): void {} + + async flush(): Promise {} + + async stop(): Promise {} +} + +export class UnresolvedTtsProvider extends UnresolvedProvider implements TtsProvider { + constructor(requestedId: string, reason: UnresolvedReason) { + super('tts', requestedId, reason); + } + + speak(_text: string, _options: TtsSpeakOptions): AsyncIterable { + const error = this.refuse(); + // Hand-rolled rather than an async generator: a generator whose body only + // throws has no `yield` in it, which is a lint error and a fair one. The + // refusal has to arrive on the FIRST `next()`, which is what the session + // service awaits. + return { + [Symbol.asyncIterator]: () => ({ + next: () => Promise.reject(error), + }), + }; + } + + cancel(): void {} +} + +export class UnresolvedBrainProvider extends UnresolvedProvider implements BrainProvider { + constructor(requestedId: string, reason: UnresolvedReason) { + super('brain', requestedId, reason); + } + + async route(_input: string, _context: unknown): Promise { + throw this.refuse(); + } + + async converse(_agentText: string, _context: unknown): Promise { + throw this.refuse(); + } +} diff --git a/src/main/acappella/router/conductor-agent.ts b/src/main/acappella/router/conductor-agent.ts new file mode 100644 index 0000000000..665ead21f2 --- /dev/null +++ b/src/main/acappella/router/conductor-agent.ts @@ -0,0 +1,309 @@ +/** + * The Conductor as a real Maestro agent. + * + * The third Brain option, for people who want routing decided by something that + * can actually reason about their projects rather than by a 1.7B classifier. It + * is a normal batch agent run: the routing context plus the utterance go in, a + * `RouteDecision` comes back through the same schema and the same validator as + * every other Brain, so nothing downstream can tell which one answered. + * + * Two properties this file exists to hold: + * + * **It never blocks the floor.** A real agent can be mid-turn when the user + * speaks, and an agent that takes ninety seconds to answer is not a router. So + * there is exactly one request in flight, a hard deadline on it, and a second + * one is refused with a spoken "the Conductor is busy" instead of being queued + * behind something the user has already stopped caring about. + * + * **SSH is explicit.** `ProcessManager.spawn` does NOT wrap for SSH - callers + * do, which is why {@link wrapSpawnWithSsh} is called here. A configured remote + * that cannot be resolved throws rather than silently running the Conductor on + * the local machine: the user opted into a remote, and a routing prompt carries + * the names and paths of everything they have open. + * + * (`groomContext` is deliberately not reused. It is the same spawn-and-collect + * shape, but its runs live in a registry that `cancelAllGroomingSessions()` + * empties, and a Conductor decision cancelled because someone summarised a + * context in another window would be an unexplainable misroute.) + */ + +import { CONDUCTOR_AGENT_BRAIN_PROVIDER_ID } from '../../../shared/acappella/provider-catalog'; +import { VoiceProviderError } from '../../../shared/acappella/provider-errors'; +import type { + BrainProvider, + VoiceConverseContext, + VoiceRouteContext, +} from '../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../shared/acappella/route-decision'; +import { generateUUID } from '../../../shared/uuid'; +import type { AgentDetector } from '../../agents'; +import { applyAgentConfigOverrides, buildAgentArgs } from '../../utils/agent-args'; +import { logger } from '../../utils/logger'; +import { wrapSpawnWithSsh } from '../../utils/ssh-spawn-wrapper'; +import { + buildConverseUserPrompt, + buildRouteUserPrompt, + converseSystemPrompt, + limitSpokenReply, + parseRouteDecision, + routeSystemPrompt, +} from '../providers/brain-prompt'; + +const LOG_CONTEXT = 'ACappella'; + +/** + * How long a routing turn may take before it is abandoned. + * + * Twenty seconds is already far outside what voice tolerates; it exists to bound + * a hung process, not to be waited out. Anyone who finds this generous should + * be running a local Brain. + */ +export const CONDUCTOR_AGENT_TIMEOUT_MS = 20_000; + +/** The process manager surface this provider uses. Structural, so tests can fake it. */ +export interface ConductorProcessManager { + spawn(config: Record): { pid: number; success?: boolean } | null; + on(event: string, handler: (...args: any[]) => void): void; + off(event: string, handler: (...args: any[]) => void): void; + kill(sessionId: string): void; +} + +/** The SSH remote settings adapter `wrapSpawnWithSsh` reads. */ +type SshStore = Parameters[2]; + +export interface ConductorAgentOptions { + processManager: ConductorProcessManager; + agentDetector: AgentDetector; + /** Which agent runs the Conductor, and where. */ + agentType: string; + cwd: string; + /** SSH remote for the Conductor agent, when the user configured one. */ + sshRemoteConfig?: { enabled: boolean; remoteId: string | null; workingDirOverride?: string }; + /** Required when `sshRemoteConfig.enabled`. Its absence is a loud failure. */ + sshStore?: SshStore; + agentConfigValues?: Record; + customEnvVars?: Record; + modelId?: string; + timeoutMs?: number; +} + +export class ConductorAgentBrain implements BrainProvider { + readonly id = CONDUCTOR_AGENT_BRAIN_PROVIDER_ID; + readonly label = 'Conductor agent'; + readonly tier = 'local' as const; + + private readonly options: ConductorAgentOptions; + /** + * The one request in flight. + * + * A single slot rather than a queue: by the time a queued utterance reached + * the front, the conversation it belonged to would be over. Refusing is the + * behaviour a person can work with. + */ + private inFlight: Promise | null = null; + + constructor(options: ConductorAgentOptions) { + this.options = options; + } + + /** True while a decision is being computed. Read by the HUD and by tests. */ + get isBusy(): boolean { + return this.inFlight !== null; + } + + async route(input: string, context: VoiceRouteContext): Promise { + const raw = await this.ask( + [routeSystemPrompt(), '', buildRouteUserPrompt(input, context)].join('\n') + ); + return parseRouteDecision(raw, context, input); + } + + async converse(agentText: string, context: VoiceConverseContext): Promise { + const raw = await this.ask( + [converseSystemPrompt(), '', buildConverseUserPrompt(agentText, context)].join('\n') + ); + return limitSpokenReply(raw, context.maxSentences); + } + + // -- Internals ----------------------------------------------------------- + + private async ask(prompt: string): Promise { + if (this.inFlight) { + throw new VoiceProviderError('The Conductor is busy. Say that again in a moment.', { + kind: 'busy', + providerId: this.id, + }); + } + + const run = this.run(prompt); + this.inFlight = run; + try { + return await run; + } finally { + this.inFlight = null; + } + } + + private async run(prompt: string): Promise { + const { processManager, agentDetector, agentType, cwd } = this.options; + + const agent = await agentDetector.getAgent(agentType); + if (!agent || !agent.available) { + throw new VoiceProviderError( + `The Conductor agent '${agentType}' is not available. Pick another Brain in Voice Setup.`, + { kind: 'unavailable', providerId: this.id } + ); + } + + const baseArgs = buildAgentArgs(agent, { + baseArgs: agent.args ?? [], + prompt, + cwd, + // A router reads; it does not edit. Read-only also means no workspace + // lock, so the Conductor can think while the agents it routes to work. + readOnlyMode: true, + modelId: this.options.modelId, + }); + const resolved = applyAgentConfigOverrides(agent, baseArgs, { + agentConfigValues: this.options.agentConfigValues ?? {}, + sessionCustomEnvVars: this.options.customEnvVars, + readOnlyMode: true, + }); + + let spawnConfig: Record = { + command: agent.command, + args: resolved.args, + cwd, + prompt, + customEnvVars: resolved.effectiveCustomEnvVars, + promptArgs: agent.promptArgs, + noPromptSeparator: agent.noPromptSeparator, + }; + + const ssh = this.options.sshRemoteConfig; + if (ssh?.enabled) { + if (!this.options.sshStore) { + // Loud, not local. The user asked for a remote; running here instead + // would put their whole roster on a machine they did not choose. + throw new VoiceProviderError( + 'The Conductor agent is configured for an SSH remote that could not be resolved.', + { kind: 'unavailable', providerId: this.id } + ); + } + const wrapped = await wrapSpawnWithSsh( + { + command: agent.command, + args: resolved.args, + cwd, + prompt, + customEnvVars: resolved.effectiveCustomEnvVars, + promptArgs: agent.promptArgs, + noPromptSeparator: agent.noPromptSeparator, + agentBinaryName: agent.binaryName, + }, + ssh, + this.options.sshStore + ); + if (!wrapped.sshRemoteUsed) { + throw new VoiceProviderError( + 'The Conductor agent is configured for an SSH remote that could not be resolved.', + { kind: 'unavailable', providerId: this.id } + ); + } + spawnConfig = { + command: wrapped.command, + args: wrapped.args, + cwd: wrapped.cwd, + prompt: wrapped.prompt, + customEnvVars: wrapped.customEnvVars, + sshStdinScript: wrapped.sshStdinScript, + sshRemoteCommand: wrapped.sshRemoteCommand, + sshRemoteId: ssh.remoteId ?? undefined, + }; + } + + const sessionId = `acappella-conductor-${generateUUID()}`; + return this.collect(processManager, sessionId, { + ...spawnConfig, + sessionId, + toolType: agentType, + readOnlyMode: true, + }); + } + + /** + * Spawn and collect stdout until the process exits or the deadline passes. + * + * A timeout kills the process rather than only rejecting: an abandoned agent + * left running would still be holding a model, a token budget, and possibly a + * remote shell. + */ + private collect( + processManager: ConductorProcessManager, + sessionId: string, + spawnConfig: Record + ): Promise { + return new Promise((resolve, reject) => { + let output = ''; + let settled = false; + + const finish = (fn: () => void): void => { + if (settled) return; + settled = true; + clearTimeout(deadline); + processManager.off('data', onData); + processManager.off('exit', onExit); + fn(); + }; + + const onData = (id: string, chunk: string): void => { + if (id === sessionId && typeof chunk === 'string') output += chunk; + }; + + const onExit = (id: string): void => { + if (id !== sessionId) return; + finish(() => resolve(output)); + }; + + const deadline = setTimeout(() => { + finish(() => { + try { + processManager.kill(sessionId); + } catch (error) { + logger.warn( + `Could not stop the Conductor agent: ${(error as Error).message}`, + LOG_CONTEXT + ); + } + reject( + new VoiceProviderError('The Conductor agent did not answer in time.', { + kind: 'timeout', + providerId: this.id, + }) + ); + }); + }, this.options.timeoutMs ?? CONDUCTOR_AGENT_TIMEOUT_MS); + deadline.unref?.(); + + processManager.on('data', onData); + processManager.on('exit', onExit); + + const result = processManager.spawn(spawnConfig); + if (!result) { + finish(() => + reject( + new VoiceProviderError('The Conductor agent could not be started.', { + kind: 'unavailable', + providerId: this.id, + }) + ) + ); + } + }); + } +} + +/** Sugar matching the rest of A Cappella's factories. */ +export function createConductorAgentBrain(options: ConductorAgentOptions): ConductorAgentBrain { + return new ConductorAgentBrain(options); +} diff --git a/src/main/acappella/router/conductor-router.ts b/src/main/acappella/router/conductor-router.ts new file mode 100644 index 0000000000..ce094dd22c --- /dev/null +++ b/src/main/acappella/router/conductor-router.ts @@ -0,0 +1,375 @@ +/** + * The Conductor router: the decision layer between a transcript and a dispatch. + * + * It is a DECORATOR over whichever Brain the registry resolved, not a fourth + * Brain. The wrapped provider still does the inference - locally under a GBNF + * grammar, or hosted under a structured-output schema - and this file owns the + * four things that must behave identically whichever one is running: + * + * 1. **The context.** One assembler builds it, bounded and cached, and the + * recall shortlist is computed here rather than by asking a 1.7B model to + * read sixty tab summaries. + * 2. **Validation.** Every decision is checked against the roster it will run + * on, whatever produced it. A grammar guarantees a well-formed id; only the + * roster knows whether it is a real one, and the user can close a tab while + * the model is thinking. + * 3. **Recovery.** A rejected decision gets ONE constrained retry with the + * errors fed back. A second failure does not become a guess: it becomes a + * spoken question. + * 4. **Honesty about confidence.** Below the threshold the router asks rather + * than dispatches. Sending someone's spoken instruction to the wrong + * repository is worse than taking two seconds to ask which one they meant. + * + * The decorator keeps the wrapped provider's id, label and tier, so the + * `provider-state` event still names the engine that is really running. A router + * that renamed itself in the HUD would be the silent-substitution failure this + * subsystem is built to prevent, wearing a different hat. + */ + +import type { RosterAgent } from '../../../shared/acappella/protocol'; +import type { + BrainProvider, + VoiceConverseContext, + VoiceRouteContext, +} from '../../../shared/acappella/providers'; +import type { RouteDecision } from '../../../shared/acappella/route-decision'; +import { isClarification, routeTargetSessionId } from '../../../shared/acappella/route-decision'; +import { generateUUID } from '../../../shared/uuid'; +import { logger } from '../../utils/logger'; +import { validateRouteDecision } from './grammar'; +import { getRoutingContext, type RoutingContext } from './routing-context'; +import { recordRoutingTurn } from './routing-log'; +import { + narrowRosterForRecall, + rankRecallCandidates, + resolveRecall, + type RecallCandidate, +} from './tab-recall'; + +const LOG_CONTEXT = 'ACappella'; + +/** + * Below this, ask instead of dispatching. + * + * 0.55 rather than 0.5 because the models routing here are calibrated loosely + * and a bare majority is not a belief. It is the one number worth tuning against + * the routing log's hit rate, which is why the log records confidence per turn. + */ +export const DEFAULT_CONFIDENCE_THRESHOLD = 0.55; + +export interface ConductorRouterOptions { + /** The resolved Brain. Its inference, this file's rules. */ + brain: BrainProvider; + confidenceThreshold?: number; + /** Injected in tests; production reads the cached assembler. */ + loadContext?: (recentUtterances: string[]) => Promise; + /** Injected in tests so the log is not a file. */ + record?: typeof recordRoutingTurn; + now?: () => number; +} + +/** The router, plus the read-only bits the HUD and the log need. */ +export interface ConductorRouter extends BrainProvider { + /** Routing-log id of the most recent turn, or null before the first one. */ + lastTurnId(): string | null; +} + +export function createConductorRouter(options: ConductorRouterOptions): ConductorRouter { + const brain = options.brain; + const threshold = options.confidenceThreshold ?? DEFAULT_CONFIDENCE_THRESHOLD; + const loadContext = options.loadContext ?? ((recent) => getRoutingContext(recent)); + const record = options.record ?? recordRoutingTurn; + const now = options.now ?? (() => Date.now()); + + let lastTurnId: string | null = null; + + return { + id: brain.id, + label: brain.label, + tier: brain.tier, + lastTurnId: () => lastTurnId, + + async route(input: string, context: VoiceRouteContext): Promise { + const startedAt = now(); + const enriched = await enrichContext(input, context, loadContext); + const roster = enriched.context.roster; + + let retries = 0; + let decision = await brain.route(input, enriched.context); + let validation = validateRouteDecision(decision, roster); + + if (!validation.ok) { + // One retry, with the reasons. A rejected decision is recoverable + // information, not a dead turn: the model usually fixes an id it + // invented when it is told which ids exist. + retries = 1; + logger.warn(`Route decision rejected: ${validation.errors.join('; ')}`, LOG_CONTEXT); + decision = await brain.route(input, { + ...enriched.context, + retryNotes: validation.errors, + }); + validation = validateRouteDecision(decision, roster); + } + + if (!validation.ok) { + // Twice rejected. The conductor takes it and asks, which is the only + // outcome that is neither a guess nor silence. + logger.warn(`Route decision rejected twice: ${validation.errors.join('; ')}`, LOG_CONTEXT); + decision = fallbackClarification(input, roster, enriched.candidates); + } else { + decision = applyRecallPolicy(decision, roster, context); + decision = applyConfidencePolicy(decision, roster, enriched.candidates, threshold); + } + + lastTurnId = record({ + id: generateUUID(), + utterance: input, + decision, + brainProviderId: brain.id, + latencyMs: now() - startedAt, + contextChars: enriched.contextChars, + droppedTabs: enriched.droppedTabs, + retries, + }); + + return decision; + }, + + converse(agentText: string, context: VoiceConverseContext): Promise { + // Nothing to add: reshaping an answer for the ear is the Brain's own job + // and has no roster, no ids, and nothing to validate. + return brain.converse(agentText, context); + }, + }; +} + +// --------------------------------------------------------------------------- +// Context +// --------------------------------------------------------------------------- + +interface EnrichedContext { + context: VoiceRouteContext; + candidates: RecallCandidate[]; + contextChars: number; + droppedTabs: number; +} + +/** + * Replace the caller's roster with the assembled one and shortlist the recall + * candidates. + * + * The assembler is authoritative because it is the only thing that knows about + * snoozed and closed tabs, carries the per-tab topics, and enforces the size + * cap. When it cannot be built - no store in a test, a store read that threw - + * the caller's roster is used unchanged rather than failing the turn: routing on + * a thinner context is a worse decision, and routing on nothing is no decision. + */ +async function enrichContext( + input: string, + context: VoiceRouteContext, + loadContext: (recentUtterances: string[]) => Promise +): Promise { + let assembled: RoutingContext | null = null; + try { + assembled = await loadContext(context.recentUtterances ?? []); + } catch (error) { + logger.warn(`Routing context unavailable: ${(error as Error).message}`, LOG_CONTEXT); + } + + const roster = assembled?.agents ?? context.roster; + const activeAgentSessionId = + context.activeAgentSessionId ?? assembled?.activeAgentSessionId ?? null; + + const candidates = rankRecallCandidates(input, roster, { activeAgentSessionId }); + + return { + context: { + ...context, + // Only the shortlisted prior tabs, plus everything still open: a model + // handed sixty topic lines picks the one it saw most recently. + roster: narrowRosterForRecall(roster, candidates), + activeAgentSessionId, + }, + candidates, + contextChars: assembled?.serializedChars ?? 0, + droppedTabs: assembled?.droppedTabs ?? 0, + }; +} + +// --------------------------------------------------------------------------- +// Policy +// --------------------------------------------------------------------------- + +/** + * Turn a recall of a closed tab into an offer. + * + * The executor knows how to wake a snoozed tab and how to reopen a closed one, + * but reopening is the user's call: the alternative the router must never take + * is quietly opening a fresh tab, because the reply then has no memory of the + * conversation the user asked to return to and they find that out by reading it. + */ +function applyRecallPolicy( + decision: RouteDecision, + roster: readonly RosterAgent[], + context: VoiceRouteContext +): RouteDecision { + if (decision.tabAction !== 'recall' || isClarification(decision)) return decision; + + const resolution = resolveRecall(decision, roster, { + // The user has already been asked once and this utterance is their answer. + confirmed: Boolean(context.clarification), + }); + + if (resolution.kind === 'offer') { + return { ...decision, clarify: resolution.question }; + } + return decision; +} + +/** + * Ask rather than guess when the decision is not confident enough to act on. + * + * The question names the alternatives, because "which agent?" makes the user + * repeat their whole sentence while "the backend agent or the API agent?" is + * answered in two words. + */ +function applyConfidencePolicy( + decision: RouteDecision, + roster: readonly RosterAgent[], + candidates: readonly RecallCandidate[], + threshold: number +): RouteDecision { + if (isClarification(decision) || decision.confidence >= threshold) return decision; + + const question = buildDisambiguation(decision, roster, candidates); + if (!question) return decision; + return { ...decision, clarify: question }; +} + +/** + * One spoken line offering the two most plausible targets. + * + * Returns null when there is nothing to disambiguate between - a single agent, + * or a conductor-targeted utterance - because asking "the backend agent?" of + * someone who only has one agent is worse than a low-confidence dispatch. + */ +function buildDisambiguation( + decision: RouteDecision, + roster: readonly RosterAgent[], + candidates: readonly RecallCandidate[] +): string | null { + if (roster.length < 2) return null; + + if (decision.tabAction === 'recall' && candidates.length >= 2) { + const [first, second] = candidates; + return `${tabLabel(first)} or ${tabLabel(second)}?`; + } + + // The agent the model leaned toward comes first so the likely answer is the + // first thing the user hears. With no such agent - a conductor target, or an + // id that was rejected - the two most plausible are simply the first two. + const targetId = routeTargetSessionId(decision.target); + const chosen = roster.find((agent) => agent.sessionId === targetId) ?? roster[0]; + const other = roster.find((agent) => agent.sessionId !== chosen.sessionId); + if (!other) return null; + + return `${chosen.name} or ${other.name}?`; +} + +// --------------------------------------------------------------------------- +// Correction +// --------------------------------------------------------------------------- + +/** + * Phrases that mean "that went to the wrong place", rather than being a request. + * + * Matched on the WHOLE utterance, not as a substring: "no, not that one" is a + * correction and "no, not that one, use the other endpoint" is a sentence about + * endpoints. A correction is a short interjection by nature, so requiring the + * whole utterance to be one is both the accurate rule and the safe one - a false + * positive silently moves a prompt the user never asked to move. + */ +const CORRECTION_PHRASES = [ + 'no the other one', + 'no not that one', + 'not that one', + 'the other one', + 'wrong tab', + 'wrong agent', + 'wrong one', +]; + +/** True when the utterance is a correction of the last dispatch and nothing else. */ +export function isCorrectionUtterance(text: string): boolean { + const normalized = text + .toLowerCase() + .replace(/[^a-z ]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + return CORRECTION_PHRASES.includes(normalized); +} + +/** What a correction should do, once one has been recognised. */ +export type CorrectionPlan = + | { kind: 'move'; agentSessionId: string } + | { kind: 'ask'; question: string }; + +/** + * Where a correction moves the prompt. + * + * With exactly one alternative there is nothing to ask about. With several, + * asking is the only honest move: "the other one" does not name anything, and + * guessing a second time after guessing wrong the first time is how a user + * decides the feature does not work. + */ +export function planCorrection( + roster: readonly RosterAgent[], + fromAgentSessionId: string +): CorrectionPlan { + const alternatives = roster.filter((agent) => agent.sessionId !== fromAgentSessionId); + if (alternatives.length === 1) { + return { kind: 'move', agentSessionId: alternatives[0].sessionId }; + } + if (alternatives.length === 0) { + return { kind: 'ask', question: 'There is nowhere else to send that.' }; + } + return { + kind: 'ask', + question: `${alternatives + .slice(0, 3) + .map((agent) => agent.name) + .join(', or ')}?`, + }; +} + +function tabLabel(candidate: RecallCandidate): string { + return candidate.tab.name ?? candidate.tab.topic ?? `the tab on ${candidate.agentName}`; +} + +/** + * The decision of last resort: a question, targeted at the conductor. + * + * Reached only when the model produced something unusable twice. It carries the + * user's own words as the prompt so that, if they answer, the request is intact. + */ +function fallbackClarification( + input: string, + roster: readonly RosterAgent[], + candidates: readonly RecallCandidate[] +): RouteDecision { + const named = buildDisambiguation( + { target: 'conductor', tabAction: 'current', prompt: input, confidence: 0 }, + roster, + candidates + ); + + return { + target: 'conductor', + tabAction: 'current', + prompt: input.trim(), + confidence: 0, + clarify: named + ? `I did not catch which one you meant. ${named}` + : 'Which agent should take that?', + }; +} diff --git a/src/main/acappella/router/conversation-buffer.ts b/src/main/acappella/router/conversation-buffer.ts new file mode 100644 index 0000000000..588322cdc5 --- /dev/null +++ b/src/main/acappella/router/conversation-buffer.ts @@ -0,0 +1,107 @@ +/** + * Conversation buffer - what has been said while a task takes shape. + * + * A Cappella used to treat every utterance as a command: you spoke, an agent was + * chosen, a prompt was sent. That works for "run the tests" and fails for the way + * people actually arrive at a request, which is a couple of sentences of thinking + * out loud before anything is actually asked for. + * + * So the Conductor can now reply instead of dispatching, and this is the memory + * that makes those replies coherent: each turn, both halves of the exchange, fed + * back to the Brain so it can see the shape of the thing being worked out rather + * than one sentence in isolation. + * + * Three properties, each of which was a way to get this wrong: + * + * - **It is CLEARED on dispatch.** Once the task has been sent, the discussion + * that produced it is finished. Carrying it forward would make the next + * request arrive wearing the last one's context, which is how "now do the + * same for the other repo" becomes a second copy of the first job. + * - **It is capped by turns AND by characters.** A conversation feeds a model + * on every routing turn, so an uncapped one is a prompt that grows without + * limit and a routing latency that grows with it. + * - **It holds text, not decisions.** What the Brain needs is what was said. A + * buffer of `RouteDecision`s would tempt a later reader into re-dispatching + * one, which is exactly the bug the clearing rule above exists to prevent. + */ + +/** Who said one line. */ +export type ConversationRole = 'user' | 'conductor'; + +export interface ConversationTurn { + role: ConversationRole; + text: string; +} + +export interface ConversationBufferConfig { + /** + * Turns retained, counting both halves. Ten is about five exchanges, which is + * far more than the two or three it usually takes to land on a request. + */ + maxTurns: number; + /** + * Total characters retained. The real guard: `maxTurns` alone lets ten + * rambling paragraphs through, and this is a prompt that a model reads on + * every routing turn. + */ + maxChars: number; +} + +export const DEFAULT_CONVERSATION_BUFFER_CONFIG: ConversationBufferConfig = { + maxTurns: 10, + maxChars: 4_000, +}; + +export class ConversationBuffer { + private readonly config: ConversationBufferConfig; + private turns: ConversationTurn[] = []; + + constructor(config: Partial = {}) { + this.config = { + maxTurns: Math.max(0, config.maxTurns ?? DEFAULT_CONVERSATION_BUFFER_CONFIG.maxTurns), + maxChars: Math.max(0, config.maxChars ?? DEFAULT_CONVERSATION_BUFFER_CONFIG.maxChars), + }; + } + + /** True while something has been said that no agent has been told about. */ + get active(): boolean { + return this.turns.length > 0; + } + + /** The exchange so far, oldest first. A copy: callers must not mutate it. */ + get history(): ConversationTurn[] { + return [...this.turns]; + } + + /** Record one half of the exchange. Empty text is ignored. */ + add(role: ConversationRole, text: string): void { + const line = text.trim(); + if (!line) return; + this.turns.push({ role, text: line }); + this.trim(); + } + + /** + * Forget the conversation. + * + * Called on dispatch, and whenever the floor closes. Both are the same fact: + * the discussion that was building toward a request is over. + */ + clear(): void { + this.turns = []; + } + + /** Oldest turns fall off first, by count and then by total size. */ + private trim(): void { + if (this.turns.length > this.config.maxTurns) { + this.turns = this.turns.slice(this.turns.length - this.config.maxTurns); + } + let total = this.turns.reduce((sum, turn) => sum + turn.text.length, 0); + while (total > this.config.maxChars && this.turns.length > 1) { + // Never below one turn: the thing just said is the least droppable part + // of the context, even when it is long enough to blow the budget alone. + total -= this.turns[0].text.length; + this.turns.shift(); + } + } +} diff --git a/src/main/acappella/router/grammar.ts b/src/main/acappella/router/grammar.ts new file mode 100644 index 0000000000..dae6dc24f7 --- /dev/null +++ b/src/main/acappella/router/grammar.ts @@ -0,0 +1,341 @@ +/** + * Grammar-constrained routing: the shared `RouteDecision` JSON Schema compiled + * into something a model cannot violate. + * + * A router that asks a 1.7B model for JSON and hopes gets three failure modes + * for free: a fenced code block, an invented `tabAction`, and a `sessionId` for + * an agent that does not exist. The first is a parsing problem. The third is the + * one that matters - a spoken instruction landing in the wrong repository - and + * no amount of prompt wording fixes it, because the model is sampling from a + * distribution that contains plausible-looking ids. + * + * So the schema is compiled ONCE into a node tree, and that tree is rendered two + * ways: + * + * - {@link CompiledRouteGrammar.gbnf} - a GBNF grammar for `node-llama-cpp`. + * llama.cpp masks the sampler against it, so the local Brain is structurally + * incapable of emitting malformed JSON, an out-of-set enum, or an id that is + * not in the roster. + * - {@link CompiledRouteGrammar.validate} - the same rules as a validator, for + * hosted Brains (whose structured-output modes are a request, not a + * guarantee) and for anything that reaches the executor. + * + * One tree, two renderers, deliberately: a grammar and a validator maintained + * separately drift, and the drift is invisible until the day the grammar allows + * something the validator rejects and a turn dies for no visible reason. + * + * The id sets are injected rather than described in the prompt. "Never invent an + * id" is an instruction; `"a1" | "a2" | "a3"` is a constraint. + */ + +import type { RosterAgent } from '../../../shared/acappella/protocol'; +import type { RouteDecision } from '../../../shared/acappella/route-decision'; +import { + isConversationalReply, + ROUTE_DECISION_JSON_SCHEMA, +} from '../../../shared/acappella/route-decision'; + +// --------------------------------------------------------------------------- +// The node tree +// --------------------------------------------------------------------------- + +/** + * The subset of JSON Schema `RouteDecision` uses. Everything outside it throws + * at compile time rather than being skipped: a construct the compiler silently + * ignored would produce a grammar that permits more than the schema does, which + * is the one outcome worse than a compile failure. + */ +type GrammarNode = + | { kind: 'object'; properties: Array<{ name: string; node: GrammarNode; required: boolean }> } + | { kind: 'string' } + | { kind: 'literal'; value: string } + | { kind: 'enum'; values: readonly string[] } + | { kind: 'number'; minimum?: number; maximum?: number } + | { kind: 'union'; options: GrammarNode[] }; + +/** Thrown when the schema grows a construct the compiler does not model. */ +export class UnsupportedSchemaError extends Error { + constructor(detail: string) { + super(`RouteDecision schema uses an unsupported construct: ${detail}`); + this.name = 'UnsupportedSchemaError'; + } +} + +type JsonSchemaNode = Record; + +function compileNode(schema: JsonSchemaNode, path: string): GrammarNode { + if (Array.isArray(schema.oneOf)) { + const options = (schema.oneOf as JsonSchemaNode[]).map((option, index) => + compileNode(option, `${path}.oneOf[${index}]`) + ); + return { kind: 'union', options }; + } + + const type = schema.type; + if (type === 'object') { + if (schema.additionalProperties !== false) { + // An open object cannot be expressed as a closed grammar, and pretending + // otherwise would let the model add fields nothing validates. + throw new UnsupportedSchemaError(`${path} must set additionalProperties: false`); + } + const properties = (schema.properties ?? {}) as Record; + const required = new Set((schema.required as string[] | undefined) ?? []); + const entries = Object.entries(properties).map(([name, child]) => ({ + name, + node: compileNode(child, `${path}.${name}`), + required: required.has(name), + })); + if (entries.length === 0) throw new UnsupportedSchemaError(`${path} has no properties`); + if (!entries[0].required) { + // The emitted grammar puts the separating comma INSIDE each optional + // group, which only works when something required comes first. + throw new UnsupportedSchemaError(`${path}'s first property must be required`); + } + return { kind: 'object', properties: entries }; + } + + if (type === 'string') { + if (typeof schema.const === 'string') return { kind: 'literal', value: schema.const }; + if (Array.isArray(schema.enum)) return { kind: 'enum', values: schema.enum as string[] }; + return { kind: 'string' }; + } + + if (type === 'number') { + return { + kind: 'number', + minimum: typeof schema.minimum === 'number' ? schema.minimum : undefined, + maximum: typeof schema.maximum === 'number' ? schema.maximum : undefined, + }; + } + + throw new UnsupportedSchemaError(`${path} has type '${String(type)}'`); +} + +// --------------------------------------------------------------------------- +// GBNF +// --------------------------------------------------------------------------- + +/** Shared lexical rules, emitted once per grammar. */ +const GBNF_PRELUDE = [ + 'ws ::= [ \\t\\n]*', + 'hex ::= [0-9a-fA-F]', + 'char ::= [^"\\\\] | "\\\\" (["\\\\/bfnrt] | "u" hex hex hex hex)', + 'string ::= "\\"" char* "\\""', + 'number ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? ([eE] [-+]? [0-9]+)?', +].join('\n'); + +/** A JSON string literal as a GBNF terminal: the quotes are part of the match. */ +function gbnfStringLiteral(value: string): string { + const escaped = value.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); + return `"\\"${escaped}\\""`; +} + +function nodeToGbnf(node: GrammarNode): string { + switch (node.kind) { + case 'string': + return 'string'; + case 'number': + return 'number'; + case 'literal': + return gbnfStringLiteral(node.value); + case 'enum': + if (node.values.length === 0) { + // An empty id set means "no agents are running". Emitting an empty + // alternation would be a grammar that matches nothing at all, so the + // field falls back to a free string and validation refuses the value. + return 'string'; + } + return `(${node.values.map(gbnfStringLiteral).join(' | ')})`; + case 'union': + return `(${node.options.map(nodeToGbnf).join(' | ')})`; + case 'object': { + const parts = node.properties.map((property, index) => { + const body = `${gbnfStringLiteral(property.name)} ws ":" ws ${nodeToGbnf(property.node)}`; + const separated = index === 0 ? body : `"," ws ${body}`; + return property.required ? separated : `(${separated})?`; + }); + return `"{" ws ${parts.join(' ')} ws "}"`; + } + } +} + +// --------------------------------------------------------------------------- +// Validation +// --------------------------------------------------------------------------- + +export interface GrammarValidation { + ok: boolean; + /** Every problem, not just the first: a retry prompt is better for knowing all of them. */ + errors: string[]; +} + +function validateNode(node: GrammarNode, value: unknown, path: string, errors: string[]): void { + switch (node.kind) { + case 'string': + if (typeof value !== 'string') errors.push(`${path} must be a string`); + return; + case 'literal': + if (value !== node.value) errors.push(`${path} must be "${node.value}"`); + return; + case 'enum': + if (typeof value !== 'string' || !node.values.includes(value)) { + errors.push( + node.values.length === 0 + ? `${path} has no valid values right now` + : `${path} must be one of ${node.values.map((v) => `"${v}"`).join(', ')}` + ); + } + return; + case 'number': + if (typeof value !== 'number' || !Number.isFinite(value)) { + errors.push(`${path} must be a number`); + return; + } + if (node.minimum !== undefined && value < node.minimum) { + errors.push(`${path} must be at least ${node.minimum}`); + } + if (node.maximum !== undefined && value > node.maximum) { + errors.push(`${path} must be at most ${node.maximum}`); + } + return; + case 'union': { + // A union reports its own failure rather than every branch's: "target is + // not one of the allowed shapes" is readable, and eight nested branch + // errors are not. + const matched = node.options.some((option) => { + const branch: string[] = []; + validateNode(option, value, path, branch); + return branch.length === 0; + }); + if (!matched) errors.push(`${path} does not match any allowed shape`); + return; + } + case 'object': { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + errors.push(`${path} must be an object`); + return; + } + const record = value as Record; + const known = new Set(node.properties.map((property) => property.name)); + for (const key of Object.keys(record)) { + if (!known.has(key)) errors.push(`${path}.${key} is not an allowed field`); + } + for (const property of node.properties) { + const child = record[property.name]; + if (child === undefined) { + if (property.required) errors.push(`${path}.${property.name} is required`); + continue; + } + validateNode(property.node, child, `${path}.${property.name}`, errors); + } + return; + } + } +} + +// --------------------------------------------------------------------------- +// Public surface +// --------------------------------------------------------------------------- + +/** The id sets a decision is allowed to name. Empty means "unconstrained". */ +export interface RouteGrammarScope { + sessionIds?: readonly string[]; + tabIds?: readonly string[]; +} + +export interface CompiledRouteGrammar { + /** GBNF text for `node-llama-cpp`'s `createGrammar`. */ + readonly gbnf: string; + /** The roster-constrained JSON Schema, for hosted structured outputs. */ + readonly schema: Record; + /** The same rules as the grammar, applied to an already-parsed value. */ + validate(value: unknown): GrammarValidation; +} + +/** + * Narrow the shared schema to the ids that actually exist right now. + * + * Exported because hosted providers need the schema itself: OpenAI takes it as + * `json_schema`, Anthropic as a tool's `input_schema`. Both go through the same + * narrowing, so a hosted Brain gets the same "you may not invent an id" + * constraint the local one does, and both are validated afterwards regardless. + */ +export function routeDecisionSchema(scope: RouteGrammarScope = {}): Record { + const schema = structuredClone(ROUTE_DECISION_JSON_SCHEMA) as unknown as JsonSchemaNode; + const properties = schema.properties as Record; + + if (scope.sessionIds && scope.sessionIds.length > 0) { + const target = properties.target as { oneOf: JsonSchemaNode[] }; + const agentShape = target.oneOf.find((option) => option.type === 'object'); + const agentProperties = agentShape?.properties as Record | undefined; + if (agentProperties?.sessionId) { + agentProperties.sessionId = { type: 'string', enum: [...scope.sessionIds] }; + } + } + + if (scope.tabIds && scope.tabIds.length > 0) { + properties.tabId = { type: 'string', enum: [...scope.tabIds] }; + } + + return schema as Record; +} + +/** Compile the schema, narrowed to `scope`, into a grammar and its validator. */ +export function compileRouteDecisionGrammar(scope: RouteGrammarScope = {}): CompiledRouteGrammar { + const schema = routeDecisionSchema(scope); + const root = compileNode(schema as JsonSchemaNode, 'decision'); + + return { + gbnf: `root ::= ${nodeToGbnf(root)}\n${GBNF_PRELUDE}\n`, + schema, + validate(value: unknown): GrammarValidation { + const errors: string[] = []; + validateNode(root, value, 'decision', errors); + return { ok: errors.length === 0, errors }; + }, + }; +} + +/** The ids a roster makes legal, in the shape {@link compileRouteDecisionGrammar} wants. */ +export function rosterScope(roster: readonly RosterAgent[]): RouteGrammarScope { + return { + sessionIds: roster.map((agent) => agent.sessionId), + tabIds: roster.flatMap((agent) => agent.tabs.map((tab) => tab.id)), + }; +} + +/** + * Validate a decision against the roster it will be executed on. + * + * Run for EVERY provider, grammar-constrained or not. A grammar guarantees a + * well-formed id and a structured-output mode guarantees a shape; neither + * guarantees that the agent still exists, because the user can close a tab while + * the model is thinking. This is the last check before anything is dispatched. + */ +export function validateRouteDecision( + decision: RouteDecision, + roster: readonly RosterAgent[] +): GrammarValidation { + const grammar = compileRouteDecisionGrammar(rosterScope(roster)); + // Round-tripped so `undefined` optional fields disappear the way they do on + // the wire, rather than tripping the "is not an allowed field" check. + const result = grammar.validate(JSON.parse(JSON.stringify(decision)) as unknown); + const errors = [...result.errors]; + + // A conversational reply reaches no agent, so its tab fields are decoration: + // the model still has to emit `tabAction` to satisfy the grammar, and holding + // it to a tab id nobody will read would reject a perfectly good spoken line. + if (decision.tabAction === 'recall' && !isConversationalReply(decision)) { + const targetId = typeof decision.target === 'string' ? null : decision.target.sessionId; + const agent = roster.find((candidate) => candidate.sessionId === targetId); + if (!decision.tabId) { + errors.push('decision.tabId is required when tabAction is "recall"'); + } else if (agent && !agent.tabs.some((tab) => tab.id === decision.tabId)) { + // A tab id that belongs to a DIFFERENT agent passes the flat id check and + // would then fail at dispatch, after the user was told where it went. + errors.push(`decision.tabId "${decision.tabId}" is not a tab on "${agent.name}"`); + } + } + + return { ok: errors.length === 0, errors }; +} diff --git a/src/main/acappella/router/index.ts b/src/main/acappella/router/index.ts new file mode 100644 index 0000000000..caa8c76220 --- /dev/null +++ b/src/main/acappella/router/index.ts @@ -0,0 +1,71 @@ +/** + * The Conductor router: context assembly, grammar-constrained decisions, recall + * matching, and the routing log. + * + * Everything here sits between a settled transcript and the dispatch executor. + * Nothing here performs a dispatch or touches a window - that is + * `../dispatch/route-executor.ts`, and the separation is what lets the routing + * rules be tested without an Electron window. + */ + +export { + createConductorRouter, + isCorrectionUtterance, + planCorrection, + DEFAULT_CONFIDENCE_THRESHOLD, + type ConductorRouter, + type ConductorRouterOptions, + type CorrectionPlan, +} from './conductor-router'; +export { + ConductorAgentBrain, + createConductorAgentBrain, + CONDUCTOR_AGENT_TIMEOUT_MS, + type ConductorAgentOptions, + type ConductorProcessManager, +} from './conductor-agent'; +export { + compileRouteDecisionGrammar, + rosterScope, + routeDecisionSchema, + validateRouteDecision, + UnsupportedSchemaError, + type CompiledRouteGrammar, + type GrammarValidation, + type RouteGrammarScope, +} from './grammar'; +export { + buildRoutingContext, + buildRoutingRoster, + deriveTabTopic, + getRoutingContext, + invalidateRoutingContext, + serializeRoutingContext, + MAX_CONTEXT_CHARS, + type RoutingContext, + type RoutingContextSources, +} from './routing-context'; +export { + flushRoutingLog, + lastRoutingTurn, + loadRoutingLog, + noteRoutingOutcome, + readRoutingLog, + recordRoutingTurn, + resetRoutingLog, + routingQuality, + setRoutingLogPath, + MAX_ENTRIES, + type RoutingLogEntry, + type RoutingOutcome, + type RoutingQuality, +} from './routing-log'; +export { + narrowRosterForRecall, + rankRecallCandidates, + resolveRecall, + DEFAULT_RECALL_LIMIT, + type RecallCandidate, + type RecallRankingOptions, + type RecallResolution, +} from './tab-recall'; diff --git a/src/main/acappella/router/routing-context.ts b/src/main/acappella/router/routing-context.ts new file mode 100644 index 0000000000..df798117aa --- /dev/null +++ b/src/main/acappella/router/routing-context.ts @@ -0,0 +1,378 @@ +/** + * The routing context: everything the Conductor needs to decide where an + * utterance goes, and nothing else. + * + * Three properties this file exists to hold: + * + * **No second summarizer.** A tab's topic is derived from data the app already + * produced - the name the tab-naming pipeline generated + * (`src/main/ipc/handlers/tabNaming.ts`), the opening message of the + * conversation, and the session synopsis the history manager already writes. A + * routing turn that had to summarise a dozen tabs first would be slower than + * looking at the screen, which defeats the point of speaking in the first place. + * + * **Bounded.** The context is capped in serialized size and degrades by dropping + * the least recently used tabs, so a user with two hundred tabs routes as fast as + * a user with four. A prompt that grows without limit does not fail loudly: it + * gets slower, and then the model starts ignoring the middle of it. + * + * **Cached.** Rebuilding it reads the sessions store and the history files. + * Doing that inside the turn would put disk I/O between a finished sentence and + * any visible response, so it is built once and invalidated when the roster or a + * tab changes. + */ + +import type { RosterAgent, RosterTab, RosterTabState } from '../../../shared/acappella/protocol'; +import type { StoredSession } from '../../stores/types'; +import { getSessionsStore } from '../../stores/getters'; +import { truncateCommand } from '../../../shared/formatters'; +import { serializeRoster } from '../providers/brain-prompt'; + +/** Serialized characters the Brain is allowed to see. Roughly 1.5k tokens. */ +export const MAX_CONTEXT_CHARS = 6000; + +/** Topic lines are one clause, not a paragraph. */ +const MAX_TOPIC_CHARS = 90; + +/** Agents whose history file is read for a synopsis, most recently active first. */ +const MAX_SYNOPSIS_AGENTS = 8; + +/** How long a built context is trusted when nothing announced a change. */ +const CACHE_TTL_MS = 15_000; + +// --------------------------------------------------------------------------- +// Shapes +// --------------------------------------------------------------------------- + +export interface RoutingContext { + /** The roster, enriched with per-agent status and per-tab topic. */ + agents: RosterAgent[]; + /** The agent the desktop is showing, when the store knows one. */ + activeAgentSessionId: string | null; + /** Oldest first. The voice conversation, not the agent transcripts. */ + recentUtterances: string[]; + /** Tabs left out to stay under the cap. Reported, never silent. */ + droppedTabs: number; + /** Size of `serializeRoutingContext(this)`, so the cap is measurable. */ + serializedChars: number; +} + +/** Everything the assembler reads, injectable so the rules are testable. */ +export interface RoutingContextSources { + getSessions?: () => StoredSession[]; + getActiveSessionId?: () => string | null; + /** + * The session synopsis material, keyed by agent id. Async and injected + * because the real one reads the history files, and a unit test of the + * bounding rules should not need a userData directory. + */ + getSynopses?: (sessionIds: string[]) => Promise>; + maxChars?: number; +} + +// --------------------------------------------------------------------------- +// Building +// --------------------------------------------------------------------------- + +/** + * The roster as the router sees it: open tabs, plus the snoozed and closed ones + * recall has to be able to reach. + * + * Shared with the dispatch executor, which builds its roster from the same + * function so the Brain and the executor can never disagree about what exists. + */ +export function buildRoutingRoster(sessions: StoredSession[]): RosterAgent[] { + return sessions + .filter((session) => session && typeof session.id === 'string') + .map((session) => ({ + sessionId: session.id, + name: session.name ?? '', + agentType: session.toolType ?? '', + cwd: session.cwd ?? '', + tabs: buildRoutingTabs(session), + })); +} + +function buildRoutingTabs(session: StoredSession): RosterTab[] { + const open = readTabRecords(session.aiTabs) + // A hidden tab is a cross-agent consult the user has never opened. It is a + // data container, not a conversation they can be sent back to. + .filter((tab) => tab.hidden !== true) + .map((tab) => toRosterTab(tab, 'open')); + + const snoozed = readTabRecords(session.snoozedTabs) + .map((entry) => entry.tab) + .filter((tab): tab is Record => !!tab && typeof tab.id === 'string') + .map((tab) => toRosterTab(tab, 'snoozed')); + + // Only AI entries: the closed-tab history is unified across file, terminal and + // browser tabs, and a voice session can address none of those. + const closed = readTabRecords(session.unifiedClosedTabHistory) + .filter((entry) => entry.type === 'ai') + .map((entry) => entry.tab) + .filter((tab): tab is Record => !!tab && typeof tab.id === 'string') + .map((tab) => toRosterTab(tab, 'closed')); + + const seen = new Set(); + return [...open, ...snoozed, ...closed].filter((tab) => { + if (seen.has(tab.id)) return false; + seen.add(tab.id); + return true; + }); +} + +/** Defensive read of an array-of-records field off a loosely typed stored session. */ +function readTabRecords(value: unknown): Array> { + if (!Array.isArray(value)) return []; + return value.filter( + (entry): entry is Record => !!entry && typeof entry === 'object' + ); +} + +function toRosterTab(tab: Record, state: RosterTabState): RosterTab { + const name = typeof tab.name === 'string' && tab.name.length > 0 ? tab.name : null; + return { + id: String(tab.id), + name, + lastActiveAt: tabLastActiveAt(tab), + state, + topic: deriveTabTopic(tab, name), + }; +} + +/** + * Best available "when did this tab last do anything". `AITab` has no such + * field, so the last log entry's timestamp stands in, with creation time as the + * floor for a tab nobody has spoken to yet. It orders recall candidates and + * decides what gets dropped under the size cap, so an approximation is fine and + * a wrong `null` would not be. + */ +function tabLastActiveAt(tab: Record): number | null { + const logs = readTabRecords(tab.logs); + const lastLog = logs.length > 0 ? logs[logs.length - 1] : null; + const stamps = [tab.createdAt, lastLog?.timestamp].filter( + (value): value is number => typeof value === 'number' && Number.isFinite(value) + ); + return stamps.length > 0 ? Math.max(...stamps) : null; +} + +/** + * What this tab is about, in one clause. + * + * The opening user message wins over the tab name because the name is already + * rendered next to it: the tab-naming pipeline compressed that same message into + * three words, so repeating it as the topic would spend context on a duplicate. + * The unabbreviated opening line is what a user is actually paraphrasing six + * hours later ("the one where I asked about the migration"), and the name is the + * fallback for a tab whose transcript no longer has it. + * + * Either way it is data that already exists. No model is asked anything here. + */ +export function deriveTabTopic(tab: Record, name: string | null): string | null { + const firstUserMessage = readTabRecords(tab.logs).find( + (entry) => entry.source === 'user' && typeof entry.text === 'string' && entry.text.trim() + ); + const opening = typeof firstUserMessage?.text === 'string' ? firstUserMessage.text : ''; + const topic = opening.trim() || name || ''; + return topic ? truncateCommand(collapseWhitespace(topic), MAX_TOPIC_CHARS) : null; +} + +/** + * One line, bounded. + * + * The truncation itself is `truncateCommand` from `shared/formatters.ts` - this + * codebase already had a dozen hand-rolled `slice(0, n) + '...'` helpers that + * had drifted on whether the ellipsis counts toward the limit, and a topic that + * overran its budget would defeat the size cap it is measured against. + */ +function collapseWhitespace(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +} + +/** + * Assemble the context, bounded. + * + * Pure over its inputs so the bounding rules can be tested without a store, a + * history directory, or an Electron app object. + */ +export function buildRoutingContext(input: { + sessions: StoredSession[]; + activeSessionId?: string | null; + recentUtterances?: string[]; + synopses?: Map; + maxChars?: number; +}): RoutingContext { + const agents: RosterAgent[] = buildRoutingRoster(input.sessions).map((agent) => { + const stored = input.sessions.find((session) => session.id === agent.sessionId); + return { + ...agent, + status: typeof stored?.state === 'string' ? stored.state : '', + recentWork: input.synopses?.get(agent.sessionId) ?? null, + }; + }); + + const context: RoutingContext = { + agents, + activeAgentSessionId: input.activeSessionId ?? null, + recentUtterances: [...(input.recentUtterances ?? [])], + droppedTabs: 0, + serializedChars: 0, + }; + + return enforceSizeCap(context, input.maxChars ?? MAX_CONTEXT_CHARS); +} + +/** + * Shrink the context until it serializes under the cap, oldest tab first. + * + * Tabs are dropped rather than agents: an agent missing from the roster cannot + * be routed to at all, while a missing tab only costs a recall the user can + * repeat with more words. An agent's last remaining tab is never dropped for the + * same reason - an agent with no tabs still takes a `current` or `new`. + */ +function enforceSizeCap(context: RoutingContext, maxChars: number): RoutingContext { + context.serializedChars = serializeRoutingContext(context).length; + if (context.serializedChars <= maxChars) return context; + + // Least recently active first, so what survives is what the user was most + // recently doing - which is also what they are most likely to talk about. + const candidates = context.agents + .flatMap((agent) => agent.tabs.map((tab) => ({ agent, tab }))) + .sort((a, b) => (a.tab.lastActiveAt ?? 0) - (b.tab.lastActiveAt ?? 0)); + + for (const candidate of candidates) { + if (context.serializedChars <= maxChars) break; + if (candidate.agent.tabs.length <= 1) continue; + candidate.agent.tabs = candidate.agent.tabs.filter((tab) => tab.id !== candidate.tab.id); + context.droppedTabs += 1; + context.serializedChars = serializeRoutingContext(context).length; + } + + return context; +} + +// --------------------------------------------------------------------------- +// Serialization +// --------------------------------------------------------------------------- + +/** + * The context as the Brain reads it. + * + * Line-oriented rather than JSON: a small model follows a list better than it + * follows nested braces, and the size cap is measured against this exact string + * rather than against a guess about tokens. + */ +export function serializeRoutingContext(context: RoutingContext): string { + const lines: string[] = serializeRoster(context.agents); + + if (context.activeAgentSessionId) { + lines.push('', `The user is looking at agent ${context.activeAgentSessionId}.`); + } + + if (context.recentUtterances.length > 0) { + lines.push('', 'Earlier in this conversation:'); + for (const utterance of context.recentUtterances.slice(-5)) lines.push(`- ${utterance}`); + } + + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// Cache +// --------------------------------------------------------------------------- + +let cached: { context: RoutingContext; builtAt: number } | null = null; + +/** + * Drop the cached context. + * + * Called when the roster or a tab changes. The TTL exists as a backstop for the + * changes nothing announces (a rename that never reached this process), not as + * the primary invalidation: a fifteen-second-stale roster would route to an + * agent the user just closed. + */ +export function invalidateRoutingContext(): void { + cached = null; +} + +/** + * The routing context for this turn, cached. + * + * `recentUtterances` is passed per call rather than cached, because it changes + * on every turn while the expensive half - sessions, tabs, synopses - does not. + */ +export async function getRoutingContext( + recentUtterances: string[] = [], + sources: RoutingContextSources = {} +): Promise { + const now = Date.now(); + if (cached && now - cached.builtAt < CACHE_TTL_MS) { + return { ...cached.context, recentUtterances: [...recentUtterances] }; + } + + const getSessions = sources.getSessions ?? readStoredSessions; + const sessions = getSessions(); + const activeSessionId = (sources.getActiveSessionId ?? readActiveSessionId)(); + const synopses = await (sources.getSynopses ?? readSessionSynopses)( + mostRecentSessionIds(sessions) + ); + + const context = buildRoutingContext({ + sessions, + activeSessionId, + recentUtterances, + synopses, + maxChars: sources.maxChars, + }); + cached = { context, builtAt: now }; + return context; +} + +/** The agents worth spending a history read on: the ones touched most recently. */ +function mostRecentSessionIds(sessions: StoredSession[]): string[] { + return [...sessions] + .sort((a, b) => (b.lastActivityTime ?? 0) - (a.lastActivityTime ?? 0)) + .slice(0, MAX_SYNOPSIS_AGENTS) + .map((session) => session.id); +} + +function readStoredSessions(): StoredSession[] { + return getSessionsStore().get('sessions', []); +} + +function readActiveSessionId(): string | null { + return getSessionsStore().get('activeSessionId') ?? null; +} + +/** + * The newest synopsis per agent, straight out of the history manager. + * + * The history manager already holds the summary of the last thing each agent + * finished - the same sentence the History panel shows - so "what has this agent + * been doing" costs a file read rather than an inference. A history file that + * cannot be read is skipped: a missing synopsis makes routing slightly worse and + * a thrown error makes the turn fail. + */ +async function readSessionSynopses(sessionIds: string[]): Promise> { + const { getHistoryManager } = await import('../../history-manager'); + const manager = getHistoryManager(); + const synopses = new Map(); + + await Promise.all( + sessionIds.map(async (sessionId) => { + try { + const entries = await manager.getEntries(sessionId); + // Newest first, and only a real summary: an empty one would render as a + // dangling "recently:" line that tells the model nothing. + const summary = entries.find((entry) => entry.summary?.trim())?.summary; + if (summary) { + synopses.set(sessionId, truncateCommand(collapseWhitespace(summary), MAX_TOPIC_CHARS)); + } + } catch { + /* no history for this agent, or it could not be read */ + } + }) + ); + + return synopses; +} diff --git a/src/main/acappella/router/routing-log.ts b/src/main/acappella/router/routing-log.ts new file mode 100644 index 0000000000..6b414e8056 --- /dev/null +++ b/src/main/acappella/router/routing-log.ts @@ -0,0 +1,268 @@ +/** + * The routing log: what the Conductor decided, and whether it was right. + * + * "It sent that to the wrong agent" is unanswerable without this file. The + * decision is gone the moment the tab changes, the roster it was made against + * has already moved on, and the user remembers the outcome rather than the + * utterance. So every routing turn is recorded with the four things that make a + * misroute diagnosable - what was said, how much context the Brain had, what it + * chose, and how sure it was - plus what happened next. + * + * The outcome is the point. A decision the user immediately corrected is a miss + * even though nothing errored, and a decision that asked a question instead of + * guessing is neither a hit nor a miss. Without those two distinctions the log + * says "100% dispatched" for a router that is wrong half the time, so + * {@link routingQuality} counts them separately and the hit rate excludes + * corrections rather than hiding them. + * + * Storage follows the app's existing conventions: one atomically written JSON + * file under `userData`, capped at {@link MAX_ENTRIES}, serialized through a + * keyed write queue like the history manager's per-session files. Utterances are + * truncated: this is a routing log, not a transcript of everything the user has + * ever said in their office. + */ + +import * as path from 'path'; +import { app } from 'electron'; + +import type { RouteDecision } from '../../../shared/acappella/route-decision'; +import { routeTargetSessionId } from '../../../shared/acappella/route-decision'; +import { truncateCommand } from '../../../shared/formatters'; +import { atomicWriteJson, createKeyedWriteQueue } from '../../utils/atomic-json-store'; +import { logger } from '../../utils/logger'; + +const LOG_CONTEXT = 'ACappella'; + +/** Entries retained. Enough to measure a session's routing, small enough to be free. */ +export const MAX_ENTRIES = 200; + +/** Utterances are truncated to this before anything is written to disk. */ +const MAX_UTTERANCE_CHARS = 200; + +/** Writes are batched: a routing turn must not wait on a file. */ +const FLUSH_DELAY_MS = 2000; + +/** + * What became of one decision. + * + * - `dispatched` - it reached an agent and a tab. + * - `clarified` - the router asked instead of guessing. Not a miss. + * - `corrected` - it was dispatched and then the user moved it. A miss. + * - `failed` - the dispatch itself could not be performed. + */ +export type RoutingOutcome = 'dispatched' | 'clarified' | 'corrected' | 'failed'; + +/** One routing turn, flattened so the file reads without cross-referencing. */ +export interface RoutingLogEntry { + id: string; + at: number; + utterance: string; + /** Serialized size of the context the Brain saw, and what was left out of it. */ + contextChars: number; + droppedTabs: number; + brainProviderId: string; + targetSessionId: string | null; + tabAction: RouteDecision['tabAction']; + tabId?: string; + tabName?: string; + confidence: number; + clarify?: string; + latencyMs: number; + outcome: RoutingOutcome; + /** Free text for a failure reason, or where a correction moved the prompt. */ + detail?: string; + /** How many constrained retries this turn needed. 0 for the common case. */ + retries?: number; +} + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +let entries: RoutingLogEntry[] = []; +let flushTimer: ReturnType | null = null; +let loaded = false; + +const writeQueue = createKeyedWriteQueue(); + +/** Injectable in tests: the real path needs an Electron app object. */ +let logFilePath: string | null = null; + +function resolveLogFilePath(): string { + if (logFilePath) return logFilePath; + logFilePath = path.join(app.getPath('userData'), 'acappella', 'routing-log.json'); + return logFilePath; +} + +/** Point the log at a different file. Test seam; production uses `userData`. */ +export function setRoutingLogPath(filePath: string | null): void { + logFilePath = filePath; +} + +// --------------------------------------------------------------------------- +// Recording +// --------------------------------------------------------------------------- + +/** + * Record one routing turn. Returns the entry id so the outcome can be attached + * once it is known - the decision is logged before it is executed, so a dispatch + * that never comes back still leaves evidence of what was attempted. + */ +export function recordRoutingTurn(input: { + id: string; + utterance: string; + decision: RouteDecision; + brainProviderId: string; + latencyMs: number; + contextChars?: number; + droppedTabs?: number; + outcome?: RoutingOutcome; + retries?: number; +}): string { + const entry: RoutingLogEntry = { + id: input.id, + at: Date.now(), + utterance: truncate(input.utterance, MAX_UTTERANCE_CHARS), + contextChars: input.contextChars ?? 0, + droppedTabs: input.droppedTabs ?? 0, + brainProviderId: input.brainProviderId, + targetSessionId: routeTargetSessionId(input.decision.target), + tabAction: input.decision.tabAction, + tabId: input.decision.tabId, + tabName: input.decision.tabName, + confidence: input.decision.confidence, + clarify: input.decision.clarify, + latencyMs: input.latencyMs, + outcome: input.outcome ?? (input.decision.clarify ? 'clarified' : 'dispatched'), + retries: input.retries, + }; + + entries.push(entry); + if (entries.length > MAX_ENTRIES) entries = entries.slice(-MAX_ENTRIES); + scheduleFlush(); + return entry.id; +} + +/** + * Attach the real outcome to a turn already recorded. + * + * A no-op for an id that has aged out of the ring, which is deliberate: an + * outcome arriving for a turn two hundred decisions ago is not worth resurrecting + * the entry for, and re-adding it would put it out of order. + */ +export function noteRoutingOutcome(id: string, outcome: RoutingOutcome, detail?: string): void { + const entry = entries.find((candidate) => candidate.id === id); + if (!entry) return; + entry.outcome = outcome; + if (detail) entry.detail = detail; + scheduleFlush(); +} + +/** The log, newest last. A copy: callers must not be able to rewrite history. */ +export function readRoutingLog(): RoutingLogEntry[] { + return entries.map((entry) => ({ ...entry })); +} + +/** The most recent turn, for the HUD's "why did it go there" line. */ +export function lastRoutingTurn(): RoutingLogEntry | null { + return entries.length > 0 ? { ...entries[entries.length - 1] } : null; +} + +export interface RoutingQuality { + turns: number; + dispatched: number; + clarified: number; + corrected: number; + failed: number; + /** + * Dispatches the user did not have to correct, over all dispatches. + * + * Clarifications are excluded from both halves: asking is the correct + * behaviour below the confidence threshold, and counting it as either a hit + * or a miss would make the threshold impossible to tune. + */ + hitRate: number | null; + /** Mean routing latency in ms, over turns that produced a decision. */ + meanLatencyMs: number | null; +} + +/** Aggregate the log into the numbers the evaluation doc reports. */ +export function routingQuality(): RoutingQuality { + const counts = { dispatched: 0, clarified: 0, corrected: 0, failed: 0 }; + for (const entry of entries) counts[entry.outcome] += 1; + + const decided = counts.dispatched + counts.corrected; + const latencies = entries.map((entry) => entry.latencyMs).filter((ms) => ms > 0); + + return { + turns: entries.length, + ...counts, + hitRate: decided > 0 ? counts.dispatched / decided : null, + meanLatencyMs: + latencies.length > 0 + ? Math.round(latencies.reduce((sum, ms) => sum + ms, 0) / latencies.length) + : null, + }; +} + +// --------------------------------------------------------------------------- +// Persistence +// --------------------------------------------------------------------------- + +/** Load the log from disk once, at first use. A missing file is an empty log. */ +export async function loadRoutingLog(): Promise { + if (loaded) return; + loaded = true; + try { + const { readFile } = await import('fs/promises'); + const raw = await readFile(resolveLogFilePath(), 'utf-8'); + const parsed: unknown = JSON.parse(raw); + if (Array.isArray(parsed)) entries = (parsed as RoutingLogEntry[]).slice(-MAX_ENTRIES); + } catch { + /* no log yet, or it is unreadable: start fresh rather than fail a turn */ + } +} + +function scheduleFlush(): void { + if (flushTimer) return; + flushTimer = setTimeout(() => { + flushTimer = null; + void flushRoutingLog(); + }, FLUSH_DELAY_MS); + // A pending log write must never be the reason the app stays alive. + flushTimer.unref?.(); +} + +/** Write the log now. Called by the debounced flush and on shutdown. */ +export async function flushRoutingLog(): Promise { + const snapshot = readRoutingLog(); + try { + // Inside the guard: resolving the path needs an Electron app object, and a + // host that has none (a test, a headless spawn) must not turn a best-effort + // log write into an unhandled rejection. + const filePath = resolveLogFilePath(); + await writeQueue.enqueue(filePath, async () => { + const { mkdir } = await import('fs/promises'); + await mkdir(path.dirname(filePath), { recursive: true }); + await atomicWriteJson(filePath, snapshot); + }); + } catch (error) { + // A log that cannot be written must not take a voice session with it. + logger.warn(`Could not write the routing log: ${(error as Error).message}`, LOG_CONTEXT); + } +} + +/** Drop everything. Test seam, and the "clear routing history" action. */ +export function resetRoutingLog(): void { + entries = []; + loaded = false; + if (flushTimer) { + clearTimeout(flushTimer); + flushTimer = null; + } +} + +/** One line, bounded. The truncation is the shared helper, never a new ladder. */ +function truncate(text: string, limit: number): string { + return truncateCommand(text.replace(/\s+/g, ' ').trim(), limit); +} diff --git a/src/main/acappella/router/tab-recall.ts b/src/main/acappella/router/tab-recall.ts new file mode 100644 index 0000000000..bb5a93329c --- /dev/null +++ b/src/main/acappella/router/tab-recall.ts @@ -0,0 +1,296 @@ +/** + * Tab recall: turning "back to the auth thing" into one specific conversation. + * + * Recall is the feature that makes voice worth using with more than two tabs + * open, and it is the one that cannot be solved by stuffing everything into the + * prompt. A user with sixty tabs has sixty topic lines, most of which are noise + * for any given utterance, and a small model handed all of them picks the one it + * saw most recently rather than the one that matches. + * + * So the shortlist is built HERE, with cheap local signals - word overlap + * against the tab name and topic, recency, a bias toward the agent already in + * play, and a mention of the project path - and only the top few go to the Brain + * to be chosen between. Ranking is a scoring problem; choosing is a language + * problem. Neither is good at the other's job. + * + * The two states that are not "open" are handled explicitly, because both have a + * wrong answer that looks like success: + * - A SNOOZED tab that is focused without being woken leaves the user staring + * at a tab strip that does not contain the tab they were told they are in. + * - A CLOSED tab that is quietly replaced by a fresh one loses the transcript + * the user was asking to return to, and they find out by reading a reply + * that has no memory of the conversation. + * + * (The subsequence matcher in `src/renderer/utils/search.ts` is deliberately not + * reused: it scores a typed prefix against a command name, pulls in React, and + * lives in the renderer. Spoken recall is whole-word overlap across a phrase.) + */ + +import type { RosterAgent, RosterTab } from '../../../shared/acappella/protocol'; +import type { RouteDecision } from '../../../shared/acappella/route-decision'; +import { routeTargetSessionId } from '../../../shared/acappella/route-decision'; + +/** How many candidates the Brain is asked to choose between. */ +export const DEFAULT_RECALL_LIMIT = 5; + +/** A tab this recently active is treated as fully fresh. Two hours. */ +const RECENCY_FULL_MS = 2 * 60 * 60_000; + +/** Beyond this, recency contributes nothing. A week. */ +const RECENCY_ZERO_MS = 7 * 24 * 60 * 60_000; + +/** + * Words that carry no recall signal. + * + * Deliberately short. A long stop list starts removing the words that DO + * identify a tab ("test", "new", "build"), and the scorer already discounts a + * term that appears in half the tabs by requiring more than one match to win. + */ +const STOP_WORDS = new Set([ + 'a', + 'about', + 'and', + 'back', + 'go', + 'in', + 'is', + 'it', + 'me', + 'my', + 'of', + 'on', + 'one', + 'that', + 'the', + 'thing', + 'to', + 'we', + 'what', + 'where', + 'with', + 'you', +]); + +export interface RecallCandidate { + agentSessionId: string; + agentName: string; + tab: RosterTab; + /** Higher is better. Comparable only within one ranking call. */ + score: number; + /** Why it scored, in words, for the routing log and for debugging a misroute. */ + reasons: string[]; +} + +export interface RecallRankingOptions { + /** The agent already in play. Its tabs get a small, deliberate bias. */ + activeAgentSessionId?: string | null; + limit?: number; + /** Now, injectable so recency scoring is deterministic in tests. */ + now?: number; +} + +/** + * Rank every tab in the roster against the utterance. + * + * Returns at most `limit` candidates, best first, and only ones that scored at + * all: a zero-score candidate is noise, and padding the shortlist to a fixed + * length is how an unrelated tab ends up in front of the model. + */ +export function rankRecallCandidates( + utterance: string, + roster: readonly RosterAgent[], + options: RecallRankingOptions = {} +): RecallCandidate[] { + const terms = termsOf(utterance); + const now = options.now ?? Date.now(); + const limit = options.limit ?? DEFAULT_RECALL_LIMIT; + + const candidates: RecallCandidate[] = []; + for (const agent of roster) { + const pathBonus = mentionsProjectPath(terms, agent.cwd) ? 0.5 : 0; + const activeBonus = agent.sessionId === options.activeAgentSessionId ? 0.25 : 0; + + for (const tab of agent.tabs) { + const reasons: string[] = []; + const overlap = overlapScore(terms, tab); + if (overlap > 0) reasons.push(`matches "${tab.name ?? tab.topic ?? tab.id}"`); + + const recency = recencyScore(tab.lastActiveAt, now); + if (recency > 0.5) reasons.push('recently active'); + if (pathBonus) reasons.push(`project path mentioned (${agent.cwd})`); + if (activeBonus) reasons.push('same agent as the current turn'); + + // Overlap is weighted highest on purpose: recency alone would make recall + // mean "the tab before this one", which the user can already reach by + // saying nothing. + const score = overlap * 3 + recency + pathBonus + activeBonus; + if (overlap === 0 && pathBonus === 0) continue; + + candidates.push({ + agentSessionId: agent.sessionId, + agentName: agent.name, + tab, + score: Math.round(score * 1000) / 1000, + reasons, + }); + } + } + + return candidates + .sort((a, b) => b.score - a.score || (b.tab.lastActiveAt ?? 0) - (a.tab.lastActiveAt ?? 0)) + .slice(0, limit); +} + +/** + * The roster the Brain should see for a recall-shaped utterance: every agent, + * but only the shortlisted tabs. + * + * Agents are kept whole even when none of their tabs shortlisted, because the + * utterance may not be a recall at all and dropping an agent would make it + * unroutable. Only the tab lists shrink. + */ +export function narrowRosterForRecall( + roster: readonly RosterAgent[], + candidates: readonly RecallCandidate[] +): RosterAgent[] { + if (candidates.length === 0) return [...roster]; + const keep = new Set(candidates.map((candidate) => candidate.tab.id)); + return roster.map((agent) => ({ + ...agent, + tabs: agent.tabs.filter( + // A tab that is open and current is always kept: `current` is the common + // action and it needs the tab the user is looking at to still be listed. + (tab) => keep.has(tab.id) || (tab.state ?? 'open') === 'open' + ), + })); +} + +// --------------------------------------------------------------------------- +// Scoring +// --------------------------------------------------------------------------- + +function termsOf(text: string): string[] { + return text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((term) => term.length > 1 && !STOP_WORDS.has(term)); +} + +/** Fraction of the utterance's terms that appear in the tab's name or topic. */ +function overlapScore(terms: string[], tab: RosterTab): number { + if (terms.length === 0) return 0; + const haystack = new Set(termsOf(`${tab.name ?? ''} ${tab.topic ?? ''}`)); + if (haystack.size === 0) return 0; + const hits = terms.filter((term) => haystack.has(term)).length; + return hits / terms.length; +} + +/** 1 for a tab touched in the last couple of hours, decaying to 0 over a week. */ +function recencyScore(lastActiveAt: number | null, now: number): number { + if (!lastActiveAt) return 0; + const age = now - lastActiveAt; + if (age <= RECENCY_FULL_MS) return 1; + if (age >= RECENCY_ZERO_MS) return 0; + return 1 - (age - RECENCY_FULL_MS) / (RECENCY_ZERO_MS - RECENCY_FULL_MS); +} + +/** True when the utterance names a directory from the agent's project path. */ +function mentionsProjectPath(terms: string[], cwd: string): boolean { + if (!cwd || terms.length === 0) return false; + const segments = new Set(termsOf(cwd.replace(/[\\/]/g, ' '))); + return terms.some((term) => segments.has(term)); +} + +// --------------------------------------------------------------------------- +// Resolution +// --------------------------------------------------------------------------- + +/** + * What the dispatch has to do to honour a recall. + * + * `offer` is not a failure: it is the only honest answer for a closed tab, whose + * transcript is retained but whose reopening is a decision the user gets to make. + */ +export type RecallResolution = + | { kind: 'focus'; agentSessionId: string; tab: RosterTab } + | { kind: 'wake'; agentSessionId: string; tab: RosterTab } + | { kind: 'reopen'; agentSessionId: string; tab: RosterTab } + | { kind: 'offer'; agentSessionId: string; tab: RosterTab; question: string } + | { kind: 'missing'; tabId: string | undefined }; + +export interface RecallResolutionOptions { + /** + * The user has already been offered this reopen and answered. Set on the turn + * after an `offer`, which is what turns the second pass into an action rather + * than the same question again. + */ + confirmed?: boolean; +} + +/** + * Resolve a `recall` decision against the roster it will run on. + * + * Called by the dispatch executor, which then performs exactly one of the four + * outcomes. Nothing here mutates anything: a resolution is a plan. + */ +export function resolveRecall( + decision: RouteDecision, + roster: readonly RosterAgent[], + options: RecallResolutionOptions = {} +): RecallResolution { + const targetId = routeTargetSessionId(decision.target); + const found = findTab(roster, decision.tabId, targetId); + if (!found) return { kind: 'missing', tabId: decision.tabId }; + + const { agent, tab } = found; + switch (tab.state ?? 'open') { + case 'snoozed': + // Woken as part of the dispatch, not focused and left hidden: the tab is + // not in the tab strip, so "I'm in the auth tab" would be a lie. + return { kind: 'wake', agentSessionId: agent.sessionId, tab }; + case 'closed': + return options.confirmed + ? { kind: 'reopen', agentSessionId: agent.sessionId, tab } + : { + kind: 'offer', + agentSessionId: agent.sessionId, + tab, + question: `${describeTab(tab)} is closed. Should I reopen it?`, + }; + default: + return { kind: 'focus', agentSessionId: agent.sessionId, tab }; + } +} + +/** + * Find the tab a recall names. + * + * The target agent is searched first, then the rest of the roster: a Brain that + * picked the right tab and the wrong agent has still identified the conversation, + * and refusing that would be pedantry the user hears as a failure. + */ +function findTab( + roster: readonly RosterAgent[], + tabId: string | undefined, + targetSessionId: string | null +): { agent: RosterAgent; tab: RosterTab } | null { + if (!tabId) return null; + const ordered = + targetSessionId === null + ? [...roster] + : [ + ...roster.filter((agent) => agent.sessionId === targetSessionId), + ...roster.filter((agent) => agent.sessionId !== targetSessionId), + ]; + + for (const agent of ordered) { + const tab = agent.tabs.find((candidate) => candidate.id === tabId); + if (tab) return { agent, tab }; + } + return null; +} + +/** How a tab is referred to out loud. */ +function describeTab(tab: RosterTab): string { + return tab.name ? `"${tab.name}"` : 'that conversation'; +} diff --git a/src/main/acappella/runtime/native-loader.ts b/src/main/acappella/runtime/native-loader.ts new file mode 100644 index 0000000000..c476817e22 --- /dev/null +++ b/src/main/acappella/runtime/native-loader.ts @@ -0,0 +1,401 @@ +/** + * The one place A Cappella's native runtimes are imported. + * + * Three rules, and the file exists because all three are easy to break by + * accident anywhere else: + * + * 1. **Nothing native is imported at module load.** Every import in here is a + * dynamic `import()` inside a function. A user with the Encore Feature off + * never loads a single native symbol, and a user with it on pays for + * llama.cpp only when a session actually needs the Brain. A top-level + * `import 'node-llama-cpp'` anywhere else in the codebase would undo that + * silently, which is why concrete providers must come through here. + * 2. **A failure is structured, never an opaque dlopen string.** + * "Error: Cannot open shared object file" tells a user nothing and tells a + * support report less. Every failure comes back as a + * {@link NativeRuntimeUnavailable} carrying the runtime, the module, the + * platform, the arch, a classified reason, and the underlying message. + * 3. **A failed load never crashes the app.** {@link tryLoadNativeRuntime} + * returns a result object; {@link loadNativeRuntime} throws a typed error a + * caller can catch. Nothing here escapes as an unhandled rejection. + * + * The last failure per runtime is REMEMBERED, which is what lets the capability + * gate say "the ONNX Runtime will not load on this machine" without loading + * anything itself. The gate runs on every Settings render; making it dlopen a + * few hundred megabytes of inference engine to draw a panel would be the exact + * opposite of rule 1. + */ + +import { + getNativeRuntime, + nativePlatformKey, + type NativeRuntimeDescriptor, + type NativeRuntimeId, +} from '../../../shared/acappella/native-runtimes'; +import { isWindows } from '../../../shared/platformDetection'; + +/** + * Why a runtime is not usable. + * + * Classified rather than collapsed into one "load failed" because the user's + * next action differs for every one of them: wait for the feature to ship, + * reinstall the app, use a different machine, or install a redistributable. + */ +export type NativeRuntimeFailureKind = + | 'not-a-dependency' + | 'unsupported-platform' + | 'module-not-found' + | 'load-failed'; + +/** A runtime that will not load, and everything needed to say why. */ +export interface NativeRuntimeUnavailable { + /** Discriminator, so this can travel inside a union without being mistaken for a module. */ + readonly kind: 'runtime-unavailable'; + readonly runtimeId: NativeRuntimeId; + readonly moduleId: string; + readonly platform: string; + readonly arch: string; + readonly failure: NativeRuntimeFailureKind; + /** One sentence, written for a person. */ + readonly message: string; + /** What the caller can do about it. */ + readonly suggestedAction: string; + /** The underlying error text, kept verbatim for support reports. */ + readonly detail?: string; +} + +/** The typed throw. Callers that prefer a result object use {@link tryLoadNativeRuntime}. */ +export class NativeRuntimeUnavailableError extends Error { + readonly info: NativeRuntimeUnavailable; + + constructor(info: NativeRuntimeUnavailable) { + super(info.message); + this.name = 'NativeRuntimeUnavailableError'; + this.info = info; + } +} + +export type NativeRuntimeResult = + | { ok: true; module: T } + | { ok: false; error: NativeRuntimeUnavailable }; + +/** + * A real dynamic import that survives transpilation. + * + * `tsconfig.main.json` emits CommonJS, and TypeScript rewrites `import()` into + * `require()` there. That would be fine for a classic addon and fatal for + * `node-llama-cpp`, which is ESM-only. Going through `new Function` keeps a + * genuine dynamic import in the emitted output. It is also why the specifier is + * always a variable: a literal would make TypeScript try to resolve a package + * that is deliberately not installed yet. + */ +const dynamicImport: (specifier: string) => Promise = new Function( + 'specifier', + 'return import(specifier);' +) as (specifier: string) => Promise; + +/** Injected in tests. Production always uses the real import. */ +let importer: (specifier: string) => Promise = dynamicImport; + +/** In-flight and settled loads, so a second caller does not dlopen twice. */ +const loaded = new Map>(); + +/** The last failure per runtime, for the capability gate and the debug package. */ +const failures = new Map(); + +/** + * Replace the module importer. Tests only. + * + * Exists because the alternative is a test suite that either installs a + * gigabyte of inference engines or cannot test the failure paths at all, and the + * failure paths are the entire point of this module. + */ +export function __setNativeImporter(fn: ((specifier: string) => Promise) | null): void { + importer = fn ?? dynamicImport; +} + +/** Forget every cached module and failure. Tests, and the "retry" affordance. */ +export function resetNativeRuntimes(): void { + loaded.clear(); + failures.clear(); +} + +/** True when this runtime has already been loaded in this process. */ +export function isNativeRuntimeLoaded(id: NativeRuntimeId): boolean { + return loaded.has(id); +} + +/** + * Drop a runtime from the cache. + * + * Node has no unload, so the native library stays resident for the life of the + * process. What this buys is a fresh attempt: after a self-test, or after a user + * installs the thing that was missing, the next load actually retries instead of + * replaying a cached rejection. + */ +export function unloadNativeRuntime(id: NativeRuntimeId): void { + loaded.delete(id); + failures.delete(id); +} + +/** + * The last known failure for a runtime, or null when it has never failed here. + * + * Deliberately does NOT attempt a load: this is what the capability gate reads, + * and the gate must stay a cheap disk-and-settings question. + */ +export function lastNativeRuntimeFailure(id: NativeRuntimeId): NativeRuntimeUnavailable | null { + return failures.get(id) ?? null; +} + +/** Every remembered failure, in registry order. For the debug package. */ +export function allNativeRuntimeFailures(): NativeRuntimeUnavailable[] { + return [...failures.values()]; +} + +/** + * Why this runtime will not load, WITHOUT loading it. Null when it should. + * + * The difference from {@link lastNativeRuntimeFailure} is the difference between + * "has this already gone wrong here" and "will this work here", and the second + * is the only one a capability gate can act on. Two of the reasons a runtime + * cannot load are knowable from the registry alone - the package is not a + * dependency of this build, or there is no binary for this platform - and a gate + * that only reads remembered failures reports those runtimes as FINE until + * something attempts a load and fails. + * + * That gap is not theoretical. On a fresh boot nothing has attempted anything, + * so readiness came back "everything satisfied, start a session" for slots whose + * runtime is not in the build at all; the user got a green button, downloaded + * gigabytes of models on its say-so, and the session then died mid-flight in a + * provider's `start()`. The same call after any load attempt said the opposite, + * which made readiness depend on the order the app happened to do things in. + * + * Side-effect free on purpose: it does NOT record into the remembered failures, + * so asking the question cannot make the debug package report a failure nobody + * ever hit. + */ +export function knownNativeRuntimeUnavailability( + id: NativeRuntimeId +): NativeRuntimeUnavailable | null { + const remembered = failures.get(id); + if (remembered) return remembered; + + const descriptor = getNativeRuntime(id); + if (!descriptor) return unknownRuntime(id); + return declineBeforeLoading(descriptor); +} + +/** + * Load a native runtime, or throw {@link NativeRuntimeUnavailableError}. + * + * Resolves to the module's namespace object. Callers cast to their own minimal + * structural type rather than importing the package's types, which is what keeps + * the package out of every other file's import graph. + */ +export async function loadNativeRuntime(id: NativeRuntimeId): Promise { + const result = await tryLoadNativeRuntime(id); + if (!result.ok) throw new NativeRuntimeUnavailableError(result.error); + return result.module; +} + +/** + * Load a native runtime, reporting failure as a value. + * + * Never rejects. A runtime that cannot load is a capability the app does not + * have, not an exception the app should die on. + */ +export async function tryLoadNativeRuntime( + id: NativeRuntimeId +): Promise> { + const descriptor = getNativeRuntime(id); + if (!descriptor) { + return { ok: false, error: unknownRuntime(id) }; + } + + const cached = loaded.get(id); + if (cached) { + try { + return { ok: true, module: (await cached) as T }; + } catch { + // The rejection was already classified and remembered on the first + // attempt; replay it rather than re-deriving it from a stale error. + return { + ok: false, + error: failures.get(id) ?? classify(descriptor, new Error('load failed')), + }; + } + } + + const declined = declineBeforeLoading(descriptor); + if (declined) { + failures.set(id, declined); + return { ok: false, error: declined }; + } + + const attempt = importer(descriptor.moduleId); + loaded.set(id, attempt); + + try { + const module = (await attempt) as T; + failures.delete(id); + return { ok: true, module }; + } catch (error) { + // Drop the cache entry so a later attempt (after the user installs the + // missing piece) is a real retry rather than a replayed rejection. + loaded.delete(id); + const classified = classify(descriptor, error); + failures.set(id, classified); + return { ok: false, error: classified }; + } +} + +/** + * The two failures that are knowable without touching the module at all. + * + * Checked first because attempting the import would produce a MODULE_NOT_FOUND + * in both cases, and "the package is not installed yet" and "your platform has + * no build" are completely different answers to give a user. + */ +function declineBeforeLoading( + descriptor: NativeRuntimeDescriptor +): NativeRuntimeUnavailable | null { + if (!descriptor.declared) { + return { + kind: 'runtime-unavailable', + runtimeId: descriptor.id, + moduleId: descriptor.moduleId, + platform: process.platform, + arch: process.arch, + failure: 'not-a-dependency', + message: `${descriptor.label} is not part of this build yet.`, + // No "for this slot": the detail this rides behind already names the slot, + // and `readinessErrorMessage` hoists a shared recovery to the end of a + // multi-slot refusal, where a singular "this slot" would be wrong. + suggestedAction: 'Use a hosted provider or the mock tier until the local runtime ships.', + }; + } + + const key = nativePlatformKey(process.platform, process.arch); + if (!key) { + return { + kind: 'runtime-unavailable', + runtimeId: descriptor.id, + moduleId: descriptor.moduleId, + platform: process.platform, + arch: process.arch, + failure: 'unsupported-platform', + message: `${descriptor.label} has no build for ${process.platform}-${process.arch}.`, + suggestedAction: `Switch this slot to a hosted provider: there is no local ${descriptor.label} binary for this platform.`, + }; + } + + if (descriptor.prebuilds[key] === 'unavailable') { + return { + kind: 'runtime-unavailable', + runtimeId: descriptor.id, + moduleId: descriptor.moduleId, + platform: process.platform, + arch: process.arch, + failure: 'unsupported-platform', + message: `${descriptor.label} is not shipped for ${key}.`, + suggestedAction: `Switch this slot to a hosted provider: there is no local ${descriptor.label} binary for this platform.`, + }; + } + + return null; +} + +/** Turn whatever the module system threw into something a person can act on. */ +function classify(descriptor: NativeRuntimeDescriptor, error: unknown): NativeRuntimeUnavailable { + const detail = error instanceof Error ? error.message : String(error); + const code = (error as NodeJS.ErrnoException | null)?.code; + const missing = code === 'MODULE_NOT_FOUND' || code === 'ERR_MODULE_NOT_FOUND'; + + if (missing) { + return { + kind: 'runtime-unavailable', + runtimeId: descriptor.id, + moduleId: descriptor.moduleId, + platform: process.platform, + arch: process.arch, + failure: 'module-not-found', + message: `${descriptor.label} is missing from this installation (${descriptor.moduleId}).`, + // A missing module in a packaged app means the binary never made it into + // the bundle, which a user cannot repair from inside the app. + suggestedAction: 'Reinstall Maestro, then run the voice self-test again.', + detail, + }; + } + + if (isMissingSystemLibrary(detail)) { + return { + kind: 'runtime-unavailable', + runtimeId: descriptor.id, + moduleId: descriptor.moduleId, + platform: process.platform, + arch: process.arch, + failure: 'load-failed', + message: `${descriptor.label} loaded but a system library it needs is missing.`, + // Windows reports this as a bare "The specified module could not be + // found", naming the addon rather than the DLL it actually wanted, which + // reads like a corrupt install. It is almost always the Visual C++ + // runtime, and naming it is the difference between a fix and a reinstall + // that changes nothing. + suggestedAction: isWindows() + ? 'Install the Microsoft Visual C++ Redistributable (x64), then run the voice self-test again.' + : 'A shared library this runtime depends on is missing from the system. Run the voice self-test and include the result in a bug report.', + detail, + }; + } + + return { + kind: 'runtime-unavailable', + runtimeId: descriptor.id, + moduleId: descriptor.moduleId, + platform: process.platform, + arch: process.arch, + failure: 'load-failed', + message: `${descriptor.label} failed to load on ${process.platform}-${process.arch}.`, + suggestedAction: + 'Run the voice self-test in Settings > Plugins > A Cappella > Models and include the result in a bug report.', + detail, + }; +} + +/** + * Whether a load error is the OS saying a dependent shared library is absent. + * + * Matched by message because that is all the platforms give: Windows returns + * error 126 as text, and the Unix loaders name the missing object. This is the + * one load failure with a specific user-facing fix, so it is worth telling apart + * from the generic case. + */ +function isMissingSystemLibrary(detail: string): boolean { + return ( + /The specified module could not be found/i.test(detail) || + /error (?:code )?126\b/i.test(detail) || + /0xc000007b/i.test(detail) || + /cannot open shared object file/i.test(detail) || + /(?:Library|image) not loaded/i.test(detail) + ); +} + +function unknownRuntime(id: NativeRuntimeId): NativeRuntimeUnavailable { + return { + kind: 'runtime-unavailable', + runtimeId: id, + moduleId: String(id), + platform: process.platform, + arch: process.arch, + failure: 'module-not-found', + message: `Unknown native runtime "${id}".`, + suggestedAction: 'This is a bug: the runtime registry has no descriptor for that id.', + }; +} + +/** One line for a log or a support report. */ +export function describeRuntimeUnavailable(info: NativeRuntimeUnavailable): string { + const detail = info.detail ? ` (${info.detail})` : ''; + return `${info.moduleId} [${info.failure}] on ${info.platform}-${info.arch}: ${info.message}${detail}`; +} diff --git a/src/main/acappella/runtime/runtime-installer.ts b/src/main/acappella/runtime/runtime-installer.ts new file mode 100644 index 0000000000..fa465e4816 --- /dev/null +++ b/src/main/acappella/runtime/runtime-installer.ts @@ -0,0 +1,319 @@ +/** + * Download, verify, and lay out a native runtime payload. + * + * The transaction, in the order it must happen: + * + * 1. Stream the tarball to `.staging/payload.tgz`, hashing as it arrives. + * 2. Compare the hash to the catalog. A mismatch deletes everything and stops. + * 3. Extract into `.staging/`, keeping only this platform's subtree. + * 4. Prove the binary the artifact promised is really there. + * 5. Replace `/` with the staging directory, then write the manifest. + * + * **Nothing is visible at the install path until step 5.** That ordering is the + * whole design: `isRuntimeInstalled()` answers by reading the manifest, so the + * manifest is the commit record of this transaction and is written last, after + * the bytes are proven and in place. A killed app leaves a staging directory, + * which the next install deletes, rather than a half-extracted engine that passes + * an existence check and detonates later inside a dlopen. + * + * **Why the hash is checked before extraction, not after.** These payloads carry + * executable code that the app will dlopen. Unpacking unverified bytes onto disk + * and checking afterwards means the window where a tampered archive exists on the + * user's machine is a window where it can be executed by something else. Verify + * first, unpack second, and there is no window. + * + * The extraction FILTER is the other half of the size argument in + * `runtime-artifacts.ts`: the ONNX Runtime tarball carries five platforms, and + * only the running one is written to disk. That logic is + * {@link shouldKeepArchiveEntry}, kept pure and separately tested, because a + * filter that is wrong in the permissive direction silently costs a user 220 MB + * and a filter that is wrong in the strict direction produces an install that is + * missing its binary. + */ + +import { createHash } from 'crypto'; +import { createWriteStream } from 'fs'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { Readable } from 'stream'; +import { pipeline } from 'stream/promises'; +import * as tar from 'tar'; + +import type { NativeRuntimeId } from '../../../shared/acappella/native-runtimes'; +import type { NativeRuntimeArtifact } from '../../../shared/acappella/runtime-artifacts'; +import { logger } from '../../utils/logger'; +import { + artifactForThisPlatform, + runtimeDir, + runtimeStagingDir, + writeRuntimeManifest, + type RuntimeManifest, +} from './runtime-store'; + +const LOG_CONTEXT = 'ACappella'; + +/** The tarball's name inside the staging directory. Deleted after extraction. */ +const PAYLOAD_FILENAME = 'payload.tgz'; + +/** Progress push interval. Matches the model downloader's ~4 Hz, for one cadence. */ +export const RUNTIME_PROGRESS_INTERVAL_MS = 250; + +export type RuntimeInstallPhase = 'downloading' | 'verifying' | 'extracting' | 'done'; + +export interface RuntimeInstallProgress { + runtimeId: NativeRuntimeId; + phase: RuntimeInstallPhase; + /** Bytes downloaded so far. Zero outside the download phase. */ + bytes: number; + /** Total bytes expected, from the catalog. */ + totalBytes: number; +} + +export type RuntimeProgressListener = (progress: RuntimeInstallProgress) => void; + +export type FetchLike = (url: string, init?: RequestInit) => Promise; + +export interface RuntimeInstallOptions { + /** Injected in tests. Production uses global fetch. */ + fetchImpl?: FetchLike; + onProgress?: RuntimeProgressListener; + signal?: AbortSignal; +} + +/** A download whose bytes are not what the catalog promised. Never retried. */ +export class RuntimeHashMismatchError extends Error { + readonly expected: string; + readonly actual: string; + + constructor(runtimeId: NativeRuntimeId, expected: string, actual: string) { + super( + `Downloaded ${runtimeId} runtime does not match the expected checksum. ` + + `Expected ${expected}, got ${actual}.` + ); + this.name = 'RuntimeHashMismatchError'; + this.expected = expected; + this.actual = actual; + } +} + +/** An archive that extracted without producing the binary it promised. */ +export class RuntimeBinaryMissingError extends Error { + constructor(runtimeId: NativeRuntimeId, binary: string) { + super(`The ${runtimeId} runtime payload did not contain ${binary}.`); + this.name = 'RuntimeBinaryMissingError'; + } +} + +/** + * Should this archive entry be written to disk? + * + * Pure, and separated from the extraction so it can be tested against real + * archive paths with no tarball and no filesystem. Three jobs: + * + * - Drop the leading `package/` that every npm tarball carries + * (`stripComponents`), so `keep` is written in terms the artifact's other + * paths already use rather than repeating the prefix everywhere. + * - Keep an entry only when it is inside one of the `keep` prefixes. Matching + * is on path SEGMENTS, so `bin/napi-v6/darwin/arm64` cannot also admit a + * sibling directory whose name merely starts with `arm64`. + * - Refuse anything that climbs out of the root. node-tar guards this too; + * the check is repeated here because this function is the one place that has + * both the path and the intent, and a traversal that reaches a dlopen target + * is the worst failure this module could have. + * + * @param archivePath Entry path exactly as it appears in the archive. + * @returns The path to write, relative to the install root, or null to skip. + */ +export function shouldKeepArchiveEntry( + archivePath: string, + stripComponents: number, + keep: readonly string[] +): string | null { + // Archive paths are POSIX regardless of the platform unpacking them. + const segments = archivePath.split('/').filter((segment) => segment.length > 0); + if (segments.length <= stripComponents) return null; + + const stripped = segments.slice(stripComponents); + if (stripped.some((segment) => segment === '..' || segment === '.')) return null; + + const relative = stripped.join('/'); + for (const prefix of keep) { + const prefixSegments = prefix.split('/').filter((segment) => segment.length > 0); + if (prefixSegments.length > stripped.length) continue; + const matches = prefixSegments.every((segment, index) => stripped[index] === segment); + if (matches) return relative; + } + return null; +} + +/** + * Install a runtime for the platform this process is running on. + * + * Resolves to the manifest that was written. Throws on every failure, because a + * caller that asked for an install wants to know why it did not happen; the + * READINESS question is `isRuntimeInstalled()`, and it is deliberately somewhere + * else so that asking it can never start a download. + */ +export async function installNativeRuntime( + id: NativeRuntimeId, + options: RuntimeInstallOptions = {} +): Promise { + const artifact = artifactForThisPlatform(id); + if (!artifact) { + throw new Error( + `There is no downloadable ${id} runtime for ${process.platform}-${process.arch}.` + ); + } + + const staging = runtimeStagingDir(id); + // A staging directory here is the wreckage of an install that was killed. + // Removing it beats resuming into it: the tarball's hash covers the whole + // file, and half a tarball plus a fresh tail is not a file anyone verified. + await fs.rm(staging, { recursive: true, force: true }); + await fs.mkdir(staging, { recursive: true }); + + try { + const payload = path.join(staging, PAYLOAD_FILENAME); + await downloadPayload(artifact, payload, options); + await extractPayload(artifact, payload, staging); + await fs.rm(payload, { force: true }); + + const binary = path.join(staging, artifact.binary); + if (!(await pathExists(binary))) { + throw new RuntimeBinaryMissingError(id, artifact.binary); + } + + // Promote. The old directory goes first: rename onto an existing + // directory fails on every platform, and leaving the previous version + // half-merged with the new one is how a stale binary survives an upgrade. + const target = runtimeDir(id); + await fs.rm(target, { recursive: true, force: true }); + await fs.rename(staging, target); + + const manifest: RuntimeManifest = { + runtimeId: id, + version: versionFromUrl(artifact.url), + platform: artifact.platform, + sourceUrl: artifact.url, + sha256: artifact.sha256, + entry: artifact.entry, + binary: artifact.binary, + installedAt: Date.now(), + bytes: await directoryBytes(target), + }; + await writeRuntimeManifest(id, manifest); + + options.onProgress?.({ + runtimeId: id, + phase: 'done', + bytes: artifact.bytes, + totalBytes: artifact.bytes, + }); + logger.info(`Installed the ${id} voice runtime from ${artifact.url}`, LOG_CONTEXT); + return manifest; + } catch (error) { + // Leave nothing behind. A failed install that leaves a staging directory + // is disk the user cannot see and cannot reclaim from the UI. + await fs.rm(staging, { recursive: true, force: true }); + throw error; + } +} + +/** Stream the tarball to disk, hashing as it goes, and verify before returning. */ +async function downloadPayload( + artifact: NativeRuntimeArtifact, + destination: string, + options: RuntimeInstallOptions +): Promise { + const fetchImpl = options.fetchImpl ?? (globalThis.fetch as FetchLike); + const response = await fetchImpl(artifact.url, { signal: options.signal }); + if (!response.ok || !response.body) { + throw new Error(`Downloading ${artifact.url} failed with HTTP ${response.status}.`); + } + + const hash = createHash('sha256'); + let received = 0; + let lastEmit = 0; + + const source = Readable.fromWeb(response.body as never); + source.on('data', (chunk: Buffer) => { + hash.update(chunk); + received += chunk.length; + const now = Date.now(); + if (now - lastEmit < RUNTIME_PROGRESS_INTERVAL_MS) return; + lastEmit = now; + options.onProgress?.({ + runtimeId: artifact.runtimeId, + phase: 'downloading', + bytes: received, + totalBytes: artifact.bytes, + }); + }); + + await pipeline(source, createWriteStream(destination)); + + options.onProgress?.({ + runtimeId: artifact.runtimeId, + phase: 'verifying', + bytes: received, + totalBytes: artifact.bytes, + }); + + const actual = hash.digest('hex'); + if (actual !== artifact.sha256) { + await fs.rm(destination, { force: true }); + throw new RuntimeHashMismatchError(artifact.runtimeId, artifact.sha256, actual); + } +} + +/** Unpack the verified tarball, writing only this platform's subtree. */ +async function extractPayload( + artifact: NativeRuntimeArtifact, + payload: string, + staging: string +): Promise { + await tar.x({ + file: payload, + cwd: staging, + strip: artifact.stripComponents, + // node-tar calls this with the archive path, before `strip` is applied, + // which is exactly what `shouldKeepArchiveEntry` expects: it does its own + // stripping so the same call can also return the destination path and be + // tested without a tarball. + filter: (entryPath: string) => + shouldKeepArchiveEntry(entryPath, artifact.stripComponents, artifact.keep) !== null, + }); +} + +/** `.../mac-arm64-metal-3.20.0.tgz` -> `3.20.0`. Empty when it does not parse. */ +function versionFromUrl(url: string): string { + const match = /-(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?)\.tgz$/.exec(url); + return match?.[1] ?? ''; +} + +async function pathExists(target: string): Promise { + try { + await fs.access(target); + return true; + } catch { + return false; + } +} + +async function directoryBytes(target: string): Promise { + let total = 0; + const entries = await fs.readdir(target, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + const child = path.join(target, entry.name); + if (entry.isDirectory()) { + total += await directoryBytes(child); + continue; + } + try { + total += (await fs.stat(child)).size; + } catch { + // Raced with a delete. A size report is not worth failing over. + } + } + return total; +} diff --git a/src/main/acappella/runtime/runtime-selftest.ts b/src/main/acappella/runtime/runtime-selftest.ts new file mode 100644 index 0000000000..6c47a4136e --- /dev/null +++ b/src/main/acappella/runtime/runtime-selftest.ts @@ -0,0 +1,288 @@ +/** + * "Run voice self-test" - the answer to a bug report that says voice does not + * work. + * + * Without this, that report is a guess: the microphone, three native runtimes, + * four model downloads, and a code-signing story all fail in ways that look + * identical from the outside ("nothing happens when I press the button"). The + * self-test loads each runtime the way the real providers will, runs a trivial + * operation against it, and reports per-runtime pass/fail with timings, plus the + * microphone permission, in one structure that goes straight into a support + * report through the debug package. + * + * Three properties it must keep: + * + * - **It proves the load path, not a stub.** Each probe goes through + * `native-loader.ts`, so a self-test that passes means the same dlopen the + * provider will do actually succeeded in this installation, on this + * platform, from this code-signed bundle. + * - **It loads no model.** A probe touches the module's own surface (its + * version, its constructor) and nothing on disk. A self-test that needed + * 1.4 GB of models could not be run by the person who most needs to run it. + * - **It cannot hang.** Every probe races a timeout, because "the button did + * nothing" is precisely the bug being diagnosed, and a diagnostic that + * reproduces it is not a diagnostic. + */ + +import { + NATIVE_RUNTIMES, + nativePlatformKey, + type NativePlatformKey, + type NativePrebuildAvailability, + type NativeRuntimeDescriptor, + type NativeRuntimeId, +} from '../../../shared/acappella/native-runtimes'; +import type { MicPermission } from '../../../shared/acappella/protocol'; +import { getMicPermission } from '../permissions/mic-permission'; +import { + tryLoadNativeRuntime, + unloadNativeRuntime, + type NativeRuntimeFailureKind, + type NativeRuntimeResult, +} from './native-loader'; + +/** + * `skipped` is not a soft failure: it means the runtime is not part of this + * build yet, which is a true and useful thing for a support report to say. It is + * reported separately from `fail` so a bug report cannot read as three broken + * runtimes when nothing is broken. + */ +export type RuntimeSelfTestStatus = 'pass' | 'fail' | 'skipped'; + +export interface RuntimeSelfTestEntry { + runtimeId: NativeRuntimeId; + moduleId: string; + label: string; + status: RuntimeSelfTestStatus; + /** Wall-clock for the load plus the probe. The number that shows a slow dlopen. */ + durationMs: number; + /** What the probe found (a version string) or why it failed. */ + detail?: string; + failure?: NativeRuntimeFailureKind | 'probe-failed' | 'timeout'; + /** How this runtime's binary is meant to arrive on this platform. */ + prebuild: NativePrebuildAvailability | 'unsupported-platform'; +} + +export interface RuntimeSelfTestReport { + /** Epoch millis. Stamped by the caller's clock, so it matches the log around it. */ + ranAt: number; + platform: string; + arch: string; + /** Null on a platform Maestro ships no installer for. */ + platformKey: NativePlatformKey | null; + entries: RuntimeSelfTestEntry[]; + /** True when nothing FAILED. A skipped runtime does not fail the run. */ + passed: boolean; + microphone: { + permission: MicPermission; + /** True when the OS prompt has not been shown yet, so the state is not a refusal. */ + canPrompt: boolean; + }; +} + +/** + * A minimal structural view of each runtime's surface. + * + * Structural rather than imported types on purpose: importing + * `node-llama-cpp`'s types here would put the package back into the static + * import graph, which is the exact thing `native-loader.ts` exists to prevent. + */ +interface LlamaModule { + getLlama?: unknown; +} +interface WhisperModule { + Whisper?: unknown; +} +interface OnnxModule { + InferenceSession?: unknown; + env?: { versions?: Record }; +} + +/** How long one runtime gets before it is called hung. */ +const PROBE_TIMEOUT_MS = 15_000; + +/** + * The trivial operation per runtime. + * + * Each returns a short detail string on success and THROWS on failure. They + * deliberately check the export the provider will actually call, so a package + * that loads but has moved its API (a version bump nobody meant to take) fails + * here rather than mid-session. + */ +const PROBES: Record Promise> = { + llama: async (module) => { + const llama = module as LlamaModule; + if (typeof llama.getLlama !== 'function') { + throw new Error('node-llama-cpp loaded but exposes no getLlama()'); + } + return 'getLlama() present'; + }, + whisper: async (module) => { + const whisper = module as WhisperModule; + if (typeof whisper.Whisper !== 'function') { + throw new Error('smart-whisper loaded but exposes no Whisper constructor'); + } + return 'Whisper constructor present'; + }, + onnx: async (module) => { + const onnx = module as OnnxModule; + if (!onnx.InferenceSession) { + throw new Error('onnxruntime-node loaded but exposes no InferenceSession'); + } + const version = onnx.env?.versions?.common; + return version ? `ONNX Runtime ${version}` : 'InferenceSession present'; + }, +}; + +export interface RunSelfTestOptions { + /** Injected in tests. Defaults to the real lazy loader. */ + loadRuntime?: (id: NativeRuntimeId) => Promise>; + /** Injected in tests. Defaults to the real OS query, which never prompts. */ + readMicPermission?: () => { state: MicPermission; canPrompt: boolean }; + /** Per-runtime timeout. Lowered in tests. */ + timeoutMs?: number; + /** Injected in tests so a fake clock can produce deterministic timings. */ + now?: () => number; +} + +/** + * Run the whole self-test. Never throws: every failure is a row in the report, + * because a diagnostic that can itself blow up gives the user nothing. + */ +export async function runSelfTest( + options: RunSelfTestOptions = {} +): Promise { + const load = options.loadRuntime ?? tryLoadNativeRuntime; + const readMic = + options.readMicPermission ?? + (() => { + const info = getMicPermission(); + return { state: info.state, canPrompt: info.canPrompt }; + }); + const now = options.now ?? Date.now; + const timeoutMs = options.timeoutMs ?? PROBE_TIMEOUT_MS; + const platformKey = nativePlatformKey(process.platform, process.arch); + + const entries: RuntimeSelfTestEntry[] = []; + for (const descriptor of NATIVE_RUNTIMES) { + entries.push(await testRuntime(descriptor, { load, now, timeoutMs, platformKey })); + } + + const mic = readMic(); + return { + ranAt: now(), + platform: process.platform, + arch: process.arch, + platformKey, + entries, + passed: entries.every((entry) => entry.status !== 'fail'), + microphone: { permission: mic.state, canPrompt: mic.canPrompt }, + }; +} + +async function testRuntime( + descriptor: NativeRuntimeDescriptor, + deps: { + load: (id: NativeRuntimeId) => Promise>; + now: () => number; + timeoutMs: number; + platformKey: NativePlatformKey | null; + } +): Promise { + const started = deps.now(); + const base = { + runtimeId: descriptor.id, + moduleId: descriptor.moduleId, + label: descriptor.label, + prebuild: deps.platformKey + ? descriptor.prebuilds[deps.platformKey] + : ('unsupported-platform' as const), + }; + + const result = await withTimeout(deps.load(descriptor.id), deps.timeoutMs); + + if (result === TIMED_OUT) { + return { + ...base, + status: 'fail', + failure: 'timeout', + durationMs: deps.now() - started, + detail: `Loading ${descriptor.moduleId} did not finish within ${deps.timeoutMs} ms.`, + }; + } + + if (!result.ok) { + // A runtime that is not a dependency yet is not a broken installation, and + // a support report that says "fail" for it would send someone hunting a bug + // that does not exist. + const skipped = result.error.failure === 'not-a-dependency'; + return { + ...base, + status: skipped ? 'skipped' : 'fail', + failure: result.error.failure, + durationMs: deps.now() - started, + detail: result.error.detail + ? `${result.error.message} (${result.error.detail})` + : result.error.message, + }; + } + + try { + const detail = await withTimeout(PROBES[descriptor.id](result.module), deps.timeoutMs); + if (detail === TIMED_OUT) { + return { + ...base, + status: 'fail', + failure: 'timeout', + durationMs: deps.now() - started, + detail: `${descriptor.moduleId} loaded but its probe did not finish within ${deps.timeoutMs} ms.`, + }; + } + // Dropped from the loader cache only on success. A FAILED runtime keeps its + // remembered failure, which is what the capability gate reads to explain a + // blocked slot after the self-test has been run. + unloadNativeRuntime(descriptor.id); + return { ...base, status: 'pass', durationMs: deps.now() - started, detail }; + } catch (error) { + return { + ...base, + status: 'fail', + failure: 'probe-failed', + durationMs: deps.now() - started, + detail: error instanceof Error ? error.message : String(error), + }; + } +} + +/** Sentinel rather than a rejection, so a timeout is not confused with a load error. */ +const TIMED_OUT = Symbol('timed-out'); + +async function withTimeout(promise: Promise, ms: number): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((resolve) => { + timer = setTimeout(() => resolve(TIMED_OUT), ms); + // Never hold the process open for a diagnostic. + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +/** The report as lines, for a log or a pasted bug report. */ +export function formatSelfTestReport(report: RuntimeSelfTestReport): string { + const header = `A Cappella voice self-test on ${report.platform}-${report.arch}: ${ + report.passed ? 'PASS' : 'FAIL' + }`; + const rows = report.entries.map( + (entry) => + ` ${entry.status.toUpperCase().padEnd(7)} ${entry.label} (${entry.moduleId}) ${entry.durationMs} ms${ + entry.detail ? ` - ${entry.detail}` : '' + }` + ); + return [header, ...rows, ` Microphone: ${report.microphone.permission}`].join('\n'); +} diff --git a/src/main/acappella/runtime/runtime-store.ts b/src/main/acappella/runtime/runtime-store.ts new file mode 100644 index 0000000000..4953bb234a --- /dev/null +++ b/src/main/acappella/runtime/runtime-store.ts @@ -0,0 +1,269 @@ +/** + * Where a downloaded native runtime lives, and whether it is really there. + * + * userData/runtimes/acappella// + * manifest.json - what was installed, from where, and when + * dist/ bins/ bin/ ... - the kept subtree of the payload + * + * Deliberately a SIBLING of the model store rather than a directory inside it. + * The two have different lifetimes and different reasons to be deleted: "reclaim + * the disk my models are using" must not silently uninstall the engines, and a + * runtime replaced on version bump must not disturb a 1 GB model that is still + * current. Keeping them apart makes each delete mean one thing. + * + * The invariant, inherited from the model store because it is the property that + * matters: **a runtime counts as installed only when its manifest exists AND the + * binary the artifact promised is on disk.** A manifest alone is a claim; the + * binary is the thing a dlopen needs. Checking both is what stops a half-extracted + * payload from being reported as ready and then dying inside the loader, where + * the error names a shared library rather than anything a user can act on. + */ + +import { app } from 'electron'; +import * as fs from 'fs/promises'; +import * as path from 'path'; + +import { + getNativeRuntime, + nativePlatformKey, + type NativeRuntimeId, +} from '../../../shared/acappella/native-runtimes'; +import { + nativeRuntimeArtifact, + type NativeRuntimeArtifact, +} from '../../../shared/acappella/runtime-artifacts'; + +/** Directory under userData. Also what the "remove runtimes" flow deletes. */ +export const ACAPPELLA_RUNTIMES_DIRNAME = path.join('runtimes', 'acappella'); + +/** Written last, on success only, so its presence means the install finished. */ +export const RUNTIME_MANIFEST_FILENAME = 'manifest.json'; + +/** Extraction target while a payload is being laid out. Promoted by rename. */ +export const RUNTIME_STAGING_SUFFIX = '.staging'; + +/** + * What a completed runtime install recorded about itself. + * + * Self-contained on purpose, exactly as `ModelManifest` is: it has to stay + * readable after the catalog moves to a new version, so it repeats the version + * and the source rather than pointing at a catalog row that may since have + * changed underneath it. That is what lets `isStale()` be a comparison rather + * than a guess. + */ +export interface RuntimeManifest { + runtimeId: NativeRuntimeId; + /** The npm version this payload came from. Compared against the pin. */ + version: string; + platform: string; + sourceUrl: string; + /** SHA-256 of the downloaded tarball, before extraction. */ + sha256: string; + /** Module path to import, relative to the install directory. */ + entry: string; + /** Native binary that must exist, relative to the install directory. */ + binary: string; + /** Epoch ms the install completed. */ + installedAt: number; + /** Bytes the extracted subtree occupies. Not the download size. */ + bytes: number; +} + +/** + * Resolve the Maestro data dir the same way every other store does. + * + * `MAESTRO_USER_DATA` has to keep working, and it only does if nothing invents a + * second way to ask this question. See the identical note in `model-store.ts`. + */ +function dataDir(): string { + if (process.env.MAESTRO_USER_DATA) return path.resolve(process.env.MAESTRO_USER_DATA); + return app.getPath('userData'); +} + +/** Root every downloaded runtime lives under. */ +export function runtimesRoot(): string { + return path.join(dataDir(), ACAPPELLA_RUNTIMES_DIRNAME); +} + +/** + * Install directory for one runtime. + * + * Whitelisted against the registry rather than sanitised, for the same reason + * `modelDir()` is: ids arrive from IPC, and this path is handed to a recursive + * delete. Only ids that name a real runtime are accepted, so a traversal attempt + * fails as an unknown id long before it becomes a path. + */ +export function runtimeDir(id: NativeRuntimeId): string { + if (!getNativeRuntime(id)) throw new Error(`UnknownVoiceRuntime: ${id}`); + return path.join(runtimesRoot(), id); +} + +/** Where a payload is extracted before it is promoted. Never imported from. */ +export function runtimeStagingDir(id: NativeRuntimeId): string { + return runtimeDir(id) + RUNTIME_STAGING_SUFFIX; +} + +/** + * Absolute path of a file inside a runtime's install directory. + * + * Escape-checked. Artifact paths are authored in this repo, but the one mistake + * that turns an install into a disaster is a relative path that climbs out of its + * root, so it is proven rather than trusted. + */ +export function runtimeFilePath(id: NativeRuntimeId, relativePath: string): string { + const dir = runtimeDir(id); + const resolved = path.resolve(dir, relativePath); + if (resolved !== dir && !resolved.startsWith(dir + path.sep)) { + throw new Error(`UnsafeRuntimePath: ${relativePath}`); + } + return resolved; +} + +function manifestPath(id: NativeRuntimeId): string { + return path.join(runtimeDir(id), RUNTIME_MANIFEST_FILENAME); +} + +/** The payload this platform would install for a runtime, or null when none. */ +export function artifactForThisPlatform(id: NativeRuntimeId): NativeRuntimeArtifact | null { + const key = nativePlatformKey(process.platform, process.arch); + if (!key) return null; + return nativeRuntimeArtifact(id, key); +} + +async function exists(target: string): Promise { + try { + await fs.access(target); + return true; + } catch { + return false; + } +} + +/** Read a runtime's manifest, or null when it is absent or unreadable. */ +export async function readRuntimeManifest(id: NativeRuntimeId): Promise { + try { + const raw = await fs.readFile(manifestPath(id), 'utf8'); + const parsed = JSON.parse(raw) as RuntimeManifest; + // A manifest that does not name its own runtime is a manifest from a + // different install that was copied or renamed into place. Refusing it is + // cheaper than trusting it and loading the wrong engine. + return parsed.runtimeId === id ? parsed : null; + } catch { + return null; + } +} + +/** + * Is this runtime installed and usable? + * + * Both halves are required. See the module header: a manifest is a claim about + * the past, and the binary is what the loader will actually reach for. + */ +export async function isRuntimeInstalled(id: NativeRuntimeId): Promise { + const manifest = await readRuntimeManifest(id); + if (!manifest) return false; + return exists(path.join(runtimeDir(id), manifest.binary)); +} + +/** + * The absolute module path to import for an installed runtime, or null. + * + * Null rather than a throw, because "not downloaded yet" is the ordinary state of + * this feature and every caller has to handle it anyway. The loader turns the + * null into a classified, actionable refusal. + */ +export async function installedRuntimeEntry(id: NativeRuntimeId): Promise { + const manifest = await readRuntimeManifest(id); + if (!manifest) return null; + + const entry = path.join(runtimeDir(id), manifest.entry); + const binary = path.join(runtimeDir(id), manifest.binary); + if (!(await exists(entry)) || !(await exists(binary))) return null; + return entry; +} + +/** + * True when what is installed no longer matches what this build expects. + * + * Compared against the artifact's HASH, not only its version. A version string + * is what someone edits; the hash is what was actually downloaded, so comparing + * it catches a re-published tarball and a hand-edited manifest alike. An + * uninstalled runtime is not stale, it is absent, and the two want different + * words in front of the user. + */ +export async function isRuntimeStale(id: NativeRuntimeId): Promise { + const manifest = await readRuntimeManifest(id); + if (!manifest) return false; + + const artifact = artifactForThisPlatform(id); + if (!artifact) return false; + return manifest.sha256 !== artifact.sha256; +} + +/** Recursive size of a directory in bytes. Missing directories count as zero. */ +async function directorySize(target: string): Promise { + let total = 0; + // A missing directory is zero, not an error: the footprint of something that + // was never installed is a legitimate question with an obvious answer. + const entries = await fs.readdir(target, { withFileTypes: true }).catch(() => []); + + for (const entry of entries) { + const child = path.join(target, entry.name); + if (entry.isDirectory()) { + total += await directorySize(child); + continue; + } + try { + const stat = await fs.stat(child); + total += stat.size; + } catch { + // A file that vanished between readdir and stat contributes nothing. + // A footprint report is not worth failing over a race with a delete. + } + } + return total; +} + +/** Disk a single installed runtime occupies. Zero when it is not installed. */ +export async function runtimeFootprint(id: NativeRuntimeId): Promise { + return directorySize(runtimeDir(id)); +} + +/** Disk every downloaded runtime occupies, including any abandoned staging. */ +export async function runtimesFootprint(): Promise { + return directorySize(runtimesRoot()); +} + +/** + * Record a finished install. + * + * Written LAST by the installer and never before, which is the whole reason + * `isRuntimeInstalled()` can be a cheap question: the manifest's existence is the + * commit point of the install transaction. + */ +export async function writeRuntimeManifest( + id: NativeRuntimeId, + manifest: RuntimeManifest +): Promise { + await fs.mkdir(runtimeDir(id), { recursive: true }); + await fs.writeFile(manifestPath(id), JSON.stringify(manifest, null, 2), 'utf8'); +} + +/** + * Delete one runtime, including any staging directory left by a failed install. + * + * Safe to call on a runtime that was never installed. Note that the loader keeps + * the native library resident for the life of the process (Node has no unload), + * so removing a runtime that has already been loaded frees the disk but not the + * memory until restart. Callers that present this to a user should say so rather + * than implying the engine is gone. + */ +export async function removeRuntime(id: NativeRuntimeId): Promise { + await fs.rm(runtimeDir(id), { recursive: true, force: true }); + await fs.rm(runtimeStagingDir(id), { recursive: true, force: true }); +} + +/** Delete every downloaded runtime. The "reclaim this disk" action. */ +export async function removeAllRuntimes(): Promise { + await fs.rm(runtimesRoot(), { recursive: true, force: true }); +} diff --git a/src/main/acappella/speech/agent-output-tap.ts b/src/main/acappella/speech/agent-output-tap.ts new file mode 100644 index 0000000000..dcd01a2164 --- /dev/null +++ b/src/main/acappella/speech/agent-output-tap.ts @@ -0,0 +1,464 @@ +/** + * The streaming tap on a dispatched agent's output. + * + * A voice turn cannot wait for an agent to finish writing. Four hundred lines of + * implementation detail take a minute to produce and the user is standing there + * in silence for all of it, so the tap follows the SAME process events the + * desktop transcript follows and hands the translator coherent pieces as they + * appear. The first spoken word therefore lands while the agent is still typing. + * + * Three rules this file exists to keep: + * + * - **One listener path.** It subscribes to the process manager's own emitter, + * the one `src/main/process-listeners/` already uses. A second listener path, + * or a poll of the transcript, would drift from what the user sees on screen + * and would double-count every chunk. + * - **Nothing unspeakable escapes.** Tool calls, diffs, code fences, file + * listings, spinner frames, and raw ANSI are dropped HERE rather than being + * left for the translator to notice, because a translator handed a diff + * spends a model call deciding not to read it out. + * - **Silence is never the answer.** An agent that errored or went quiet + * produces a short honest status chunk. A voice interface with nothing to say + * is indistinguishable from one that is broken. + * + * Free of Electron and of the process manager's concrete type: the emitter + * arrives as {@link AgentOutputSource} so the suite can drive it with a bare + * EventEmitter. + */ + +import { buildProcessSessionId } from '../../dispatch-callbacks/dispatch-callback-registry'; +import { extractTextFromStreamJson } from '../../group-chat/output-parser'; +import { stripAnsiCodes } from '../../../shared/stringUtils'; + +/** Handler shape for the untyped `EventEmitter` the process manager really is. */ +type ProcessEventHandler = (...args: unknown[]) => void; + +/** + * The slice of `ProcessManager` the tap needs. Narrow on purpose: the tap + * listens and never spawns, kills, or writes. + */ +export interface AgentOutputSource { + on(event: string, handler: ProcessEventHandler): unknown; + off(event: string, handler: ProcessEventHandler): unknown; +} + +/** + * What kind of thing the chunk is. + * + * - `text` - a completed thought from the middle of a reply. + * - `final` - the tail, flushed when the agent finished its turn. + * - `status` - the tap speaking for itself: an error, or an agent gone quiet. + * Never model output, so the translator passes it straight + * through rather than paying a hop to rephrase a failure. + */ +export type AgentOutputChunkKind = 'text' | 'final' | 'status'; + +export interface AgentOutputChunk { + agentSessionId: string; + tabId: string; + kind: AgentOutputChunkKind; + /** Speech-safe prose. Never a diff, a path listing, or a spinner frame. */ + text: string; + ts: number; +} + +export interface AgentOutputTapOptions { + source: AgentOutputSource; + /** One coherent piece of the reply, in order. */ + onChunk: (chunk: AgentOutputChunk) => void; + /** Agent type for the stream-json parser, when the caller knows it. */ + getAgentType?: (agentSessionId: string) => string | undefined; + /** Quiet for this long with a turn open and the tap says so. */ + hangMs?: number; + /** Smallest run of prose emitted as its own chunk mid-reply. */ + minChunkChars?: number; + now?: () => number; + setTimeoutFn?: (fn: () => void, ms: number) => ReturnType; + clearTimeoutFn?: (handle: ReturnType) => void; +} + +/** An agent that has said nothing for this long is worth a word. */ +const DEFAULT_HANG_MS = 20_000; + +/** + * Below this, a finished thought waits for the next one instead of becoming its + * own spoken chunk. Two hundred characters is roughly a spoken sentence and a + * half: short enough that the first words come fast, long enough that the + * translator is not called once per "Okay.". + */ +const DEFAULT_MIN_CHUNK_CHARS = 200; + +interface WatchEntry { + agentSessionId: string; + tabId: string; + processSessionId: string; + /** Complete-line buffer: filtering is line-oriented and chunks split anywhere. */ + partialLine: string; + /** Speech-safe prose waiting to reach `minChunkChars` or a paragraph break. */ + speech: string; + /** Inside a fenced code block, which can span many `data` events. */ + inFence: boolean; + /** Something has been emitted for this turn, so a hang notice is not the first word. */ + emitted: boolean; + hangTimer: ReturnType | null; + hangAnnounced: boolean; +} + +export class AgentOutputTap { + private readonly options: AgentOutputTapOptions; + private readonly hangMs: number; + private readonly minChunkChars: number; + private readonly now: () => number; + private readonly setTimeoutFn: (fn: () => void, ms: number) => ReturnType; + private readonly clearTimeoutFn: (handle: ReturnType) => void; + + /** Keyed by the composite process session id, which is what the events carry. */ + private readonly watched = new Map(); + private readonly registered: [string, ProcessEventHandler][] = []; + private disposed = false; + + constructor(options: AgentOutputTapOptions) { + this.options = options; + this.hangMs = options.hangMs ?? DEFAULT_HANG_MS; + this.minChunkChars = options.minChunkChars ?? DEFAULT_MIN_CHUNK_CHARS; + this.now = options.now ?? Date.now; + this.setTimeoutFn = options.setTimeoutFn ?? ((fn, ms) => setTimeout(fn, ms)); + this.clearTimeoutFn = options.clearTimeoutFn ?? ((handle) => clearTimeout(handle)); + + this.listen('data', (sessionId: string, data: string) => this.handleData(sessionId, data)); + this.listen('agent-error', (sessionId: string, error: { message?: string }) => + this.handleError(sessionId, error?.message) + ); + this.listen('query-complete', (sessionId: string) => this.handleTurnEnd(sessionId)); + this.listen('exit', (sessionId: string, code: number) => this.handleExit(sessionId, code)); + } + + /** + * Follow one dispatched tab. Re-watching the same tab restarts its buffers, + * which is what a second voice turn into the same tab should do. + */ + watch(params: { agentSessionId: string; tabId: string }): void { + if (this.disposed) return; + const processSessionId = buildProcessSessionId(params.agentSessionId, params.tabId); + this.stopEntry(this.watched.get(processSessionId)); + + const entry: WatchEntry = { + agentSessionId: params.agentSessionId, + tabId: params.tabId, + processSessionId, + partialLine: '', + speech: '', + inFence: false, + emitted: false, + hangTimer: null, + hangAnnounced: false, + }; + this.watched.set(processSessionId, entry); + this.armHangTimer(entry); + } + + /** Stop following one tab, dropping whatever it had buffered. */ + unwatch(params: { agentSessionId: string; tabId: string }): void { + const processSessionId = buildProcessSessionId(params.agentSessionId, params.tabId); + this.stopEntry(this.watched.get(processSessionId)); + this.watched.delete(processSessionId); + } + + /** True while any tab is being followed. */ + get isWatching(): boolean { + return this.watched.size > 0; + } + + /** Drop every subscription and every buffer. Safe to repeat. */ + dispose(): void { + if (this.disposed) return; + this.disposed = true; + for (const entry of this.watched.values()) this.stopEntry(entry); + this.watched.clear(); + for (const [event, handler] of this.registered) this.options.source.off(event, handler); + this.registered.length = 0; + } + + // -- Events -------------------------------------------------------------- + + private handleData(processSessionId: string, data: string): void { + const entry = this.watched.get(processSessionId); + if (!entry || typeof data !== 'string') return; + + this.armHangTimer(entry); + + entry.partialLine += toPlainText(data, this.options.getAgentType?.(entry.agentSessionId)); + const lines = entry.partialLine.split('\n'); + // The last element is whatever came after the final newline: an unfinished + // line that the next `data` event completes. + entry.partialLine = lines.pop() ?? ''; + + for (const line of lines) this.consumeLine(entry, line); + this.drain(entry, false); + } + + private handleError(processSessionId: string, message?: string): void { + const entry = this.watched.get(processSessionId); + if (!entry) return; + // The agent's own failure text, said plainly. Going silent here is the + // worst option: the user has no screen and would wait forever. + this.flush(entry); + this.emit( + entry, + 'status', + message?.trim() ? `It hit an error: ${oneLine(message)}` : 'It hit an error.' + ); + this.stopEntry(entry); + } + + private handleTurnEnd(processSessionId: string): void { + const entry = this.watched.get(processSessionId); + if (!entry) return; + this.finishTurn(entry); + } + + private handleExit(processSessionId: string, code: number): void { + const entry = this.watched.get(processSessionId); + if (!entry) return; + + this.finishTurn(entry); + if (code !== 0 && !entry.emitted) { + this.emit(entry, 'status', `It stopped without answering, exit code ${code}.`); + } + this.watched.delete(processSessionId); + } + + /** Flush the tail, close the turn, and stop the hang clock. */ + private finishTurn(entry: WatchEntry): void { + if (entry.partialLine) { + this.consumeLine(entry, entry.partialLine); + entry.partialLine = ''; + } + this.flush(entry); + this.stopEntry(entry); + } + + // -- Filtering ----------------------------------------------------------- + + /** + * One complete line, kept or dropped. + * + * Line-oriented rather than whole-buffer, because a `data` event splits + * anywhere - including in the middle of a code fence - and a filter that + * re-examined the whole buffer each time would re-decide earlier lines with + * different context. + */ + private consumeLine(entry: WatchEntry, rawLine: string): void { + // A carriage return is a spinner redrawing itself in place. Only the last + // frame is even a candidate, and it is dropped below along with the rest. + const line = rawLine.split('\r').pop() ?? ''; + const trimmed = line.trim(); + + if (isFenceDelimiter(trimmed)) { + entry.inFence = !entry.inFence; + return; + } + if (entry.inFence) return; + + if (!trimmed) { + // A blank line is a paragraph boundary: the completed thought before it is + // the natural place to cut a spoken chunk. + if (entry.speech.trim()) entry.speech += '\n\n'; + return; + } + + if (isUnspeakableLine(trimmed)) return; + + const prose = toProse(trimmed); + if (!prose) return; + + entry.speech += entry.speech && !entry.speech.endsWith('\n') ? ` ${prose}` : prose; + } + + // -- Emission ------------------------------------------------------------ + + /** + * Emit whatever is ready. + * + * Mid-reply the cut is a paragraph boundary, so a chunk is a completed thought + * rather than a sentence torn off a list. A run that has passed + * `minChunkChars` with no paragraph in sight is cut at its last sentence + * boundary instead: an agent writing one long block should not buy silence. + */ + private drain(entry: WatchEntry, final: boolean): void { + if (final) { + const text = entry.speech.trim(); + entry.speech = ''; + if (text) this.emit(entry, 'final', text); + return; + } + + let boundary = entry.speech.indexOf('\n\n'); + while (boundary !== -1) { + const piece = entry.speech.slice(0, boundary).trim(); + entry.speech = entry.speech.slice(boundary + 2); + if (piece) this.emit(entry, 'text', piece); + boundary = entry.speech.indexOf('\n\n'); + } + + if (entry.speech.trim().length < this.minChunkChars) return; + const cut = lastSentenceBoundary(entry.speech); + if (cut <= 0) return; + + const piece = entry.speech.slice(0, cut).trim(); + entry.speech = entry.speech.slice(cut); + if (piece) this.emit(entry, 'text', piece); + } + + private flush(entry: WatchEntry): void { + this.drain(entry, true); + } + + private emit(entry: WatchEntry, kind: AgentOutputChunkKind, text: string): void { + entry.emitted = true; + this.options.onChunk({ + agentSessionId: entry.agentSessionId, + tabId: entry.tabId, + kind, + text: oneLine(text), + ts: this.now(), + }); + } + + // -- Hang detection ------------------------------------------------------ + + /** + * An agent that has produced nothing for `hangMs` is reported once. + * + * Once, not on a repeat, because the honest fact is "it is taking a while" and + * saying it every twenty seconds turns a slow turn into a nagging one. The + * clock is re-armed on every byte of output, so a working agent never trips it. + */ + private armHangTimer(entry: WatchEntry): void { + if (entry.hangTimer) this.clearTimeoutFn(entry.hangTimer); + if (entry.hangAnnounced) return; + entry.hangTimer = this.setTimeoutFn(() => { + entry.hangTimer = null; + // Checked inside the callback as well as before arming: a timer that has + // already fired can still be held by whoever scheduled it. + if (entry.hangAnnounced) return; + entry.hangAnnounced = true; + this.emit(entry, 'status', 'It is still working on that one.'); + }, this.hangMs); + } + + private stopEntry(entry: WatchEntry | undefined): void { + if (!entry) return; + if (entry.hangTimer) this.clearTimeoutFn(entry.hangTimer); + entry.hangTimer = null; + } + + private listen(event: string, handler: (...args: T) => void): void { + // The emitter is a bare `EventEmitter`, so its handlers are untyped by + // construction; the cast is where the typed handlers above meet it. + const bound = handler as unknown as ProcessEventHandler; + this.options.source.on(event, bound); + this.registered.push([event, bound]); + } +} + +export function createAgentOutputTap(options: AgentOutputTapOptions): AgentOutputTap { + return new AgentOutputTap(options); +} + +// --------------------------------------------------------------------------- +// Text extraction +// --------------------------------------------------------------------------- + +/** + * Raw process output as plain text. + * + * Stream-json agents go through the parser the group chat already uses, so tool + * calls and usage records never reach the filter as text at all. A PTY agent + * (Terminal, and any agent running in its own TUI) produces no JSON, so the raw + * bytes are used - with ANSI stripped by `stripAnsiCodes()` rather than by a + * second regex that would drift from it. + */ +function toPlainText(raw: string, agentType?: string): string { + const stripped = stripAnsiCodes(raw); + if (!looksLikeStreamJson(stripped)) return stripped; + + const extracted = extractTextFromStreamJson(stripped, agentType); + // An empty extraction means the chunk held only tool calls or bookkeeping, + // which is exactly what should not be spoken. Falling back to the raw JSON + // here would read a tool_use payload aloud. + return extracted ? `${extracted}\n` : ''; +} + +function looksLikeStreamJson(text: string): boolean { + return /^\s*\{/m.test(text); +} + +// --------------------------------------------------------------------------- +// Line filters +// --------------------------------------------------------------------------- + +/** ``` or ~~~, with or without a language tag. */ +function isFenceDelimiter(line: string): boolean { + return /^(?:```|~~~)/.test(line); +} + +/** Unified-diff furniture, from `git diff` and from an agent narrating an edit. */ +const DIFF_LINE = /^(?:diff --git |index [0-9a-f]{7,}|@@ |\+\+\+ |--- |[+-](?=\S))/; + +/** Braille and block spinners, progress bars, and box drawing. */ +const SPINNER_OR_RULE = /^[\s─-╿▀-▟⠀-⣿|+=_.*#[\]()<>/\\-]+$/u; + +/** + * TUI gutter markers for a tool call or its result. + * + * Deliberately not `•` or `●`: those are list bullets in most agents' output, + * and a bullet line is content the translator should be allowed to summarise + * rather than furniture to drop. They are handled as list markers in + * {@link toProse} instead. + */ +const TOOL_GUTTER = /^[⏺✦✻·⎿└├⠀-⣿]\s/u; + +/** A bare path, with or without a line number. Nothing else on the line. */ +const BARE_PATH = /^[~.]?[\w@.-]*(?:\/[\w@.-]+)+(?::\d+(?::\d+)?)?$/; + +/** A progress readout leading with its own percentage. */ +const PROGRESS_READOUT = /^\d{1,3}(?:\.\d+)?%/; + +/** True for a line that must never reach a speaker. */ +function isUnspeakableLine(line: string): boolean { + if (DIFF_LINE.test(line)) return true; + if (TOOL_GUTTER.test(line)) return true; + if (BARE_PATH.test(line)) return true; + if (PROGRESS_READOUT.test(line)) return true; + // Ordered last: it is the broadest, and a line made only of punctuation and box + // characters has no words in it by construction. + return SPINNER_OR_RULE.test(line); +} + +/** + * A surviving line, with the markdown furniture removed. + * + * Only the leading markers, and only the ones that are pure decoration: the + * translator prompt handles voice, and stripping inline emphasis here would + * fight `stripMarkdown()` downstream over the same text. + */ +function toProse(line: string): string { + return line + .replace(/^#{1,6}\s+/, '') + .replace(/^>\s+/, '') + .replace(/^[-*+•●]\s+/u, '') + .replace(/^\d+[.)]\s+/, '') + .trim(); +} + +/** Index just past the last sentence-ending punctuation, or -1. */ +function lastSentenceBoundary(text: string): number { + const match = /[.!?](?=\s)(?![\s\S]*[.!?]\s)/.exec(text); + return match ? match.index + 1 : -1; +} + +/** Collapse to one line: a newline in spoken text is read as a pause that is not there. */ +function oneLine(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +} diff --git a/src/main/acappella/speech/background-announcer.ts b/src/main/acappella/speech/background-announcer.ts new file mode 100644 index 0000000000..ca74b5a9bf --- /dev/null +++ b/src/main/acappella/speech/background-announcer.ts @@ -0,0 +1,142 @@ +/** + * Background completions: an agent finishing long after its voice turn ended. + * + * The tempting implementation - speak it the moment it lands - is wrong in a way + * that is obvious the first time it happens to you: you are mid-sentence with one + * agent and a different one starts talking over both of you. So a completion is + * QUEUED and delivered at the next natural pause, which is the moment the floor + * is open and nothing is being said or dispatched. + * + * Every announcement names its source, using the same idea as the toast system's + * `sourceAgent` label: the user has no screen and no tab bar, so "the migration + * is done" is unusable unless it says which agent finished it. That is one + * identity concept across both surfaces rather than a second one invented here. + */ + +import type { BackgroundAnnouncementSetting } from '../../../shared/acappella/announcements'; +import { shouldSpeakBackgroundCompletions } from '../../../shared/acappella/announcements'; +import type { VoiceScope } from '../../../shared/acappella/protocol'; + +export interface BackgroundCompletion { + agentSessionId: string; + /** The label spoken and shown. Same concept as a toast's `sourceAgent`. */ + agentName: string; + tabId?: string; + /** One line of what it finished, from the synopsis the history manager wrote. */ + summary?: string; +} + +export interface BackgroundAnnouncement extends BackgroundCompletion { + /** The sentence to speak, source named. */ + text: string; + queuedAt: number; +} + +export interface BackgroundAnnouncerOptions { + /** What the session is bound to. Read per queue: the scope can change. */ + getScope: () => VoiceScope; + getSetting: () => BackgroundAnnouncementSetting | undefined; + /** The agent the current voice turn is about. Its own completion is not "background". */ + getForegroundAgentSessionId?: () => string | null; + /** Longest backlog held. Beyond it the oldest are dropped. */ + queueLimit?: number; + now?: () => number; +} + +/** + * Nobody wants six announcements at once. Past this the OLDEST go, because a + * completion the user has been waiting on for ten minutes has already been + * overtaken by the ones behind it. + */ +const DEFAULT_QUEUE_LIMIT = 5; + +export class BackgroundAnnouncer { + private readonly options: BackgroundAnnouncerOptions; + private readonly queueLimit: number; + private readonly now: () => number; + private pending: BackgroundAnnouncement[] = []; + + constructor(options: BackgroundAnnouncerOptions) { + this.options = options; + this.queueLimit = Math.max(1, options.queueLimit ?? DEFAULT_QUEUE_LIMIT); + this.now = options.now ?? Date.now; + } + + get queued(): BackgroundAnnouncement[] { + return [...this.pending]; + } + + /** + * An agent finished. + * + * @returns the queued announcement, or `null` when it was declined: the + * setting is off for this scope, or the agent is the one the current + * turn is already about, in which case its reply is the answer and an + * announcement would say the same thing twice. + */ + queue(completion: BackgroundCompletion): BackgroundAnnouncement | null { + if (!shouldSpeakBackgroundCompletions(this.options.getSetting(), this.options.getScope())) { + return null; + } + if (this.options.getForegroundAgentSessionId?.() === completion.agentSessionId) return null; + + const announcement: BackgroundAnnouncement = { + ...completion, + text: announcementText(completion), + queuedAt: this.now(), + }; + + this.pending.push(announcement); + if (this.pending.length > this.queueLimit) { + this.pending = this.pending.slice(-this.queueLimit); + } + return announcement; + } + + /** + * Take the next announcement, if this is a natural pause. + * + * `atPause` is the caller's answer to "is the floor quiet right now", which + * only the session knows. Passing false is not an error: it is the ordinary + * case of a completion landing mid-conversation, and the announcement simply + * waits. + */ + take(atPause: boolean): BackgroundAnnouncement | null { + if (!atPause) return null; + return this.pending.shift() ?? null; + } + + /** Drop the backlog. Called when the session ends: it belonged to that session. */ + clear(): void { + this.pending = []; + } +} + +export function createBackgroundAnnouncer( + options: BackgroundAnnouncerOptions +): BackgroundAnnouncer { + return new BackgroundAnnouncer(options); +} + +/** + * The spoken sentence, source first. + * + * Source first rather than last because the listener has to know who is talking + * before they can make sense of what was done, and because an announcement is + * arriving out of nowhere in the middle of a different conversation. + */ +export function announcementText(completion: BackgroundCompletion): string { + const name = completion.agentName.trim() || 'another agent'; + const summary = completion.summary?.replace(/\s+/g, ' ').trim(); + const article = /\bagent\b/i.test(name) ? name : `the ${name} agent`; + return summary ? `${article} finished ${lowerFirst(summary)}` : `${article} finished.`; +} + +/** "Fixed the auth bug" reads as "finished fixed the auth bug" otherwise. */ +function lowerFirst(text: string): string { + const trimmed = text.replace(/\.$/, ''); + // Only when the second character is lowercase: "API rate limiting" must keep + // its capital, and a sentence that starts with an acronym is common here. + if (/^[A-Z][a-z]/.test(trimmed)) return `${trimmed[0].toLowerCase()}${trimmed.slice(1)}.`; + return `${trimmed}.`; +} diff --git a/src/main/acappella/speech/barge-in.ts b/src/main/acappella/speech/barge-in.ts new file mode 100644 index 0000000000..d9d4befaae --- /dev/null +++ b/src/main/acappella/speech/barge-in.ts @@ -0,0 +1,190 @@ +/** + * Barge-in, finished end to end. + * + * The audio pipeline already decides WHEN (`audio/audio-pipeline.ts`: duck on a + * candidate frame, fire on a confirmed `speech-start`). This file owns WHAT + * HAPPENS NEXT, in an order that matters, because the four things that have to + * be torn down live in four different places and three of them are invisible to + * the user: + * + * 1. **Duck**, within about 20 ms. The only step the user can hear, so it goes + * first. Everything after it is bookkeeping done into a quiet room. + * 2. **Flush the playback queue.** Audio already handed to the host is a + * sentence the user has decided not to listen to. + * 3. **Cancel the in-flight synthesis.** A provider mid-sentence will keep + * billing and keep emitting chunks otherwise. + * 4. **Cancel the translator stream.** The rewrite behind the synthesis is a + * second in-flight model call, and it is the one people forget: cancelling + * TTS alone leaves a Brain writing sentences for a turn that is over. + * + * Then the floor reopens with the pre-roll included, which is what captures the + * first word of the interruption rather than the second. + * + * Two invariants worth stating out loud: + * + * - **What was HEARD is not what was QUEUED.** The conversation memory records + * only the sentences that reached the speaker. A model told it already said + * something the user never heard will refer back to it, and the user will + * have no idea what it means. + * - **Barge-in keeps the floor; the stop word goes cold.** They are different + * verbs and this file never ends a session. Conflating them makes talking + * over the assistant hang up on it, which is the single most annoying failure + * a voice interface has. The stop word lives in `wake/stop-word.ts`. + */ + +import type { InterruptSource } from '../../../shared/acappella/protocol'; +import type { SpeechRunResult } from './speech-scheduler'; + +/** The teardown, in the order it must happen. Reported for the suite and for tracing. */ +export type BargeInStep = 'duck' | 'flush' | 'cancel-speech' | 'cancel-translation' | 'listening'; + +export interface BargeInOutcome { + source: InterruptSource; + utteranceId: string | null; + /** Sentences the user actually heard. */ + spoken: string[]; + /** Sentences that were queued or mid-synthesis and never reached them. */ + unspoken: string[]; + /** Steps performed, in order. */ + steps: BargeInStep[]; + at: number; +} + +export interface BargeInControllerOptions { + /** Drop playback gain. Called first, with the fast ramp. */ + duck: (gain: number, rampMs: number) => void; + /** Discard audio already queued in the host. */ + flushPlayback: () => void; + /** Cut the speech run off mid-sentence. Returns what was heard against what was not. */ + cancelSpeech: () => SpeechRunResult | null; + /** Abort the translator stream feeding the run. */ + cancelTranslation: () => void; + /** Reopen the floor. The pre-roll is drained by the audio pipeline on the way in. */ + toListening: (outcome: BargeInOutcome) => void; + /** Remember what was actually heard, for the conversation memory. */ + rememberSpoken?: (sentences: string[]) => void; + /** Gain playback is ducked to. */ + duckGain?: number; + /** Ramp for the duck. Fast enough to feel instant, slow enough not to click. */ + duckRampMs?: number; + /** Dead time after speech starts during which a barge-in is refused. */ + guardMs?: number; + now?: () => number; +} + +/** Quiet, not silent: the user should still be able to tell it was speaking. */ +const DEFAULT_DUCK_GAIN = 0.15; + +/** About 20 ms. Below the threshold where a gain change reads as a step. */ +const DEFAULT_DUCK_RAMP_MS = 20; + +/** + * The self-interrupt guard. + * + * Echo cancellation is good, not perfect, and the first moments of playback are + * when it is worst: the canceller has no reference signal for a sentence that has + * only just started. Without a guard the assistant's own first syllable trips the + * detector and it interrupts itself, which looks exactly like a crash. 250 ms is + * long enough to cover the AEC's convergence and short enough that a user + * genuinely talking over the first word is still heard - they will still be + * talking when it expires. + */ +const DEFAULT_GUARD_MS = 250; + +export class BargeInController { + private readonly options: BargeInControllerOptions; + private readonly duckGain: number; + private readonly duckRampMs: number; + private readonly guardMs: number; + private readonly now: () => number; + + /** When the current speech run started, or null when nothing is speaking. */ + private speechStartedAt: number | null = null; + + constructor(options: BargeInControllerOptions) { + this.options = options; + this.duckGain = clamp01(options.duckGain ?? DEFAULT_DUCK_GAIN); + this.duckRampMs = Math.max(0, options.duckRampMs ?? DEFAULT_DUCK_RAMP_MS); + this.guardMs = Math.max(0, options.guardMs ?? DEFAULT_GUARD_MS); + this.now = options.now ?? Date.now; + } + + /** The assistant started speaking. Opens the guard window. */ + noteSpeechStarted(): void { + this.speechStartedAt = this.now(); + } + + /** The assistant stopped speaking. There is no floor left to barge into. */ + noteSpeechEnded(): void { + this.speechStartedAt = null; + } + + /** + * False while the guard window is open, or when nothing is speaking. + * + * The guard applies to VOICE only. A button press carries no ambiguity about + * who pressed it, so refusing one because the assistant started talking 80 ms + * ago would be a dead control rather than a protection. + */ + canInterrupt(source: InterruptSource = 'voice'): boolean { + if (this.speechStartedAt === null) return false; + if (source !== 'voice') return true; + return this.now() - this.speechStartedAt >= this.guardMs; + } + + /** + * Take the floor back. + * + * @returns `null` when there was nothing to interrupt or the guard window is + * still open, so a self-interrupt is a no-op rather than an error. + */ + trigger(source: InterruptSource = 'voice'): BargeInOutcome | null { + if (!this.canInterrupt(source)) return null; + + const steps: BargeInStep[] = []; + + // Heard first, because it is the only step the user perceives. + this.options.duck(this.duckGain, this.duckRampMs); + steps.push('duck'); + + this.options.flushPlayback(); + steps.push('flush'); + + const result = this.options.cancelSpeech(); + steps.push('cancel-speech'); + + // The rewrite behind the synthesis. Cancelling TTS alone leaves a Brain + // writing sentences for a turn that is already over. + this.options.cancelTranslation(); + steps.push('cancel-translation'); + + const outcome: BargeInOutcome = { + source, + utteranceId: result?.utteranceId ?? null, + spoken: result?.spoken ?? [], + unspoken: result?.unspoken ?? [], + steps, + at: this.now(), + }; + + // Only what reached the speaker. The queued half never happened. + if (outcome.spoken.length > 0) this.options.rememberSpoken?.(outcome.spoken); + + this.speechStartedAt = null; + // Pushed before the callback runs, so a listener reading `outcome.steps` + // sees the same list the caller gets back rather than one step short. + steps.push('listening'); + this.options.toListening(outcome); + + return outcome; + } +} + +export function createBargeInController(options: BargeInControllerOptions): BargeInController { + return new BargeInController(options); +} + +function clamp01(value: number): number { + if (!Number.isFinite(value)) return 0; + return value < 0 ? 0 : value > 1 ? 1 : value; +} diff --git a/src/main/acappella/speech/conversational-translator.ts b/src/main/acappella/speech/conversational-translator.ts new file mode 100644 index 0000000000..f174c93f43 --- /dev/null +++ b/src/main/acappella/speech/conversational-translator.ts @@ -0,0 +1,245 @@ +/** + * The conversational translator: agent prose in, spoken conversation out. + * + * It lives OUTSIDE the agent, and that is the whole design. The obvious + * alternative - injecting a "be brief, you are being read aloud" instruction + * into the agent's system prompt - fails three ways at once: Claude, Codex, and + * Droid each honour it differently, it pollutes a transcript the user also reads + * on screen, and Terminal has no system prompt to inject into. A translator on + * the outside works identically for every agent and leaves the written record + * exactly as the agent wrote it. + * + * Latency is hidden the way every good voice assistant hides it: the tap + * (`agent-output-tap.ts`) delivers a completed thought while the agent is still + * writing, this file rewrites that piece alone, and the scheduler + * (`speech-scheduler.ts`) starts speaking it. The first spoken word therefore + * costs one short rewrite rather than a whole reply plus a whole rewrite. A Brain + * that implements `converseStream` shortens it again by emitting sentences as + * they are written, but the streaming seam is an optimisation on top of that, not + * the thing that makes it fast. + */ + +import type { BrainProvider, VoiceConverseContext } from '../../../shared/acappella/providers'; +import { + splitCompleteSentences, + splitIntoSpokenSentences, +} from '../../../shared/acappella/sentences'; +import { stripMarkdown } from '../../../shared/markdown'; +import type { AgentOutputChunk } from './agent-output-tap'; + +/** Spoken budget per rewritten chunk. Two sentences and an offer of detail. */ +const DEFAULT_MAX_SENTENCES = 2; + +/** Spoken lines carried between turns so a reply can refer back to what it said. */ +const DEFAULT_MEMORY_LIMIT = 8; + +/** + * Longest reply that is allowed to skip the model entirely. + * + * "Yes, the tests pass." does not need a translation hop, and paying one costs a + * round trip in the single place a user notices latency most - the short answer + * they expected to be instant. + */ +const DEFAULT_PASSTHROUGH_CHARS = 140; + +/** Anything markdown-shaped, path-shaped, or code-shaped disqualifies a passthrough. */ +const NOT_CONVERSATIONAL = + /[`*_#|]|\n|https?:\/\/|\w+\/[\w./-]+|\b\w+\.(?:ts|tsx|js|jsx|py|go|rs|json|md|yaml|yml)\b|\{|\}|=>/; + +/** + * Source longer than this had detail worth offering, whether or not the rewrite + * thought to offer it. + */ +const DEFAULT_DETAIL_OFFER_CHARS = 400; + +/** The offer, when the rewrite did not make one. Answered by `drill-down.ts`. */ +const DETAIL_OFFER = 'Want the details?'; + +/** An offer already made: a question, or the offer phrasing in any tense. */ +const ALREADY_OFFERS = /\?\s*$|\b(?:want|shall I|should I|tell you more|walk you through)\b/i; + +export interface ConversationalTranslatorOptions { + brain: BrainProvider; + maxSentences?: number; + /** Spoken lines retained as conversation memory. */ + memoryLimit?: number; + passthroughChars?: number; + /** Source longer than this gets an offer of detail appended when none was made. */ + detailOfferChars?: number; +} + +/** One rewrite in progress. Sentences arrive in order and stop early on abort. */ +export interface TranslationRequest { + agentSessionId: string; + tabId: string; + text: string; + kind: AgentOutputChunk['kind']; + /** Aborting stops the iteration and cancels the provider call behind it. */ + signal?: AbortSignal; +} + +export class ConversationalTranslator { + private readonly brain: BrainProvider; + private readonly maxSentences: number; + private readonly memoryLimit: number; + private readonly passthroughChars: number; + private readonly detailOfferChars: number; + + /** What the user actually HEARD, oldest first. Not what was queued. */ + private spokenMemory: string[] = []; + + /** Rewrites that skipped the model, for the suite and for the latency report. */ + private passthroughs = 0; + private translations = 0; + + constructor(options: ConversationalTranslatorOptions) { + this.brain = options.brain; + this.maxSentences = options.maxSentences ?? DEFAULT_MAX_SENTENCES; + this.memoryLimit = options.memoryLimit ?? DEFAULT_MEMORY_LIMIT; + this.passthroughChars = options.passthroughChars ?? DEFAULT_PASSTHROUGH_CHARS; + this.detailOfferChars = options.detailOfferChars ?? DEFAULT_DETAIL_OFFER_CHARS; + } + + /** Rewrites that reached the Brain, and rewrites that did not. */ + get stats(): { translations: number; passthroughs: number } { + return { translations: this.translations, passthroughs: this.passthroughs }; + } + + /** What has been said out loud this conversation, oldest first. */ + get memory(): string[] { + return [...this.spokenMemory]; + } + + /** + * Record what was actually spoken. + * + * Called by the speech scheduler with the sentences that reached the speaker, + * never with the ones that were queued and then cut off by a barge-in: a model + * told it already said something the user never heard will refer back to it, + * and the user will have no idea what it means. + */ + rememberSpoken(sentences: readonly string[]): void { + for (const sentence of sentences) { + const line = sentence.trim(); + if (line) this.spokenMemory.push(line); + } + if (this.spokenMemory.length > this.memoryLimit) { + this.spokenMemory = this.spokenMemory.slice(-this.memoryLimit); + } + } + + /** Forget the conversation. Called when the voice session ends. */ + reset(): void { + this.spokenMemory = []; + this.passthroughs = 0; + this.translations = 0; + } + + /** + * Rewrite one chunk, yielding complete sentences as they are produced. + * + * Sentences rather than deltas because a sentence is the unit TTS synthesises + * and the unit barge-in cuts at, and because `splitCompleteSentences()` is the + * one splitter the whole protocol agrees on. + */ + async *translate(request: TranslationRequest): AsyncIterable { + const source = request.text.trim(); + if (!source) return; + + // A status line is the tap speaking for itself about an error or a stall. It + // is already one honest spoken sentence, and handing it to a model would buy + // a round trip and a chance of the failure being softened into ambiguity. + if (request.kind === 'status' || this.isAlreadyConversational(source)) { + this.passthroughs += 1; + for (const sentence of splitIntoSpokenSentences(source).slice(0, this.maxSentences)) { + if (request.signal?.aborted) return; + yield sentence; + } + return; + } + + this.translations += 1; + const context: VoiceConverseContext = { + agentSessionId: request.agentSessionId, + tabId: request.tabId, + maxSentences: this.maxSentences, + recentSpoken: this.memory, + signal: request.signal, + }; + + let spoken = 0; + let last = ''; + for await (const raw of this.rewrite(source, context)) { + if (request.signal?.aborted) return; + // Enforced here rather than trusted from the model: every backend is asked + // for plain speech and every backend eventually returns a bullet anyway, + // and an asterisk read aloud is the one defect nobody forgives. + const sentence = stripMarkdown(raw).replace(/\s+/g, ' ').trim(); + if (!sentence) continue; + last = sentence; + yield sentence; + if (++spoken >= this.maxSentences) break; + } + + // The offer of detail is the point of the whole layer: a long answer becomes + // a headline plus a door back into it, served instantly from `drill-down.ts`. + // Only added when the rewrite did not think to make one. + if ( + spoken > 0 && + source.length >= this.detailOfferChars && + !ALREADY_OFFERS.test(last) && + !request.signal?.aborted + ) { + yield DETAIL_OFFER; + } + } + + // -- Internals ----------------------------------------------------------- + + /** + * The provider call, streaming when the provider can and buffered when it + * cannot. Both paths yield the same thing, so nothing downstream branches on + * which Brain is running. + */ + private async *rewrite(source: string, context: VoiceConverseContext): AsyncIterable { + const stream = this.brain.converseStream?.bind(this.brain); + if (!stream) { + const whole = await this.brain.converse(source, context); + yield* splitIntoSpokenSentences(whole.trim()); + return; + } + + let buffer = ''; + for await (const delta of stream(source, context)) { + if (context.signal?.aborted) return; + buffer += delta; + const { sentences, rest } = splitCompleteSentences(buffer); + buffer = rest; + for (const sentence of sentences) yield sentence; + } + + // The tail has no punctuation coming after it, so it is a sentence now. + const tail = buffer.trim(); + if (tail && !context.signal?.aborted) yield* splitIntoSpokenSentences(tail); + } + + /** + * True when the agent already wrote something a person would say. + * + * Short, one or two sentences, and free of every shape that has to be reworded + * for the ear. The test is deliberately conservative in the direction of + * translating: a needless hop costs a few hundred milliseconds, while a diff + * that slipped through as "already conversational" gets read aloud. + */ + private isAlreadyConversational(text: string): boolean { + if (text.length > this.passthroughChars) return false; + if (NOT_CONVERSATIONAL.test(text)) return false; + return splitIntoSpokenSentences(text).length <= this.maxSentences; + } +} + +export function createConversationalTranslator( + options: ConversationalTranslatorOptions +): ConversationalTranslator { + return new ConversationalTranslator(options); +} diff --git a/src/main/acappella/speech/drill-down.ts b/src/main/acappella/speech/drill-down.ts new file mode 100644 index 0000000000..ef49f4ed7f --- /dev/null +++ b/src/main/acappella/speech/drill-down.ts @@ -0,0 +1,213 @@ +/** + * Drill-down: "tell me more", served from what the agent already wrote. + * + * The translator's whole job is to offer the detail rather than deliver it, which + * only works if taking up the offer is instant. Re-asking the agent would cost a + * full turn and, worse, would produce a DIFFERENT answer - the user would be told + * about work that has moved on since the sentence they are asking about. + * + * So the real, untranslated output of the last turn is retained in a per-turn + * buffer and follow-ups are answered from it. That costs one string per turn and + * makes "tell me more" free. + * + * "Show me" is deliberately not a spoken answer. Anything the user wants to SEE - + * a diff, a file, a test run - is better delivered by putting it on screen, and + * reading a path character by character is the single worst thing this feature + * could do with a request to look at something. + */ + +import { getBasename } from '../../../shared/formatters'; +import { splitIntoSpokenSentences } from '../../../shared/acappella/sentences'; + +/** + * What a follow-up is asking for. + * + * - `more` - the next slice of detail. + * - `repeat` - say the last thing again, unchanged. + * - `file` - which file was that. + * - `show` - put it on screen instead of saying it. + */ +export type DrillDownIntent = 'more' | 'repeat' | 'file' | 'show'; + +export interface DrillDownTurn { + agentSessionId: string; + tabId: string; + /** The agent's real output, untranslated. What every follow-up is served from. */ + detail: string; + /** What was actually said out loud about it. */ + spoken: string[]; +} + +export type DrillDownResponse = + | { kind: 'speak'; text: string } + | { kind: 'focus'; agentSessionId: string; tabId: string; path?: string } + | { kind: 'none' }; + +export interface DetailBufferOptions { + /** Sentences of detail served per "tell me more". */ + sentencesPerSlice?: number; +} + +/** Enough to be worth asking for, short enough to interrupt. */ +const DEFAULT_SENTENCES_PER_SLICE = 3; + +// --------------------------------------------------------------------------- +// Intent +// --------------------------------------------------------------------------- + +/** + * Follow-up phrasings, most specific first. + * + * Ordering matters: "show me the file" is a `show`, not a `file`, because the + * answer to it is a focused tab rather than a spoken path. Matching `file` + * first would speak a path at someone who asked to look at one. + */ +const INTENT_PATTERNS: [DrillDownIntent, RegExp][] = [ + // Anchored on a demonstrative on purpose: a bare "open" or "show" is a routing + // utterance ("open a new tab", "show me the backlog"), and treating it as a + // follow-up would swallow a real request into the last turn's buffer. + [ + 'show', + /\b(?:show|open|pull up|bring up|display|let me see)\s+(?:me\s+)?(?:that|it|this|the (?:file|diff|code|change|error|test|tab))\b/, + ], + [ + 'repeat', + /\b(?:say (?:that|it) again|repeat (?:that|it)?|read that again|what did you say|come again)\b/, + ], + [ + 'file', + /\b(?:what|which)\s+(?:was\s+)?the\s+(?:file|path|test|error)\b|\bwhich file\b|\bwhat file\b/, + ], + [ + 'more', + /\b(?:tell me more|more detail|the details|go on|keep going|what else|elaborate|expand on that|and then)\b|^more$/, + ], +]; + +/** The intent behind a follow-up utterance, or null when it is a fresh request. */ +export function detectDrillDownIntent(utterance: string): DrillDownIntent | null { + const text = utterance + .toLowerCase() + .replace(/[^\p{L}\p{N}\s]/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (!text) return null; + for (const [intent, pattern] of INTENT_PATTERNS) { + if (pattern.test(text)) return intent; + } + return null; +} + +// --------------------------------------------------------------------------- +// Buffer +// --------------------------------------------------------------------------- + +/** Paths as agents write them: with a slash, or with a known-ish extension. */ +const PATH_TOKEN = /(?:[\w@.-]+\/)+[\w@-]+(?:\.[\w@-]+)*|\b[\w-]{2,}\.[a-z]{1,5}\b/g; + +export class DetailBuffer { + private readonly sentencesPerSlice: number; + private turn: DrillDownTurn | null = null; + /** How far through the detail successive "tell me more" calls have read. */ + private cursor = 0; + + constructor(options: DetailBufferOptions = {}) { + this.sentencesPerSlice = Math.max(1, options.sentencesPerSlice ?? DEFAULT_SENTENCES_PER_SLICE); + } + + /** Retain a turn. Replaces the previous one and rewinds the read cursor. */ + record(turn: DrillDownTurn): void { + this.turn = turn; + this.cursor = 0; + } + + /** Add to what was said out loud about the retained turn. */ + noteSpoken(sentences: readonly string[]): void { + if (!this.turn) return; + this.turn.spoken = [...this.turn.spoken, ...sentences.filter((s) => s.trim())]; + } + + /** Drop the buffer. Called when the voice session ends. */ + clear(): void { + this.turn = null; + this.cursor = 0; + } + + get hasTurn(): boolean { + return this.turn !== null; + } + + /** + * Answer a follow-up from the retained output. Never dispatches a new agent + * turn, which is the entire point. + */ + serve(intent: DrillDownIntent): DrillDownResponse { + const turn = this.turn; + if (!turn) return { kind: 'none' }; + + switch (intent) { + case 'repeat': + return turn.spoken.length > 0 + ? { kind: 'speak', text: turn.spoken.join(' ') } + : { kind: 'none' }; + case 'more': + return this.serveMore(turn); + case 'file': + return this.serveFile(turn); + case 'show': + // No speech: a request to look at something is answered on screen. + return { + kind: 'focus', + agentSessionId: turn.agentSessionId, + tabId: turn.tabId, + path: firstPath(turn.detail), + }; + } + } + + private serveMore(turn: DrillDownTurn): DrillDownResponse { + const sentences = splitIntoSpokenSentences(turn.detail); + if (this.cursor >= sentences.length) { + return { kind: 'speak', text: "That's everything it said." }; + } + const slice = sentences.slice(this.cursor, this.cursor + this.sentencesPerSlice); + this.cursor += slice.length; + return { kind: 'speak', text: slice.join(' ') }; + } + + private serveFile(turn: DrillDownTurn): DrillDownResponse { + const path = firstPath(turn.detail); + if (!path) return { kind: 'speak', text: 'It did not name a file.' }; + return { kind: 'speak', text: `It was ${speakPath(path)}.` }; + } +} + +export function createDetailBuffer(options?: DetailBufferOptions): DetailBuffer { + return new DetailBuffer(options); +} + +// --------------------------------------------------------------------------- +// Paths +// --------------------------------------------------------------------------- + +/** The first path-shaped token in the detail, or undefined. */ +export function firstPath(detail: string): string | undefined { + PATH_TOKEN.lastIndex = 0; + const match = PATH_TOKEN.exec(detail); + return match?.[0]; +} + +/** + * A path, said the way a person would say it. + * + * The basename and its extension, never the directories: "the speech scheduler + * file" is what a colleague says, and `s-r-c-slash-m-a-i-n-slash` is what a + * screen reader says. Reusing `getBasename()` rather than splitting on `/` here + * keeps Windows paths working, which the naive version did not. + */ +export function speakPath(path: string): string { + const base = getBasename(path) || path; + const dot = base.lastIndexOf('.'); + if (dot <= 0) return base; + return `${base.slice(0, dot)} dot ${base.slice(dot + 1)}`; +} diff --git a/src/main/acappella/speech/index.ts b/src/main/acappella/speech/index.ts new file mode 100644 index 0000000000..b5e40ced31 --- /dev/null +++ b/src/main/acappella/speech/index.ts @@ -0,0 +1,63 @@ +/** + * The speech half of A Cappella: what an agent wrote, turned into what a person + * hears. + * + * The pieces compose in one direction and each is useful alone, which is why + * they are separate modules rather than one "speech manager": + * + * tap -> translator -> scheduler -> audio + * ^ | + * +-- barge-in --+ (cancels both, keeps the floor) + * + * `drill-down.ts` sits beside the tap holding the untranslated output, and + * `background-announcer.ts` sits outside the turn entirely, waiting for a pause. + */ + +export { + AgentOutputTap, + createAgentOutputTap, + type AgentOutputChunk, + type AgentOutputChunkKind, + type AgentOutputSource, + type AgentOutputTapOptions, +} from './agent-output-tap'; +export { + BackgroundAnnouncer, + announcementText, + createBackgroundAnnouncer, + type BackgroundAnnouncement, + type BackgroundAnnouncerOptions, + type BackgroundCompletion, +} from './background-announcer'; +export { + BargeInController, + createBargeInController, + type BargeInControllerOptions, + type BargeInOutcome, + type BargeInStep, +} from './barge-in'; +export { + ConversationalTranslator, + createConversationalTranslator, + type ConversationalTranslatorOptions, + type TranslationRequest, +} from './conversational-translator'; +export { + DetailBuffer, + createDetailBuffer, + detectDrillDownIntent, + firstPath, + speakPath, + type DetailBufferOptions, + type DrillDownIntent, + type DrillDownResponse, + type DrillDownTurn, +} from './drill-down'; +export { + SpeechScheduler, + createSpeechScheduler, + type SpeechRunEndReason, + type SpeechRunResult, + type SpeechSchedulerEvents, + type SpeechSchedulerOptions, +} from './speech-scheduler'; diff --git a/src/main/acappella/speech/send-phrase.ts b/src/main/acappella/speech/send-phrase.ts new file mode 100644 index 0000000000..3365206fb6 --- /dev/null +++ b/src/main/acappella/speech/send-phrase.ts @@ -0,0 +1,107 @@ +/** + * Send phrases - the spoken "that's it, go". + * + * A voice request has no Enter key. Without one, the only way to know a person + * has finished is to wait for silence, and the wait is always wrong: short + * enough to feel responsive and it cuts them off mid-thought, long enough to let + * them think and every finished request sits there. A phrase removes the guess - + * you say when you are done, and the pause becomes a backstop rather than the + * mechanism. + * + * **Matched on the transcript, not on audio frames.** The stop word is the + * opposite (see `wake/stop-word.ts`): it has to be heard while text-to-speech is + * mid-sentence, so it runs on a local classifier over raw microphone frames. + * A send phrase is said at the end of ordinary dictation, when nothing is + * playing and the recogniser is already producing text, and matching text buys + * three things that audio cannot: + * + * 1. It works with EVERY recogniser. The audio detector needs openWakeWord, + * which is a separate model download; this needs nothing. + * 2. It can be STRIPPED. "fix the auth bug, good to go" has to reach the agent + * as "fix the auth bug" - the send signal is not part of the request, and + * an audio-level match cannot remove words from a transcript it never saw. + * 3. It matches what was actually transcribed, so a phrase the recogniser + * renders as "that's it" rather than "thats it" still fires. + * + * **Anchored to the end, always.** "That's it, the bug is in the auth module" is + * someone agreeing with you and then talking; "fix the auth module, that's it" + * is someone finishing. Only the second may send, and the difference is entirely + * position. A contains-match here would send the moment anyone said "go ahead" + * in the middle of a sentence. + */ + +// Re-exported rather than declared here: the settings panel needs the same list +// to seed its input, and the renderer cannot import from the main process. +export { DEFAULT_SEND_PHRASES } from '../../../shared/acappella/voice-controls'; +import { DEFAULT_SEND_PHRASES } from '../../../shared/acappella/voice-controls'; + +/** + * Casing, punctuation and spacing removed. + * + * Recognisers differ on all three - "That's it." and "thats it" are the same + * intent - and the apostrophe is the one that matters most, because whether a + * transcript contains one is a property of the engine rather than of the + * speaker. + */ +export function normalisePhrase(text: string): string { + return text + .toLowerCase() + .replace(/[‘’']/g, '') + .replace(/[^a-z0-9]+/g, ' ') + .trim(); +} + +export interface SendPhraseMatch { + /** What is left of the utterance once the send phrase is removed. */ + text: string; + /** The phrase that fired, as configured. */ + phrase: string; +} + +/** + * Find a send phrase at the end of `text`. + * + * @returns the remaining request and the phrase, or null when nothing matched. + */ +export function matchSendPhrase( + text: string, + phrases: readonly string[] = DEFAULT_SEND_PHRASES +): SendPhraseMatch | null { + const normalised = normalisePhrase(text); + if (!normalised) return null; + + // Longest first, so "that's it then" cannot be beaten to the match by a + // shorter phrase that happens to be a suffix of it. + const candidates = [...phrases] + .map((phrase) => ({ phrase, normalised: normalisePhrase(phrase) })) + .filter((entry) => entry.normalised.length > 0) + .sort((a, b) => b.normalised.length - a.normalised.length); + + for (const candidate of candidates) { + if (normalised === candidate.normalised) { + // The whole turn was the signal: everything already buffered is the + // request, and this utterance adds nothing to it. + return { text: '', phrase: candidate.phrase }; + } + const suffix = ` ${candidate.normalised}`; + if (!normalised.endsWith(suffix)) continue; + + // Trim the ORIGINAL text rather than returning the normalised head: + // normalising destroys the user's capitals and punctuation, and this string + // becomes the prompt an agent receives. + const words = candidate.normalised.split(' ').length; + return { text: dropTrailingWords(text, words), phrase: candidate.phrase }; + } + + return null; +} + +/** + * Remove `count` words from the end of the ORIGINAL text, then any punctuation + * the send phrase was hanging off ("fix the bug, good to go" -> "fix the bug"). + */ +function dropTrailingWords(text: string, count: number): string { + const tokens = text.trim().split(/\s+/); + const kept = tokens.slice(0, Math.max(0, tokens.length - count)); + return kept.join(' ').replace(/[\s,;:.!-]+$/, ''); +} diff --git a/src/main/acappella/speech/speech-scheduler.ts b/src/main/acappella/speech/speech-scheduler.ts new file mode 100644 index 0000000000..d44e398b8a --- /dev/null +++ b/src/main/acappella/speech/speech-scheduler.ts @@ -0,0 +1,461 @@ +/** + * The sentence-streaming speech scheduler. + * + * One speech run is a queue of sentences, each moving through synthesize and + * then play. The scheduler's whole reason to exist is the seam between those two + * verbs: sentence N+1 is synthesized while sentence N is still audible, so there + * is no gap between them, and no more than `lookahead` sentences are synthesized + * beyond what has actually been heard, so a barge-in throws away one sentence of + * paid-for audio rather than a paragraph of it. + * + * It owns three things nothing else should duplicate: + * + * - **Segmentation.** Through `splitCompleteSentences()`, the one splitter in + * `src/shared/acappella/sentences.ts`. `v1.2.3`, `src/main/index.ts`, `99.5`, + * and `e.g.` are exactly what agents write, and a second splitter that got + * any of them wrong would cut a spoken sentence in half. + * - **The protocol events.** `speak-start`, one `speak-sentence` per sentence + * with its text, and `speak-end`. The live transcript and the phone render + * from those, so what they show is what is being said by construction. + * - **The end reason.** `interrupted` is not `completed`. Collapsing them makes + * a turn the user talked over indistinguishable from one they listened to, + * which is the difference the conversation memory is built on. + */ + +import type { TtsChunk, TtsProvider } from '../../../shared/acappella/providers'; +import { + splitCompleteSentences, + splitIntoSpokenSentences, +} from '../../../shared/acappella/sentences'; + +/** + * How a run ended. + * + * - `completed` - every queued sentence was spoken, cap included. + * - `interrupted` - the user took the floor back. Distinct on purpose. + * - `error` - the provider failed mid-run. + */ +export type SpeechRunEndReason = 'completed' | 'interrupted' | 'error'; + +export interface SpeechRunResult { + utteranceId: string; + reason: SpeechRunEndReason; + /** Sentences that reached the speaker, in order. This is what was HEARD. */ + spoken: string[]; + /** Sentences that were queued and never reached it. */ + unspoken: string[]; + /** The run hit the per-turn cap and wrapped up rather than reading on. */ + capped: boolean; +} + +export interface SpeechSchedulerEvents { + onStart: (event: { + utteranceId: string; + sentenceCount: number; + ttsProviderId: string; + /** True while more sentences are still being written. `sentenceCount` is a lower bound. */ + streaming: boolean; + }) => void; + onSentence: (event: { utteranceId: string; index: number; text: string }) => void; + onEnd: (result: SpeechRunResult) => void; + /** One synthesised chunk, for the audio sink. Never broadcast. */ + onChunk?: (chunk: TtsChunk) => void; + /** A classified provider failure. Anything unexpected is rethrown to Sentry. */ + onError?: (error: Error) => void; +} + +export interface SpeechSchedulerOptions extends SpeechSchedulerEvents { + tts: TtsProvider; + /** Hard cap on sentences spoken per turn. */ + maxSentencesPerTurn?: number; + /** Sentences allowed to be synthesized beyond the one being heard. */ + lookahead?: number; + /** Longest queue held before `push()` starts dropping. */ + queueLimit?: number; + /** Said instead of an abrupt cut when the cap is reached. */ + wrapUpText?: string; + /** + * The voice and rate to synthesize with, read fresh for EVERY sentence. + * + * A getter rather than a value because that is what makes the settings apply + * live: a user who drags the speed slider mid-reply hears the change on the + * next sentence instead of on the next session. Reading it once at + * construction would pin the whole session to whatever was configured when it + * started, which is the "restart to hear your own setting" behaviour this + * exists to avoid. + */ + speechOptions?: () => { voiceId?: string; rate?: number }; +} + +/** + * Roughly forty seconds of speech. Long enough for a real answer, short enough + * that a runaway agent summary cannot hold the floor while the user waits for a + * gap to talk into. + */ +const DEFAULT_MAX_SENTENCES_PER_TURN = 6; + +/** One sentence ahead: no gap, and no more than one wasted synthesis on barge-in. */ +const DEFAULT_LOOKAHEAD = 1; + +/** A queue longer than this is an agent that will never be listened to in full. */ +const DEFAULT_QUEUE_LIMIT = 64; + +const DEFAULT_WRAP_UP = "There's more, ask me for the details."; + +export class SpeechScheduler { + private readonly options: SpeechSchedulerOptions; + private readonly maxSentencesPerTurn: number; + private readonly lookahead: number; + private readonly queueLimit: number; + private readonly wrapUpText: string; + + private utteranceId: string | null = null; + private queue: string[] = []; + /** Text after the last complete sentence, waiting for the rest of it. */ + private partial = ''; + private spoken: string[] = []; + /** + * Sentences being synthesized right now, oldest first. + * + * This is the no-gap mechanism, and it is why synthesis is not simply awaited + * in a loop: `lookahead + 1` sentences are in flight at once and are DELIVERED + * strictly in order, so the audio for sentence two is already made by the time + * sentence one finishes playing. Serial synthesis would leave a provider round + * trip of silence between every pair of sentences. + */ + private pipeline: { sentence: string; chunks: Promise }[] = []; + /** Sentences handed to the provider this run, delivered or not. */ + private started = 0; + /** Sentences whose audio has reached the sink. */ + private delivered = 0; + private capped = false; + private closed = false; + private running = false; + private ended = false; + private endReason: SpeechRunEndReason = 'completed'; + private settle: (() => void) | null = null; + private wake: (() => void) | null = null; + private lastResult: SpeechRunResult | null = null; + + constructor(options: SpeechSchedulerOptions) { + this.options = options; + this.maxSentencesPerTurn = Math.max( + 1, + options.maxSentencesPerTurn ?? DEFAULT_MAX_SENTENCES_PER_TURN + ); + this.lookahead = Math.max(0, options.lookahead ?? DEFAULT_LOOKAHEAD); + this.queueLimit = Math.max(1, options.queueLimit ?? DEFAULT_QUEUE_LIMIT); + this.wrapUpText = options.wrapUpText ?? DEFAULT_WRAP_UP; + } + + get isSpeaking(): boolean { + return this.utteranceId !== null && !this.ended; + } + + get activeUtteranceId(): string | null { + return this.ended ? null : this.utteranceId; + } + + /** + * Open a run. + * + * `speak-start` is emitted here with the sentences known so far, which for a + * streaming turn is usually zero: the announced count is a lower bound and the + * `streaming` flag says so, because the alternative is holding the first + * sentence back until the whole reply exists and losing the entire point of + * the pipeline. + */ + begin(utteranceId: string, seed = ''): void { + this.utteranceId = utteranceId; + this.queue = []; + this.partial = ''; + this.spoken = []; + this.pipeline = []; + this.started = 0; + this.delivered = 0; + this.capped = false; + this.closed = false; + this.ended = false; + this.endReason = 'completed'; + + if (seed) this.enqueue(splitIntoSpokenSentences(seed)); + + this.options.onStart({ + utteranceId, + sentenceCount: this.queue.length, + ttsProviderId: this.options.tts.id, + streaming: !this.closed, + }); + this.pump(); + } + + /** + * Add text to the run. Only complete sentences are queued; the tail is held + * until the rest of it arrives, so nothing is synthesized twice. + */ + push(text: string): void { + if (!this.utteranceId || this.closed || this.ended) return; + this.partial += this.partial ? ` ${text}` : text; + const { sentences, rest } = splitCompleteSentences(this.partial); + this.partial = rest; + this.enqueue(sentences); + this.pump(); + } + + /** + * One whole sentence, already segmented by the translator. Bypasses the + * partial buffer: the translator yields sentences, not deltas. + */ + pushSentence(sentence: string): void { + if (!this.utteranceId || this.closed || this.ended) return; + this.enqueue([sentence]); + this.pump(); + } + + /** No more text is coming. The run ends once the queue drains. */ + close(): void { + if (!this.utteranceId || this.ended) return; + this.closed = true; + if (this.partial.trim()) { + this.enqueue(splitIntoSpokenSentences(this.partial)); + this.partial = ''; + } + this.wake?.(); + this.pump(); + } + + /** + * The user took the floor. Cuts the provider off mid-sentence and reports what + * was heard against what was not. + * + * Idempotent, because a barge-in and a speech run finishing on its own can race + * and the loser must not emit a second `speak-end`. + */ + cancel(reason: SpeechRunEndReason = 'interrupted'): SpeechRunResult | null { + if (!this.utteranceId || this.ended) return null; + this.endReason = reason; + this.options.tts.cancel(); + this.wake?.(); + return this.finish(reason); + } + + /** Resolves when the run has ended, however it ended. */ + async drained(): Promise { + if (this.ended || !this.utteranceId) return null; + await new Promise((resolve) => { + this.settle = resolve; + }); + return this.lastResult; + } + + // -- Internals ----------------------------------------------------------- + + private enqueue(sentences: readonly string[]): void { + for (const raw of sentences) { + const sentence = raw.trim(); + if (!sentence) continue; + if (this.queue.length >= this.queueLimit) { + // The queue is already longer than anyone will listen to. Dropping the + // newest is right: the oldest is the headline, and the cap below will + // wrap the turn up long before this matters. + return; + } + this.queue.push(sentence); + } + } + + /** Start the worker, or wake the one already parked waiting for text. */ + private pump(): void { + // Before the `running` check, not after: a worker asleep on an empty queue IS + // running, and returning early here is how a pushed sentence was never spoken. + this.wake?.(); + if (this.running || this.ended || !this.utteranceId) return; + this.running = true; + void this.run().finally(() => { + this.running = false; + }); + } + + /** + * Fill the synthesis pipeline, then deliver its head in order. + * + * Two halves on purpose. Filling runs ahead so audio for the next sentence + * exists before the current one finishes; delivering is strictly in order so + * the user hears the reply in the order it was written. Merging them - the + * obvious `await speak(sentence)` loop - reintroduces a provider round trip of + * silence between every pair of sentences, which is precisely what this module + * exists to remove. + */ + private async run(): Promise { + const utteranceId = this.utteranceId; + if (!utteranceId) return; + + while (!this.ended) { + this.fill(utteranceId); + + const head = this.pipeline[0]; + if (!head) { + if (this.started >= this.maxSentencesPerTurn) { + await this.speakWrapUp(utteranceId); + this.finish('completed'); + return; + } + if (this.closed && this.queue.length === 0 && !this.partial.trim()) { + this.finish(this.endReason); + return; + } + // Nothing to synthesize and more text is coming: wait to be woken rather + // than spinning, and never end a run on a gap in the translation. + await this.sleep(); + continue; + } + + let chunks: TtsChunk[]; + try { + chunks = await head.chunks; + } catch (error) { + this.options.onError?.(error as Error); + this.finish('error'); + return; + } + + // A cancelled run's stragglers belong to a floor the user already took + // back; `cancel()` has emitted `speak-end` and moved on. + if (this.ended || this.utteranceId !== utteranceId) return; + + // The event fires before the audio reaches the sink, so the sentence is on + // screen by the time it is audible rather than after it. + this.options.onSentence({ utteranceId, index: this.delivered, text: head.sentence }); + + // A barge-in can arrive from INSIDE that event: the transcript rendering a + // sentence is exactly what a user talks over. Checked before the shift, so + // a sentence that was announced but never audible is reported as unspoken + // rather than vanishing from both lists. + if (this.ended || this.utteranceId !== utteranceId) return; + + this.pipeline.shift(); + this.spoken.push(head.sentence); + this.delivered += 1; + for (const chunk of chunks) this.options.onChunk?.(chunk); + } + } + + /** + * Start synthesis for as many sentences as the lookahead allows. + * + * `lookahead + 1` in flight: one being delivered and `lookahead` being made + * ahead of it. Higher would buy nothing (the user cannot get further ahead + * than one sentence of listening) and would throw away more paid-for audio on + * every barge-in. + */ + private fill(utteranceId: string): void { + while ( + this.pipeline.length <= this.lookahead && + this.started < this.maxSentencesPerTurn && + this.queue.length > 0 + ) { + const sentence = this.queue.shift() as string; + this.started += 1; + const chunks = this.synthesize(utteranceId, sentence); + // A prefetched sentence can reject long before the delivery loop reaches + // it - a provider that failed for the whole run fails every sentence in + // flight. Attaching a handler here is what keeps that from surfacing as an + // unhandled rejection; the delivery loop still sees the rejection when it + // awaits the same promise, and still ends the run with `error`. + void chunks.catch(() => {}); + this.pipeline.push({ sentence, chunks }); + } + } + + /** The voice and rate for the sentence about to be synthesized. Read live. */ + private speakOptions(utteranceId: string): { + utteranceId: string; + voiceId?: string; + rate?: number; + } { + return { utteranceId, ...(this.options.speechOptions?.() ?? {}) }; + } + + /** Collect one sentence's audio. Rejections are handled by the delivery loop. */ + private async synthesize(utteranceId: string, sentence: string): Promise { + const chunks: TtsChunk[] = []; + for await (const chunk of this.options.tts.speak(sentence, this.speakOptions(utteranceId))) { + chunks.push(chunk); + } + return chunks; + } + + /** + * The cap, said out loud rather than performed silently. + * + * Stopping mid-answer with no explanation reads as a crash to someone who + * cannot see the screen, so the turn ends on an offer instead. + */ + private async speakWrapUp(utteranceId: string): Promise { + if (this.capped) return; + // Nothing left to offer: the run happened to end exactly on the cap, and + // promising details that do not exist is worse than stopping. + if (this.queue.length === 0 && !this.partial.trim() && this.closed) return; + + this.capped = true; + this.options.onSentence({ utteranceId, index: this.delivered, text: this.wrapUpText }); + try { + for await (const chunk of this.options.tts.speak( + this.wrapUpText, + this.speakOptions(utteranceId) + )) { + if (this.ended) return; + this.options.onChunk?.(chunk); + } + } catch (error) { + this.options.onError?.(error as Error); + return; + } + this.spoken.push(this.wrapUpText); + this.delivered += 1; + } + + private finish(reason: SpeechRunEndReason): SpeechRunResult | null { + if (this.ended || !this.utteranceId) return this.lastResult; + this.ended = true; + + const result: SpeechRunResult = { + utteranceId: this.utteranceId, + reason, + spoken: [...this.spoken], + // Everything still being synthesized plus everything still queued: the user + // heard none of it, so the conversation memory must not claim they did. + unspoken: [ + ...this.pipeline.map((entry) => entry.sentence), + ...this.queue, + ...splitIntoSpokenSentences(this.partial), + ], + capped: this.capped, + }; + + this.lastResult = result; + this.queue = []; + this.partial = ''; + this.pipeline = []; + this.options.onEnd(result); + + const settle = this.settle; + this.settle = null; + settle?.(); + return result; + } + + /** Park until something happens: a push, a close, a cancel, or a played sentence. */ + private sleep(): Promise { + return new Promise((resolve) => { + this.wake = () => { + this.wake = null; + resolve(); + }; + }); + } +} + +export function createSpeechScheduler(options: SpeechSchedulerOptions): SpeechScheduler { + return new SpeechScheduler(options); +} diff --git a/src/main/acappella/speech/utterance-composer.ts b/src/main/acappella/speech/utterance-composer.ts new file mode 100644 index 0000000000..fef03b96e4 --- /dev/null +++ b/src/main/acappella/speech/utterance-composer.ts @@ -0,0 +1,294 @@ +/** + * Utterance composer - assembles the fragments of one thought into one request. + * + * A recogniser endpoints on silence (700 ms, see `audio/vad.ts`), but people do + * not. "Look at the auth module..." *thinks* "...and tell me why the refresh is + * failing" endpoints twice, and without this the session routed and dispatched + * BOTH halves: the agent got a fragment, started answering it, and then received + * a second request that only made sense joined to the first. + * + * So a settled fragment is held rather than dispatched. Another fragment inside + * the settle window joins it and restarts the clock; silence past the window + * means the thought is finished and the whole thing goes as one request. + * + * **The cost is honest and deliberate.** A request that really was complete now + * waits `settleMs` before anything happens. That is the trade this component + * exists to make: dead air before a correct dispatch beats an agent working on + * half a sentence. It is tunable for exactly that reason, and a settle of 0 + * restores the old behaviour for anyone who wants it. + * + * Pure and timer-driven, with no session, provider, or transport knowledge, so + * it can be tested against fake timers rather than against a microphone. + */ + +import { DEFAULT_SEND_PHRASES, matchSendPhrase } from './send-phrase'; + +/** How long to wait, and the backstop that stops it waiting forever. */ +export interface UtteranceComposerConfig { + /** + * Silence after a fragment before the thought counts as finished. + * + * On top of the recogniser's own endpoint silence, not instead of it: at the + * defaults a dispatch happens ~1.6 s after you stop making noise. Zero + * disables composition entirely and dispatches every fragment on arrival. + */ + settleMs: number; + /** + * Hard cap on how long one thought may be assembled for. + * + * A backstop against a room noisy enough to keep producing fragments, never a + * normal path - firing it mid-sentence splits the thought, which is the very + * thing this module exists to prevent, so it is set far beyond any real + * sentence rather than close to one. + */ + maxHoldMs: number; + /** + * Phrases that end dictation immediately, said at the END of a turn. + * + * A voice request has no Enter key, and {@link settleMs} is a guess at when + * someone stopped talking. A phrase removes the guess: the pause becomes a + * backstop for the times you forget to say it, rather than the mechanism. + * Empty disables them. See `speech/send-phrase.ts`. + */ + sendPhrases: readonly string[]; +} + +export const DEFAULT_UTTERANCE_COMPOSER_CONFIG: UtteranceComposerConfig = { + settleMs: 900, + maxHoldMs: 30_000, + sendPhrases: DEFAULT_SEND_PHRASES, +}; + +/** One assembled thought, with the parts it was built from. */ +export interface ComposedUtterance { + text: string; + /** + * The LOWEST confidence of any fragment. + * + * The assembled utterance is only as trustworthy as its worst part: averaging + * would let one clear fragment vouch for a mumbled one, and the whole thing is + * dispatched as a single request. + */ + confidence: number; + /** Summed speech duration of the fragments, when they reported one. */ + durationMs?: number; + /** How many recogniser finals were joined. 1 means nothing was coalesced. */ + fragments: number; + /** + * What ended this thought: a send phrase as configured, `'release'` for a + * push-to-talk key coming up, or absent when the settle timer fired. + * + * Worth distinguishing: a client can say "sending" the instant you ask for it, + * rather than after a pause that looks like nothing happening. + */ + sentBy?: string; +} + +export interface UtteranceComposerOptions extends Partial { + /** The assembled thought, once the user has stopped adding to it. */ + onSettled: (utterance: ComposedUtterance) => void; + /** + * A fragment joined the buffer and the clock restarted. + * + * The HUD renders this as a growing partial: without it the transcript blanks + * between fragments and a composing session looks like one that stopped + * listening. + */ + onComposing?: (text: string) => void; +} + +interface Buffered { + parts: string[]; + confidence: number; + durationMs: number; + sawDuration: boolean; +} + +export class UtteranceComposer { + private readonly config: UtteranceComposerConfig; + private readonly onSettled: (utterance: ComposedUtterance) => void; + private readonly onComposing?: (text: string) => void; + + private buffer: Buffered | null = null; + /** Set by {@link armImmediateSettle}: the next fragment ends the thought. */ + private immediateSettle = false; + private settleTimer: ReturnType | null = null; + private holdTimer: ReturnType | null = null; + private disposed = false; + + constructor(options: UtteranceComposerOptions) { + const settleMs = Math.max(0, options.settleMs ?? DEFAULT_UTTERANCE_COMPOSER_CONFIG.settleMs); + const maxHoldMs = Math.max(0, options.maxHoldMs ?? DEFAULT_UTTERANCE_COMPOSER_CONFIG.maxHoldMs); + this.config = { + settleMs, + // The cap can never be tighter than the wait it is backstopping. A 30 s + // hold under a 30 s cap fires the cap first and splits the thought - the + // exact failure the hold exists to prevent - because the cap starts on the + // first fragment while the settle restarts on every one. The multiple is + // slack for the fragments themselves, not a tuned number. + maxHoldMs: maxHoldMs === 0 ? 0 : Math.max(maxHoldMs, settleMs * 4), + sendPhrases: options.sendPhrases ?? DEFAULT_UTTERANCE_COMPOSER_CONFIG.sendPhrases, + }; + this.onSettled = options.onSettled; + this.onComposing = options.onComposing; + } + + /** True while a thought is being assembled. */ + get composing(): boolean { + return this.buffer !== null; + } + + /** What has been collected so far. Empty when nothing is buffered. */ + get pending(): string { + return this.buffer ? this.buffer.parts.join(' ') : ''; + } + + /** Take one recogniser final. It may or may not be the whole thought. */ + add(text: string, confidence: number, durationMs?: number): void { + if (this.disposed) return; + let fragment = text.trim(); + // An empty final is the recogniser reporting silence. Joining it would put a + // stray space in the prompt and restart the clock for nothing. + if (!fragment) return; + + // "fix the auth bug, good to go" is a request and a send signal in one + // breath. The signal is removed before the rest is buffered, because what + // survives here becomes the prompt an agent receives. + const send = matchSendPhrase(fragment, this.config.sendPhrases); + if (send) { + fragment = send.text.trim(); + if (fragment) this.append(fragment, confidence, durationMs); + // Said with nothing buffered and nothing else in the sentence, this is a + // send signal for a request that does not exist. Dispatching an empty + // prompt would be worse than ignoring it. + if (!this.buffer) return; + this.settle(send.phrase); + return; + } + + this.append(fragment, confidence, durationMs); + + // The user let go of the key before this fragment arrived: it is the tail of + // the sentence they had already finished saying. + if (this.immediateSettle) { + this.settle('release'); + return; + } + + // Zero settle is "compose nothing": dispatch on arrival, which is what the + // session did before this module existed. + if (this.config.settleMs === 0) { + this.settle(); + return; + } + + this.onComposing?.(this.pending); + this.restartSettleTimer(); + this.startHoldTimer(); + } + + /** + * End the thought as soon as the recogniser has finished delivering it. + * + * For a release gesture - letting go of a push-to-talk key - where the user + * has already said they are done but the words may still be in flight. The + * recogniser is flushed at the same moment, and its final can land either side + * of this call, so: + * + * - a fragment arriving after this settles immediately, tail included; + * - if nothing arrives, what is already buffered settles now. + * + * Settling only the current buffer would send the sentence minus its last few + * words, which is the failure a release gesture must never produce. + */ + armImmediateSettle(): void { + if (this.disposed) return; + if (this.buffer) { + // Nothing may be in flight at all, so the buffer must not be left waiting + // on a fragment that never comes. A late final still settles on arrival, + // as its own thought, which is the honest reading of words spoken after + // the user said they were finished. + this.settle('release'); + } + // Set AFTER the settle, which clears it: the flag has to survive the flush + // of what was already buffered so a tail arriving afterwards is still sent + // on arrival rather than sitting out a settle window the user has answered. + this.immediateSettle = true; + } + + /** + * Settle now, whatever the clock says. + * + * For the moments that end a thought by decree rather than by silence: a stop + * word, a hotkey release, the floor closing under it. + */ + flush(): void { + if (this.disposed || !this.buffer) return; + this.settle(); + } + + /** Drop everything buffered. A barge-in or a new session, not an endpoint. */ + cancel(): void { + this.clearTimers(); + this.buffer = null; + this.immediateSettle = false; + } + + dispose(): void { + this.disposed = true; + this.cancel(); + } + + /** Add one fragment to the buffer, creating it when this is the first. */ + private append(fragment: string, confidence: number, durationMs?: number): void { + if (!this.buffer) { + this.buffer = { parts: [], confidence, durationMs: 0, sawDuration: false }; + } + this.buffer.parts.push(fragment); + this.buffer.confidence = Math.min(this.buffer.confidence, confidence); + if (typeof durationMs === 'number') { + this.buffer.durationMs += durationMs; + this.buffer.sawDuration = true; + } + } + + private settle(sentBy?: string): void { + const buffer = this.buffer; + this.clearTimers(); + this.buffer = null; + this.immediateSettle = false; + if (!buffer) return; + + this.onSettled({ + text: buffer.parts.join(' '), + confidence: buffer.confidence, + durationMs: buffer.sawDuration ? buffer.durationMs : undefined, + fragments: buffer.parts.length, + sentBy, + }); + } + + private restartSettleTimer(): void { + if (this.settleTimer !== null) clearTimeout(this.settleTimer); + this.settleTimer = setTimeout(() => { + this.settleTimer = null; + this.settle(); + }, this.config.settleMs); + } + + /** Started once per thought, and deliberately NOT restarted by a fragment. */ + private startHoldTimer(): void { + if (this.holdTimer !== null || this.config.maxHoldMs === 0) return; + this.holdTimer = setTimeout(() => { + this.holdTimer = null; + this.settle(); + }, this.config.maxHoldMs); + } + + private clearTimers(): void { + if (this.settleTimer !== null) clearTimeout(this.settleTimer); + if (this.holdTimer !== null) clearTimeout(this.holdTimer); + this.settleTimer = null; + this.holdTimer = null; + } +} diff --git a/src/main/acappella/telemetry/turn-metrics.ts b/src/main/acappella/telemetry/turn-metrics.ts new file mode 100644 index 0000000000..252102b139 --- /dev/null +++ b/src/main/acappella/telemetry/turn-metrics.ts @@ -0,0 +1,183 @@ +/** + * Per-turn latency, broken down by hop. + * + * "Voice feels slow" is the report this file exists to answer. Without a + * breakdown it is unanswerable: the same sentence covers a whisper decode on a + * cold CPU, a rate-limited API retrying twice, a local Brain reloading a model it + * unloaded four minutes ago, and a TTS provider that will not start speaking + * until it has synthesised the whole reply. Those have nothing in common except + * the symptom, and guessing between them wastes an afternoon. + * + * So every turn records the same six spans, stamped against the provider + * configuration that produced them, and the last one is readable from a + * developer panel. A report becomes "first partial took 2.4 s on whisper-local" + * rather than a feeling. + * + * Timings are formatted with `formatDuration()` from + * `src/shared/performance-metrics.ts`. There is no second duration helper in + * here, deliberately: this codebase already had a dozen and they had drifted. + */ + +import { formatDuration } from '../../../shared/performance-metrics'; +import type { VoicePipelineShape, VoiceProviderRole } from '../../../shared/acappella/providers'; + +/** How many turns are kept. Enough to see a pattern, small enough to be free. */ +const HISTORY_LIMIT = 20; + +/** + * The hops of one turn, in the order they happen. + * + * Named for what the USER experiences at each boundary, not for the function + * that runs there: `speech-end to first partial` is the gap where a person is + * looking at a screen that has not reacted yet, and that is the number worth + * arguing about. + */ +export const TURN_SPANS = [ + 'firstPartial', + 'finalTranscript', + 'routeDecision', + 'agentFirstToken', + 'firstSpokenSentence', + 'total', +] as const; + +export type TurnSpan = (typeof TURN_SPANS)[number]; + +export const TURN_SPAN_LABELS: Record = { + firstPartial: 'Speech end to first partial', + finalTranscript: 'Final transcript', + routeDecision: 'Route decision', + agentFirstToken: 'Agent first token', + firstSpokenSentence: 'First spoken sentence', + total: 'Total turn', +}; + +/** The provider trio a turn ran on, so timings can be compared across configs. */ +export interface TurnConfiguration { + pipeline: VoicePipelineShape; + providerIds: Record; +} + +export interface TurnMetrics { + turnId: string; + startedAt: number; + configuration: TurnConfiguration; + /** + * Milliseconds from the START of the turn to each milestone. Absent when the + * milestone never happened, which is itself information: a turn with a route + * decision and no spoken sentence failed somewhere specific. + */ + spans: Partial>; +} + +/** One finished turn, with the per-hop deltas a person actually reads. */ +export interface TurnBreakdown extends TurnMetrics { + /** Time spent IN each hop, rather than time since the turn began. */ + deltas: Array<{ span: TurnSpan; label: string; ms: number; formatted: string }>; +} + +/** + * Records one turn. + * + * A class per turn rather than a global with a "current turn" pointer, because + * turns overlap: a superseded turn's provider callbacks keep arriving after the + * user has moved on, and a shared mutable current-turn would attribute the old + * turn's late transcript to the new turn's timeline. + */ +export class TurnTimer { + private readonly spans: Partial> = {}; + + constructor( + readonly turnId: string, + readonly configuration: TurnConfiguration, + private readonly now: () => number = Date.now, + readonly startedAt: number = now() + ) {} + + /** + * Stamp a milestone. The FIRST stamp for a span wins: a second partial is not + * the first partial, and overwriting would quietly turn this into a + * most-recent-event log. + */ + mark(span: TurnSpan): void { + if (this.spans[span] !== undefined) return; + this.spans[span] = this.now() - this.startedAt; + } + + /** Close the turn and produce its record. */ + finish(): TurnMetrics { + this.mark('total'); + return { + turnId: this.turnId, + startedAt: this.startedAt, + configuration: this.configuration, + spans: { ...this.spans }, + }; + } +} + +/** + * The rolling window of finished turns. + * + * Module state rather than an injected store: there is one voice session at a + * time by construction (a single floor), and a developer panel asking "what did + * the last turn do" must not have to be threaded through the session service to + * get an answer. + */ +const history: TurnMetrics[] = []; + +export function recordTurn(metrics: TurnMetrics): void { + history.push(metrics); + if (history.length > HISTORY_LIMIT) history.shift(); +} + +export function lastTurn(): TurnBreakdown | null { + const metrics = history[history.length - 1]; + return metrics ? describeTurn(metrics) : null; +} + +/** Every retained turn, oldest first. */ +export function turnHistory(): TurnBreakdown[] { + return history.map(describeTurn); +} + +export function resetTurnMetrics(): void { + history.length = 0; +} + +/** + * Turn cumulative marks into per-hop durations. + * + * The cumulative form is what gets recorded (each mark is one subtraction at the + * moment it happens, which is all a hot path should do); the per-hop form is what + * a person reads, because "the route decision took 1.9 s" is actionable and "the + * route decision landed 2.4 s in" is arithmetic homework. + */ +export function describeTurn(metrics: TurnMetrics): TurnBreakdown { + const deltas: TurnBreakdown['deltas'] = []; + let previous = 0; + + for (const span of TURN_SPANS) { + const at = metrics.spans[span]; + if (at === undefined) continue; + const ms = Math.max(0, at - previous); + previous = at; + deltas.push({ + span, + label: TURN_SPAN_LABELS[span], + // `total` is the whole turn, not the gap after the last milestone: it is + // the one span a reader expects to equal the sum of the others. + ms: span === 'total' ? at : ms, + formatted: formatDuration(span === 'total' ? at : ms), + }); + } + + return { ...metrics, deltas }; +} + +/** One line per hop, for a log or a support bundle. */ +export function formatTurnBreakdown(breakdown: TurnBreakdown): string { + const config = `${breakdown.configuration.pipeline}: ${breakdown.configuration.providerIds.stt} / ${breakdown.configuration.providerIds.brain} / ${breakdown.configuration.providerIds.tts}`; + const lines = breakdown.deltas.map((delta) => ` ${delta.label}: ${delta.formatted}`); + return [config, ...lines].join('\n'); +} diff --git a/src/main/acappella/transport/ice-config.ts b/src/main/acappella/transport/ice-config.ts new file mode 100644 index 0000000000..25a4cf434b --- /dev/null +++ b/src/main/acappella/transport/ice-config.ts @@ -0,0 +1,266 @@ +/** + * ICE, NAT traversal, and TURN, stated honestly. + * + * Three paths a phone can reach the desktop on, in the order they are tried and + * in decreasing order of how good they are: + * + * 1. **Host candidates.** Same WiFi, or an overlay network like Tailscale or + * ZeroTier where both machines already have a routable address for each + * other. Nothing in the path, single-digit milliseconds, connects + * instantly, needs no infrastructure at all. This is the common case and it + * is the case this design is optimised for. + * 2. **STUN.** Both ends learn their public mapping and punch through their + * NATs. Media is still direct. Works for most home routers. + * 3. **TURN.** A relay in the middle forwards every packet. It is the only + * thing that works behind carrier-grade NAT, which is what a phone on + * cellular is behind, and it is therefore not an exotic fallback: **if you + * want voice to work on a walk, you need a TURN server.** Somebody has to + * run it and somebody has to pay for its bandwidth. Pretending otherwise is + * how a feature ships that works in the office and nowhere else. + * + * The thing that is NOT a path: the Cloudflare quick tunnel in + * `src/main/tunnel-manager.ts`. It is an HTTP(S) reverse proxy. It can carry the + * signaling WebSocket, and it does. It cannot carry the media, because the media + * is UDP between two peers and the tunnel terminates TCP at Cloudflare. See + * {@link TUNNEL_MEDIA_NOTE}, which is the copy shown in Settings rather than a + * comment nobody reads. + */ + +import { getLocalIpAddressSync, listLocalIpv4Addresses } from '../../utils/networkUtils'; +import type { DeviceCandidateType } from '../../../shared/acappella/device-protocol'; +import type { IceServerConfig } from '../../../shared/acappella/webrtc-host'; + +// --------------------------------------------------------------------------- +// Settings +// --------------------------------------------------------------------------- + +export interface TurnSettings { + enabled: boolean; + /** `turn:host:3478` or `turns:host:5349`. Multiple, comma-free, one per entry. */ + url: string; + username: string; + credential: string; +} + +export interface IceTransportSettings { + /** + * STUN servers, in preference order. Empty is legal and means "LAN and + * overlay only", which is a reasonable choice for someone who never leaves + * their own network and does not want their address reflected off anyone. + */ + stunUrls: string[]; + turn: TurnSettings; + /** + * Advertise host candidates. On by default and essentially always right; the + * switch exists because a machine with a dozen virtual interfaces gathers a + * dozen useless candidates, which slows the connection down. + */ + hostCandidates: boolean; + /** + * Refuse anything but a relay. + * + * `RTCConfiguration.iceTransportPolicy: 'relay'`. Not a performance setting - + * it is a privacy one: a direct connection tells the far end your address, + * and forcing a relay hides both ends from each other. Costs latency, so it + * is off by default and labelled for what it is. + */ + forceRelay: boolean; +} + +/** + * Google's public STUN servers. + * + * A default rather than a recommendation, and it is stated in the settings copy: + * using them tells Google the IP address of anything that connects. They are + * here because a STUN-less default silently fails for most home users, and a + * feature that fails silently is worse than one that discloses an IP and says + * so. Both are removable and neither is required on a LAN. + */ +export const DEFAULT_STUN_URLS: readonly string[] = [ + 'stun:stun.l.google.com:19302', + 'stun:stun1.l.google.com:19302', +]; + +export const DEFAULT_ICE_SETTINGS: IceTransportSettings = { + stunUrls: [...DEFAULT_STUN_URLS], + turn: { enabled: false, url: '', username: '', credential: '' }, + hostCandidates: true, + forceRelay: false, +}; + +/** The one paragraph a user has to read before blaming the tunnel. Shown in Settings. */ +export const TUNNEL_MEDIA_NOTE = + 'The Cloudflare quick tunnel that serves the browser interface cannot carry voice audio. ' + + 'It is an HTTPS reverse proxy and the audio leg is a direct UDP connection between the two ' + + 'devices, so the media path is separate from the tunnel rather than borrowed from it. On the ' + + 'same network (or a Tailscale-style overlay) the connection is direct and needs nothing else. ' + + 'Off your network, a phone on cellular sits behind carrier-grade NAT and genuinely requires a ' + + 'TURN relay.'; + +// The label map moved to `shared/acappella/device-protocol.ts` so the renderer's +// device list and this file read the same words. Re-exported rather than +// re-pointed at every call site: the labels belong to the candidate type, and +// whoever already has this module has no reason to need a second import. +export { CANDIDATE_TYPE_LABELS } from '../../../shared/acappella/device-protocol'; + +// --------------------------------------------------------------------------- +// Reading stored settings +// --------------------------------------------------------------------------- + +function asStringList(value: unknown, fallback: readonly string[]): string[] { + if (!Array.isArray(value)) return [...fallback]; + const urls = value.filter( + (entry): entry is string => typeof entry === 'string' && !!entry.trim() + ); + return urls.map((url) => url.trim()); +} + +function asString(value: unknown): string { + return typeof value === 'string' ? value.trim() : ''; +} + +/** + * Widen whatever is on disk into a complete configuration. + * + * Sanitised rather than validated, for the same reason as the floor-control + * config: these numbers and strings come from a settings pane, and a typo must + * not be able to throw somewhere that leaves a device unable to connect with no + * explanation. + */ +export function readIceSettings(stored: unknown): IceTransportSettings { + const raw = (stored ?? {}) as Record; + const turnRaw = (raw.turn ?? {}) as Record; + const url = asString(turnRaw.url); + return { + stunUrls: asStringList(raw.stunUrls, DEFAULT_STUN_URLS), + turn: { + // A TURN server switched on with no URL is off, whatever the flag says: + // the alternative is an ICE configuration that throws on an empty `urls`. + enabled: turnRaw.enabled === true && !!url, + url, + username: asString(turnRaw.username), + credential: asString(turnRaw.credential), + }, + hostCandidates: raw.hostCandidates !== false, + forceRelay: raw.forceRelay === true, + }; +} + +// --------------------------------------------------------------------------- +// Building the configuration +// --------------------------------------------------------------------------- + +/** + * The `RTCIceServer[]` for a peer connection. + * + * TURN is appended last so ICE tries the free paths first; the browser's own + * candidate-pair prioritisation does the rest. + */ +export function buildIceServers(settings: IceTransportSettings): IceServerConfig[] { + const servers: IceServerConfig[] = []; + if (settings.stunUrls.length > 0) servers.push({ urls: [...settings.stunUrls] }); + if (settings.turn.enabled && settings.turn.url) { + servers.push({ + urls: settings.turn.url, + username: settings.turn.username, + credential: settings.turn.credential, + }); + } + return servers; +} + +/** `iceTransportPolicy` for a peer, which is the only thing `forceRelay` changes. */ +export function iceTransportPolicy(settings: IceTransportSettings): 'all' | 'relay' { + return settings.forceRelay ? 'relay' : 'all'; +} + +/** + * What a configuration can and cannot reach, in one sentence, for the settings + * pane. Written as a statement of fact rather than a warning, because the user + * needs to decide whether to run a TURN server and cannot decide that from a + * yellow triangle. + */ +export function describeIceReach(settings: IceTransportSettings): string { + if (settings.forceRelay) { + return settings.turn.enabled + ? 'Relay only. Every connection goes through your TURN server, including ones on this network.' + : 'Relay only is on but no TURN server is configured, so no device can connect at all.'; + } + if (settings.turn.enabled) { + return 'This network, overlay networks, most home NATs through STUN, and cellular through your TURN relay.'; + } + if (settings.stunUrls.length > 0) { + return 'This network, overlay networks, and most home NATs. Cellular will not connect without a TURN server.'; + } + return 'This network and overlay networks only. Nothing outside them will connect without STUN.'; +} + +// --------------------------------------------------------------------------- +// Candidates +// --------------------------------------------------------------------------- + +/** + * Collapse an ICE candidate type onto the three words a person can act on. + * + * `prflx` joins `srflx` under `stun`: both mean the media is direct and a + * reflexive address was involved, and the difference between them is an ICE + * implementation detail nobody outside this file should have to hold. + */ +export function classifyCandidateType(candidateType: string | undefined): DeviceCandidateType { + switch (candidateType) { + case 'host': + return 'lan'; + case 'srflx': + case 'prflx': + return 'stun'; + case 'relay': + return 'relay'; + default: + return 'unknown'; + } +} + +/** + * The winning pair's type, from the LOCAL and REMOTE candidate types. + * + * A relay on either end means the media is relayed, so the worse of the two is + * the honest answer: a device list that said "direct" because our end happened + * to gather a host candidate would be describing a path the audio is not taking. + */ +export function classifyCandidatePair( + localType: string | undefined, + remoteType: string | undefined +): DeviceCandidateType { + const local = classifyCandidateType(localType); + const remote = classifyCandidateType(remoteType); + if (local === 'unknown' || remote === 'unknown') return 'unknown'; + const rank: Record, number> = { + lan: 0, + stun: 1, + relay: 2, + }; + return rank[local] >= rank[remote] ? local : remote; +} + +/** + * Addresses a device could be told to try, best first. + * + * Includes overlay-network addresses (Tailscale hands out 100.64.0.0/10) as + * first-class entries rather than filtering them out as "not a real LAN": an + * overlay is exactly the zero-infrastructure remote case this transport is + * happiest on, and a QR code that omitted the Tailscale address would send a + * user to TURN for a connection they could have had directly. + */ +export function listHostCandidates(): string[] { + const primary = getLocalIpAddressSync(); + const all = listLocalIpv4Addresses(); + const ordered = [primary, ...all].filter((ip) => ip && ip !== 'localhost'); + return Array.from(new Set(ordered)); +} + +/** True for the CGNAT block Tailscale and friends allocate out of. */ +export function isOverlayAddress(ip: string): boolean { + const parts = ip.split('.').map(Number); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part))) return false; + return parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127; +} diff --git a/src/main/acappella/transport/index.ts b/src/main/acappella/transport/index.ts new file mode 100644 index 0000000000..ca927d060a --- /dev/null +++ b/src/main/acappella/transport/index.ts @@ -0,0 +1,418 @@ +/** + * The A Cappella transport, assembled. + * + * Five objects that each know one thing, wired together here so none of them has + * to know about Electron: + * + * - `../pairing/pairing-service.ts` - who is allowed to connect. + * - `../pairing/discovery.ts` - how they find this machine. + * - `./signaling.ts` - the offer/answer/candidate exchange, on the existing + * authenticated WebSocket. + * - `./remote-session.ts` - what it means for a device to hold the microphone. + * - `../../../renderer/acappella-audio/peer-connection.ts` - the peer itself, + * in the hidden audio window, reached from here over IPC. + * + * This module is the only one that knows the audio host is a `BrowserWindow` and + * that settings come from an electron-store. Everything below it is testable + * with a fake socket and a fake clock. + */ + +import * as path from 'path'; + +import type { DeviceMessage } from '../../../shared/acappella/device-protocol'; +import { isACappellaEnabled } from '../../../shared/acappella/feature-flag'; +import type { VoiceOrigin, VoiceScope } from '../../../shared/acappella/protocol'; +import { + ACAPPELLA_WEBRTC_COMMAND_CHANNEL, + DEFAULT_REMOTE_AUDIO_CONFIG, + type IceProbeResult, + type PeerQualityStats, + type RemoteAudioConfig, + type WebRtcHostCommand, + type WebRtcHostEvent, +} from '../../../shared/acappella/webrtc-host'; +import { logger } from '../../utils/logger'; +import { generateUUID } from '../../../shared/uuid'; +import { DiscoveryService, manualEntryHint, type DiscoveryStatus } from '../pairing/discovery'; +import { + PairingService, + type PairedDeviceView, + type PairingOffer, + type PairingRequest, +} from '../pairing/pairing-service'; +import { + buildIceServers, + listHostCandidates, + readIceSettings, + type IceTransportSettings, +} from './ice-config'; +import { RemoteSessionCoordinator, type RemoteFloor } from './remote-session'; +import { SignalingService, type SignalingServerMessage } from './signaling'; + +const LOG_CONTEXT = 'ACappella'; + +/** Settings blob key A Cappella keeps everything under. Mirrors the other readers. */ +const ACAPPELLA_SETTINGS_KEY = 'acappella'; + +/** How long a Test Connection run waits for candidates before reporting. */ +const ICE_PROBE_TIMEOUT_MS = 5000; + +/** What a device sees in the pairing QR code. */ +export interface PairingPayload { + /** Always `maestro-acappella`, so a scanner can reject an unrelated QR code. */ + kind: 'maestro-acappella'; + /** Wire protocol version, so an old client fails with a sentence. */ + v: number; + /** Addresses to try, best first: LAN, then any overlay network. */ + hosts: string[]; + port: number; + /** The server token, which is what gets the device onto the WebSocket at all. */ + token: string; + code: string; + expiresAt: number; + fingerprint: string; +} + +export interface ACappellaTransportDeps { + settingsStore: { + get: (key: string, defaultValue?: unknown) => unknown; + onDidChange?: (key: string, callback: (value: unknown) => void) => void; + }; + /** Where `devices.json` lives. Usually `app.getPath('userData')`. */ + userDataPath: string; + /** The audio host's webContents, or null when the window is not open. */ + sendToAudioHost: (command: WebRtcHostCommand) => void; + /** The one floor controller, from the hotkey installation. */ + acquireFloor: (scope: VoiceScope, origin?: VoiceOrigin) => RemoteFloor; + /** The live voice session, or null before one has ever been built. */ + getSession: () => import('./remote-session').RemoteVoiceSession | null; + /** The web server's security token and port, for the QR code and the fingerprint. */ + getServerToken: () => string | null; + getServerPort: () => number | null; + getAppVersion: () => string; + getMachineName: () => string; + /** The device list changed. Pushed to every window so the panel repaints. */ + onDevicesChanged?: () => void; + /** A device is asking to pair, or the request went away. */ + onPairingRequest?: (request: PairingRequest | null) => void; +} + +/** How a device is doing right now, joined onto its stored record for the UI. */ +export interface DeviceStatus extends PairedDeviceView { + online: boolean; + holdsFloor: boolean; + quality: PeerQualityStats | null; +} + +export class ACappellaTransport { + readonly pairing: PairingService; + readonly signaling: SignalingService; + readonly discovery: DiscoveryService; + private readonly remote: RemoteSessionCoordinator | null = null; + private readonly deps: ACappellaTransportDeps; + private readonly quality = new Map(); + private readonly probes = new Map void>(); + private readonly names = new Map(); + + constructor(deps: ACappellaTransportDeps) { + this.deps = deps; + this.pairing = new PairingService({ + filePath: path.join(deps.userDataPath, 'acappella', 'devices.json'), + hostSecret: deps.getServerToken() ?? undefined, + }); + this.pairing.onChange(() => { + void this.refreshNames(); + deps.onDevicesChanged?.(); + }); + this.pairing.onPairingRequest((request) => deps.onPairingRequest?.(request)); + + this.signaling = new SignalingService({ + pairing: this.pairing, + peerHost: { + acceptOffer: (params) => this.send({ kind: 'accept-offer', ...params }), + addIceCandidate: (deviceId, candidate) => + this.send({ kind: 'add-ice-candidate', deviceId, candidate }), + closePeer: (deviceId, reason) => this.send({ kind: 'close-peer', deviceId, reason }), + }, + getIceSettings: () => this.iceSettings(), + getAudioConfig: () => this.audioConfig(), + onDeviceOnline: (deviceId) => { + deps.onDevicesChanged?.(); + logger.debug(`Device ${deviceId} online`, LOG_CONTEXT); + }, + onDeviceOffline: (deviceId, reason) => { + this.quality.delete(deviceId); + this.remote?.handleDisconnected(deviceId, reason); + deps.onDevicesChanged?.(); + }, + }); + + const session = deps.getSession(); + if (session) { + this.remote = new RemoteSessionCoordinator({ + session, + acquireFloor: (scope, origin) => deps.acquireFloor(scope, origin), + sink: { + send: (deviceId, message) => this.send({ kind: 'send', deviceId, message }), + broadcast: (message) => this.send({ kind: 'broadcast', message }), + }, + getDeviceName: (deviceId) => this.names.get(deviceId) ?? '', + onFloorChange: (holder) => { + // The audio host gates which remote track reaches the capture + // pipeline, so it needs to be told before the next frame arrives. + this.send({ kind: 'set-floor-holder', deviceId: holder === 'local' ? null : holder }); + deps.onDevicesChanged?.(); + }, + }); + } + + this.discovery = new DiscoveryService({ + getPort: deps.getServerPort, + getName: deps.getMachineName, + getAppVersion: deps.getAppVersion, + getFingerprint: () => this.pairing.fingerprint(), + }); + + void this.refreshNames(); + } + + dispose(): void { + this.signaling.dispose(); + this.remote?.dispose(); + void this.discovery.stop(); + } + + /** + * Stand down because the Encore Feature was switched off. + * + * Not `dispose()`, and the difference matters: the transport is constructed + * once per process (see `initACappellaTransport`, called from handler + * registration at boot), so disposing it here would mean switching the feature + * back on did nothing until the next restart. This releases every resource the + * flag actually promises are gone - the advert, the live connections, and any + * half-finished pairing - while leaving the object able to serve again. + * + * Devices are disconnected rather than revoked. A user switching the feature + * off is saying "stop", not "forget my phone", and re-pairing a phone because + * a checkbox was toggled is a punishment for reading the settings screen. + */ + standDown(): void { + void this.discovery.stop(); + this.pairing.cancelPairing(); + this.disconnectAll('A Cappella was switched off on the desktop'); + } + + /** + * Whether the Encore Feature is on right now. + * + * Read straight from settings on every call so the signaling adapter and the + * IPC handlers cannot disagree about it, and so a device that was mid-handshake + * when the flag flipped is refused rather than served. + */ + featureEnabled(): boolean { + return isACappellaEnabled(this.deps.settingsStore); + } + + // -- Pairing ------------------------------------------------------------- + + /** Open a pairing window and build the payload the QR code encodes. */ + startPairing(): PairingPayload | null { + const token = this.deps.getServerToken(); + const port = this.deps.getServerPort(); + if (!token || !port) return null; + const offer = this.pairing.startPairing(); + return this.payloadFor(offer, token, port); + } + + currentPairingPayload(): PairingPayload | null { + const offer = this.pairing.currentOffer(); + const token = this.deps.getServerToken(); + const port = this.deps.getServerPort(); + if (!offer || !token || !port) return null; + return this.payloadFor(offer, token, port); + } + + private payloadFor(offer: PairingOffer, token: string, port: number): PairingPayload { + return { + kind: 'maestro-acappella', + v: 1, + hosts: listHostCandidates(), + port, + token, + code: offer.code, + expiresAt: offer.expiresAt, + fingerprint: offer.fingerprint, + }; + } + + /** The sentence to show when discovery is off or unavailable. */ + manualHint(): string { + return manualEntryHint(listHostCandidates(), this.deps.getServerPort()); + } + + discoveryStatus(): DiscoveryStatus { + return this.discovery.status; + } + + // -- Device list --------------------------------------------------------- + + async listDevices(): Promise { + const devices = await this.pairing.list(); + return devices.map((device) => ({ + ...device, + online: this.signaling.isOnline(device.id), + holdsFloor: this.remote?.floorHolder === device.id, + quality: this.quality.get(device.id) ?? null, + })); + } + + /** Revoke and tear down. The signaling service is subscribed to the event. */ + async revokeDevice(deviceId: string): Promise { + return this.pairing.revoke(deviceId); + } + + async revokeAllDevices(): Promise { + return this.pairing.revokeAll(); + } + + /** Drop every live connection without revoking anything. */ + disconnectAll(reason = 'the desktop disconnected all devices'): void { + for (const deviceId of this.signaling.onlineDeviceIds()) { + this.signaling.closeDevice(deviceId, reason); + } + } + + // -- Peer events ---------------------------------------------------------- + + /** One event from the audio host's peer registry. */ + handleHostEvent(event: WebRtcHostEvent): void { + switch (event.kind) { + case 'answer': + this.signaling.deliverAnswer(event.deviceId, event.answer); + return; + case 'ice-candidate': + this.signaling.deliverIceCandidate(event.deviceId, event.candidate); + return; + case 'connection-state': + this.remote?.handlePeerState(event.deviceId, event.state); + if (event.state === 'connected') { + void this.pairing.noteConnected( + event.deviceId, + this.quality.get(event.deviceId)?.candidateType ?? 'unknown' + ); + } + this.deps.onDevicesChanged?.(); + return; + case 'stats': + this.quality.set(event.stats.deviceId, event.stats); + this.deps.onDevicesChanged?.(); + return; + case 'message': + this.handleDeviceMessage(event.deviceId, event.message); + return; + case 'peer-error': + this.signaling.deliverPeerError(event.deviceId, event.message); + return; + case 'ice-probe-result': { + const resolve = this.probes.get(event.probeId); + this.probes.delete(event.probeId); + resolve?.(event.result); + return; + } + } + } + + private handleDeviceMessage(deviceId: string, message: DeviceMessage): void { + if (message.type === 'hello') { + // The name a device calls itself is only ever a display string, so it is + // taken as the device says it - but the identity that matters was settled + // at authentication, and nothing here can change it. + this.names.set(deviceId, message.identity.name); + this.deps.onDevicesChanged?.(); + } + this.remote?.handleDeviceMessage(deviceId, message); + } + + // -- Test Connection ------------------------------------------------------ + + /** + * Gather candidates against the configured servers and report what actually + * came back. + * + * Resolves `unknown` if the audio host never answers, rather than hanging: the + * button has to give a verdict, and "no answer" is one. + */ + testConnection(): Promise { + const probeId = generateUUID(); + const settings = this.iceSettings(); + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.probes.delete(probeId); + resolve({ + host: false, + stun: false, + relay: false, + best: 'unknown', + error: 'The audio engine did not answer. Start a voice session and try again.', + }); + }, ICE_PROBE_TIMEOUT_MS * 2); + this.probes.set(probeId, (result) => { + clearTimeout(timer); + resolve(result); + }); + this.send({ + kind: 'probe-ice', + probeId, + iceServers: buildIceServers(settings), + timeoutMs: ICE_PROBE_TIMEOUT_MS, + }); + }); + } + + // -- Signaling plumbing --------------------------------------------------- + + registerClient(params: { + clientId: string; + send: (message: SignalingServerMessage) => void; + remoteAddress?: string; + }): void { + this.signaling.register(params); + } + + handleSignalMessage(clientId: string, payload: unknown): Promise { + return this.signaling.handleMessage(clientId, payload); + } + + handleClientDisconnect(clientId: string): void { + this.signaling.handleDisconnect(clientId); + } + + // -- Internals ------------------------------------------------------------ + + private iceSettings(): IceTransportSettings { + const blob = (this.deps.settingsStore.get(ACAPPELLA_SETTINGS_KEY, {}) ?? {}) as { + ice?: unknown; + }; + return readIceSettings(blob.ice); + } + + private audioConfig(): RemoteAudioConfig { + const blob = (this.deps.settingsStore.get(ACAPPELLA_SETTINGS_KEY, {}) ?? {}) as { + remoteAudio?: Partial; + }; + return { ...DEFAULT_REMOTE_AUDIO_CONFIG, ...(blob.remoteAudio ?? {}) }; + } + + private send(command: WebRtcHostCommand): void { + this.deps.sendToAudioHost(command); + } + + private async refreshNames(): Promise { + for (const device of await this.pairing.list()) this.names.set(device.id, device.name); + } +} + +export { ACAPPELLA_WEBRTC_COMMAND_CHANNEL }; +export * from './ice-config'; +export * from './remote-session'; +export * from './signaling'; diff --git a/src/main/acappella/transport/remote-session.ts b/src/main/acappella/transport/remote-session.ts new file mode 100644 index 0000000000..77ef73b816 --- /dev/null +++ b/src/main/acappella/transport/remote-session.ts @@ -0,0 +1,336 @@ +/** + * What it means for a phone to hold the microphone. + * + * The single most important property of this file is what it does NOT do. A + * remote utterance is not a second pipeline, a second router, or a second voice. + * The phone presses talk, this coordinator presses the SAME + * `FloorController` the desktop hotkey presses, and the session that opens runs + * the identical STT, routing, dispatch, translation, and TTS as a sentence + * spoken at the keyboard. The only thing that differs is `VoiceOrigin`, which + * exists so the desktop HUD can say which microphone is open rather than to + * branch on. + * + * Three rules it enforces: + * + * **One floor.** There is one microphone stream feeding one recogniser, so two + * devices talking at once cannot be mixed into one utterance without producing a + * transcript of neither. The rule is **last press wins**: a device that presses + * talk takes the floor, whoever had it, and the displaced device is told + * immediately with `takenOverBy` so its button snaps back rather than lying. Any + * other rule ends with a user pressing talk on the phone in their hand and + * nothing happening because a laptop in another room is holding the floor - and + * every device here was individually approved by the person doing the pressing. + * + * **A stale release cannot close a live floor.** Only the CURRENT holder's + * release does anything. Without this, a device that just lost the floor sending + * its release (which it will, a few milliseconds later) would shut the + * microphone of the device that just took it. + * + * **A dropped connection ends the session, cleanly.** Not "eventually", and + * never by leaving a `speaking` state nobody will finish or a floor nobody will + * close. `disconnected` is deliberately not that trigger - ICE reports it during + * an ordinary WiFi-to-LTE handover, which is the normal case on a walk - but + * `failed`, `closed`, and a lost signaling socket are. + * + * Wake word and stop word stay on whichever device is capturing. That is the + * standing rule that no audio leaves a device before the wake phrase fires, and + * it falls out of the design rather than being enforced here: the phone's + * microphone is not sent anywhere until the phone opens the floor. + */ + +import { + deviceChannelForMessage, + type DeviceCandidateType, + type DeviceMessage, +} from '../../../shared/acappella/device-protocol'; +import type { VoiceEvent, VoiceOrigin, VoiceScope } from '../../../shared/acappella/protocol'; +import type { PeerConnectionState } from '../../../shared/acappella/webrtc-host'; +import { isTerminalPeerState } from '../../../shared/acappella/webrtc-host'; +import { logger } from '../../utils/logger'; + +const LOG_CONTEXT = 'ACappella'; + +/** The slice of `FloorController` a remote device drives. */ +export interface RemoteFloor { + press(source?: 'remote-device'): Promise; + release(source?: 'remote-device'): Promise; + close(reason: 'session-ended' | 'shutdown' | 'toggle'): Promise; + readonly isFloorOpen: boolean; +} + +/** The slice of `VoiceSessionService` this coordinator needs. */ +export interface RemoteVoiceSession { + subscribe(listener: (event: VoiceEvent) => void): () => void; + interrupt(source: 'voice' | 'client-button'): boolean; + hardStop(source: 'voice' | 'client-button', phrase?: string): Promise; + stopSession(reason: 'user' | 'shutdown' | 'error'): Promise; + getState(): string; +} + +/** Sends protocol messages to devices. In production, the audio host's peers. */ +export interface RemoteMessageSink { + send(deviceId: string, message: DeviceMessage): void; + broadcast(message: DeviceMessage): void; +} + +export interface RemoteSessionCoordinatorOptions { + /** + * The one floor controller, configured for `origin` before the press. + * + * A function rather than a value because the floor's scope and origin are set + * immediately before a press by whoever is pressing - the same pattern the + * hotkeys use, and the reason there is one state machine rather than one per + * surface. + */ + acquireFloor: (scope: VoiceScope, origin: VoiceOrigin) => RemoteFloor; + session: RemoteVoiceSession; + sink: RemoteMessageSink; + /** Name for a device id, for the HUD line and the takeover message. */ + getDeviceName: (deviceId: string) => string; + /** Set when a device leaves the floor for any reason. Used by the device list. */ + onFloorChange?: (holder: string | null) => void; +} + +/** Who holds the floor. `'local'` is this machine's own microphone. */ +export type FloorHolder = 'local' | string; + +export class RemoteSessionCoordinator { + private readonly options: RemoteSessionCoordinatorOptions; + private readonly connected = new Set(); + private holder: FloorHolder | null = null; + private readonly unsubscribe: () => void; + /** Serialises floor changes so a press and a release cannot interleave. */ + private queue: Promise = Promise.resolve(); + + constructor(options: RemoteSessionCoordinatorOptions) { + this.options = options; + this.unsubscribe = options.session.subscribe((event) => this.handleVoiceEvent(event)); + } + + dispose(): void { + this.unsubscribe(); + this.connected.clear(); + this.holder = null; + } + + /** Who has the microphone right now, or null when nobody does. */ + get floorHolder(): FloorHolder | null { + return this.holder; + } + + /** Resolves once every queued floor change has run. Tests and shutdown use it. */ + whenSettled(): Promise { + return this.queue; + } + + // -- Connection lifecycle ------------------------------------------------- + + /** A device's peer connection came up. */ + handleConnected(deviceId: string): void { + this.connected.add(deviceId); + this.publishFloorState(); + } + + /** + * A peer connection changed state. + * + * Only terminal states end anything. `disconnected` is ICE noticing a network + * change, and a walk out of WiFi range recovers from it within seconds; ending + * the session there would hang up on the exact user this transport exists for. + */ + handlePeerState(deviceId: string, state: PeerConnectionState): void { + if (state === 'connected') { + this.handleConnected(deviceId); + return; + } + if (!isTerminalPeerState(state)) return; + this.handleDisconnected(deviceId, `the connection to ${this.name(deviceId)} ${state}`); + } + + /** + * A device is gone: signaling closed, peer failed, or the pairing was revoked. + * + * If it was holding the floor, the session ends here rather than being left to + * a timeout. An orphaned `speaking` state talks to an empty room, and an + * orphaned open floor is a microphone nobody knows is on. + */ + handleDisconnected(deviceId: string, reason: string): void { + this.connected.delete(deviceId); + if (this.holder !== deviceId) { + this.publishFloorState(); + return; + } + void this.enqueue(async () => { + logger.info(`Remote floor lost: ${reason}`, LOG_CONTEXT); + this.holder = null; + // Cancel speech first, then close the session. In that order because the + // reverse leaves a sentence in flight after the session that owns it has + // gone: the TTS chunks are already queued in the audio host and the thing + // that cancels them is the interrupt, not the stop. + this.options.session.interrupt('client-button'); + await this.options.session.stopSession('user'); + this.options.onFloorChange?.(null); + this.publishFloorState(); + }); + } + + // -- Inbound device messages --------------------------------------------- + + /** One protocol message from a device. Unknown or unauthorised ones are dropped. */ + handleDeviceMessage(deviceId: string, message: DeviceMessage): void { + switch (message.type) { + case 'floor': + if (message.action === 'press') this.requestFloor(deviceId, message.scope); + else this.releaseFloor(deviceId); + return; + case 'interrupt': + // Barge-in keeps the floor; the stop word ends the session. Both are + // refused from a device that is not holding the floor, because + // interrupting a conversation you are not in is not a thing. + if (this.holder !== deviceId) return; + if (message.kind === 'stop-word') void this.options.session.hardStop('client-button'); + else this.options.session.interrupt('client-button'); + return; + case 'audio-level': + case 'hello': + case 'link-quality': + // Handled by the peer host and the device list, not by the floor. + return; + default: + return; + } + } + + /** + * A device pressed talk. + * + * Last press wins. The displaced holder is told before the new session starts, + * so its button lets go while the takeover is happening rather than after. + */ + requestFloor(deviceId: string, scope?: VoiceScope): Promise { + return this.enqueue(async () => { + if (this.holder === deviceId) return; + const previous = this.holder; + this.holder = deviceId; + + if (previous && previous !== 'local') { + this.options.sink.send(previous, { + type: 'floor-state', + holder: deviceId, + isSelf: false, + takenOverBy: this.name(deviceId), + }); + } + + const origin: VoiceOrigin = { + kind: 'remote', + deviceId, + deviceName: this.name(deviceId), + }; + const floor = this.options.acquireFloor(scope ?? { kind: 'conductor' }, origin); + // `press()` on the shared controller: the session it opens is a normal + // session in every respect, which is the point. + await floor.press('remote-device'); + this.options.onFloorChange?.(deviceId); + this.publishFloorState(); + }); + } + + /** A device let go. Ignored unless that device is the one holding the floor. */ + releaseFloor(deviceId: string): Promise { + return this.enqueue(async () => { + if (this.holder !== deviceId) return; + const origin: VoiceOrigin = { + kind: 'remote', + deviceId, + deviceName: this.name(deviceId), + }; + const floor = this.options.acquireFloor({ kind: 'conductor' }, origin); + await floor.release('remote-device'); + // The floor stays credited to the device until the session actually ends: + // in tap-to-toggle a release is a no-op, and in hold-to-talk the session + // lives on to answer. `listen-stop` is what clears the holder. + this.publishFloorState(); + }); + } + + /** The desktop took the floor back. Every device is told it is not holding it. */ + takeLocalFloor(): void { + const previous = this.holder; + this.holder = 'local'; + if (previous && previous !== 'local') { + this.options.sink.send(previous, { + type: 'floor-state', + holder: 'local', + isSelf: false, + takenOverBy: 'this computer', + }); + } + this.options.onFloorChange?.('local'); + this.publishFloorState(); + } + + // -- Outbound ------------------------------------------------------------- + + /** + * Forward one session event to every connected device, on the channel the + * protocol table says it belongs on. + * + * Every device sees the whole stream, including while another device holds the + * floor, because a phone in a pocket still has to be able to show what the Mac + * is doing. Only the microphone is exclusive. + */ + private handleVoiceEvent(event: VoiceEvent): void { + if (this.connected.size > 0) { + this.options.sink.broadcast({ type: 'voice-event', event }); + } + + // The floor is released by the SESSION ending, not by the release message, + // so this is where a remote holder stops being the holder. + if (event.type === 'listen-stop' || event.type === 'stop-word') { + if (this.holder && this.holder !== 'local') { + this.holder = null; + this.options.onFloorChange?.(null); + this.publishFloorState(); + } + } + if (event.type === 'listen-start' && event.origin?.kind === 'local') { + // A local wake or hotkey opened the floor without going through this + // coordinator. Reflecting it keeps the phones honest rather than leaving + // them showing a floor that moved without telling them. + if (this.holder !== 'local') { + this.holder = 'local'; + this.publishFloorState(); + } + } + } + + private publishFloorState(): void { + for (const deviceId of this.connected) { + this.options.sink.send(deviceId, { + type: 'floor-state', + holder: this.holder, + isSelf: this.holder === deviceId, + }); + } + } + + private name(deviceId: string): string { + return this.options.getDeviceName(deviceId) || 'a paired device'; + } + + private enqueue(action: () => Promise): Promise { + const next = this.queue.then(action).catch((error: Error) => { + logger.error(`Remote floor failure: ${error.message}`, LOG_CONTEXT); + }); + this.queue = next; + return next; + } +} + +/** + * Which channel a message goes out on, re-exported so a sink implementation does + * not have to reach into the protocol module to find out. + */ +export { deviceChannelForMessage }; +export type { DeviceCandidateType }; diff --git a/src/main/acappella/transport/signaling.ts b/src/main/acappella/transport/signaling.ts new file mode 100644 index 0000000000..3d1d0f7039 --- /dev/null +++ b/src/main/acappella/transport/signaling.ts @@ -0,0 +1,540 @@ +/** + * WebRTC signaling for A Cappella, carried on the existing authenticated + * WebSocket at `/$TOKEN/ws`. + * + * There is no second port and no second auth surface. A device reaches the same + * socket the browser interface uses, so it has already cleared the server token + * before a single A Cappella message is looked at; this service adds the + * per-device layer on top of that: + * + * 1. **Pairing** (`pair-claim`, `pair-poll`) is the only thing an unpaired + * device may say. It buys a request that a human on the desktop has to + * approve. See `../pairing/pairing-service.ts`. + * 2. **Authentication** (`auth`) exchanges the long-lived device token for a + * signaling session. A revoked device fails here, every time, because the + * check is a lookup rather than a cached decision. + * 3. **Signaling** (`offer`, `ice-candidate`) is refused outright until step 2 + * has succeeded on THIS socket. A connection is never inherited. + * + * Renegotiation is a first-class path, not an edge case. A phone walking from + * WiFi to LTE re-offers on the same authenticated socket and the peer is updated + * in place, so the media leg survives the handover. That is also why offers are + * rate limited rather than allowed once: the legitimate case sends several over + * a session, and the abusive case sends hundreds. + * + * Free of Fastify and of Electron. The socket is a `send` callback, the peer is + * an injected {@link SignalingPeerHost}, and the clock is an option, so the + * whole protocol runs in a test with no network at all. + */ + +import type { + IceCandidatePayload, + SessionDescriptionPayload, +} from '../../../shared/acappella/webrtc-host'; +import type { RemoteAudioConfig } from '../../../shared/acappella/webrtc-host'; +import { DEFAULT_REMOTE_AUDIO_CONFIG } from '../../../shared/acappella/webrtc-host'; +import { + DEVICE_PROTOCOL_VERSION, + MIN_SUPPORTED_DEVICE_PROTOCOL_VERSION, + negotiateProtocolVersion, +} from '../../../shared/acappella/device-protocol'; +import { + ACAPPELLA_SIGNAL_MESSAGE, + type SignalingClientMessage, + type SignalingErrorCode, + type SignalingServerMessage, +} from '../../../shared/acappella/signaling-protocol'; +import { logger } from '../../utils/logger'; +import type { PairingService } from '../pairing/pairing-service'; +import { buildIceServers, type IceTransportSettings } from './ice-config'; + +const LOG_CONTEXT = 'ACappella'; + +/** + * The wire shapes moved to `shared/acappella/signaling-protocol.ts` when a + * second client appeared that has to speak them. Re-exported here because this + * is the module every desktop-side caller already imports them from, and + * because the shapes and the parser that enforces them should still read as one + * thing. + */ +export { + ACAPPELLA_SIGNAL_MESSAGE, + type SignalingClientMessage, + type SignalingServerMessage, + type SignalingErrorCode, +}; + +/** + * How many offers one device may send per {@link OFFER_RATE_WINDOW_MS}. + * + * Sized for the real workload: an initial offer plus a renegotiation on every + * network change. Six a minute covers a bus ride through four cell handovers and + * still stops a client stuck in a reconnect loop from rebuilding a peer + * connection fifty times a second. + */ +export const OFFER_RATE_LIMIT = 6; +export const OFFER_RATE_WINDOW_MS = 60_000; + +/** + * How many failed `auth` attempts one socket gets before it is cut off. + * + * The token is 32 random bytes, so this is not really about guessing; it is + * about a client with a stale credential retrying in a tight loop. + */ +export const AUTH_ATTEMPT_LIMIT = 5; + +// --------------------------------------------------------------------------- +// Seams +// --------------------------------------------------------------------------- + +/** + * The peer connection, wherever it actually lives. In production this forwards + * to the hidden audio window over `acappella:webrtc-command`. + */ +export interface SignalingPeerHost { + acceptOffer(params: { + deviceId: string; + offer: SessionDescriptionPayload; + iceServers: ReturnType; + audio: RemoteAudioConfig; + }): void; + addIceCandidate(deviceId: string, candidate: IceCandidatePayload): void; + closePeer(deviceId: string, reason: string): void; +} + +export interface SignalingServiceOptions { + pairing: PairingService; + peerHost: SignalingPeerHost; + /** Read fresh per offer, so a settings change applies to the next connection. */ + getIceSettings: () => IceTransportSettings; + getAudioConfig?: () => RemoteAudioConfig; + now?: () => number; + /** A device authenticated. The remote-session coordinator binds here. */ + onDeviceOnline?: (deviceId: string) => void; + /** A device's signaling session ended, for any reason. */ + onDeviceOffline?: (deviceId: string, reason: string) => void; +} + +/** One connected socket, from this service's point of view. */ +interface SignalingSession { + clientId: string; + send: (message: SignalingServerMessage) => void; + deviceId: string | null; + protocolVersion: number; + remoteAddress?: string; + authAttempts: number; + /** Epoch ms of each accepted offer inside the current window. */ + offerTimes: number[]; +} + +// --------------------------------------------------------------------------- +// Service +// --------------------------------------------------------------------------- + +export class SignalingService { + private readonly options: SignalingServiceOptions; + private readonly sessions = new Map(); + /** deviceId -> clientId. One live signaling session per device. */ + private readonly byDevice = new Map(); + private readonly disposers: Array<() => void> = []; + + constructor(options: SignalingServiceOptions) { + this.options = options; + // Revocation has to reach a LIVE connection, which is the whole reason the + // pairing service publishes it as an event instead of a flag. + this.disposers.push( + options.pairing.onRevoke((deviceId, reason) => this.closeDevice(deviceId, reason)) + ); + } + + dispose(): void { + for (const dispose of this.disposers) dispose(); + this.disposers.length = 0; + for (const clientId of [...this.sessions.keys()]) { + this.handleDisconnect(clientId); + } + } + + /** + * A socket connected. Nothing is trusted yet beyond the server token. + * + * Idempotent: the WebSocket route registers lazily, on the first A Cappella + * message rather than on connect, so this runs again for every message on an + * already-registered socket. Rebuilding the session there would silently + * discard the device it had authenticated. + */ + register(params: { + clientId: string; + send: (message: SignalingServerMessage) => void; + remoteAddress?: string; + }): void { + const existing = this.sessions.get(params.clientId); + if (existing) { + existing.send = params.send; + return; + } + this.sessions.set(params.clientId, { + clientId: params.clientId, + send: params.send, + deviceId: null, + protocolVersion: DEVICE_PROTOCOL_VERSION, + remoteAddress: params.remoteAddress, + authAttempts: 0, + offerTimes: [], + }); + } + + /** A socket went away. */ + handleDisconnect(clientId: string): void { + const session = this.sessions.get(clientId); + if (!session) return; + this.sessions.delete(clientId); + if (!session.deviceId) return; + if (this.byDevice.get(session.deviceId) === clientId) { + this.byDevice.delete(session.deviceId); + } + this.options.peerHost.closePeer(session.deviceId, 'signaling closed'); + this.options.onDeviceOffline?.(session.deviceId, 'disconnected'); + } + + /** Is this device currently signaling? Used by the device list. */ + isOnline(deviceId: string): boolean { + return this.byDevice.has(deviceId); + } + + onlineDeviceIds(): string[] { + return [...this.byDevice.keys()]; + } + + /** + * End a device's session now: peer down, socket told, coordinator informed. + * + * The order matters. The device is told BEFORE the peer is closed so the + * message goes out over a socket that is still open; a `closed` frame written + * after the teardown is a frame nobody receives. + */ + closeDevice(deviceId: string, reason: string): void { + const clientId = this.byDevice.get(deviceId); + this.byDevice.delete(deviceId); + if (clientId) { + const session = this.sessions.get(clientId); + if (session) { + session.send({ op: 'closed', reason }); + session.deviceId = null; + session.offerTimes = []; + } + } + this.options.peerHost.closePeer(deviceId, reason); + this.options.onDeviceOffline?.(deviceId, reason); + } + + // -- Inbound ------------------------------------------------------------- + + /** One `acappella_signal` payload from a client. */ + async handleMessage(clientId: string, payload: unknown): Promise { + const session = this.sessions.get(clientId); + if (!session) return; + const message = parseClientMessage(payload); + if (!message) { + session.send({ op: 'error', code: 'malformed', message: 'Unrecognised signaling message.' }); + return; + } + + switch (message.op) { + case 'pair-claim': + this.handlePairClaim(session, message); + return; + case 'pair-poll': + this.handlePairPoll(session, message); + return; + case 'auth': + await this.handleAuth(session, message); + return; + case 'offer': + this.handleOffer(session, message); + return; + case 'ice-candidate': + this.handleIceCandidate(session, message); + return; + case 'bye': + if (session.deviceId) this.closeDevice(session.deviceId, 'the device disconnected'); + return; + } + } + + /** The peer answered. Forwarded to whichever socket owns that device. */ + deliverAnswer(deviceId: string, sdp: SessionDescriptionPayload): void { + this.sendToDevice(deviceId, { op: 'answer', sdp }); + } + + /** A locally gathered candidate, trickled out as soon as it exists. */ + deliverIceCandidate(deviceId: string, candidate: IceCandidatePayload): void { + this.sendToDevice(deviceId, { op: 'ice-candidate', candidate }); + } + + /** The peer failed in a way the device needs to hear about. */ + deliverPeerError(deviceId: string, message: string): void { + this.sendToDevice(deviceId, { op: 'error', code: 'peer-failed', message }); + } + + // -- Handlers ------------------------------------------------------------ + + private handlePairClaim( + session: SignalingSession, + message: Extract + ): void { + const result = this.options.pairing.claim({ + code: message.code, + name: message.name, + platform: message.platform, + appVersion: message.appVersion, + remoteAddress: session.remoteAddress, + }); + if (result.status === 'rejected') { + session.send({ + op: 'pair-rejected', + reason: result.reason, + message: pairingRejectionMessage(result.reason), + }); + return; + } + session.send({ + op: 'pair-pending', + requestId: result.requestId, + expiresAt: result.expiresAt, + }); + } + + private handlePairPoll( + session: SignalingSession, + message: Extract + ): void { + const result = this.options.pairing.redeem(message.requestId); + switch (result.status) { + case 'pending': + session.send({ op: 'pair-pending', requestId: message.requestId, expiresAt: 0 }); + return; + case 'approved': + session.send({ op: 'pair-approved', deviceId: result.deviceId, token: result.token }); + return; + case 'denied': + session.send({ op: 'pair-denied' }); + return; + case 'expired': + session.send({ + op: 'pair-rejected', + reason: 'expired', + message: pairingRejectionMessage('expired'), + }); + return; + } + } + + private async handleAuth( + session: SignalingSession, + message: Extract + ): Promise { + const negotiation = negotiateProtocolVersion(message.protocolVersion); + if (!negotiation.ok) { + // Version first, before the credential is even looked at: a client that + // cannot be talked to correctly should be told THAT, rather than being + // authenticated into a session where it will misbehave silently. + session.send({ op: 'error', code: 'protocol-version', message: negotiation.message }); + return; + } + + if (session.authAttempts >= AUTH_ATTEMPT_LIMIT) { + session.send({ op: 'error', code: 'rate-limited', message: 'Too many failed attempts.' }); + return; + } + + const device = await this.options.pairing.authenticate(message.deviceId, message.token); + if (!device) { + session.authAttempts += 1; + session.send({ + op: 'auth-failed', + reason: 'unauthorized', + message: 'This device is not paired with this computer, or its pairing was revoked.', + }); + return; + } + + // One live signaling session per device. A second login displaces the first + // rather than running two, because two sockets claiming one device would + // both be told to hold the floor. + const previous = this.byDevice.get(device.id); + if (previous && previous !== session.clientId) { + const stale = this.sessions.get(previous); + if (stale) { + stale.send({ op: 'closed', reason: 'this device connected again from somewhere else' }); + stale.deviceId = null; + } + this.options.peerHost.closePeer(device.id, 'replaced by a newer connection'); + } + + session.deviceId = device.id; + session.protocolVersion = negotiation.version; + session.authAttempts = 0; + session.offerTimes = []; + this.byDevice.set(device.id, session.clientId); + + const iceSettings = this.options.getIceSettings(); + session.send({ + op: 'authenticated', + deviceId: device.id, + protocolVersion: negotiation.version, + iceServers: buildIceServers(iceSettings), + iceTransportPolicy: iceSettings.forceRelay ? 'relay' : 'all', + audio: this.options.getAudioConfig?.() ?? DEFAULT_REMOTE_AUDIO_CONFIG, + }); + this.options.onDeviceOnline?.(device.id); + logger.info(`Device '${device.name}' authenticated for A Cappella signaling`, LOG_CONTEXT); + } + + private handleOffer( + session: SignalingSession, + message: Extract + ): void { + if (!session.deviceId) { + session.send({ + op: 'error', + code: 'not-authenticated', + message: 'Authenticate this device before sending an offer.', + }); + return; + } + if (!this.consumeOfferAllowance(session)) { + session.send({ + op: 'error', + code: 'rate-limited', + message: `Too many connection attempts. Wait a moment and try again.`, + }); + return; + } + + const iceSettings = this.options.getIceSettings(); + this.options.peerHost.acceptOffer({ + deviceId: session.deviceId, + offer: message.sdp, + iceServers: buildIceServers(iceSettings), + audio: this.options.getAudioConfig?.() ?? DEFAULT_REMOTE_AUDIO_CONFIG, + }); + } + + private handleIceCandidate( + session: SignalingSession, + message: Extract + ): void { + if (!session.deviceId) { + session.send({ + op: 'error', + code: 'not-authenticated', + message: 'Authenticate this device before trickling candidates.', + }); + return; + } + this.options.peerHost.addIceCandidate(session.deviceId, message.candidate); + } + + // -- Internals ----------------------------------------------------------- + + /** + * Sliding window rather than a fixed one: a fixed window lets a client send + * its whole allowance at 59 s and again at 61 s, which is exactly the burst + * the limit exists to stop. + */ + private consumeOfferAllowance(session: SignalingSession): boolean { + const now = (this.options.now ?? Date.now)(); + session.offerTimes = session.offerTimes.filter((at) => now - at < OFFER_RATE_WINDOW_MS); + if (session.offerTimes.length >= OFFER_RATE_LIMIT) return false; + session.offerTimes.push(now); + return true; + } + + private sendToDevice(deviceId: string, message: SignalingServerMessage): void { + const clientId = this.byDevice.get(deviceId); + if (!clientId) return; + this.sessions.get(clientId)?.send(message); + } +} + +// --------------------------------------------------------------------------- +// Parsing +// --------------------------------------------------------------------------- + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Validate an inbound payload into a message, or null. + * + * Everything here arrives from another machine, so nothing is trusted: an + * unknown `op`, a missing field, or a wrong type all produce null and one + * `malformed` reply rather than an exception inside a socket handler. + */ +export function parseClientMessage(payload: unknown): SignalingClientMessage | null { + if (!isRecord(payload)) return null; + switch (payload.op) { + case 'pair-claim': + if (typeof payload.code !== 'string') return null; + return { + op: 'pair-claim', + code: payload.code, + name: typeof payload.name === 'string' ? payload.name : '', + platform: typeof payload.platform === 'string' ? payload.platform : '', + appVersion: typeof payload.appVersion === 'string' ? payload.appVersion : undefined, + }; + case 'pair-poll': + if (typeof payload.requestId !== 'string') return null; + return { op: 'pair-poll', requestId: payload.requestId }; + case 'auth': + if (typeof payload.deviceId !== 'string' || typeof payload.token !== 'string') return null; + return { + op: 'auth', + deviceId: payload.deviceId, + token: payload.token, + protocolVersion: + typeof payload.protocolVersion === 'number' + ? payload.protocolVersion + : MIN_SUPPORTED_DEVICE_PROTOCOL_VERSION - 1, + }; + case 'offer': { + const sdp = payload.sdp; + if (!isRecord(sdp) || typeof sdp.sdp !== 'string') return null; + return { op: 'offer', sdp: { type: 'offer', sdp: sdp.sdp } }; + } + case 'ice-candidate': { + const candidate = payload.candidate; + if (!isRecord(candidate) || typeof candidate.candidate !== 'string') return null; + return { + op: 'ice-candidate', + candidate: { + candidate: candidate.candidate, + sdpMid: typeof candidate.sdpMid === 'string' ? candidate.sdpMid : null, + sdpMLineIndex: + typeof candidate.sdpMLineIndex === 'number' ? candidate.sdpMLineIndex : null, + usernameFragment: + typeof candidate.usernameFragment === 'string' ? candidate.usernameFragment : null, + }, + }; + } + case 'bye': + return { op: 'bye' }; + default: + return null; + } +} + +function pairingRejectionMessage(reason: string): string { + switch (reason) { + case 'unknown-code': + return 'That pairing code does not match the one on the desktop.'; + case 'already-used': + return 'That pairing code has already been used. Start pairing again on the desktop.'; + case 'busy': + return 'Another device is already waiting to be approved on the desktop.'; + default: + return 'That pairing code has expired. Start pairing again on the desktop.'; + } +} diff --git a/src/main/acappella/voice-session-service.ts b/src/main/acappella/voice-session-service.ts new file mode 100644 index 0000000000..c2dcf99a11 --- /dev/null +++ b/src/main/acappella/voice-session-service.ts @@ -0,0 +1,1880 @@ +/** + * A Cappella headless voice session service. + * + * Owns the whole session: lifecycle, the state machine from + * `src/shared/acappella/session-state.ts`, the monotonic `seq` counter, and the + * subscriber fan-out. Every client (the desktop HUD today, the iPhone later) + * sees the identical event stream; none of them holds authoritative state. + * + * Two rules this file exists to enforce, both inherited by every later phase: + * - Nothing here may reference a BrowserWindow, the DOM, or a React store. The + * session is transport-agnostic because the phone is a peer client, not a + * port of the desktop UI. See docs/architecture/acappella/decisions/adr-002-main-process-session.md. + * - No concrete provider is ever imported. The trio arrives at construction so + * Phase 05 can swap Whisper/Kokoro/OpenAI in without touching this file. + */ + +import type { + DispatchAction, + InterruptSource, + MicState, + RosterAgent, + SpeakEndReason, + VoiceEvent, + VoiceEventBase, + VoiceEventPayload, + VoiceEventType, + VoiceOrigin, + VoiceScope, + VoiceSessionErrorCode, + VoiceWindowId, + WakeSource, +} from '../../shared/acappella/protocol'; +import type { RouteDecision } from '../../shared/acappella/route-decision'; +import { + isClarification, + isConversationalReply, + routeTargetSessionId, +} from '../../shared/acappella/route-decision'; +import { isCorrectionUtterance, planCorrection } from './router/conductor-router'; +import type { + SttCallbacks, + SttProvider, + TtsChunk, + VoicePipelineShape, + VoiceProviderTrio, + VoiceRouteContext, +} from '../../shared/acappella/providers'; +import { + isVoiceProviderError, + type VoiceProviderError, +} from '../../shared/acappella/provider-errors'; +import { recordTurn, TurnTimer } from './telemetry/turn-metrics'; +import { + audioHostErrorToSessionError, + type AudioHostErrorCode, +} from '../../shared/acappella/audio-host'; +import { readinessErrorMessage, type VoiceReadiness } from '../../shared/acappella/readiness'; +import type { VoiceSessionState } from '../../shared/acappella/session-state'; +import { + assertVoiceStateTransition, + canTransitionVoiceState, +} from '../../shared/acappella/session-state'; +import type { BackgroundAnnouncementSetting } from '../../shared/acappella/announcements'; +import { generateUUID } from '../../shared/uuid'; +import { logger } from '../utils/logger'; +import { captureException } from '../utils/sentry'; +import type { AgentOutputChunk } from './speech/agent-output-tap'; +import { BackgroundAnnouncer, type BackgroundCompletion } from './speech/background-announcer'; +import { BargeInController } from './speech/barge-in'; +import { ConversationalTranslator } from './speech/conversational-translator'; +import { DetailBuffer, detectDrillDownIntent } from './speech/drill-down'; +import { UtteranceComposer, type UtteranceComposerConfig } from './speech/utterance-composer'; +import { ConversationBuffer } from './router/conversation-buffer'; +import { SpeechScheduler, type SpeechRunResult } from './speech/speech-scheduler'; + +const LOG_CONTEXT = 'ACappella'; + +/** Spoken replies stay short by default: nobody wants a diff read aloud. */ +const DEFAULT_MAX_SPOKEN_SENTENCES = 2; + +/** How many utterances the Brain gets as "back to the auth one" context. */ +const DEFAULT_UTTERANCE_HISTORY = 8; + +// --------------------------------------------------------------------------- +// Public shapes +// --------------------------------------------------------------------------- + +/** A single event delivered to one subscriber. Subscribers never mutate it. */ +export type VoiceEventListener = (event: VoiceEvent) => void; + +/** + * Why a session ended. Maps onto `ListenStopEvent.reason` on the way out. + * + * `timeout` is the idle backstop in `audio/floor-control.ts`: a forgotten open + * microphone going cold on its own. It is deliberately not the same reason as + * `user`, because "you stopped me" and "you walked away" are different facts and + * only one of them is worth telling the user about. + */ +export type VoiceStopReason = 'user' | 'stop-word' | 'timeout' | 'replaced' | 'shutdown' | 'error'; + +/** What the dispatch executor actually did, echoed as a `dispatch` event. */ +export interface VoiceDispatchResult { + agentSessionId: string; + agentName: string; + tabId: string; + tabName?: string; + action: DispatchAction; + promptSent: boolean; +} + +/** + * Thrown by a route executor for a KNOWN dispatch failure (the renderer did not + * answer within its timeout, the recalled tab is gone). Anything else thrown by + * an executor is a bug and bubbles to Sentry unchanged. + */ +export class VoiceDispatchError extends Error { + constructor(message: string) { + super(message); + this.name = 'VoiceDispatchError'; + } +} + +/** + * Performs a `RouteDecision` against the renderer. Injected rather than imported + * because main has no tab authority: the executor forwards `remote:*` messages + * and waits for the renderer to confirm. + */ +export type VoiceRouteExecutor = ( + decision: RouteDecision, + context: { roster: RosterAgent[]; scope: VoiceScope } +) => Promise; + +/** + * The slice of `speech/agent-output-tap.ts` this file drives. + * + * A narrow interface rather than the class, for the same reason no concrete + * provider is imported here: the tap reaches into the process manager and the + * per-agent output parsers, and a session service that imported it would drag + * both into every context that only wants to speak a string. The owner + * constructs the tap with `pushAgentOutput` as its sink and hands the two verbs + * over. See `src/main/ipc/handlers/acappella.ts`. + */ +export interface AgentReplyStream { + watch(params: { agentSessionId: string; tabId: string }): void; + unwatch(params: { agentSessionId: string; tabId: string }): void; +} + +/** Where a "show me" lands. Main has no tab authority, so this is injected. */ +export type VoiceFocusTarget = (target: { + agentSessionId: string; + tabId: string; + path?: string; +}) => void; + +export interface VoiceSessionServiceOptions { + /** The active provider trio. Resolved by `providers/provider-registry.ts`. */ + providers: VoiceProviderTrio; + /** + * Which pipeline shape the trio came from. Recorded with every turn's timings + * so a latency report can be compared against the right configuration; nothing + * in this file branches on it, which is the whole point of the two shapes + * sharing one interface. + */ + pipelineShape?: VoicePipelineShape; + /** Current agents and their tabs. Defaults to an empty roster. */ + getRoster?: () => RosterAgent[] | Promise; + /** Executes route decisions. Absent until the executor is wired. */ + executeRoute?: VoiceRouteExecutor; + /** Spoken-form budget handed to the translator for one rewritten chunk. */ + maxSpokenSentences?: number; + /** Hard cap on sentences spoken per turn, across every chunk of one reply. */ + maxSentencesPerTurn?: number; + /** + * The voice and rate the user chose, read fresh for every sentence. + * + * A getter, not a value, so the Settings sliders take effect on the NEXT + * SPOKEN SENTENCE rather than on the next session. A voice assistant that + * needs restarting to change its own speed is one nobody will ever tune. + */ + getSpeechOptions?: () => { voiceId?: string; rate?: number }; + /** Utterances retained for `VoiceRouteContext.recentUtterances`. */ + utteranceHistoryLimit?: number; + /** + * How long a settled fragment is held before it counts as a finished thought. + * + * A getter, not a value, so tuning it applies to the NEXT THOUGHT rather than + * the next session - a person who finds it too eager notices mid-conversation. + * See `speech/utterance-composer.ts` for the trade it makes. + */ + getUtteranceComposerConfig?: () => Partial; + /** + * Whether the Conductor may talk with the user instead of dispatching. + * + * A getter, read per turn, so switching it applies to the next thing said + * rather than to the next session. Absent means command mode: every utterance + * is routed, which is what A Cappella did before conversation existed. + */ + getConversationalMode?: () => boolean; + /** + * The live tap on a dispatched agent's output. + * + * Present means a reply is spoken AS IT IS WRITTEN: the tap hands over a + * completed thought, the translator rewrites that piece alone, and the + * scheduler starts speaking it while the agent is still typing the rest. + * Absent means the session waits for a whole reply through + * {@link VoiceSessionService.submitAgentReply}, which is the mock tier and the + * dev harness. + */ + agentReplyStream?: AgentReplyStream; + /** Puts a tab, and optionally a file, on screen for a spoken "show me". */ + focusTarget?: VoiceFocusTarget; + /** + * Whether an agent finishing outside the current turn is announced out loud. + * `auto` (the default) is on for the Conductor and off inside an agent scope. + */ + getBackgroundAnnouncementSetting?: () => BackgroundAnnouncementSetting | undefined; + /** + * Drop playback gain for a barge-in, ahead of the flush. + * + * Optional because the audio pipeline already ducks on a CANDIDATE frame, + * before the detector has confirmed anything - which is what makes the duck + * feel instant. This is the confirmed-barge-in duck for a host that has no + * pipeline in front of it (a client button, the phone), and a no-op otherwise. + */ + duckPlayback?: (gain: number, rampMs: number) => void; + /** Discard audio already queued in the host. Same optionality as `duckPlayback`. */ + flushPlayback?: () => void; + /** + * Dead time after speech starts during which a VOICE barge-in is refused. + * + * Zero disables it. Only voice is guarded: echo cancellation is at its worst in + * the first moments of playback, so the assistant's own first syllable can trip + * the detector and it interrupts itself. A button press has no such ambiguity. + */ + bargeInGuardMs?: number; + /** + * One chunk of synthesised speech, as it comes off the TTS provider. + * + * The audio bridge turns these into `play` commands for the audio host. It is + * a callback rather than an event because audio is the one thing in this + * pipeline that must NOT be broadcast: `speak-sentence` goes to every client so + * they can render the text, while the samples go to exactly one output device. + * Chunks with no audio behind them (the mock tier) are still delivered - what + * to do with `format: 'none'` is the sink's call, not this file's. + */ + onSpeechChunk?: (chunk: TtsChunk) => void; + /** + * The capability gate, consulted before the microphone is touched. + * + * A verdict rather than a provider, deliberately: the service refuses to start + * when a required slot is unsatisfied and says which one. It does NOT ask + * anything to pick a replacement. Routing audio to a cloud API the user did not + * choose is both an unasked-for charge and a privacy break, so the "recovery" + * for a missing local model is a stated error, not a substitution. + * + * Absent means "no gate", which is the mock tier: nothing to be missing. + */ + checkReadiness?: () => VoiceReadiness | Promise; + /** + * The body of the `provider-state` event, from whoever resolved the pipeline. + * + * A supplier rather than a value because a slot can be substituted, and only + * the registry knows what was requested; and a supplier rather than an import + * because this file must never learn how providers are chosen. + */ + getProviderState?: () => Omit, 'type'> | null; +} + +/** Everything a client needs to catch up after `get-state`. */ +export interface VoiceSessionSnapshot { + sessionId: string | null; + state: VoiceSessionState; + scope: VoiceScope | null; + /** + * Which microphone is holding the session, for a client that joined after the + * `listen-start` went out. `local` while idle. + */ + origin: VoiceOrigin; + /** + * The window whose HUD owns this session, for a window that reloaded or opened + * mid-session and so never saw the `wake`. Null while idle. + */ + windowId: VoiceWindowId; + /** Last `seq` emitted. A client whose next event skips this has lost events. */ + seq: number; + startedAt: number | null; + providerIds: { stt: string; tts: string; brain: string }; + /** + * The last routing decision, so a client that joined mid-session can show + * where the last thing went and how sure the router was. Null until the first + * utterance of the session has been routed. + */ + lastDecision: RouteDecision | null; + /** Where that decision actually landed, once it was performed. */ + lastDispatch: VoiceDispatchResult | null; +} + +/** The body of an event before the service stamps `sessionId`, `seq`, and `ts`. */ +type VoiceEventBody = Omit< + Extract, + keyof VoiceEventBase | 'type' +>; + +// --------------------------------------------------------------------------- +// Service +// --------------------------------------------------------------------------- + +export class VoiceSessionService { + private readonly providers: VoiceProviderTrio; + private readonly getRoster: () => RosterAgent[] | Promise; + private readonly executeRoute?: VoiceRouteExecutor; + private readonly maxSpokenSentences: number; + private readonly maxSentencesPerTurn?: number; + private readonly getSpeechOptions?: () => { voiceId?: string; rate?: number }; + private readonly utteranceHistoryLimit: number; + private readonly onSpeechChunk?: (chunk: TtsChunk) => void; + private readonly agentReplyStream?: AgentReplyStream; + private readonly focusTarget?: VoiceFocusTarget; + private readonly checkReadiness?: () => VoiceReadiness | Promise; + private readonly getProviderState?: () => Omit< + VoiceEventPayload<'provider-state'>, + 'type' + > | null; + + private readonly pipelineShape: VoicePipelineShape; + + /** + * The speech half, from `speech/`. Composed here because this is the only + * object that knows all four facts they need between them: what state the + * session is in, whose turn it is, what was actually heard, and when the floor + * is quiet. + */ + private readonly translator: ConversationalTranslator; + private readonly detail = new DetailBuffer(); + /** Assembles the fragments of one thought. See `speech/utterance-composer.ts`. */ + private readonly composer: UtteranceComposer; + /** What has been said while a task takes shape. See `router/conversation-buffer.ts`. */ + private readonly conversation = new ConversationBuffer(); + /** Whether the Conductor may talk back rather than dispatch. Read per turn. */ + private readonly conversationalMode: () => boolean; + private readonly announcer: BackgroundAnnouncer; + private readonly bargeIn: BargeInController; + + private readonly listeners = new Set(); + + private state: VoiceSessionState = 'idle'; + /** A teardown is in flight. Guards the await inside `stopSession`. */ + private stopping = false; + private sessionId: string | null = null; + private scope: VoiceScope | null = null; + /** + * Which microphone is holding the session. Set at start and never mid-session: + * a floor that changes hands starts a NEW session, so an origin that mutated + * under a live session would be describing a handover that cannot happen. + */ + private origin: VoiceOrigin = { kind: 'local' }; + /** + * The window whose HUD owns this session. Set at start, for the same reason + * `origin` is: the surface a session belongs to cannot change under it. + */ + private windowId: VoiceWindowId = null; + private seq = 0; + private startedAt: number | null = null; + + private recentUtterances: string[] = []; + + /** + * The question the router asked out loud, waiting for an answer. + * + * Consumed by the next utterance and cleared, so an abandoned question does + * not silently reinterpret an unrelated sentence three turns later. + */ + private pendingClarification: { question: string; utterance: string } | null = null; + + /** The last decision, for the HUD's "why did it go there" line. */ + private lastDecision: RouteDecision | null = null; + + /** The last dispatch, so a correction has something to move. */ + private lastDispatch: { decision: RouteDecision; result: VoiceDispatchResult } | null = null; + + /** + * Bumped on every utterance and on every teardown. A provider callback whose + * turn no longer matches is a straggler from a superseded turn and is + * dropped: async providers resolve after the user has already moved on. + */ + private turn = 0; + /** The speech run currently on the floor, or null when nothing is speaking. */ + private activeUtteranceId: string | null = null; + + /** The scheduler driving the run on the floor. One per run, never reused. */ + private scheduler: SpeechScheduler | null = null; + /** + * Suppresses the scheduler's end handling for a run being torn down. + * + * A stop word and a session teardown cancel speech on their way out and own + * the events themselves; without this the scheduler would emit a `speak-end` + * into a session that is already closing and try to hand the floor back. + */ + private speechTeardown = false; + /** A provider failure reported by the scheduler, read when the run ends. */ + private speechError: Error | null = null; + /** Cancels the translator stream feeding the run. Barge-in's fourth step. */ + private translationAbort: AbortController | null = null; + + /** The agent turn the tap is following, or null when nothing is being streamed. */ + private streamTarget: { agentSessionId: string; tabId: string; turn: number } | null = null; + /** + * Translations run one at a time, chained. + * + * The tap emits chunks from a process event while the previous chunk's rewrite + * is still in flight, and a spoken reply whose second thought overtakes its + * first is worse than a slow one. + */ + private streamChain: Promise = Promise.resolve(); + /** The untranslated output of the turn being spoken, for `drill-down.ts`. */ + private streamDetail = ''; + /** The turn's first-token timing is marked once, on the first chunk, not per chunk. */ + private streamMarked = false; + + /** + * Timings for the turn being spoken now. + * + * Started at the DETECTOR's endpoint rather than at the transcript, because the + * decode between those two moments is the hop most often to blame and the one a + * transcript-anchored timer cannot see. Null between turns. + */ + private timer: TurnTimer | null = null; + + constructor(options: VoiceSessionServiceOptions) { + this.providers = options.providers; + this.getRoster = options.getRoster ?? (() => []); + this.executeRoute = options.executeRoute; + this.maxSpokenSentences = options.maxSpokenSentences ?? DEFAULT_MAX_SPOKEN_SENTENCES; + this.maxSentencesPerTurn = options.maxSentencesPerTurn; + this.getSpeechOptions = options.getSpeechOptions; + this.utteranceHistoryLimit = options.utteranceHistoryLimit ?? DEFAULT_UTTERANCE_HISTORY; + this.onSpeechChunk = options.onSpeechChunk; + this.checkReadiness = options.checkReadiness; + this.getProviderState = options.getProviderState; + this.pipelineShape = options.pipelineShape ?? 'cascade'; + this.agentReplyStream = options.agentReplyStream; + this.focusTarget = options.focusTarget; + + this.translator = new ConversationalTranslator({ + brain: this.providers.brain, + maxSentences: this.maxSpokenSentences, + }); + this.announcer = new BackgroundAnnouncer({ + getScope: () => this.scope ?? { kind: 'conductor' }, + getSetting: () => options.getBackgroundAnnouncementSetting?.(), + getForegroundAgentSessionId: () => this.streamTarget?.agentSessionId ?? null, + }); + this.conversationalMode = () => options.getConversationalMode?.() ?? false; + this.composer = new UtteranceComposer({ + // Read through the getter so a settings change takes effect on the next + // thought rather than on the next session: a person who finds it too + // eager will tune it mid-conversation, which is when they notice. + ...options.getUtteranceComposerConfig?.(), + onSettled: ({ text, confidence, durationMs }) => { + // The floor can close between the last fragment and the settle. Every + // path that closes it cancels the composer, so this is belt and braces + // rather than a known race - but a fragment dispatched into a dead + // session is exactly the bug this component exists to prevent. + if (this.state !== 'listening') return; + void this.runTurn(text, confidence, durationMs); + }, + // A growing partial, so the transcript does not blank between the halves + // of one sentence and make a composing session look like a stalled one. + onComposing: (text) => this.emit('partial-transcript', { text, stability: 0.95 }), + }); + this.bargeIn = new BargeInController({ + // Both default to no-ops: the audio pipeline ducks on a candidate frame + // and the audio bridge flushes on a non-complete `speak-end`, so a host + // with those in front of it has already done these two steps. + duck: (gain, rampMs) => options.duckPlayback?.(gain, rampMs), + flushPlayback: () => options.flushPlayback?.(), + cancelSpeech: () => this.scheduler?.cancel('interrupted') ?? null, + cancelTranslation: () => this.abortTranslation(), + // Deliberately no `rememberSpoken`: the scheduler's end handler is the one + // owner of the conversation memory, and a second writer here would record + // every interrupted turn twice. + toListening: () => this.finishBargeIn(), + guardMs: options.bargeInGuardMs, + }); + } + + // -- Subscription -------------------------------------------------------- + + /** Subscribe to the event stream. Returns the unsubscribe function. */ + subscribe(listener: VoiceEventListener): () => void { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + } + + getState(): VoiceSessionState { + return this.state; + } + + /** + * The recogniser this session is feeding, or null when there is no session. + * + * The audio pipeline reads this per frame rather than capturing the provider, + * so a trio rebuilt between sessions cannot leave frames going into a stopped + * recogniser. Null while idle is the whole point: audio that arrives with no + * session behind it is dropped and counted, never buffered. + */ + getActiveStt(): SttProvider | null { + return this.state === 'idle' ? null : this.providers.stt; + } + + getSnapshot(): VoiceSessionSnapshot { + return { + sessionId: this.sessionId, + state: this.state, + scope: this.scope, + origin: this.origin, + windowId: this.windowId, + seq: this.seq, + startedAt: this.startedAt, + providerIds: { + stt: this.providers.stt.id, + tts: this.providers.tts.id, + brain: this.providers.brain.id, + }, + lastDecision: this.lastDecision, + lastDispatch: this.lastDispatch?.result ?? null, + }; + } + + // -- Lifecycle ----------------------------------------------------------- + + /** + * Open a session in `scope`. An already-running session is stopped first, so + * waking with a different scope switches rather than stacking. + * + * A provider that cannot start is a classified `provider-unavailable` error, + * not a throw: the snapshot comes back in the `error` state. + */ + async startSession(params: { + scope: VoiceScope; + source?: WakeSource; + /** + * Which microphone is holding this session. Defaults to this machine's. + * + * It travels with the session rather than being looked up, because by the + * time a `listen-start` is read the floor may already have moved: a client + * showing "your iPhone is holding the microphone" has to be describing the + * session it was told about, not whatever is true a second later. + */ + origin?: VoiceOrigin; + /** + * The window whose HUD owns this session. Null (the default) means no + * window claimed it, and the primary window shows it. + * + * Resolved by the CALLER, never here: the service has no access to the IPC + * sender or the window registry, and guessing "the focused window" from in + * here would silently reassign a session started from a background window. + */ + windowId?: VoiceWindowId; + }): Promise { + if (this.state !== 'idle') { + await this.stopSession('replaced'); + } + + this.sessionId = generateUUID(); + this.scope = params.scope; + this.origin = params.origin ?? { kind: 'local' }; + this.windowId = params.windowId ?? null; + this.seq = 0; + this.startedAt = Date.now(); + this.recentUtterances = []; + // Anything half-said before this session began belongs to the last one. + this.composer.cancel(); + this.conversation.clear(); + this.activeUtteranceId = null; + this.pendingClarification = null; + this.lastDecision = null; + this.lastDispatch = null; + this.turn += 1; + + this.transition('arming'); + this.emit('wake', { + source: params.source ?? 'client-button', + scope: params.scope, + origin: this.origin, + // On the FIRST event of the session, so a window never has to render a + // HUD before it knows whether the session is its own. + windowId: this.windowId, + }); + + // Before the device, not after: a session that opened the microphone and + // then discovered it has nowhere to send the audio has already cost the user + // a recording light and an OS permission prompt for nothing. + const readiness = await this.checkReadiness?.(); + if (readiness && !readiness.canStartSession) { + const blocked = readiness.blocking[0]; + this.fail( + 'provider-unavailable', + readinessErrorMessage(readiness) || 'Voice mode is not ready.', + blocked?.providerId + ); + return this.getSnapshot(); + } + + try { + await this.providers.stt.start(this.sttCallbacks()); + } catch (error) { + if (isVoiceProviderError(error)) { + this.failFromProvider(error, this.providers.stt.id); + } else { + this.fail( + 'provider-unavailable', + `Speech provider '${this.providers.stt.id}' could not start: ${(error as Error).message}`, + this.providers.stt.id + ); + } + return this.getSnapshot(); + } + + this.transition('listening'); + this.emit('listen-start', { + scope: params.scope, + sttProviderId: this.providers.stt.id, + origin: this.origin, + }); + // Immediately after the floor opens, so a client that joined mid-session + // never has to guess which engines it is actually talking to. + this.publishProviderState(); + await this.publishRoster(); + + return this.getSnapshot(); + } + + /** End the session and release the floor. Safe to call when already idle. */ + async stopSession(reason: VoiceStopReason): Promise { + // Re-entrant by construction: this awaits the recogniser's own teardown, and + // a second caller arriving inside that window (two hotkeys, a stop word and a + // window close) would find the state still non-idle and tear the same session + // down twice. + if (this.state === 'idle' || this.stopping) return; + this.stopping = true; + try { + await this.runStopSession(reason); + } finally { + this.stopping = false; + } + } + + private async runStopSession(reason: VoiceStopReason): Promise { + this.turn += 1; + this.cancelSpeech(); + // A half-collected thought belongs to the session that is ending. Letting it + // settle afterwards would dispatch a fragment into a session nobody is in. + this.composer.cancel(); + // So does an unfinished discussion: the next session starts a new one. + this.conversation.clear(); + + try { + await this.providers.stt.stop(); + } catch (error) { + // Teardown failure must not wedge the session in a non-idle state, so + // it is reported rather than thrown. + void captureException(error as Error, { + context: 'acappella.stopSession', + providerId: this.providers.stt.id, + }); + } + + this.emit('listen-stop', { reason: reason === 'error' ? 'error' : 'stopped' }); + this.transition('idle'); + + this.sessionId = null; + this.scope = null; + this.origin = { kind: 'local' }; + this.windowId = null; + this.startedAt = null; + this.recentUtterances = []; + // A question nobody answered, and a dispatch nobody can correct any more: + // both belong to the session that just ended. + this.pendingClarification = null; + this.lastDispatch = null; + // So does the conversation: what was said out loud, the detail behind it, + // and a backlog of completions nobody is listening for any more. + this.translator.reset(); + this.detail.clear(); + this.announcer.clear(); + this.streamDetail = ''; + this.streamMarked = false; + } + + /** + * The stop word. Ends the session from wherever it is. Distinct from + * `interrupt()` on purpose: conflating the two makes talking over the + * assistant hang up on it. + */ + async hardStop(source: InterruptSource = 'voice', phrase?: string): Promise { + if (this.state === 'idle' || this.stopping) return; + this.emit('stop-word', { source, phrase }); + await this.stopSession('stop-word'); + } + + /** + * Barge-in. Cancels speech and KEEPS the floor + * (`speaking -> interrupted -> listening`). + * + * @returns `false` when nothing was speaking, so a stray button press is a + * no-op rather than an error. + */ + interrupt(source: InterruptSource = 'voice'): boolean { + if (this.state !== 'speaking') return false; + // The guard window, which applies to voice and not to a button press: echo + // cancellation is at its worst in the first moments of playback, so the + // assistant's own first syllable can otherwise interrupt it. + if (!this.bargeIn.canInterrupt(source)) return false; + + // Announced BEFORE the teardown, because the teardown's own `speak-end` is + // the consequence: a client reading the stream should see what happened and + // then what it cost, in that order. + this.emit('barge-in', { source, cancelledUtteranceId: this.activeUtteranceId ?? undefined }); + + // Duck, flush, cancel the synthesis, cancel the rewrite behind it, then + // reopen the floor. The order is the controller's, not this file's. + this.bargeIn.trigger(source); + return true; + } + + /** + * The tail of a barge-in: back to `listening` with the floor retained. + * + * Separate from `trigger()` because the state machine and the event stream are + * this file's alone, and because a barge-in that found nothing to cancel still + * has to hand the floor back. + */ + private finishBargeIn(): void { + this.stopStreaming(); + if (this.state !== 'speaking') return; + this.transition('interrupted'); + this.transition('listening'); + this.emitListenStart(); + } + + // -- Input --------------------------------------------------------------- + + /** + * The seam a real STT final transcript lands on, and the one the dev harness + * types into. Routed through the provider's `injectUtterance` so the two are + * indistinguishable downstream. + * + * @returns `false` when the session cannot take an utterance right now. + */ + submitUtterance(text: string): boolean { + if (this.state === 'speaking') { + // An utterance arriving over active speech IS the user talking over it. + this.interrupt('voice'); + } + if (this.state === 'dispatching') { + // The user moved on before the agent replied. Abandon the pending reply + // and take back the floor. + this.transition('listening'); + this.emitListenStart(); + } + + if (this.state !== 'listening') { + logger.warn(`Utterance ignored in state '${this.state}'`, LOG_CONTEXT); + return false; + } + + const inject = this.providers.stt.injectUtterance; + if (!inject) { + this.fail( + 'provider-unavailable', + `Speech provider '${this.providers.stt.id}' has no text-in seam`, + this.providers.stt.id + ); + return false; + } + + inject.call(this.providers.stt, text); + return true; + } + + /** + * A WHOLE agent reply, already finished. Reshapes it for the ear and speaks it. + * + * The text-in seam, for a caller that has the complete reply in hand: the dev + * harness, the CLI, and the Phase 10 phone. A live desktop agent does not come + * through here - it comes through {@link pushAgentOutput}, sentence by + * sentence, while it is still writing. The two share the translator and the + * scheduler, so what is spoken is identical; only the arrival is different. + * + * @returns `false` when the session was not waiting on a reply. + */ + async submitAgentReply(params: { + agentSessionId: string; + tabId: string; + text: string; + }): Promise { + if (this.state !== 'dispatching') { + logger.warn(`Agent reply ignored in state '${this.state}'`, LOG_CONTEXT); + return false; + } + + const turn = this.turn; + this.timer?.mark('agentFirstToken'); + // Nothing is arriving on the tap for this turn: the whole reply is right + // here, so a tap still following the tab would only double-speak it. + this.stopStreaming(); + + const sentences: string[] = []; + try { + for await (const sentence of this.translator.translate({ + agentSessionId: params.agentSessionId, + tabId: params.tabId, + text: params.text, + kind: 'final', + })) { + sentences.push(sentence); + } + } catch (error) { + if (!isVoiceProviderError(error)) throw error; + this.failFromProvider(error, error.providerId); + return true; + } + if (!this.isCurrentTurn(turn)) return false; + + const spokenText = sentences.join(' '); + this.emit('agent-reply', { + agentSessionId: params.agentSessionId, + tabId: params.tabId, + text: params.text, + spokenText, + }); + // The real output, retained so "tell me more" is instant and costs nothing. + this.detail.record({ + agentSessionId: params.agentSessionId, + tabId: params.tabId, + detail: params.text, + spoken: [], + }); + + if (sentences.length === 0) { + // Nothing worth speaking. Take the floor back rather than opening a + // speech run with no sentences in it. + this.transition('listening'); + this.emitListenStart(); + return true; + } + + this.transition('speaking'); + try { + await this.speak(spokenText, turn); + } catch (error) { + // A streaming voice can throw mid-iteration. Without this the rejection + // leaves through the caller (an IPC handler) and the session sits in + // `speaking` holding a floor nothing will ever hand back. + if (isVoiceProviderError(error)) this.failFromProvider(error, error.providerId); + else this.closeFloorOnUnexpectedError(error as Error, 'acappella.speak'); + } + return true; + } + + /** + * One coherent piece of a dispatched agent's output, as it is written. + * + * The sink for `speech/agent-output-tap.ts`, and the thing that makes the first + * spoken word land while the agent is still typing: the chunk is rewritten on + * its own and its sentences go straight to the scheduler, rather than the whole + * reply being waited for and then rewritten in one hop. + * + * Chunks are translated one at a time, in arrival order. A spoken reply whose + * second thought overtakes its first is worse than a slow one. + */ + pushAgentOutput(chunk: AgentOutputChunk): void { + const target = this.streamTarget; + if (!target) return; + if (chunk.agentSessionId !== target.agentSessionId || chunk.tabId !== target.tabId) return; + if (!this.isCurrentTurn(target.turn)) { + // The user has moved on. Stop following the tab rather than speaking an + // answer to a question they have already replaced. + this.stopStreaming(); + return; + } + if (this.state !== 'dispatching' && this.state !== 'speaking') return; + + // Retained untranslated, which is what every follow-up is served from. + this.streamDetail += this.streamDetail ? ` ${chunk.text}` : chunk.text; + + this.streamChain = this.streamChain + .then(() => this.speakChunk(chunk, target.turn)) + .catch((error: Error) => { + if (isVoiceProviderError(error)) this.failFromProvider(error, error.providerId); + else this.closeFloorOnUnexpectedError(error, 'acappella.pushAgentOutput'); + }); + } + + /** + * An agent finished outside the current voice turn. + * + * Queued rather than spoken: interrupting the conversation you are having with + * one agent because a different one finished is the failure this exists to + * prevent. It is released at the next natural pause, source named. + * + * @returns `false` when it was declined - the setting is off for this scope, or + * it is the agent the current turn is already about. + */ + noteAgentCompletion(completion: BackgroundCompletion): boolean { + if (!this.sessionId) return false; + const queued = this.announcer.queue(completion) !== null; + // The floor may already be quiet, in which case there is no later pause to + // wait for and holding it back would be silence for no reason. + if (queued) this.deliverBackgroundAnnouncement(); + return queued; + } + + // -- Audio telemetry ----------------------------------------------------- + + /** + * Publish one meter update, already downsampled by + * `audio/level-meter.ts`. The service does no rate limiting of its own: the + * meter owns the window, and a second opinion here would only mean two places + * deciding how often a client hears about the same number. + * + * Dropped when no session is open - `emit` needs an envelope to stamp, and a + * level that belongs to no session belongs nowhere. + */ + publishAudioLevel(level: number, speech: boolean): void { + this.emit('audio-level', { level: clampLevel(level), speech }); + } + + /** + * Publish the microphone's state, as projected by `audio/mic-state.ts`. + * + * Every transition goes out, including the benign ones. The failure this + * exists to prevent is a client showing a listening indicator over a + * microphone that will never produce a transcript, and that failure is silent + * by construction: a denied permission and a quiet room look identical from + * the event stream unless something says otherwise. + */ + publishMicState(state: MicState): void { + this.emit('mic-state', { ...state }); + } + + /** + * The microphone could not be opened, or was taken away mid-session. + * + * Parks the session in `error` rather than leaving it listening, because a + * listening indicator over a device that will never produce a transcript is + * the worst outcome this feature has: the user has no screen to read and hears + * nothing back. `recoverable` comes from the classified host code, so the HUD + * can offer a privacy-settings button for the failures a user can actually fix + * and stay quiet about the ones they cannot. + */ + reportAudioCaptureFailure(code: AudioHostErrorCode, message: string): void { + const translated = audioHostErrorToSessionError({ kind: 'mic-error', code, message }); + this.fail(translated.code, translated.message, undefined, translated.recoverable); + } + + /** + * Announce which engines are live. + * + * The body is supplied by whoever resolved the pipeline, because the honest + * answer includes what the user ASKED for and this file deliberately never + * learns that: it is handed a trio and has no idea whether it is the configured + * one. Without a supplier this is a no-op, which is the mock tier's case. + */ + publishProviderState(): void { + const state = this.getProviderState?.(); + if (!state) return; + this.emit('provider-state', state); + } + + /** Re-read the roster and push it to every client. */ + async publishRoster(): Promise { + const roster = await this.getRoster(); + this.emit('agent-roster', { agents: roster }); + return roster; + } + + /** Stop the session and drop every subscriber. Called on app shutdown. */ + async dispose(): Promise { + await this.stopSession('shutdown'); + this.listeners.clear(); + } + + // -- Turn pipeline ------------------------------------------------------- + + /** + * The detector heard the user stop talking. + * + * The zero point for the turn's timings. Wired from the audio pipeline, which + * is the only place that instant is known: everything downstream sees the + * consequences (a flush, then a transcript) rather than the moment itself. + */ + noteSpeechEnd(): void { + if (this.state !== 'listening') return; + this.timer = new TurnTimer(generateUUID(), { + pipeline: this.pipelineShape, + providerIds: { + stt: this.providers.stt.id, + tts: this.providers.tts.id, + brain: this.providers.brain.id, + }, + }); + } + + private sttCallbacks(): SttCallbacks { + return { + onPartial: (text, stability) => { + if (this.state !== 'listening') return; + this.timer?.mark('firstPartial'); + this.emit('partial-transcript', { text, stability }); + }, + onFinal: (text, confidence, durationMs) => { + if (this.state !== 'listening') return; + // Composed only for a recogniser that listens to a room. A text-in + // provider's utterance was already delimited by whoever typed it and + // pressed send, so holding it would be latency in exchange for nothing. + if (!this.providers.stt.acceptsAudio) { + void this.runTurn(text, confidence, durationMs); + return; + } + this.composer.add(text, confidence, durationMs); + }, + onError: (error) => { + this.failFromProvider(error, this.providers.stt.id); + }, + }; + } + + /** + * One utterance, end to end: transcript, routing, dispatch. Only the three + * classified failure modes become `session-error` events. + * + * Anything else is a bug and goes to Sentry explicitly rather than by + * bubbling: this runs from a provider callback with no caller to bubble to, + * so an escaping rejection would arrive at the process handler stripped of + * the session context, and would leave the HUD frozen mid-turn. + */ + private async runTurn(text: string, confidence: number, durationMs?: number): Promise { + const turn = ++this.turn; + + try { + this.timer?.mark('finalTranscript'); + this.emit('final-transcript', { text, confidence, durationMs }); + this.transition('transcribing'); + + const utterance = text.trim(); + if (!utterance) { + this.transition('listening'); + this.emitListenStart(); + return; + } + + // A diagnostic recogniser's turn ENDS here, at the transcript. + // + // The microphone check hears real audio and reports how long you spoke + // for, which is exactly what proves the device and the capture graph + // work - and it is not something anyone said. Routing it onward sent a + // live agent prompts like "Echo utterance 4: 1.5s of speech.", which the + // agent then answered, at the user's expense, having been told nothing. + // The floor reopens so the meter keeps working turn after turn. + if (this.providers.stt.transcribesSpeech === false) { + this.transition('listening'); + this.emitListenStart(); + return; + } + + this.rememberUtterance(utterance); + // Before routing, so the Brain sees this turn as part of the exchange + // rather than as a sentence with the exchange sitting behind it. + if (this.conversationalMode()) this.conversation.add('user', utterance); + + // "Tell me more" is not a request, so it never reaches the router or the + // agent: it is served from the real output of the last turn, which makes + // it instant and makes it about the answer the user actually heard. + if (await this.serveFollowUp(utterance, turn)) return; + if (!this.isCurrentTurn(turn)) return; + + this.transition('routing'); + + const roster = await this.publishRoster(); + if (!this.isCurrentTurn(turn)) return; + + // A correction is not a request, so it never reaches the Brain: "no, the + // other one" routed as an utterance becomes a prompt, and the agent it + // lands in has no idea what it refers to. + if (this.lastDispatch && isCorrectionUtterance(utterance)) { + await this.runCorrection(roster, turn); + return; + } + + const startedAt = Date.now(); + const context = this.routeContext(roster); + const decision = await this.providers.brain.route(utterance, context); + if (!this.isCurrentTurn(turn)) return; + + const targetId = routeTargetSessionId(decision.target); + if (targetId && !roster.some((agent) => agent.sessionId === targetId)) { + this.fail('no-agent-matched', `No agent with id '${targetId}' is running`); + return; + } + + this.timer?.mark('routeDecision'); + this.lastDecision = decision; + this.emit('route-decision', { + decision, + brainProviderId: this.providers.brain.id, + latencyMs: Date.now() - startedAt, + }); + + if (isClarification(decision)) { + // The router is not sure enough to act. Ask, remember what the question + // was about, and hand the floor straight back: the answer arrives as the + // next utterance and routes the ORIGINAL request. + await this.askForClarification(decision, context, utterance, turn); + return; + } + + // Talking, not sending. The floor comes straight back, so the next thing + // said continues the same exchange rather than starting a new one. + if (isConversationalReply(decision)) { + await this.speakConversationalReply(decision.reply ?? '', turn); + return; + } + + this.transition('dispatching'); + await this.dispatch(decision, roster, turn); + } catch (error) { + // A provider that predicted its own failure is announced, not reported: it + // has a message written for the user and a recovery to go with it. + // Anything else is a bug and keeps the Sentry path. + if (isVoiceProviderError(error)) { + this.failFromProvider(error, error.providerId); + return; + } + this.closeFloorOnUnexpectedError(error as Error, 'acappella.runTurn'); + } + } + + /** + * The user says the utterance is over: send whatever has been collected. + * + * The hold-to-talk release, and anything else that ends dictation by decree. + * The recogniser is flushed separately (`audio-bridge.endUtterance`), and that + * alone is not enough: the flush produces a final, the composer BUFFERS it, + * and the request would then sit waiting out a settle window the user has + * already answered by letting go of the key. + * + * Ordering is the whole subtlety. A flushed final can arrive after this call + * returns, so settling only what is buffered right now would send the sentence + * minus its last few words. `armImmediateSettle()` makes the composer settle + * on the NEXT fragment instead, and settles what it already holds if none + * arrives. + */ + endUtteranceNow(): void { + this.composer.armImmediateSettle(); + } + + /** + * Say one conversational line and hand the floor straight back. + * + * Deliberately the SAME speak path a clarification uses, rather than a second + * one: what differs between the two is why the Conductor is talking, not how + * the words reach the room. A parallel path would be a second place for the + * barge-in guard, the volume, and the voice to drift. + */ + private async speakConversationalReply(reply: string, turn: number): Promise { + const line = reply.trim(); + if (!line) { + // A reply with nothing in it would strand the session in `routing` with + // the microphone shut. Treat it as a turn that produced nothing and give + // the floor back. + this.transition('listening'); + this.emitListenStart(); + return; + } + + // Recorded before it is spoken: speech can be interrupted, and what the + // Conductor MEANT to say is still what the user heard it start saying. + this.conversation.add('conductor', line); + + this.transition('speaking'); + try { + await this.speak(line, turn); + } catch (error) { + if (isVoiceProviderError(error)) this.failFromProvider(error, error.providerId); + else this.closeFloorOnUnexpectedError(error as Error, 'acappella.speakConversationalReply'); + } + } + + /** + * Perform the decision and announce what happened. The session stays in + * `dispatching` afterwards, holding the turn open for the agent's reply. + */ + private async dispatch( + decision: RouteDecision, + roster: RosterAgent[], + turn: number + ): Promise { + if (!this.executeRoute) { + this.fail('dispatch-failed', 'No route executor is configured for this session'); + return; + } + + let result: VoiceDispatchResult; + try { + result = await this.executeRoute(decision, { + roster, + scope: this.scope ?? { kind: 'conductor' }, + }); + } catch (error) { + // Only the executor's own classified failure is an event. Anything else + // is a bug and belongs in Sentry. + if (!(error instanceof VoiceDispatchError)) throw error; + this.fail('dispatch-failed', error.message); + return; + } + + if (!this.isCurrentTurn(turn)) return; + // Remembered so "no, the other one" has something to move, and so the HUD + // can show where the last thing went. + this.lastDispatch = { decision, result }; + // The discussion that produced this request is finished. Carrying it into + // the next one is how "now do the same for the other repo" arrives wearing + // the last job's context and becomes a second copy of it. + this.conversation.clear(); + this.emit('dispatch', result); + // Follow the tab from here, so the reply is spoken as it is written rather + // than after it is finished. + this.beginStreaming(result, turn); + } + + /** + * Ask the disambiguation out loud and take the floor back. + * + * The pending clarification is remembered rather than the question being + * re-derived next turn, because the ANSWER is a fragment: "the API one" routed + * on its own creates a tab called "the API one". The original utterance rides + * along so the next turn routes the request the user actually made. + */ + private async askForClarification( + decision: RouteDecision, + context: VoiceRouteContext, + utterance: string, + turn: number + ): Promise { + const question = decision.clarify?.trim(); + if (!question) return; + + this.pendingClarification = { + question, + // A clarification of a clarification is still about the ORIGINAL request. + utterance: context.clarification?.utterance ?? utterance, + }; + + this.transition('speaking'); + try { + await this.speak(question, turn); + } catch (error) { + if (isVoiceProviderError(error)) this.failFromProvider(error, error.providerId); + else this.closeFloorOnUnexpectedError(error as Error, 'acappella.askForClarification'); + } + } + + /** + * Move the last dispatch somewhere else, on the user's say-so. + * + * Its own event rather than a second `dispatch`: the two mean opposite things + * to the routing log, and a correction counted as a hit would make a router + * that is wrong half the time look perfect. + */ + private async runCorrection(roster: RosterAgent[], turn: number): Promise { + const previous = this.lastDispatch; + if (!previous) return; + + const plan = planCorrection(roster, previous.result.agentSessionId); + if (plan.kind === 'ask') { + await this.askForClarification( + { ...previous.decision, clarify: plan.question }, + this.routeContext(roster), + previous.decision.prompt, + turn + ); + return; + } + + this.transition('dispatching'); + await this.correctTo(plan.agentSessionId, roster, turn, 'voice'); + } + + /** + * Re-dispatch the last prompt to a different agent. + * + * The prompt is the one that was actually sent, not the raw utterance: the + * user is moving a request they already made, and re-deriving it would send + * the wrong agent a differently worded question. + * + * @returns false when there is nothing to correct. + */ + async correctLastDispatch( + agentSessionId: string, + source: InterruptSource = 'client-button' + ): Promise { + if (!this.lastDispatch) return false; + if (this.state !== 'listening' && this.state !== 'dispatching' && this.state !== 'speaking') { + return false; + } + if (this.state === 'speaking') this.interrupt(source); + + const roster = await this.publishRoster(); + const turn = ++this.turn; + this.walkTo(['transcribing', 'routing', 'dispatching']); + + return this.correctTo(agentSessionId, roster, turn, source); + } + + /** + * Walk the session to the last state in `path`, one legal edge at a time. + * + * A correction arrives from a button rather than from a turn, and a background + * announcement arrives from another agent entirely, so both can start in + * `listening` with the whole transcribe-and-route path still in front of them. + * The machine has no shortcut edge and should not grow one for a case that is + * three legal transitions away. + */ + private walkTo(path: readonly VoiceSessionState[]): void { + const destination = path[path.length - 1]; + for (const next of path) { + if (this.state === destination) return; + if (canTransitionVoiceState(this.state, next)) this.transition(next); + } + } + + private async correctTo( + agentSessionId: string, + roster: RosterAgent[], + turn: number, + source: InterruptSource + ): Promise { + const previous = this.lastDispatch; + if (!previous) return false; + + const decision: RouteDecision = { + target: { sessionId: agentSessionId }, + // The corrected target's own current tab: the tab id from the wrong agent + // means nothing on the right one. + tabAction: 'current', + prompt: previous.decision.prompt, + confidence: 1, + }; + + if (!this.executeRoute) { + this.fail('dispatch-failed', 'No route executor is configured for this session'); + return false; + } + + let result: VoiceDispatchResult; + try { + result = await this.executeRoute(decision, { + roster, + scope: this.scope ?? { kind: 'conductor' }, + }); + } catch (error) { + if (!(error instanceof VoiceDispatchError)) throw error; + this.fail('dispatch-failed', error.message); + return false; + } + + if (!this.isCurrentTurn(turn)) return false; + + this.emit('route-correction', { + fromAgentSessionId: previous.result.agentSessionId, + fromTabId: previous.result.tabId, + agentSessionId: result.agentSessionId, + agentName: result.agentName, + tabId: result.tabId, + tabName: result.tabName, + action: result.action, + promptSent: result.promptSent, + source, + }); + this.lastDispatch = { decision, result }; + return true; + } + + // -- Speech -------------------------------------------------------------- + + /** + * Speak text that is already in its final spoken form, and wait for it. + * + * Used by every caller that has the whole thing in hand: a translated reply, a + * clarifying question, a background announcement. It goes through the same + * scheduler as a streamed reply, so the cap, the no-gap synthesis, and the + * spoken-versus-queued bookkeeping are the same in both. + */ + private async speak(spokenText: string, turn: number): Promise { + const scheduler = this.beginSpeechRun({ seed: spokenText, streaming: false, turn }); + scheduler.close(); + await scheduler.drained(); + } + + /** + * Open a speech run. + * + * `streaming` says whether more sentences are still being written, and it is + * the honest answer to a client asking "how many sentences is this": for a + * streamed reply the announced count is a lower bound, because the alternative + * is holding the first sentence back until the whole reply exists and losing + * the entire point of the pipeline. + */ + private beginSpeechRun(params: { + seed?: string; + streaming: boolean; + turn: number; + }): SpeechScheduler { + const utteranceId = generateUUID(); + this.activeUtteranceId = utteranceId; + this.speechTeardown = false; + + const scheduler = new SpeechScheduler({ + tts: this.providers.tts, + maxSentencesPerTurn: this.maxSentencesPerTurn, + speechOptions: this.getSpeechOptions, + onStart: (event) => + this.emit('speak-start', { + utteranceId: event.utteranceId, + sentenceCount: event.sentenceCount, + ttsProviderId: event.ttsProviderId, + streaming: params.streaming, + }), + onSentence: (event) => { + this.timer?.mark('firstSpokenSentence'); + // Before the audio reaches the sink, so the sentence is on screen by + // the time it is audible rather than after it. + this.emit('speak-sentence', event); + }, + onChunk: (chunk) => this.onSpeechChunk?.(chunk), + onError: (error) => { + this.speechError = error; + }, + onEnd: (result) => this.completeSpeechRun(result, params.turn), + }); + + // Assigned before `begin()`, which emits and pumps synchronously: a barge-in + // landing inside that first tick must find something to cancel. + this.scheduler = scheduler; + this.bargeIn.noteSpeechStarted(); + scheduler.begin(utteranceId, params.seed); + return scheduler; + } + + /** + * A speech run ended, however it ended. + * + * The one place the conversation memory is written, and it is written from + * `spoken` alone. A model told it already said something the user never heard + * will refer back to it, and the user will have no idea what it means. + */ + private completeSpeechRun(result: SpeechRunResult, turn: number): void { + this.scheduler = null; + this.activeUtteranceId = null; + this.bargeIn.noteSpeechEnded(); + + this.translator.rememberSpoken(result.spoken); + this.detail.noteSpoken(result.spoken); + + // A stop word or a session teardown owns its own events and has already + // decided where the session is going. + if (this.speechTeardown) return; + + this.emit('speak-end', { utteranceId: result.utteranceId, reason: speakEndReason(result) }); + + // The barge-in owns the transitions on its own path (`interrupted` then + // `listening`), so this must not race it back to the floor. + if (result.reason === 'interrupted') return; + + if (result.reason === 'error') { + const error = this.speechError; + this.speechError = null; + if (error && isVoiceProviderError(error)) this.failFromProvider(error, error.providerId); + else if (error) this.closeFloorOnUnexpectedError(error, 'acappella.speak'); + return; + } + + this.stopStreaming(); + if (this.state !== 'speaking') return; + if (this.isCurrentTurn(turn)) this.closeTurnMetrics(); + this.transition('listening'); + this.emitListenStart(); + // The floor is quiet now, which is the moment a queued completion from a + // different agent has been waiting for. + this.deliverBackgroundAnnouncement(); + } + + /** + * Translate one tapped chunk and speak it. + * + * The speech run opens on the first SENTENCE rather than on the chunk: a chunk + * the translator had nothing to say about (a status the user has already heard, + * an empty rewrite) must not open a silent run and strand the session in + * `speaking`. + */ + private async speakChunk(chunk: AgentOutputChunk, turn: number): Promise { + if (!this.isCurrentTurn(turn)) return; + if (!this.streamMarked) { + this.streamMarked = true; + this.timer?.mark('agentFirstToken'); + } + + const abort = (this.translationAbort ??= new AbortController()); + const spoken: string[] = []; + + for await (const sentence of this.translator.translate({ + agentSessionId: chunk.agentSessionId, + tabId: chunk.tabId, + text: chunk.text, + kind: chunk.kind, + signal: abort.signal, + })) { + if (abort.signal.aborted || !this.isCurrentTurn(turn)) return; + const scheduler = this.ensureSpeechRun(turn); + if (!scheduler) return; + spoken.push(sentence); + scheduler.pushSentence(sentence); + } + + if (spoken.length > 0) { + // Per chunk rather than per reply, and after its sentences rather than + // before them: the sentences are what the user is already hearing, and + // holding the record back until the whole reply existed would put the + // transcript behind the audio. + this.emit('agent-reply', { + agentSessionId: chunk.agentSessionId, + tabId: chunk.tabId, + text: chunk.text, + spokenText: spoken.join(' '), + }); + } + + if (chunk.kind === 'final') this.closeStream(); + } + + /** The open run, or a new one. Null when the session cannot speak right now. */ + private ensureSpeechRun(turn: number): SpeechScheduler | null { + if (this.scheduler) return this.scheduler; + if (this.state === 'dispatching') this.transition('speaking'); + if (this.state !== 'speaking') return null; + return this.beginSpeechRun({ streaming: true, turn }); + } + + /** + * Follow a dispatched tab's output. The turn stays open until the agent + * finishes writing, which is what lets the reply be spoken as it arrives. + */ + private beginStreaming(result: VoiceDispatchResult, turn: number): void { + if (!this.agentReplyStream) return; + // A focus-only dispatch asked the agent nothing, so nothing is coming back + // and a tap on it would only pick up whatever it was already doing. + if (!result.promptSent) return; + + this.stopStreaming(); + this.streamDetail = ''; + this.streamMarked = false; + this.translationAbort = new AbortController(); + this.streamTarget = { + agentSessionId: result.agentSessionId, + tabId: result.tabId, + turn, + }; + this.agentReplyStream.watch({ + agentSessionId: result.agentSessionId, + tabId: result.tabId, + }); + } + + /** The agent finished writing. Retain what it said and let the run drain. */ + private closeStream(): void { + const target = this.streamTarget; + this.stopStreaming(); + + if (target && this.streamDetail.trim()) { + this.detail.record({ + agentSessionId: target.agentSessionId, + tabId: target.tabId, + detail: this.streamDetail, + spoken: [], + }); + } + + if (this.scheduler) { + this.scheduler.close(); + return; + } + // The whole turn produced nothing speakable. Hand the floor back rather than + // sitting in `dispatching` waiting for a reply that has already happened. + if (this.state === 'dispatching') { + this.transition('listening'); + this.emitListenStart(); + } + } + + /** Stop following the dispatched tab. Any run already on the floor is untouched. */ + private stopStreaming(): void { + const target = this.streamTarget; + if (!target) return; + this.streamTarget = null; + this.agentReplyStream?.unwatch({ + agentSessionId: target.agentSessionId, + tabId: target.tabId, + }); + } + + /** Barge-in's fourth step: the rewrite behind the synthesis. */ + private abortTranslation(): void { + this.translationAbort?.abort(); + this.translationAbort = null; + } + + /** + * Release one queued background announcement, if the floor is quiet. + * + * `listening` is the only quiet state: every other one has a turn in it, and + * "the backend agent finished the migration" arriving over the answer you are + * waiting for is exactly the interruption the queue exists to prevent. + */ + private deliverBackgroundAnnouncement(): void { + const announcement = this.announcer.take(this.state === 'listening'); + if (!announcement) return; + + const turn = ++this.turn; + this.walkTo(['transcribing', 'routing', 'speaking']); + if (this.state !== 'speaking') return; + + void this.speak(announcement.text, turn).catch((error: Error) => { + if (isVoiceProviderError(error)) this.failFromProvider(error, error.providerId); + else this.closeFloorOnUnexpectedError(error, 'acappella.backgroundAnnouncement'); + }); + } + + /** + * Serve a follow-up from the last turn's real output. + * + * It never becomes a routing decision and never costs an agent turn, which is + * the whole point: re-asking would be slow AND would produce a different + * answer, because the work has moved on since the sentence being asked about. + * + * @returns `false` when this was a fresh request rather than a follow-up. + */ + private async serveFollowUp(utterance: string, turn: number): Promise { + const intent = detectDrillDownIntent(utterance); + if (!intent) return false; + + const response = this.detail.serve(intent); + if (response.kind === 'none') return false; + + if (response.kind === 'focus') { + // Deliberately silent. Anything the user wants to SEE is answered on + // screen, and reading a path character by character is the worst thing + // this feature could do with a request to look at something. + this.focusTarget?.({ + agentSessionId: response.agentSessionId, + tabId: response.tabId, + path: response.path, + }); + this.transition('listening'); + this.emitListenStart(); + return true; + } + + this.walkTo(['routing', 'speaking']); + if (this.state !== 'speaking') return true; + try { + await this.speak(response.text, turn); + } catch (error) { + if (isVoiceProviderError(error)) this.failFromProvider(error, error.providerId); + else this.closeFloorOnUnexpectedError(error as Error, 'acappella.serveFollowUp'); + } + return true; + } + + /** + * Close the current turn's timings and file them. + * + * Only a turn that reached spoken audio is recorded: a turn abandoned by a + * barge-in has a "total" that measures how long the user waited before giving + * up, and averaging that into the latency history would make an impatient user + * look like a slow provider. + */ + private closeTurnMetrics(): void { + const timer = this.timer; + this.timer = null; + if (!timer) return; + recordTurn(timer.finish()); + } + + // -- Internals ----------------------------------------------------------- + + private routeContext(roster: RosterAgent[]): VoiceRouteContext { + const scope = this.scope ?? { kind: 'conductor' }; + // Consumed here, not on the next turn: a question that was asked and then + // abandoned must not reinterpret an unrelated sentence later on. + const clarification = this.pendingClarification ?? undefined; + this.pendingClarification = null; + + const conversational = this.conversationalMode(); + return { + roster, + scope, + activeAgentSessionId: scope.kind === 'agent' ? scope.sessionId : null, + recentUtterances: [...this.recentUtterances], + clarification, + conversational, + // Only in conversational mode: in command mode an empty array and an + // absent one mean the same thing to the Brain, and sending the field + // anyway would put a section in the prompt describing a conversation + // that is not happening. + conversation: conversational ? this.conversation.history : undefined, + }; + } + + private rememberUtterance(utterance: string): void { + this.recentUtterances.push(utterance); + if (this.recentUtterances.length > this.utteranceHistoryLimit) { + this.recentUtterances.shift(); + } + } + + private isCurrentTurn(turn: number): boolean { + return this.turn === turn && this.sessionId !== null; + } + + /** + * Cancel any speech run WITHOUT emitting. Callers own the events. + * + * Used by the paths that are ending the session (the stop word, teardown, an + * unexpected failure): they have already decided where the session is going, + * and a `speak-end` from underneath them would hand a floor back that is about + * to be released. Barge-in does not come through here - it wants the events. + */ + private cancelSpeech(): void { + this.abortTranslation(); + this.stopStreaming(); + + const scheduler = this.scheduler; + if (!scheduler) { + this.activeUtteranceId = null; + return; + } + this.speechTeardown = true; + scheduler.cancel('interrupted'); + this.speechTeardown = false; + } + + private transition(to: VoiceSessionState): void { + assertVoiceStateTransition(this.state, to); + this.state = to; + } + + private emitListenStart(): void { + this.emit('listen-start', { + scope: this.scope ?? { kind: 'conductor' }, + sttProviderId: this.providers.stt.id, + origin: this.origin, + }); + } + + /** Classified failure: announce it and park the session in `error`. */ + private fail( + code: VoiceSessionErrorCode, + message: string, + providerId?: string, + // Most codes answer this from the code alone. A capture failure does not: + // a denied permission is fixable and a machine with no audio stack is not, + // and both arrive as `audio-capture-failed`. + recoverable = code !== 'provider-unavailable' + ): void { + logger.warn(`Voice session error (${code}): ${message}`, LOG_CONTEXT); + // A provider callback can fire after teardown; there is no session left to + // move into `error` and no envelope to stamp the event with. + if (!this.sessionId || this.state === 'idle' || this.state === 'error') return; + + this.emit('session-error', { + code, + message, + recoverable, + providerId, + }); + this.transition('error'); + } + + /** + * A provider reported a failure it predicted. + * + * The error carries its own protocol code (auth, quota, network, or plain + * unavailable), its own recoverable flag, and a message written for someone + * with no screen in front of them. Collapsing all four into + * `provider-unavailable` would tell a user with an expired API key to go and + * download a model. + */ + private failFromProvider(error: Error, providerId?: string): void { + if (!isVoiceProviderError(error)) { + this.fail('provider-unavailable', error.message, providerId); + return; + } + const failure: VoiceProviderError = error; + this.fail( + failure.sessionErrorCode, + failure.message, + providerId ?? failure.providerId, + failure.recoverable + ); + } + + /** + * An unexpected exception escaped the turn. Report it with the session + * context Sentry would otherwise lose, then close the floor honestly so the + * HUD is not stuck mid-turn. + */ + private closeFloorOnUnexpectedError(error: Error, context: string): void { + logger.error(`Unexpected voice session failure: ${error.message}`, LOG_CONTEXT); + void captureException(error, { + context, + voiceSessionId: this.sessionId, + state: this.state, + }); + if (this.state === 'idle' || this.state === 'error') return; + this.cancelSpeech(); + this.emit('listen-stop', { reason: 'error' }); + this.transition('error'); + } + + /** Stamp `sessionId`, `seq`, and `ts`, then fan out to every subscriber. */ + private emit(type: T, body: VoiceEventBody): void { + if (!this.sessionId) return; + + // The spread is provably a `VoiceEvent` for each concrete `T`, but TypeScript + // cannot narrow a generic discriminant, hence the assertion. + const event = { + ...body, + type, + sessionId: this.sessionId, + seq: ++this.seq, + ts: Date.now(), + } as unknown as VoiceEvent; + + for (const listener of [...this.listeners]) { + try { + listener(event); + } catch (error) { + // One broken client must not stop the stream reaching the others. + void captureException(error as Error, { + context: 'acappella.emit', + eventType: type, + }); + } + } + } +} + +/** + * A speech run's end reason, in the protocol's words. + * + * `interrupted` and `completed` are kept apart all the way out to the client: + * collapsing them makes a turn the user talked over indistinguishable from one + * they listened to, and that difference is what the conversation memory is + * built on. A capped run still COMPLETED - it wrapped up out loud rather than + * being cut off. + */ +function speakEndReason(result: SpeechRunResult): SpeakEndReason { + if (result.reason === 'interrupted') return 'cancelled'; + return result.reason === 'error' ? 'error' : 'complete'; +} + +/** A meter value out of range is clamped rather than published: it is only a bar. */ +function clampLevel(value: number): number { + if (!Number.isFinite(value)) return 0; + return value < 0 ? 0 : value > 1 ? 1 : value; +} diff --git a/src/main/acappella/wake/stop-word.ts b/src/main/acappella/wake/stop-word.ts new file mode 100644 index 0000000000..13a69d6ffb --- /dev/null +++ b/src/main/acappella/wake/stop-word.ts @@ -0,0 +1,281 @@ +/** + * The stop word: the way to shut a voice assistant up. + * + * It runs on the SAME always-local detector as the wake word, and that placement + * is the whole design. The stop word has to be heard while text-to-speech is + * mid-sentence and while a cloud speech stream is open, which rules out waiting + * for a transcript to come back from a remote engine: the answer would arrive + * after the thing it was meant to stop had finished. A local classifier on the + * raw microphone frames is the only place this can live. + * + * **The stop word is not barge-in, and the two must never converge.** + * + * - Barge-in means "stop talking, I am still here". Speech is cancelled, the + * floor stays open, the session goes `speaking -> interrupted -> listening`, + * and the event is `barge-in`. + * - The stop word means "we are done". Speech is cancelled, playback is + * flushed, the microphone is closed, the session returns to `idle`, and the + * event is `stop-word`. + * + * They are separate modules, separate events, separate settings, and separate + * HUD feedback for one reason: every assistant that folded them together became + * one you cannot get rid of. Interrupting the machine is the most common thing a + * person does in a conversation, and it must not be the gesture that hangs up. + * + * Two phrases, always. The configurable one is whatever the user chose, and + * "nevermind" is always armed alongside it, because the moment you need the stop + * word is the moment you will not remember which words you assigned to it. + */ + +import type { AudioHostCommand } from '../../../shared/acappella/audio-host'; +import type { InterruptSource } from '../../../shared/acappella/protocol'; +import type { VoiceSessionState } from '../../../shared/acappella/session-state'; +import { + DEFAULT_STOP_PHRASE, + FALLBACK_STOP_PHRASE, +} from '../../../shared/acappella/voice-controls'; +import { logger } from '../../utils/logger'; +import { captureException } from '../../utils/sentry'; +import { DEFAULT_WAKE_SENSITIVITY, type WakeDetection, type WakePhrase } from './wake-detector'; + +const LOG_CONTEXT = 'ACappella'; + +export { + DEFAULT_STOP_PHRASE, + FALLBACK_STOP_PHRASE, +} from '../../../shared/acappella/voice-controls'; + +/** Stop phrase ids carry a prefix so a detection can be routed without a lookup. */ +export const STOP_PHRASE_PREFIX = 'stop:'; + +export const PRIMARY_STOP_PHRASE_ID = `${STOP_PHRASE_PREFIX}primary`; +export const FALLBACK_STOP_PHRASE_ID = `${STOP_PHRASE_PREFIX}nevermind`; + +/** True when a detection came from a stop phrase rather than a wake phrase. */ +export function isStopPhraseId(id: string): boolean { + return id.startsWith(STOP_PHRASE_PREFIX); +} + +export interface StopWordConfig { + /** The user's phrase. Blank falls back to {@link DEFAULT_STOP_PHRASE}. */ + phrase?: string; + /** 0 to 1, higher is easier to trigger. */ + sensitivity?: number; + /** False disarms the configurable phrase. "nevermind" stays armed regardless. */ + enabled?: boolean; +} + +/** + * The stop phrases, as the detector wants them. + * + * Their `scope` is the Conductor because a stop phrase does not open anything + * and the field has to hold something; nothing reads it, and + * {@link StopWordController.handleDetection} routes on the id. + */ +export function stopWordPhrases(config: StopWordConfig = {}): WakePhrase[] { + const sensitivity = config.sensitivity ?? DEFAULT_WAKE_SENSITIVITY; + return [ + { + id: PRIMARY_STOP_PHRASE_ID, + phrase: config.phrase?.trim() || DEFAULT_STOP_PHRASE, + scope: { kind: 'conductor' }, + sensitivity, + enabled: config.enabled !== false, + }, + { + id: FALLBACK_STOP_PHRASE_ID, + phrase: FALLBACK_STOP_PHRASE, + scope: { kind: 'conductor' }, + sensitivity, + }, + ]; +} + +/** + * Which phrases the local detector should be listening for right now. + * + * Idle means wake phrases; anything else means stop phrases. Arming both at once + * would let a wake phrase spoken mid-answer open a second session on top of the + * one already running, and arming neither would make one of the two features + * silently unavailable in half the states the session can be in. + */ +export function armedPhrases( + state: VoiceSessionState, + phrases: { wake: readonly WakePhrase[]; stop: readonly WakePhrase[] } +): WakePhrase[] { + const sessionIsCold = state === 'idle' || state === 'error'; + return sessionIsCold ? [...phrases.wake] : [...phrases.stop]; +} + +// --------------------------------------------------------------------------- +// Controller +// --------------------------------------------------------------------------- + +/** + * The slice of `VoiceSessionService` the stop word drives. + * + * `hardStop` and nothing else. Notably absent: `interrupt`. The stop word must + * not be able to reach barge-in even by accident, which is what makes "these two + * behaviours can never drift into each other" a property of the code. + */ +export interface StopWordSession { + getState(): VoiceSessionState; + /** Emits `stop-word`, cancels speech, stops the recogniser, returns to `idle`. */ + hardStop(source?: InterruptSource, phrase?: string): Promise; +} + +/** What actually happened, for the HUD and the tests. */ +export interface StopWordEventInfo { + phrase: string; + phraseId: string; + /** The state the session was in when the phrase landed. */ + from: VoiceSessionState; + score: number; + at: number; +} + +export interface StopWordControllerOptions { + session: StopWordSession; + /** + * Pushes a command to the hidden audio host. Used for exactly two things: + * discarding queued speech and closing the microphone. + */ + sendCommand?: (command: AudioHostCommand) => void; + /** The user's stop phrase settings, read per detection so a change takes effect live. */ + getConfig?: () => StopWordConfig; + /** + * The session went cold and the detector should go back to wake-word-only. + * + * A seam rather than a direct call because the detector's phrase list is owned + * by the wiring layer, and a controller that reached into it would be able to + * arm a wake phrase mid-session. + */ + onWakeWordOnly?: () => void; + /** The stop word fired. Distinct from any barge-in seam, deliberately. */ + onStopWord?: (info: StopWordEventInfo) => void; + /** Something the caller could not have awaited went wrong. Already reported. */ + onError?: (error: Error) => void; +} + +export class StopWordController { + private readonly options: StopWordControllerOptions; + /** Serialises stops, so two phrases inside one window cannot race two teardowns. */ + private queue: Promise = Promise.resolve(); + + constructor(options: StopWordControllerOptions) { + this.options = options; + } + + /** The phrases to arm while a session is running. */ + phrases(): WakePhrase[] { + return stopWordPhrases(this.options.getConfig?.() ?? {}); + } + + /** + * Route one detection from the local detector. + * + * @returns true when this was a stop phrase and was acted on, so the caller + * knows not to treat it as a wake. + */ + handleDetection(detection: WakeDetection): boolean { + if (!isStopPhraseId(detection.phraseId)) return false; + + const from = this.options.session.getState(); + if (from === 'idle') { + // Nothing to stop. Not an error: "nevermind" said into a quiet room is a + // perfectly ordinary thing for a person to do. + logger.debug('Stop phrase heard with no session running', LOG_CONTEXT); + return true; + } + + void this.enqueue(() => this.stop(detection, from)); + return true; + } + + /** Resolves once every queued stop has run. Tests and shutdown paths use it. */ + whenSettled(): Promise { + return this.queue; + } + + // -- Internals ----------------------------------------------------------- + + /** + * Go cold, in the order that keeps the room quiet. + * + * Playback is flushed FIRST. `hardStop` cancels the speech run, but chunks + * already handed to the audio host are queued in the renderer and would keep + * playing for as long as that queue is deep - which is the exact experience + * the stop word exists to end. + */ + private async stop(detection: WakeDetection, from: VoiceSessionState): Promise { + // Re-checked here, not only at the door: both stop phrases can clear on the + // same window, and the second one arrives at the front of the queue after + // the first has already taken the session down. Stopping an idle session + // again would close a microphone somebody else may have just opened. + if (this.options.session.getState() === 'idle') return; + + logger.info(`Stop word '${detection.phrase}' from ${from}`, LOG_CONTEXT); + this.send({ kind: 'flush' }); + + try { + await this.options.session.hardStop('voice', detection.phrase); + } catch (error) { + this.report(error as Error, 'acappella.stopWord.hardStop'); + } + + // The microphone closes after the session is down, so a frame in flight + // cannot arrive at a recogniser that has already been stopped. + this.send({ kind: 'stop-capture' }); + + this.notifyStopWord({ + phrase: detection.phrase, + phraseId: detection.phraseId, + from, + score: detection.score, + at: detection.at, + }); + + try { + this.options.onWakeWordOnly?.(); + } catch (error) { + this.report(error as Error, 'acappella.stopWord.onWakeWordOnly'); + } + } + + private notifyStopWord(info: StopWordEventInfo): void { + try { + this.options.onStopWord?.(info); + } catch (error) { + this.report(error as Error, 'acappella.stopWord.onStopWord'); + } + } + + private send(command: AudioHostCommand): void { + try { + this.options.sendCommand?.(command); + } catch (error) { + // A destroyed audio host must not stop the session from ending: the + // session going idle is the part the user asked for. + this.report(error as Error, 'acappella.stopWord.sendCommand'); + } + } + + private enqueue(action: () => Promise): Promise { + const next = this.queue.then(action).catch((error: Error) => { + this.report(error, 'acappella.stopWord'); + }); + this.queue = next; + return next; + } + + private report(error: Error, context: string): void { + logger.error(`Stop word failure (${context}): ${error.message}`, LOG_CONTEXT); + void captureException(error, { context }); + this.options.onError?.(error); + } +} + +/** Sugar, matching the rest of A Cappella's factories. */ +export function createStopWordController(options: StopWordControllerOptions): StopWordController { + return new StopWordController(options); +} diff --git a/src/main/acappella/wake/wake-detector.ts b/src/main/acappella/wake/wake-detector.ts new file mode 100644 index 0000000000..d09490438c --- /dev/null +++ b/src/main/acappella/wake/wake-detector.ts @@ -0,0 +1,637 @@ +/** + * The always-local wake word. + * + * openWakeWord, on `onnxruntime-node`, loaded through `native-loader.ts` and fed + * the same 16 kHz PCM frames the rest of the pipeline already produces. It runs + * whether the user picked Whisper or OpenAI for speech-to-text, and that is not + * a preference: **no audio may leave this machine until a wake phrase has + * actually fired.** An always-listening feature that streams a room to a service + * on the off chance somebody says a name is not a feature anyone should have to + * opt out of. + * + * The invariant is enforced structurally rather than by discipline. The detector + * has exactly one outward edge - `onWake` - and its scorer is typed with a + * literal `tier: 'local'`, so a hosted scorer will not compile and, if one is + * cast in anyway, the constructor throws. The detector never sees a provider, + * never holds a socket, and cannot be handed one. + * + * **It is not the speech recogniser and must not become one.** The STT engine + * stays unloaded while this runs: a wake word that keeps a 148 MB model resident + * for the life of the app is the reason "always listening" gets a bad name. The + * front end here is two small ONNX graphs and a per-phrase classifier. + * + * A global phrase plus per-agent phrases, so "hey scout" lands in that agent's + * context without a routing round trip. Every phrase carries its own sensitivity, + * because a two-syllable agent name and "hey maestro" do not false-fire at the + * same threshold, and every hit is debounced, because one spoken phrase produces + * several consecutive scoring windows over the threshold and each of them would + * otherwise be a session. + */ + +import { ACAPPELLA_AUDIO_FRAME_SAMPLES } from '../../../shared/acappella/audio-host'; +import { OPENWAKEWORD_BASE_ID } from '../../../shared/acappella/model-catalog'; +import type { VoiceScope } from '../../../shared/acappella/protocol'; +import { + DEFAULT_WAKE_DEBOUNCE_MS, + DEFAULT_WAKE_PHRASE, + DEFAULT_WAKE_SENSITIVITY, + MIN_WAKE_THRESHOLD, +} from '../../../shared/acappella/voice-controls'; +import { logger } from '../../utils/logger'; +import { captureException } from '../../utils/sentry'; +import { AudioFrameRing } from '../audio/audio-pipeline'; +import { WAKE_WORD_PROVIDER_ID } from '../models/capability-gate'; +import { modelFilePath } from '../models/model-store'; +import { loadLocalRuntime } from '../providers/local/runtime'; + +const LOG_CONTEXT = 'ACappella'; + +export { + DEFAULT_WAKE_DEBOUNCE_MS, + DEFAULT_WAKE_PHRASE, + DEFAULT_WAKE_SENSITIVITY, + MIN_WAKE_THRESHOLD, +} from '../../../shared/acappella/voice-controls'; + +/** The id of the always-present global phrase. Also its classifier file stem. */ +export const GLOBAL_WAKE_PHRASE_ID = 'global'; + +/** + * openWakeWord's hop: 1280 samples at 16 kHz, which is four of our 20 ms frames. + * The models are trained on this cadence, so it is a property of the graph and + * not a tuning knob. + */ +export const WAKE_HOP_SAMPLES = 1280; + +const FRAMES_PER_HOP = Math.ceil(WAKE_HOP_SAMPLES / ACAPPELLA_AUDIO_FRAME_SAMPLES); + +/** Int16 full scale, for the conversion to the float range the models want. */ +const INT16_SCALE = 32768; + +// --------------------------------------------------------------------------- +// Phrases +// --------------------------------------------------------------------------- + +/** One thing the detector is listening for. */ +export interface WakePhrase { + /** Stable id. Also the classifier model's file stem. */ + id: string; + /** What the user says. Display text; the classifier is what actually matches. */ + phrase: string; + /** Where a hit takes the session. The global phrase resolves to the Conductor. */ + scope: VoiceScope; + /** 0 to 1, higher is easier to trigger. Defaults to {@link DEFAULT_WAKE_SENSITIVITY}. */ + sensitivity?: number; + /** False parks the phrase without forgetting it. Defaults to true. */ + enabled?: boolean; +} + +/** The global phrase, bound to the Conductor. */ +export function globalWakePhrase( + phrase: string = DEFAULT_WAKE_PHRASE, + sensitivity?: number +): WakePhrase { + return { + id: GLOBAL_WAKE_PHRASE_ID, + phrase, + scope: { kind: 'conductor' }, + sensitivity, + }; +} + +/** A phrase bound to one agent, so saying it jumps straight into that agent's context. */ +export function agentWakePhrase( + agentSessionId: string, + phrase: string, + sensitivity?: number +): WakePhrase { + return { + id: `agent:${agentSessionId}`, + phrase, + scope: { kind: 'agent', sessionId: agentSessionId }, + sensitivity, + }; +} + +/** The score a phrase has to clear. Derived so the slider and the gate cannot disagree. */ +export function wakeThresholdFor(phrase: WakePhrase, fallback = DEFAULT_WAKE_SENSITIVITY): number { + const raw = phrase.sensitivity ?? fallback; + const sensitivity = Number.isFinite(raw) ? Math.min(1, Math.max(0, raw)) : fallback; + return Math.max(MIN_WAKE_THRESHOLD, 1 - sensitivity); +} + +// --------------------------------------------------------------------------- +// Scorer +// --------------------------------------------------------------------------- + +/** + * Scores one 80 ms hop against every armed phrase. + * + * `tier` is a literal, not a `VoiceProviderTier`. That is the compile-time half + * of the no-egress invariant: a cloud provider's tier is `'cloud'`, so a hosted + * scorer is not assignable here and the mistake is a type error rather than a + * privacy incident. The constructor checks it again at runtime for anything that + * arrives through a cast or across an IPC boundary. + */ +export interface WakePhraseScorer { + readonly tier: 'local'; + /** + * @param hop Mono float samples in [-1, 1], {@link WAKE_HOP_SAMPLES} long. + * @returns Score per phrase id, 0 to 1. Ids the scorer does not know are omitted. + */ + score(hop: Float32Array, phrases: readonly WakePhrase[]): Record; + dispose?(): void | Promise; +} + +/** + * Throws unless the scorer is local. + * + * Exported because the wiring layer resolves the scorer and should fail there, + * loudly, rather than constructing a detector that quietly does nothing. + */ +export function assertWakeScorerLocal(scorer: WakePhraseScorer): void { + if (scorer.tier !== 'local') { + throw new Error( + 'A Cappella wake word refuses a non-local scorer: no audio may leave the machine before a wake phrase fires.' + ); + } +} + +// --------------------------------------------------------------------------- +// Detection +// --------------------------------------------------------------------------- + +/** A wake phrase fired. */ +export interface WakeDetection { + phraseId: string; + /** The phrase as the user says it, for the `wake` event and the HUD. */ + phrase: string; + /** Where the session this opens is bound. */ + scope: VoiceScope; + score: number; + /** Epoch ms, so a debounce and a log line agree on when. */ + at: number; + /** + * The audio immediately around the phrase, oldest frame first. + * + * Handed to STT ahead of the live frames. Without it, "Maestro, what's the + * status" reaches the recogniser as "...what's the status", because the floor + * does not open until after the phrase has been said. + */ + preRoll: Int16Array[]; +} + +/** The pre-roll the detection carries. `AudioFrameRing` satisfies it structurally. */ +export interface WakePreRoll { + push(samples: Int16Array): void; + drain(): Int16Array[]; + clear(): void; +} + +export interface WakeDetectorOptions { + /** + * The armed phrases, read per hop rather than captured, so adding an agent + * phrase takes effect without restarting the detector. + */ + getPhrases: () => readonly WakePhrase[]; + /** + * The scorer. Omitted means `start()` builds the ONNX one; null means the + * detector runs inert, which is what a machine with no wake model does. + */ + scorer?: WakePhraseScorer | null; + /** Builds the scorer on `start()`. Defaults to {@link createOnnxWakeScorer}. */ + createScorer?: () => Promise; + /** Sensitivity for phrases that do not state one. */ + defaultSensitivity?: number; + /** Minimum gap between two hits of the same phrase. */ + debounceMs?: number; + /** + * The pre-roll buffer. + * + * Pass the AUDIO PIPELINE's ring when wiring this for real, so there is one + * buffer rather than two: a detector with its own copy would hand STT the same + * half second the pipeline is about to replay. The default exists so the + * detector is usable, and testable, on its own. + */ + preRoll?: WakePreRoll; + /** Pre-roll length when the detector builds its own ring. */ + preRollMs?: number; + /** A phrase fired. The only outward edge this module has. */ + onWake: (detection: WakeDetection) => void; + /** Injected clock, for tests. */ + now?: () => number; +} + +/** Counters. Every wake-word failure is silent, so it has to be counted to be seen. */ +export interface WakeDetectorStats { + framesReceived: number; + hopsScored: number; + detections: number; + /** Hits suppressed because the same phrase fired inside the debounce window. */ + debounced: number; + /** Throws out of `score()`. Counted, not propagated: see {@link WakeDetector.pushFrame}. */ + scoreErrors: number; +} + +// --------------------------------------------------------------------------- +// Detector +// --------------------------------------------------------------------------- + +export class WakeDetector { + private readonly options: WakeDetectorOptions; + private readonly preRoll: WakePreRoll; + private readonly debounceMs: number; + private readonly defaultSensitivity: number; + private readonly now: () => number; + + private scorer: WakePhraseScorer | null = null; + /** + * Whether `stop()` may release the scorer. + * + * A scorer the detector BUILT holds ONNX sessions and must be freed; a scorer + * the caller passed in belongs to the caller, and disposing it would leave a + * restarted detector inert with no way to say why. + */ + private ownsScorer = false; + private running = false; + /** Accumulates 20 ms frames into one 80 ms hop. */ + private readonly hop: Int16Array; + private hopFill = 0; + private readonly lastFiredAt = new Map(); + private readonly stats: WakeDetectorStats = { + framesReceived: 0, + hopsScored: 0, + detections: 0, + debounced: 0, + scoreErrors: 0, + }; + + constructor(options: WakeDetectorOptions) { + this.options = options; + if (options.scorer) assertWakeScorerLocal(options.scorer); + this.scorer = options.scorer ?? null; + this.debounceMs = Math.max(0, options.debounceMs ?? DEFAULT_WAKE_DEBOUNCE_MS); + this.defaultSensitivity = options.defaultSensitivity ?? DEFAULT_WAKE_SENSITIVITY; + this.now = options.now ?? Date.now; + this.hop = new Int16Array(WAKE_HOP_SAMPLES); + this.preRoll = + options.preRoll ?? + new AudioFrameRing( + Math.max(FRAMES_PER_HOP, Math.round((options.preRollMs ?? 500) / 20)) // 20 ms frames + ); + } + + get isRunning(): boolean { + return this.running; + } + + /** True when there is a scorer behind the detector. False means it is inert. */ + get isArmed(): boolean { + return this.running && this.scorer !== null; + } + + getStats(): Readonly { + return { ...this.stats }; + } + + /** + * Build the scorer and start consuming frames. + * + * A scorer that cannot be built is NOT an error: a machine that has not + * downloaded the wake model has a perfectly good hotkey. The detector runs + * inert and says so, and the capability gate is what tells the user why + * hands-free is unavailable. + */ + async start(): Promise { + if (this.running) return; + if (!this.scorer) { + const build = this.options.createScorer ?? createOnnxWakeScorer; + try { + const built = await build(); + if (built) assertWakeScorerLocal(built); + this.scorer = built; + this.ownsScorer = built !== null; + } catch (err) { + // Classified failures are already remembered by the native loader for + // the capability gate; anything else is a real bug and goes to Sentry. + logger.warn(`Wake word scorer unavailable: ${(err as Error).message}`, LOG_CONTEXT); + this.scorer = null; + } + } + this.running = true; + this.hopFill = 0; + this.lastFiredAt.clear(); + logger.info( + this.scorer ? 'Wake word detector armed' : 'Wake word detector running without a model', + LOG_CONTEXT + ); + } + + /** + * Stop consuming frames and release the models. + * + * The pre-roll is cleared: audio held for a wake phrase that will not now be + * spoken is audio kept for no reason. + */ + async stop(): Promise { + if (!this.running) return; + this.running = false; + this.hopFill = 0; + this.lastFiredAt.clear(); + this.preRoll.clear(); + if (!this.ownsScorer) return; + const scorer = this.scorer; + this.scorer = null; + this.ownsScorer = false; + try { + await scorer?.dispose?.(); + } catch (err) { + logger.warn(`Wake word scorer dispose failed: ${(err as Error).message}`, LOG_CONTEXT); + } + } + + /** + * One 20 ms frame. + * + * Frames go into the pre-roll first and into the hop second, so a phrase that + * fires on this hop carries the audio that produced it. Nothing else happens + * to them: this method has no path to a provider, a socket, or a file. + */ + pushFrame(samples: Int16Array): void { + if (!this.running) return; + this.stats.framesReceived += 1; + this.preRoll.push(samples); + + let offset = 0; + while (offset < samples.length) { + const take = Math.min(WAKE_HOP_SAMPLES - this.hopFill, samples.length - offset); + this.hop.set(samples.subarray(offset, offset + take), this.hopFill); + this.hopFill += take; + offset += take; + if (this.hopFill === WAKE_HOP_SAMPLES) { + this.hopFill = 0; + this.scoreHop(); + } + } + } + + // -- Internals ----------------------------------------------------------- + + private scoreHop(): void { + if (!this.scorer) return; + + const phrases = this.options.getPhrases().filter((p) => p.enabled !== false); + if (!phrases.length) return; + + const hop = new Float32Array(WAKE_HOP_SAMPLES); + for (let i = 0; i < WAKE_HOP_SAMPLES; i++) hop[i] = this.hop[i] / INT16_SCALE; + + let scores: Record; + try { + scores = this.scorer.score(hop, phrases); + } catch (err) { + // Counted rather than thrown, for the same reason the audio pipeline + // counts feed errors: this runs fifty times a second on an audio callback, + // and one bad inference must not become fifty unhandled exceptions. + this.stats.scoreErrors += 1; + if (this.stats.scoreErrors === 1) { + logger.error(`Wake word scoring failed: ${(err as Error).message}`, LOG_CONTEXT); + void captureException(err as Error, { context: 'acappella.wakeDetector.score' }); + } + return; + } + this.stats.hopsScored += 1; + + // Best match wins rather than first: two phrases can clear their thresholds + // on the same hop, and firing both would open two sessions for one sentence. + let best: { phrase: WakePhrase; score: number } | null = null; + for (const phrase of phrases) { + const score = scores[phrase.id]; + if (typeof score !== 'number' || Number.isNaN(score)) continue; + if (score < wakeThresholdFor(phrase, this.defaultSensitivity)) continue; + if (!best || score > best.score) best = { phrase, score }; + } + if (!best) return; + + const at = this.now(); + const last = this.lastFiredAt.get(best.phrase.id); + if (last !== undefined && at - last < this.debounceMs) { + this.stats.debounced += 1; + return; + } + this.lastFiredAt.set(best.phrase.id, at); + this.stats.detections += 1; + + this.emit({ + phraseId: best.phrase.id, + phrase: best.phrase.phrase, + scope: best.phrase.scope, + score: best.score, + at, + // Drained, not copied: the frames are being handed to STT, and leaving + // them in the ring would replay the same half second again when the floor + // opens. + preRoll: this.preRoll.drain(), + }); + } + + private emit(detection: WakeDetection): void { + logger.info( + `Wake phrase '${detection.phrase}' fired (${detection.score.toFixed(2)})`, + LOG_CONTEXT + ); + try { + this.options.onWake(detection); + } catch (err) { + // A subscriber's failure is not the detector's failure; swallowing it here + // keeps the always-on path alive for the next phrase. + logger.error(`Wake handler threw: ${(err as Error).message}`, LOG_CONTEXT); + void captureException(err as Error, { context: 'acappella.wakeDetector.onWake' }); + } + } +} + +/** Sugar, matching the rest of A Cappella's factories. */ +export function createWakeDetector(options: WakeDetectorOptions): WakeDetector { + return new WakeDetector(options); +} + +// --------------------------------------------------------------------------- +// The ONNX scorer +// --------------------------------------------------------------------------- + +/** The `onnxruntime-node` surface used here, structurally. */ +interface OnnxTensorCtor { + new (type: string, data: Float32Array, dims: number[]): OnnxTensor; +} + +interface OnnxTensor { + data: Float32Array; + dims: readonly number[]; +} + +interface OnnxSession { + inputNames: readonly string[]; + outputNames: readonly string[]; + run(feeds: Record): Promise>; + release?(): Promise; +} + +interface OnnxRuntime { + InferenceSession: { create(path: string): Promise }; + Tensor: OnnxTensorCtor; +} + +/** Mel frames the embedding graph consumes at once. A property of the trained model. */ +const EMBEDDING_WINDOW_MEL_FRAMES = 76; + +/** Embeddings the per-phrase classifier consumes at once. Also a property of the model. */ +const CLASSIFIER_WINDOW_EMBEDDINGS = 16; + +/** Mel bins openWakeWord's front end produces. */ +const MEL_BINS = 32; + +/** + * The real openWakeWord front end: mel spectrogram, then embedding, then one + * small classifier per phrase. + * + * Returns null when the runtime or the model files are missing, which is the + * ordinary state of a machine that has not opted into hands-free. The caller + * runs the detector inert rather than failing: the capability gate is where a + * missing model becomes a sentence the user can act on. + * + * Per-phrase classifiers are looked up by phrase id inside the installed model + * directory. A phrase with no classifier is simply never scored, so a custom + * agent phrase that has not been trained cannot fire on somebody else's model. + */ +export async function createOnnxWakeScorer(): Promise { + let ort: OnnxRuntime; + try { + ort = await loadLocalRuntime('onnx', WAKE_WORD_PROVIDER_ID); + } catch (err) { + logger.info(`Wake word runtime unavailable: ${(err as Error).message}`, LOG_CONTEXT); + return null; + } + + let melSession: OnnxSession; + let embeddingSession: OnnxSession; + try { + melSession = await ort.InferenceSession.create( + modelFilePath(OPENWAKEWORD_BASE_ID, 'melspectrogram.onnx') + ); + embeddingSession = await ort.InferenceSession.create( + modelFilePath(OPENWAKEWORD_BASE_ID, 'embedding_model.onnx') + ); + } catch (err) { + logger.info(`Wake word models unavailable: ${(err as Error).message}`, LOG_CONTEXT); + return null; + } + + const classifiers = new Map(); + /** Rolling mel frames, and rolling embeddings. Both are the model's own memory. */ + const melRing: Float32Array[] = []; + const embeddingRing: Float32Array[] = []; + /** The scores from the last completed classifier window, per phrase. */ + let latest: Record = {}; + /** One inference chain at a time; ONNX runs async and hops arrive every 80 ms. */ + let busy = false; + + async function classifierFor(phrase: WakePhrase): Promise { + const cached = classifiers.get(phrase.id); + if (cached !== undefined) return cached; + let session: OnnxSession | null = null; + try { + session = await ort.InferenceSession.create( + modelFilePath(OPENWAKEWORD_BASE_ID, `${phrase.id}.onnx`) + ); + } catch { + logger.info( + `No wake classifier for phrase '${phrase.phrase}' (${phrase.id}); it will never fire`, + LOG_CONTEXT + ); + } + classifiers.set(phrase.id, session); + return session; + } + + async function advance(hop: Float32Array, phrases: readonly WakePhrase[]): Promise { + const melOut = await melSession.run({ + [melSession.inputNames[0]]: new ort.Tensor('float32', hop, [1, hop.length]), + }); + const mel = melOut[melSession.outputNames[0]]; + for (let i = 0; i + MEL_BINS <= mel.data.length; i += MEL_BINS) { + melRing.push(mel.data.slice(i, i + MEL_BINS)); + } + while (melRing.length > EMBEDDING_WINDOW_MEL_FRAMES) melRing.shift(); + if (melRing.length < EMBEDDING_WINDOW_MEL_FRAMES) return; + + const melWindow = new Float32Array(EMBEDDING_WINDOW_MEL_FRAMES * MEL_BINS); + melRing.forEach((frame, index) => melWindow.set(frame, index * MEL_BINS)); + const embedOut = await embeddingSession.run({ + [embeddingSession.inputNames[0]]: new ort.Tensor('float32', melWindow, [ + 1, + EMBEDDING_WINDOW_MEL_FRAMES, + MEL_BINS, + 1, + ]), + }); + const embedding = embedOut[embeddingSession.outputNames[0]]; + embeddingRing.push(Float32Array.from(embedding.data)); + while (embeddingRing.length > CLASSIFIER_WINDOW_EMBEDDINGS) embeddingRing.shift(); + if (embeddingRing.length < CLASSIFIER_WINDOW_EMBEDDINGS) return; + + const width = embeddingRing[0].length; + const window = new Float32Array(CLASSIFIER_WINDOW_EMBEDDINGS * width); + embeddingRing.forEach((vector, index) => window.set(vector, index * width)); + + const next: Record = {}; + for (const phrase of phrases) { + const session = await classifierFor(phrase); + if (!session) continue; + const out = await session.run({ + [session.inputNames[0]]: new ort.Tensor('float32', window, [ + 1, + CLASSIFIER_WINDOW_EMBEDDINGS, + width, + ]), + }); + next[phrase.id] = out[session.outputNames[0]].data[0]; + } + latest = next; + } + + return { + tier: 'local', + /** + * Synchronous by contract, asynchronous underneath. + * + * The inference chain is kicked off and the PREVIOUS window's scores are + * returned. That is not a shortcut: a hop arrives every 80 ms, an ONNX chain + * takes a few milliseconds, and awaiting it inside an audio callback would + * make the microphone path wait on the CPU. One window of latency on a wake + * word is inaudible; a stalled capture is not. + */ + score(hop, phrases) { + if (!busy) { + busy = true; + void advance(hop, phrases) + .catch((err: Error) => { + logger.warn(`Wake inference failed: ${err.message}`, LOG_CONTEXT); + }) + .finally(() => { + busy = false; + }); + } + return latest; + }, + async dispose() { + latest = {}; + melRing.length = 0; + embeddingRing.length = 0; + await melSession.release?.(); + await embeddingSession.release?.(); + for (const session of classifiers.values()) await session?.release?.(); + classifiers.clear(); + }, + }; +} diff --git a/src/main/app-lifecycle/main-window-navigation.ts b/src/main/app-lifecycle/main-window-navigation.ts index aa24c8e34c..3dc1f2a1b5 100644 --- a/src/main/app-lifecycle/main-window-navigation.ts +++ b/src/main/app-lifecycle/main-window-navigation.ts @@ -2,6 +2,7 @@ import type { BrowserWindow } from 'electron'; import { logger } from '../utils/logger'; import { blocksSubframeNavigation } from '../../shared/plugins/panel-navigation'; import { parseConcertoHtmlUrl } from '../../shared/concerto-html'; +import { isAcappellaAudioHostContents } from '../acappella/audio-host-window'; const ALLOWED_APP_PERMISSIONS = new Set(['clipboard-read', 'clipboard-sanitized-write']); @@ -92,12 +93,19 @@ export function attachMainWindowNavigationGuards( // Deny most browser permission requests (camera, mic, geolocation, etc.) // Allow clipboard access for the app window only, never embedded browser tabs. + // + // The one microphone exception is A Cappella's hidden audio host window. Its + // whole purpose is capture, and permission handlers are per-session (this is + // the shared default session), so the grant is scoped to that exact + // webContents rather than loosened for every `window`-type contents. browserWindow.webContents.session.setPermissionRequestHandler( (webContents, permission, callback) => { const contentsType = webContents?.getType?.(); const isAppWindow = contentsType === 'window'; - if (isAppWindow && ALLOWED_APP_PERMISSIONS.has(permission)) { + if (permission === 'media' && isAcappellaAudioHostContents(webContents)) { + callback(true); + } else if (isAppWindow && ALLOWED_APP_PERMISSIONS.has(permission)) { callback(true); } else { if (contentsType === 'webview') { diff --git a/src/main/cue/cue-executor.ts b/src/main/cue/cue-executor.ts index 66b4057a41..2c536ab63c 100644 --- a/src/main/cue/cue-executor.ts +++ b/src/main/cue/cue-executor.ts @@ -160,7 +160,7 @@ function extractCleanStdout(rawStdout: string, toolType: string): string { * Maestro agent id stored on the event row. */ function extractProviderSessionId(rawStdout: string, toolType: string): string | null { - if (!rawStdout.trim()) return null; + if (!rawStdout?.trim()) return null; const parser = getOutputParser(toolType as ToolType); if (!parser) return null; @@ -290,7 +290,14 @@ export async function executeCuePrompt(config: CueExecutionConfig): Promise { } /** - * Strip paths, URLs, usernames, and hostnames out of arbitrary text. + * Strip paths, URLs, usernames, hostnames, and API keys out of arbitrary text. */ export function redactText(text: string): string { if (typeof text !== 'string' || text === '') { return text; } - let result = text.replace(TEXT_TARGET_RE, (match) => { + // Secrets first, and before the length cap: a support package is the one + // artefact most likely to be attached to a public issue, and an API key that + // survived into one is worse than any path this function was written for. + // The scrubber lives with the credential layer, so there is one definition of + // "looks like a key" rather than a second one drifting here. + let result = redactSecrets(text); + + result = result.replace(TEXT_TARGET_RE, (match) => { if (/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(match)) return redactUrl(match); // Emails, ssh targets, and git remotes name a person, a machine, and a // repository, so nothing but a correlation id survives. diff --git a/src/main/debug-package/collectors/voice-runtime.ts b/src/main/debug-package/collectors/voice-runtime.ts new file mode 100644 index 0000000000..fdce3ba576 --- /dev/null +++ b/src/main/debug-package/collectors/voice-runtime.ts @@ -0,0 +1,112 @@ +/** + * A Cappella Voice Runtime Collector + * + * Answers "voice does not work" with facts instead of a guess. A voice failure + * has at least four independent causes that look identical from the outside: the + * Encore Feature is off, the microphone permission is denied, a native runtime + * will not load in this build, or a model is missing. Every one of them is here, + * in one file, so a support report does not need a round trip per hypothesis. + * + * The self-test loads each native runtime and runs a trivial operation against + * it. It loads no model and touches no audio device, so generating a debug + * package stays a read-only act. + * + * It is skipped entirely while the Encore Feature is off, and that is a rule + * rather than an optimisation. "A Cappella off means no native module loads" has + * to hold structurally: it is true today only because no runtime is a declared + * dependency yet, so the loader declines before it imports anything. The moment + * the real runtimes ship, an ungated self-test would `dlopen` three inference + * engines every time anybody built a debug package for a feature they had never + * switched on - and it would populate the loader's process-wide failure memo, + * which the capability gate reads, on their behalf. + * + * The static runtime table is still reported either way. That is the half of the + * answer worth having with the feature off: which binaries this build expects, + * on this platform, and how they are meant to arrive. + * + * Privacy: no paths, no device names, no audio, no keys. Runtime ids, platform, + * timings, and a permission string. + */ + +import Store from 'electron-store'; + +import { + NATIVE_RUNTIMES, + nativePlatformKey, + type NativePrebuildAvailability, +} from '../../../shared/acappella/native-runtimes'; +import { isACappellaEnabled } from '../../../shared/acappella/feature-flag'; +import type { MicPermission } from '../../../shared/acappella/protocol'; +import { runSelfTest, type RuntimeSelfTestReport } from '../../acappella/runtime/runtime-selftest'; + +export interface VoiceRuntimeInfo { + /** The Encore Feature. When false, none of the rest is expected to work. */ + enabled: boolean; + microphone: { + permission: MicPermission; + /** True when the OS prompt has not been shown yet: not a refusal. */ + canPrompt: boolean; + }; + runtimes: Array<{ + id: string; + moduleId: string; + versionPin: string; + /** Whether the package is a dependency of this build at all. */ + declared: boolean; + /** How the binary is meant to arrive on THIS platform. */ + prebuild: NativePrebuildAvailability | 'unsupported-platform'; + requiresElectronRebuild: boolean; + }>; + /** Null when the self-test did not run. `selfTestSkipped` or `selfTestError` says why. */ + selfTest: RuntimeSelfTestReport | null; + selfTestError?: string; + /** + * Why the self-test was deliberately not run. Distinct from `selfTestError`, + * which means it ran and blew up: a reader of a support package has to be able + * to tell "we chose not to" from "it broke". + */ + selfTestSkipped?: string; +} + +export async function collectVoiceRuntime(settingsStore: Store): Promise { + const platformKey = nativePlatformKey(process.platform, process.arch); + + const runtimes = NATIVE_RUNTIMES.map((runtime) => ({ + id: runtime.id, + moduleId: runtime.moduleId, + versionPin: runtime.versionPin, + declared: runtime.declared, + prebuild: platformKey ? runtime.prebuilds[platformKey] : ('unsupported-platform' as const), + requiresElectronRebuild: runtime.requiresElectronRebuild, + })); + + const enabled = isACappellaEnabled(settingsStore); + + let selfTest: RuntimeSelfTestReport | null = null; + let selfTestError: string | undefined; + let selfTestSkipped: string | undefined; + if (!enabled) { + selfTestSkipped = + 'A Cappella is switched off in Encore Features, so no native runtime was loaded.'; + } else { + try { + selfTest = await runSelfTest(); + } catch (error) { + // runSelfTest is written not to throw, so this is belt and braces: a + // diagnostic that takes the whole debug package down with it would remove + // the one artifact the user was trying to produce. + selfTestError = error instanceof Error ? error.message : String(error); + } + } + + return { + enabled, + microphone: selfTest + ? selfTest.microphone + : { permission: 'unknown' as MicPermission, canPrompt: false }, + runtimes, + selfTest, + selfTestError, + selfTestSkipped, + }; +} diff --git a/src/main/debug-package/index.ts b/src/main/debug-package/index.ts index 6b4b120dfb..ba81e04df8 100644 --- a/src/main/debug-package/index.ts +++ b/src/main/debug-package/index.ts @@ -28,6 +28,7 @@ import { collectWindowsDiagnostics, WindowsDiagnosticsInfo, } from './collectors/windows-diagnostics'; +import { collectVoiceRuntime } from './collectors/voice-runtime'; import { createZipPackage, PackageContents } from './packager'; import { logger } from '../utils/logger'; import { AgentDetector } from '../agents'; @@ -136,6 +137,20 @@ export async function generateDebugPackage( logger.error('Failed to collect Windows diagnostics', 'DebugPackage', error); } + // Collect A Cappella voice runtime state (always included). The self-test runs + // only when the Encore Feature is on, so a package built with voice switched + // off loads no native runtime; the static runtime table is reported either + // way. No model and no audio device is touched in either case. + try { + const voiceRuntime = await collectVoiceRuntime(deps.settingsStore); + contents['voice-runtime.json'] = voiceRuntime; + filesIncluded.push('voice-runtime.json'); + } catch (error) { + const errMsg = error instanceof Error ? error.message : String(error); + errors.push(`voice-runtime: ${errMsg}`); + logger.error('Failed to collect voice runtime info', 'DebugPackage', error); + } + // Collect groups (always included) try { const groupsData = collectGroups(deps.groupsStore); @@ -314,6 +329,12 @@ export function previewDebugPackage(): { { id: 'storage', name: 'Storage Info', included: true, sizeEstimate: '< 2 KB' }, { id: 'groupChats', name: 'Group Chat Metadata', included: true, sizeEstimate: '< 5 KB' }, { id: 'batchState', name: 'Auto Run State', included: true, sizeEstimate: '< 5 KB' }, + { + id: 'voiceRuntime', + name: 'Voice Runtime and Self-Test', + included: true, + sizeEstimate: '< 2 KB', + }, ], }; } diff --git a/src/main/debug-package/packager.ts b/src/main/debug-package/packager.ts index 3b0ed8ea1f..75f328a661 100644 --- a/src/main/debug-package/packager.ts +++ b/src/main/debug-package/packager.ts @@ -14,6 +14,7 @@ export interface PackageContents { 'agents.json': unknown; 'external-tools.json': unknown; 'windows-diagnostics.json': unknown; + 'voice-runtime.json': unknown; 'sessions.json': unknown; 'groups.json': unknown; 'processes.json': unknown; @@ -49,6 +50,7 @@ This package was generated by Maestro for debugging purposes. - storage-info.json - Storage locations and sizes - group-chats.json - Group chat metadata - batch-state.json - Auto Run state +- voice-runtime.json - A Cappella native runtimes, microphone permission, and voice self-test result ## Submission diff --git a/src/main/global-hotkey-manager.ts b/src/main/global-hotkey-manager.ts index 119301c298..34bf9cfcc6 100644 --- a/src/main/global-hotkey-manager.ts +++ b/src/main/global-hotkey-manager.ts @@ -1,20 +1,46 @@ /** - * Global Hotkey Manager + * Global Hotkey Registry * - * Owns the single system-wide "show Maestro" hotkey registered via Electron's - * globalShortcut API. The setting is stored as a key array (same format as the - * in-app shortcuts) and translated to an Electron Accelerator at registration - * time so users can record the hotkey using the same capture UI they already - * know. + * Owns every system-wide hotkey Maestro registers through Electron's + * `globalShortcut` API, keyed by a stable id. Each registration lives and dies + * on its own: a combo the OS has already claimed fails that id and leaves the + * others bound. That independence is the whole reason this stopped being a + * singleton. When there was one hotkey, "did it register" was a boolean; with + * the A Cappella voice hotkeys there are three, and a shared failure path would + * mean the user losing "show Maestro" because they picked a bad voice combo. * - * Registration failures (OS already bound the combo, accelerator invalid, etc.) - * are surfaced to the renderer via `globalHotkey:registrationFailed` so the - * Settings UI can show a toast and the user can pick a different combo. + * Settings still store a key array (`['Meta','Shift','M']`), the same format the + * in-app shortcut recorder produces, and it is translated to an Electron + * Accelerator at registration time. That is deliberate: the recording UI does not + * know this file exists and must not have to. + * + * Two failure kinds are distinguished because the user's next move differs: + * + * - `os-conflict` - another application (or the OS) owns the combo. Pick a + * different one. + * - `maestro-conflict` - two Maestro hotkeys are bound to the SAME combo. Left + * to Electron the second registration simply wins or silently loses + * depending on platform, and the user is left with a key that does the wrong + * one of two things. Detected here instead, and named, so the settings UI can + * say which other hotkey is holding it. + * + * Failures are reported through `globalHotkey:registrationFailed`, which now + * carries the whole status object (id included) rather than a bare key array. */ import { app, BrowserWindow, globalShortcut } from 'electron'; import { logger } from './utils/logger'; import { isMacOS } from '../shared/platformDetection'; +import { + SHOW_MAESTRO_HOTKEY_ID, + type GlobalHotkeyFailureReason, + type GlobalHotkeyStatus, +} from '../shared/global-hotkeys'; + +const LOG_CONTEXT = 'GlobalHotkey'; + +export { SHOW_MAESTRO_HOTKEY_ID }; +export type { GlobalHotkeyFailureReason, GlobalHotkeyStatus }; /** * Translate a key array (e.g. ['Meta','Shift','M']) into an Electron @@ -59,7 +85,7 @@ export function keysToAccelerator(keys: string[]): string | null { } /** Bring the Maestro window to the foreground from any app. */ -function summonMainWindow(window: BrowserWindow): void { +export function summonMainWindow(window: BrowserWindow): void { if (window.isDestroyed()) return; if (window.isMinimized()) window.restore(); if (!window.isVisible()) window.show(); @@ -69,75 +95,253 @@ function summonMainWindow(window: BrowserWindow): void { window.focus(); } -let currentAccelerator: string | null = null; -let getWindowFn: (() => BrowserWindow | null) | null = null; +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- /** - * Register (or re-register) the global "show Maestro" hotkey. - * Pass an empty array to clear the binding. + * The slice of `globalShortcut` the registry uses. * - * @returns `true` on success, `false` if registration failed. + * Injected so the state machine can be tested without an Electron main process. + * `isRegistered` is deliberately absent: cross-Maestro conflicts are tracked here + * (Electron reports our OWN registration as taken, which would make every + * re-registration look like a conflict) and OS conflicts are whatever `register` + * says. */ -export function setGlobalShowHotkey(keys: string[]): boolean { - // Always clear the previous binding first so a typo doesn't leave a stale - // shortcut registered. - if (currentAccelerator) { - try { - globalShortcut.unregister(currentAccelerator); - } catch (err) { - logger.warn( - `Failed to unregister previous global hotkey '${currentAccelerator}': ${err}`, - 'GlobalHotkey' - ); +export interface GlobalShortcutBackend { + register(accelerator: string, callback: () => void): boolean; + unregister(accelerator: string): void; +} + +const electronBackend: GlobalShortcutBackend = { + register: (accelerator, callback) => globalShortcut.register(accelerator, callback), + unregister: (accelerator) => globalShortcut.unregister(accelerator), +}; + +/** Called whenever a registration fails, including a re-registration after a rebind. */ +export type GlobalHotkeyFailureListener = (status: GlobalHotkeyStatus) => void; + +// --------------------------------------------------------------------------- +// Registry +// --------------------------------------------------------------------------- + +export class GlobalHotkeyRegistry { + private readonly backend: GlobalShortcutBackend; + private readonly handlers = new Map void>(); + private readonly statuses = new Map(); + /** Accelerator -> the id that successfully holds it. The conflict detector. */ + private readonly owners = new Map(); + private failureListener: GlobalHotkeyFailureListener | null = null; + + constructor(backend: GlobalShortcutBackend = electronBackend) { + this.backend = backend; + } + + /** + * Teach the registry what an id does. + * + * Separate from `setKeys` because the handler comes from app wiring and the + * keys come from settings, and those two arrive at different times: the + * settings watcher can fire before, after, or without the feature that owns + * the handler ever being switched on. Defining a handler for an id that + * already has keys re-registers it, so ordering does not matter. + */ + define(id: string, handler: () => void): void { + this.handlers.set(id, handler); + const existing = this.statuses.get(id); + if (existing && !existing.registered && existing.keys.length) { + this.setKeys(id, existing.keys); } - currentAccelerator = null; } - const accelerator = keysToAccelerator(keys); - if (!accelerator) { - logger.info('Global show hotkey cleared', 'GlobalHotkey'); - return true; + /** Forget an id entirely, releasing its combo. */ + remove(id: string): void { + this.clear(id); + this.handlers.delete(id); + this.statuses.delete(id); } - try { - const ok = globalShortcut.register(accelerator, () => { - const win = getWindowFn?.(); - if (win) summonMainWindow(win); - }); + /** + * Bind (or rebind) one id. An empty array clears it. + * + * Always releases the previous accelerator first, so a typo cannot leave a + * stale combo registered, and a rebind back onto a combo this id already held + * cannot report itself as its own conflict. + */ + setKeys(id: string, keys: string[]): GlobalHotkeyStatus { + this.clear(id); + + const accelerator = keysToAccelerator(keys); + if (!accelerator) { + // An empty array is the user switching the hotkey off, not a failure. + const status: GlobalHotkeyStatus = keys.length + ? { + id, + keys, + accelerator: null, + registered: false, + reason: 'invalid-accelerator', + message: 'That combination needs a non-modifier key.', + } + : { id, keys, accelerator: null, registered: false }; + this.statuses.set(id, status); + if (status.reason) this.reportFailure(status); + else logger.info(`Global hotkey '${id}' cleared`, LOG_CONTEXT); + return status; + } + + const owner = this.owners.get(accelerator); + if (owner && owner !== id) { + const status: GlobalHotkeyStatus = { + id, + keys, + accelerator, + registered: false, + reason: 'maestro-conflict', + conflictsWith: owner, + message: `${accelerator} is already used by another Maestro hotkey (${owner}).`, + }; + this.statuses.set(id, status); + this.reportFailure(status); + return status; + } + + const handler = this.handlers.get(id); + let ok = false; + let threw: Error | null = null; + try { + // A hotkey with no handler yet is still claimed, so the combo is reserved + // and reported as bound the moment `define` supplies the behaviour. + ok = this.backend.register(accelerator, () => this.handlers.get(id)?.()); + } catch (err) { + threw = err as Error; + } + if (!ok) { + const status: GlobalHotkeyStatus = { + id, + keys, + accelerator, + registered: false, + reason: threw ? 'register-error' : 'os-conflict', + message: threw + ? `${accelerator} could not be registered: ${threw.message}` + : `${accelerator} is already in use by another application.`, + }; + this.statuses.set(id, status); + this.reportFailure(status); + return status; + } + + const status: GlobalHotkeyStatus = { id, keys, accelerator, registered: true }; + this.statuses.set(id, status); + this.owners.set(accelerator, id); + if (!handler) { + logger.debug( + `Global hotkey '${id}' bound to ${accelerator} with no handler yet`, + LOG_CONTEXT + ); + } + logger.info(`Registered global hotkey '${id}': ${accelerator}`, LOG_CONTEXT); + return status; + } + + /** Release one id's combo, keeping its handler and its recorded keys. */ + clear(id: string): void { + const current = this.statuses.get(id); + if (!current?.registered || !current.accelerator) return; + try { + this.backend.unregister(current.accelerator); + } catch (err) { logger.warn( - `Failed to register global hotkey '${accelerator}' - likely already in use by another app`, - 'GlobalHotkey' + `Failed to unregister global hotkey '${id}' (${current.accelerator}): ${err}`, + LOG_CONTEXT ); - return false; } - currentAccelerator = accelerator; - logger.info(`Registered global show hotkey: ${accelerator}`, 'GlobalHotkey'); - return true; - } catch (err) { - logger.warn( - `Error registering global hotkey '${accelerator}': ${(err as Error).message}`, - 'GlobalHotkey' - ); - return false; + if (this.owners.get(current.accelerator) === id) this.owners.delete(current.accelerator); + this.statuses.set(id, { ...current, registered: false }); } -} -/** Tear down any registered shortcut. Safe to call multiple times. */ -export function disposeGlobalHotkey(): void { - if (currentAccelerator) { + status(id: string): GlobalHotkeyStatus | null { + return this.statuses.get(id) ?? null; + } + + /** Every known id's status, for the settings UI's per-hotkey inline state. */ + allStatuses(): GlobalHotkeyStatus[] { + return [...this.statuses.values()]; + } + + /** + * Subscribe to registration failures. One listener: the only subscriber is the + * IPC bridge, and a fan-out would just be a list with one entry in it. + */ + onFailure(listener: GlobalHotkeyFailureListener | null): void { + this.failureListener = listener; + } + + /** Release everything. Safe to call more than once. */ + disposeAll(): void { + for (const id of [...this.statuses.keys()]) this.clear(id); + this.owners.clear(); + } + + private reportFailure(status: GlobalHotkeyStatus): void { + logger.warn(`Global hotkey '${status.id}' failed: ${status.message}`, LOG_CONTEXT); try { - globalShortcut.unregister(currentAccelerator); - } catch { - // Ignore - app is shutting down or shortcut wasn't registered. + this.failureListener?.(status); + } catch (err) { + // A window destroyed between the failure and the notify must not turn a + // bad key combo into an unhandled exception during startup. + logger.warn(`Global hotkey failure listener threw: ${err}`, LOG_CONTEXT); } - currentAccelerator = null; } } +// --------------------------------------------------------------------------- +// Process-wide instance +// --------------------------------------------------------------------------- + +const registry = new GlobalHotkeyRegistry(); + +/** The one registry the app uses. Voice hotkeys register against this. */ +export function getGlobalHotkeyRegistry(): GlobalHotkeyRegistry { + return registry; +} + +let getWindowFn: (() => BrowserWindow | null) | null = null; + /** - * Wire the manager to the main window getter. Called once during startup. + * Wire the registry to the main window getter and register the "show Maestro" + * behaviour. Called once during startup. */ export function initGlobalHotkey(getWindow: () => BrowserWindow | null): void { getWindowFn = getWindow; + registry.define(SHOW_MAESTRO_HOTKEY_ID, () => { + const win = getWindowFn?.(); + if (win) summonMainWindow(win); + }); +} + +/** + * Register (or re-register) the global "show Maestro" hotkey. + * Pass an empty array to clear the binding. + * + * Kept as a free function so the settings watcher in `main/index.ts` does not + * have to know about ids. + * + * @returns `true` on success (including a deliberate clear), `false` if + * registration failed. + */ +export function setGlobalShowHotkey(keys: string[]): boolean { + return !registry.setKeys(SHOW_MAESTRO_HOTKEY_ID, keys).reason; +} + +/** Tear down the "show Maestro" shortcut. Safe to call multiple times. */ +export function disposeGlobalHotkey(): void { + registry.clear(SHOW_MAESTRO_HOTKEY_ID); +} + +/** Tear down every registered shortcut. Wired to `will-quit`. */ +export function disposeAllGlobalHotkeys(): void { + registry.disposeAll(); } diff --git a/src/main/index.ts b/src/main/index.ts index 2ce1d9b08d..2f06e2448d 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -27,9 +27,10 @@ import { AgentDetector } from './agents'; import { createAgentConfigLookup } from './agents/agent-config-lookup'; import { shouldDropSentryEvent } from '../shared/sentryFilters'; import { + disposeAllGlobalHotkeys, + getGlobalHotkeyRegistry, initGlobalHotkey, setGlobalShowHotkey, - disposeGlobalHotkey, } from './global-hotkey-manager'; import { CueEngine } from './cue/cue-engine'; import { createCueSupervisorHooks } from './cue/cue-first-party'; @@ -159,6 +160,10 @@ import { closeCadenzaHudWindow, type QuitHandler, } from './app-lifecycle'; +// A Cappella's hidden audio host window (created lazily on the first voice +// session, closed here when the Encore Feature is switched off). +import { closeAcappellaAudioHostWindow } from './acappella/audio-host-window'; +import { shutdownACappellaForDisable } from './ipc/handlers/acappella'; // Multi-window registry (single source of truth for window<->session ownership) import { WindowRegistry } from './window-registry'; // Multi-window startup restore: turn the persisted MultiWindowState back into @@ -525,6 +530,17 @@ const cadenzaHudDeps = { windowRegistry, }; +// Same shape, different boot query: A Cappella's hidden audio host loads the +// renderer bundle with `?acappellaAudio`. Nothing is created here - the window +// is built on the first voice session start. +const acappellaAudioHostDeps = { + isDevelopment, + preloadPath, + rendererProductionUrl, + devServerUrl, + windowRegistry, +}; + // See src/main/cadenza-bridge/ and src/main/plugin-host-view-bridge/ for what // each of these does (Phase 5 refactoring). const { deliverCadenza } = createCadenzaDelivery({ @@ -546,6 +562,17 @@ registerCadenzaIpcHandlers({ getMainWindow: () => mainWindow, settingsStore: sto // read path to refresh plugin discovery. store.onDidChange('encoreFeatures', (encoreFeatures) => { if (encoreFeatures?.concerto !== true) closeCadenzaHudWindow(); + // Switching A Cappella off releases the microphone immediately rather than at + // quit: a hidden window holding an open capture device is exactly the thing a + // user turning the feature off is asking to be rid of. + if (encoreFeatures?.aCappella !== true) { + // The full stand-down: session, audio bridge, inference pipeline, Bonjour + // advert, and every connected device. Awaited only for its own ordering - + // the window is closed after it, because the bridge stops capture through + // the window it is about to lose, and closing first would leave the pipeline + // counting frames from a device nobody owns any more. + void shutdownACappellaForDisable().finally(() => closeAcappellaAudioHostWindow()); + } if (encoreFeatures?.plugins !== true) { pluginSandboxHost?.stopAll(); pluginGroupingRegistry?.clearAll(); @@ -2729,6 +2756,7 @@ app bootstrapStore, safeSend, windowRegistry, + acappellaAudioHostDeps, windowManager, createWebServer, wakatimeManager, @@ -2854,25 +2882,24 @@ app // any) and re-register live when the setting changes from any source // (settings UI, CLI, external file edit). initGlobalHotkey(() => mainWindow); - const initialHotkey = store.get('globalShowHotkey', []) as string[]; - if (Array.isArray(initialHotkey) && initialHotkey.length > 0) { - const ok = setGlobalShowHotkey(initialHotkey); + // One failure path for every id, so a voice hotkey the OS refused reports + // itself the same way the "show Maestro" one always has. + getGlobalHotkeyRegistry().onFailure((status) => { // intentionally not bridged: window-specific - if (!ok && mainWindow && isWebContentsAvailable(mainWindow)) { - mainWindow.webContents.send('globalHotkey:registrationFailed', initialHotkey); + if (mainWindow && isWebContentsAvailable(mainWindow)) { + mainWindow.webContents.send('globalHotkey:registrationFailed', status); } + }); + const initialHotkey = store.get('globalShowHotkey', []) as string[]; + if (Array.isArray(initialHotkey) && initialHotkey.length > 0) { + setGlobalShowHotkey(initialHotkey); } store.onDidChange('globalShowHotkey', (value) => { - const keys = Array.isArray(value) ? (value as string[]) : []; - const ok = setGlobalShowHotkey(keys); - // intentionally not bridged: window-specific - if (!ok && mainWindow && isWebContentsAvailable(mainWindow)) { - mainWindow.webContents.send('globalHotkey:registrationFailed', keys); - } + setGlobalShowHotkey(Array.isArray(value) ? (value as string[]) : []); }); // Electron auto-unregisters globalShortcuts on quit, but be explicit so the // behavior survives any future change to that policy. - app.on('will-quit', disposeGlobalHotkey); + app.on('will-quit', disposeAllGlobalHotkeys); // Flush any deep link URL that arrived before the window was ready (cold start) flushPendingDeepLink(() => mainWindow); diff --git a/src/main/ipc/bootstrap/index.ts b/src/main/ipc/bootstrap/index.ts index 7da12cd349..f0ea40dda0 100644 --- a/src/main/ipc/bootstrap/index.ts +++ b/src/main/ipc/bootstrap/index.ts @@ -54,6 +54,9 @@ import { registerTabsHandlers, registerContextTimelineHandlers, registerPianolaHandlers, + registerACappellaHandlers, + stopVoiceSessionForClosedWindow, + registerACappellaModelsHandlers, registerPluginsHandlers, registerAgentRunHandlers, registerCoworkingHandlers, @@ -305,6 +308,63 @@ export function setupIpcHandlers(deps: IpcBootstrapDependencies): void { }); } + // Register A Cappella handlers (voice sessions). Registration is free: the + // session service, its providers, and the dispatch executor are all built + // lazily on the first start-session, so an app with the Encore Feature off + // pays nothing for these channels being present. + registerACappellaHandlers({ + settingsStore: deps.settingsStore, + getMainWindow: deps.getMainWindow, + // Multi-window dispatch: a spoken instruction has to land in the window that + // owns the agent. Sending it to whichever window is "main" would activate an + // agent that window does not own, which is how a window ends up showing + // "No agents". + getWindowForSession: (agentSessionId: string) => { + const windowId = deps.windowRegistry.getWindowForSession(agentSessionId); + return windowId ? (deps.windowRegistry.get(windowId)?.browserWindow ?? null) : null; + }, + // Which window's HUD a voice session belongs to. Voice events are broadcast + // to every window like every other main -> renderer push, so this is what + // keeps a session the user opened in one window from drawing a HUD in all of + // them. A trigger with no window behind it (global hotkey, wake word, paired + // phone) lands on the focused window. + resolveVoiceWindowId: (sender) => + (sender + ? deps.windowRegistry.findBySender(sender) + : deps.windowRegistry.getFocusedAppWindow() + )?.id ?? null, + safeSend: deps.safeSend, + audioHostDeps: deps.acappellaAudioHostDeps, + // The paired-device transport rides the web server's authenticated socket, + // so the QR code cannot be produced without its token and port. + getWebServer: deps.getWebServer, + // What the `voiceCurrentAgent` hotkey binds to. The renderer is the only + // thing that knows which agent is on screen, and it persists that here on + // every switch, so main can answer without a round trip - which matters, + // because a global hotkey handler cannot await one. + getFocusedAgentSessionId: () => + (deps.sessionsStore.get('activeSessionId') as string | undefined) || null, + // The event source for the agent-output tap. It is the SAME emitter the + // desktop transcript listens to, which is what keeps what is spoken and what + // is on screen from drifting apart. + getProcessManager: () => deps.getProcessManager(), + getAgentType: (agentSessionId: string) => { + const sessions = (deps.sessionsStore.get('sessions', []) ?? []) as Array<{ + id?: string; + agentType?: string; + }>; + return sessions.find((session) => session.id === agentSessionId)?.agentType; + }, + }); + + // Register A Cappella model handlers (catalog, download, verify, disk). Also + // free: the catalog is a frozen constant and the downloader is built lazily, so + // nothing here reaches the network until the user presses Download. + registerACappellaModelsHandlers({ + settingsStore: deps.settingsStore, + safeSend: deps.safeSend, + }); + // Register Plugins handlers (community plugin subsystem, list-only in Phase 0). // The manager is constructed during core-service init above; guard for types. const pluginManager = deps.getPluginManager(); @@ -384,6 +444,11 @@ export function setupIpcHandlers(deps: IpcBootstrapDependencies): void { if ((change.type === 'name-changed' || change.type === 'panel-changed') && change.windowId) { saveWindowState(deps.windowStateStore, deps.windowRegistry, change.windowId); } + // A voice session is shown by exactly one window, so closing that window + // would otherwise leave an open microphone with no surface anywhere. + if (change.type === 'removed' && change.windowId) { + void stopVoiceSessionForClosedWindow(change.windowId); + } }); // Record aggregate multi-window usage telemetry (secondary windows opened + diff --git a/src/main/ipc/bootstrap/types.ts b/src/main/ipc/bootstrap/types.ts index d14970a526..bb96d0331a 100644 --- a/src/main/ipc/bootstrap/types.ts +++ b/src/main/ipc/bootstrap/types.ts @@ -15,6 +15,7 @@ import type { WakaTimeManager } from '../../wakatime-manager'; import type { MaestroCliManager } from '../../maestro-cli-manager'; import type { SafeSendFn } from '../../utils/safe-send'; import type { WindowRegistry } from '../../window-registry'; +import type { AudioHostWindowDeps } from '../../acappella/audio-host-window'; import type { createWindowManager } from '../../app-lifecycle'; import type { createWebServerFactory } from '../../web-server/web-server-factory'; import type { @@ -59,6 +60,8 @@ export interface IpcBootstrapDependencies { bootstrapStore: ReturnType['bootstrapStore']; safeSend: SafeSendFn; windowRegistry: WindowRegistry; + /** Bundle/preload paths the hidden A Cappella audio host window loads from. */ + acappellaAudioHostDeps: AudioHostWindowDeps; windowManager: ReturnType; createWebServer: ReturnType; wakatimeManager: WakaTimeManager; diff --git a/src/main/ipc/handlers/acappella-devices.ts b/src/main/ipc/handlers/acappella-devices.ts new file mode 100644 index 0000000000..19a6ca496c --- /dev/null +++ b/src/main/ipc/handlers/acappella-devices.ts @@ -0,0 +1,252 @@ +/** + * A Cappella paired-device IPC handlers. + * + * The transport in front of `src/main/acappella/transport/` and + * `src/main/acappella/pairing/`. Thin by the same rule as the rest of the A + * Cappella IPC layer: every policy that matters - who may pair, what a code + * buys, what revocation does - lives in those modules, and this file only turns + * channels into calls. + * + * Two things it is careful about. + * + * **A pairing payload is a credential.** `acappella:start-pairing` returns the + * server token, because that is what a device needs to reach the WebSocket at + * all. It is therefore gated on the Encore Feature and it is only ever rendered + * as a QR code the user is looking at. Nothing here stores it. + * + * **Revocation must work with the feature off.** `revoke-device`, + * `revoke-all-devices`, and `list-devices` stay callable when the flag is off, + * following the `stop-session` and `models:remove` precedent: the moment a user + * turns the feature off is exactly when they may want to cut a phone loose, and + * a control that disappears then is a control that was never trustworthy. + */ + +import { ipcMain } from 'electron'; + +import { requireACappellaEnabled } from '../../../shared/acappella/feature-flag'; +import type { IceProbeResult } from '../../../shared/acappella/webrtc-host'; +import { getACappellaTransport } from '../../acappella'; +import type { DeviceStatus, PairingPayload } from '../../acappella/transport'; +import type { DiscoveryStatus } from '../../acappella/pairing/discovery'; +import type { PairingRequest } from '../../acappella/pairing/pairing-service'; +import { + describeIceReach, + readIceSettings, + TUNNEL_MEDIA_NOTE, + type IceTransportSettings, +} from '../../acappella/transport/ice-config'; +import { withIpcErrorLogging, type CreateHandlerOptions } from '../../utils/ipcHandler'; + +const LOG_CONTEXT = '[ACappellaDevices]'; + +/** Broadcast whenever the device list or a connection state changes. */ +export const ACAPPELLA_DEVICES_CHANNEL = 'acappella:devices-changed'; + +/** Broadcast when a device asks to pair, and again (with null) when it stops asking. */ +export const ACAPPELLA_PAIRING_REQUEST_CHANNEL = 'acappella:pairing-request'; + +const handlerOpts = (operation: string): Pick => ({ + context: LOG_CONTEXT, + operation, +}); + +export interface ACappellaDeviceHandlerDependencies { + settingsStore: { + get: (key: string, defaultValue?: unknown) => unknown; + set?: (key: string, value: unknown) => void; + }; +} + +/** The Encore gate. See `src/shared/acappella/feature-flag.ts`. */ +const requireEnabled = requireACappellaEnabled; + +/** The ICE section of the A Cappella settings blob, widened. */ +export function readStoredIceSettings( + store: ACappellaDeviceHandlerDependencies['settingsStore'] +): IceTransportSettings { + const blob = (store.get('acappella', {}) ?? {}) as { ice?: unknown }; + return readIceSettings(blob.ice); +} + +export function registerACappellaDeviceHandlers(deps: ACappellaDeviceHandlerDependencies): void { + const { settingsStore } = deps; + + /** Null when the transport has never been built, which a client reads as "no devices". */ + const transport = () => getACappellaTransport(); + + ipcMain.handle( + 'acappella:start-pairing', + withIpcErrorLogging(handlerOpts('startPairing'), async (): Promise => { + requireEnabled(settingsStore); + return transport()?.startPairing() ?? null; + }) + ); + + ipcMain.handle( + 'acappella:pairing-status', + withIpcErrorLogging( + handlerOpts('pairingStatus'), + async (): Promise<{ + payload: PairingPayload | null; + request: PairingRequest | null; + discovery: DiscoveryStatus | null; + manualHint: string; + }> => { + const live = transport(); + return { + payload: live?.currentPairingPayload() ?? null, + request: live?.pairing.pendingRequest() ?? null, + discovery: live?.discoveryStatus() ?? null, + manualHint: live?.manualHint() ?? '', + }; + } + ) + ); + + ipcMain.handle( + 'acappella:cancel-pairing', + withIpcErrorLogging(handlerOpts('cancelPairing'), async (): Promise => { + transport()?.pairing.cancelPairing(); + }) + ); + + ipcMain.handle( + 'acappella:approve-device', + withIpcErrorLogging( + handlerOpts('approveDevice'), + // The affirmative action. Without this, knowing a six-character code + // would be enough to hold somebody's microphone. + async (_event, payload: unknown): Promise => { + requireEnabled(settingsStore); + const { requestId, name } = (payload ?? {}) as { requestId?: string; name?: string }; + if (typeof requestId !== 'string' || !requestId) throw new Error('InvalidPairingRequest'); + const device = await transport()?.pairing.approve(requestId, name); + return !!device; + } + ) + ); + + ipcMain.handle( + 'acappella:deny-device', + withIpcErrorLogging( + handlerOpts('denyDevice'), + async (_event, requestId: unknown): Promise => { + if (typeof requestId !== 'string' || !requestId) throw new Error('InvalidPairingRequest'); + transport()?.pairing.deny(requestId); + } + ) + ); + + // Deliberately ungated: see the module header. + ipcMain.handle( + 'acappella:list-devices', + withIpcErrorLogging( + handlerOpts('listDevices'), + async (): Promise => (await transport()?.listDevices()) ?? [] + ) + ); + + ipcMain.handle( + 'acappella:rename-device', + withIpcErrorLogging( + handlerOpts('renameDevice'), + async (_event, payload: unknown): Promise => { + const { deviceId, name } = (payload ?? {}) as { deviceId?: string; name?: string }; + if (typeof deviceId !== 'string' || typeof name !== 'string') { + throw new Error('InvalidDeviceRename'); + } + return (await transport()?.pairing.rename(deviceId, name)) ?? false; + } + ) + ); + + ipcMain.handle( + 'acappella:revoke-device', + withIpcErrorLogging( + handlerOpts('revokeDevice'), + async (_event, deviceId: unknown): Promise => { + if (typeof deviceId !== 'string' || !deviceId) throw new Error('InvalidDeviceId'); + return (await transport()?.revokeDevice(deviceId)) ?? false; + } + ) + ); + + ipcMain.handle( + 'acappella:forget-device', + withIpcErrorLogging( + handlerOpts('forgetDevice'), + async (_event, deviceId: unknown): Promise => { + if (typeof deviceId !== 'string' || !deviceId) throw new Error('InvalidDeviceId'); + return (await transport()?.pairing.forget(deviceId)) ?? false; + } + ) + ); + + ipcMain.handle( + 'acappella:revoke-all-devices', + withIpcErrorLogging( + handlerOpts('revokeAllDevices'), + async (): Promise => (await transport()?.revokeAllDevices()) ?? 0 + ) + ); + + ipcMain.handle( + 'acappella:disconnect-all-devices', + withIpcErrorLogging(handlerOpts('disconnectAllDevices'), async (): Promise => { + transport()?.disconnectAll(); + }) + ); + + ipcMain.handle( + 'acappella:ice-settings', + withIpcErrorLogging( + handlerOpts('iceSettings'), + async (): Promise<{ + settings: IceTransportSettings; + reach: string; + tunnelNote: string; + discovery: DiscoveryStatus | null; + }> => { + const settings = readStoredIceSettings(settingsStore); + return { + settings, + reach: describeIceReach(settings), + tunnelNote: TUNNEL_MEDIA_NOTE, + discovery: transport()?.discoveryStatus() ?? null, + }; + } + ) + ); + + ipcMain.handle( + 'acappella:test-connection', + withIpcErrorLogging(handlerOpts('testConnection'), async (): Promise => { + requireEnabled(settingsStore); + const live = transport(); + if (!live) { + return { + host: false, + stun: false, + relay: false, + best: 'unknown', + error: 'A Cappella has not started yet. Open a voice session and try again.', + }; + } + return live.testConnection(); + }) + ); + + ipcMain.handle( + 'acappella:set-discovery', + withIpcErrorLogging( + handlerOpts('setDiscovery'), + async (_event, enabled: unknown): Promise => { + const live = transport(); + if (!live) return null; + if (enabled === true) await live.discovery.start(); + else await live.discovery.stop(); + return live.discoveryStatus(); + } + ) + ); +} diff --git a/src/main/ipc/handlers/acappella-models.ts b/src/main/ipc/handlers/acappella-models.ts new file mode 100644 index 0000000000..3ba98a0ce8 --- /dev/null +++ b/src/main/ipc/handlers/acappella-models.ts @@ -0,0 +1,240 @@ +/** + * A Cappella model IPC handlers. + * + * The transport in front of the model store, the downloader, and the capability + * gate. Thin by the same rule as `acappella.ts`: every policy that matters lives + * in `src/main/acappella/models/`, and this file only turns channels into calls. + * + * Two properties this module is responsible for: + * + * - **Enabling the Encore Feature touches the network exactly never.** + * Registering these channels constructs nothing, opens no socket, and reads + * no remote metadata. `models:list` is a disk read against a frozen local + * catalog. The first byte of traffic in the whole subsystem is a + * `models:download` the user pressed a button to send. + * - **Disk is reclaimable after the feature is switched off.** `models:remove`, + * `models:remove-all`, and `models:footprint` stay callable with the flag + * off, following the `stop-session` precedent: a feature that hides the + * button that frees 1.4 GB the moment you stop wanting the feature is a + * feature that keeps your disk hostage. + * + * Progress is BROADCAST on `models:progress`, matching the multi-window + * invariant in `src/main/utils/safe-send.ts`. Throttling happens at the source + * (the downloader) rather than here. + */ + +import { ipcMain } from 'electron'; + +import { + VOICE_MODEL_CATALOG, + getVoiceModel, + type VoiceModelEntry, +} from '../../../shared/acappella/model-catalog'; +import type { VoiceReadiness } from '../../../shared/acappella/readiness'; +import { requireACappellaEnabled } from '../../../shared/acappella/feature-flag'; +import { resolveVoiceReadiness } from '../../acappella/models/capability-gate'; +import { + getModelDownloader, + type DownloadProgress, + type DownloadResult, +} from '../../acappella/models/model-downloader'; +import { + installPathFor, + listStatuses, + remove, + removeAll, + totalFootprint, + verify, + type ModelFootprint, + type ModelStatus, + type VerifyResult, +} from '../../acappella/models/model-store'; +import { readVoiceProviderSettings } from '../../acappella/providers/provider-registry'; +import { withIpcErrorLogging, type CreateHandlerOptions } from '../../utils/ipcHandler'; +import type { SafeSendFn } from '../../utils/safe-send'; + +const LOG_CONTEXT = '[ACappellaModels]'; + +/** The push channel download progress goes out on. */ +export const ACAPPELLA_MODEL_PROGRESS_CHANNEL = 'models:progress'; + +/** + * One catalog entry joined to what is on disk. This is the row Voice Setup + * renders: the bill of materials and its install state in one object, so the UI + * never has to correlate two lists. + */ +export interface VoiceModelListing { + entry: VoiceModelEntry; + status: ModelStatus; + /** Absolute install paths, one per catalog file, for the "where does this go" line. */ + installPaths: string[]; +} + +export interface ACappellaModelsHandlerDependencies { + settingsStore: { + get: (key: string, defaultValue?: unknown) => unknown; + }; + /** Broadcasts to every window and to the web-desktop bridge. */ + safeSend: SafeSendFn; +} + +const handlerOpts = (operation: string): Pick => ({ + context: LOG_CONTEXT, + operation, +}); + +/** The Encore gate. See `src/shared/acappella/feature-flag.ts`. */ +const requireEnabled = requireACappellaEnabled; + +/** Reject anything that is not a catalog id before it can reach a path join. */ +function requireModelId(raw: unknown): string { + if (typeof raw !== 'string' || !getVoiceModel(raw)) throw new Error('UnknownVoiceModel'); + return raw; +} + +/** + * Readiness for the current settings. + * + * Read fresh every call rather than cached: a model finishing its download, a + * file going corrupt, and an API key being pasted all change the answer, and a + * stale "not ready" is a disabled button nobody can explain. + */ +export async function readVoiceReadiness( + settingsStore: ACappellaModelsHandlerDependencies['settingsStore'] +): Promise { + const stored = (settingsStore.get('acappella', {}) ?? {}) as { handsFree?: unknown }; + // No `hasApiKey` override: keys live in the OS keychain, never in settings, so + // the gate's default (which reads the keychain) is the only correct answer. + return resolveVoiceReadiness({ + settings: readVoiceProviderSettings(settingsStore), + handsFreeEnabled: stored.handsFree === true, + }); +} + +/** + * Register the A Cappella model handlers. + * + * Wired from `setupIpcHandlers()` (src/main/ipc/bootstrap/index.ts), which is + * what the running app calls. A handler registered only through + * `registerAllHandlers()` in handlers/index.ts would be dead. + */ +export function registerACappellaModelsHandlers(deps: ACappellaModelsHandlerDependencies): void { + const { settingsStore, safeSend } = deps; + + // Subscribed once, for the life of the app. The downloader is a singleton and + // its listener set would otherwise grow one entry per registration. + getModelDownloader().onProgress((progress: DownloadProgress) => { + safeSend(ACAPPELLA_MODEL_PROGRESS_CHANNEL, progress); + }); + + const wrappedList = withIpcErrorLogging( + handlerOpts('list'), + // A pure disk read against the frozen catalog. No metadata request, no HEAD, + // no revision lookup: the catalog already knows every size and hash, which is + // the whole reason it is pinned. + async (): Promise => { + const statuses = await listStatuses(); + return VOICE_MODEL_CATALOG.map((entry, index) => ({ + entry, + status: statuses[index], + installPaths: entry.files.map((file) => installPathFor(entry, file)), + })); + } + ); + + const wrappedDownload = withIpcErrorLogging( + handlerOpts('download'), + async (rawId: unknown): Promise => + getModelDownloader().download(requireModelId(rawId)) + ); + + const wrappedPause = withIpcErrorLogging( + handlerOpts('pause'), + async (rawId: unknown): Promise => getModelDownloader().pause(requireModelId(rawId)) + ); + + const wrappedResume = withIpcErrorLogging( + handlerOpts('resume'), + async (rawId: unknown): Promise => + getModelDownloader().resume(requireModelId(rawId)) + ); + + const wrappedCancel = withIpcErrorLogging( + handlerOpts('cancel'), + async (rawId: unknown): Promise => getModelDownloader().cancel(requireModelId(rawId)) + ); + + const wrappedVerify = withIpcErrorLogging( + handlerOpts('verify'), + async (rawId: unknown): Promise => verify(requireModelId(rawId)) + ); + + const wrappedRemove = withIpcErrorLogging( + handlerOpts('remove'), + async (rawId: unknown): Promise => { + const id = requireModelId(rawId); + // Cancel first: deleting the directory under a live writer would let the + // writer recreate the file it was told to stop writing. + await getModelDownloader().cancel(id); + return remove(id); + } + ); + + const wrappedRemoveAll = withIpcErrorLogging( + handlerOpts('removeAll'), + async (): Promise => { + await getModelDownloader().cancelAll(); + return removeAll(); + } + ); + + const wrappedFootprint = withIpcErrorLogging( + handlerOpts('footprint'), + async (): Promise => totalFootprint() + ); + + const wrappedReadiness = withIpcErrorLogging( + handlerOpts('readiness'), + async (): Promise => readVoiceReadiness(settingsStore) + ); + + ipcMain.handle('models:list', async (event): Promise => { + requireEnabled(settingsStore); + return wrappedList(event); + }); + + ipcMain.handle('models:download', async (event, id: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedDownload(event, id); + }); + + ipcMain.handle('models:pause', async (event, id: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedPause(event, id); + }); + + ipcMain.handle('models:resume', async (event, id: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedResume(event, id); + }); + + // Ungated: a download in flight when the feature is switched off must still be + // stoppable, or the app keeps pulling gigabytes for a feature that is now off. + ipcMain.handle('models:cancel', wrappedCancel); + + ipcMain.handle('models:verify', async (event, id: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedVerify(event, id); + }); + + // Ungated, with `models:footprint` and `models:remove-all`: the reclaim-disk + // offer appears exactly when the Encore Feature has just been turned OFF. + ipcMain.handle('models:remove', wrappedRemove); + ipcMain.handle('models:remove-all', wrappedRemoveAll); + ipcMain.handle('models:footprint', wrappedFootprint); + + ipcMain.handle('models:readiness', async (event): Promise => { + requireEnabled(settingsStore); + return wrappedReadiness(event); + }); +} diff --git a/src/main/ipc/handlers/acappella.ts b/src/main/ipc/handlers/acappella.ts new file mode 100644 index 0000000000..0daa3166d9 --- /dev/null +++ b/src/main/ipc/handlers/acappella.ts @@ -0,0 +1,1487 @@ +/** + * A Cappella IPC Handlers + * + * The Electron transport in front of the headless voice session service. Thin + * on purpose: every rule that matters (state machine, provider substitution, + * dispatch) lives in `src/main/acappella/`, and this file only translates + * channels into calls and protocol events into a push. + * + * Three properties this module is responsible for: + * - **Nothing runs until a session is started.** Enabling the Encore Feature + * opens no device, downloads nothing, and constructs no provider: the + * service is built lazily on the first `acappella:start-session`. + * - **Provider resolution goes through the registry, always.** No concrete + * provider is imported here, so "never silently substitute a cloud provider + * for a missing local one" stays a property of the registry rather than of + * an import in the transport layer. + * - **Every client sees the same stream.** Protocol events are BROADCAST on + * `acappella:event` (all windows plus the web-desktop bridge), matching the + * multi-window invariant in `src/main/utils/safe-send.ts`. There is no + * per-window subscription list: the session is a single-floor thing, and a + * client that does not want the events simply does not listen. + * + * Gated at the handler on `encoreFeatures.aCappella`, following the Pianola + * precedent: when the flag is off every channel throws 'ACappellaDisabled' so + * the renderer can tell "feature off" from "no session". The one exception is + * `acappella:stop-session`, which stays callable so toggling the feature off + * mid-session can still release the floor. + */ + +import { app, ipcMain, type BrowserWindow, type WebContents } from 'electron'; +import { hostname } from 'os'; + +import { withIpcErrorLogging, type CreateHandlerOptions } from '../../utils/ipcHandler'; +import { logger } from '../../utils/logger'; +import type { SafeSendFn } from '../../utils/safe-send'; +import type { + InterruptSource, + RosterAgent, + VoiceEvent, + VoiceScope, + VoiceWindowId, +} from '../../../shared/acappella/protocol'; +import { getSessionsStore } from '../../stores/getters'; +import { createConductorRouter } from '../../acappella/router/conductor-router'; +import { invalidateRoutingContext } from '../../acappella/router/routing-context'; +import { + flushRoutingLog, + lastRoutingTurn, + loadRoutingLog, + noteRoutingOutcome, + readRoutingLog, + routingQuality, + type RoutingLogEntry, + type RoutingQuality, +} from '../../acappella/router/routing-log'; +import { + getMicPermission, + noteCaptureFailure, + noteCaptureStarted, + openMicSystemSettings, + requestMicPermission, + type MicPermissionInfo, +} from '../../acappella/permissions/mic-permission'; +import { + ACAPPELLA_AUDIO_COMMAND_CHANNEL, + ACAPPELLA_AUDIO_FRAME_CHANNEL, + ACAPPELLA_AUDIO_STATUS_CHANNEL, + ACAPPELLA_SYSTEM_DEFAULT_INPUT, + type AudioDeviceInfo, + type AudioFrame, + type AudioHostCommand, + type AudioHostStatus, +} from '../../../shared/acappella/audio-host'; +import { + closeAcappellaAudioHostWindow, + createRendererVoiceBridge, + createVoiceAudioBridge, + createVoiceRouteExecutor, + disposeVoiceSessionService, + ensureAcappellaAudioHostWindow, + getAcappellaAudioHostWindow, + getVoiceSessionService, + initVoiceSessionService, + isAcappellaAudioHostContents, + readAgentRoster, + type AudioHostWindowDeps, + type VoiceAudioBridge, + type VoiceSessionService, + type VoiceSessionSnapshot, +} from '../../acappella'; +import { + createAgentOutputTap, + type AgentOutputSource, + type AgentOutputTap, +} from '../../acappella/speech'; +import { readVoiceReadiness } from './acappella-models'; +import { DEFAULT_TTS_VOLUME } from '../../../shared/acappella/voice-controls'; +import { requireACappellaEnabled } from '../../../shared/acappella/feature-flag'; +import { + ACAPPELLA_SETTINGS_KEY, + buildProviderState, + pipelineKey, + readVoiceProviderSettings, + resolveVoicePipeline, + swapVoicePipeline, + type VoiceProviderResolution, + type VoiceProviderSubstitution, +} from '../../acappella/providers/provider-registry'; +import { + clearCredential, + listCredentialStates, + setCredential, + validateCredential, + type CredentialState, + type CredentialValidation, +} from '../../acappella/providers/credentials'; +import { + VOICE_CREDENTIAL_SERVICES, + type VoiceCredentialService, +} from '../../../shared/acappella/provider-catalog'; +import { lastTurn, type TurnBreakdown } from '../../acappella/telemetry/turn-metrics'; +import { installVoiceHotkeys, type VoiceHotkeyInstallation } from '../../acappella/hotkeys'; +import { describePressHoldCapability } from '../../acappella/hotkeys/press-hold'; +import type { GlobalHotkeyStatus } from '../../../shared/global-hotkeys'; +import { + createWakeDetector, + globalWakePhrase, + type WakeDetection, + type WakeDetector, + type WakePhrase, +} from '../../acappella/wake/wake-detector'; +import type { FloorControlSession } from '../../acappella/audio/floor-control'; +import { + disposeACappellaTransport, + getACappellaTransport, + initACappellaTransport, +} from '../../acappella'; +import type { ACappellaTransport } from '../../acappella/transport'; +import { + ACAPPELLA_WEBRTC_COMMAND_CHANNEL, + ACAPPELLA_WEBRTC_EVENT_CHANNEL, + type WebRtcHostEvent, +} from '../../../shared/acappella/webrtc-host'; +import { + ACAPPELLA_DEVICES_CHANNEL, + ACAPPELLA_PAIRING_REQUEST_CHANNEL, + registerACappellaDeviceHandlers, +} from './acappella-devices'; + +const LOG_CONTEXT = '[ACappella]'; + +/** The push channel every protocol event goes out on. */ +export const ACAPPELLA_EVENT_CHANNEL = 'acappella:event'; + +/** + * The wake-word tuning channel. + * + * Deliberately NOT a protocol event: the Test button in Settings runs the local + * detector with no session behind it, and inventing a voice session id so a + * settings panel can light up a dot would put a fake session in every client's + * event stream. + */ +export const ACAPPELLA_WAKE_TEST_CHANNEL = 'acappella:wake-test'; + +/** + * Push channel for the microphone list. + * + * Its own channel rather than a protocol event: the list exists with no session + * running (a settings panel has to draw the picker before anyone has spoken), + * and inventing a voice session id so Settings can populate a dropdown would put + * a fake session in every client's event stream. + */ +export const ACAPPELLA_INPUT_DEVICES_CHANNEL = 'acappella:input-devices'; + +/** What the wake-word tuning affordance pushes while it is running. */ +export interface WakeTestEvent { + phraseId: string; + phrase: string; + score: number; + at: number; +} + +/** + * What a start returns: the session snapshot plus anything the user needs to be + * told about the trio they are actually running. Substitutions travel with the + * start rather than only being logged - a silent downgrade to the mock tier is + * exactly the failure the registry exists to prevent. + */ +export interface VoiceStartSessionResult { + snapshot: VoiceSessionSnapshot; + substitutions: VoiceProviderSubstitution[]; +} + +export interface ACappellaHandlerDependencies { + settingsStore: { + get: (key: string, defaultValue?: unknown) => unknown; + /** + * Persist a setting. Optional for the same reason `onDidChange` is: tests + * pass a plain object. Without it the microphone choice cannot be saved, and + * `acappella:set-input-device` says so rather than reporting a success that + * would be forgotten on the next read. + */ + set?: (key: string, value: unknown) => void; + /** + * Live setting changes. Optional because tests pass a plain object; without + * it the voice hotkeys bind once at startup and do not follow a rebind. + */ + onDidChange?: (key: string, callback: (value: unknown) => void) => void; + }; + /** The window the dispatch executor talks to. Main has no tab authority. */ + getMainWindow: () => BrowserWindow | null; + /** + * The window that OWNS an agent, for multi-window dispatch. + * + * Agent ownership is per window while `activeSessionId` is global, so + * dispatching to whichever window is "main" would activate an agent that + * window does not own - the documented way to make a window render "No + * agents". Absent in tests and in any single-window host, where the main + * window is the right answer by construction. + */ + getWindowForSession?: (agentSessionId: string) => BrowserWindow | null; + /** + * Which window's HUD a session started from THIS trigger belongs to. + * + * `sender` is the IPC sender for a click (composer microphone, palette, Left + * Bar menu); it is absent for a trigger that has no window behind it - a + * global hotkey, a wake word, a paired phone - and the implementation then + * answers with the focused window. + * + * Every main -> renderer push is broadcast to all windows, so without this the + * same HUD rendered in every window at once and one microphone looked like + * several. Absent in tests and in any single-window host, where null means the + * primary window shows it, which is the only window there is. + */ + resolveVoiceWindowId?: (sender?: WebContents | null) => VoiceWindowId; + /** Broadcasts to every window and to the web-desktop bridge. */ + safeSend: SafeSendFn; + /** + * What the hidden audio host window needs to load the renderer bundle. Absent + * only in tests, which never want a real `BrowserWindow`; when it is absent + * the session runs without audio I/O rather than failing to start. + */ + audioHostDeps?: AudioHostWindowDeps; + /** + * The agent the user is looking at, for the `voiceCurrentAgent` hotkey. Absent + * in tests and in any host with no session store, where the agent hotkey + * refuses with a reason rather than binding a session to a guessed agent. + */ + getFocusedAgentSessionId?: () => string | null; + /** + * The process manager, as an event source for the agent-output tap. + * + * This is what makes a spoken reply arrive while the agent is still writing: + * the tap rides the SAME `data` / `query-complete` / `agent-error` events the + * desktop transcript rides. Absent in tests and before the manager is + * constructed, in which case the session falls back to waiting for a whole + * reply through `submitAgentReply`. + */ + getProcessManager?: () => AgentOutputSource | null; + /** The agent type behind a session id, for the tap's stream-json parsing. */ + getAgentType?: (agentSessionId: string) => string | undefined; + /** + * The running web server, for the paired-device transport. + * + * Its security token and port are what a phone needs to reach the signaling + * socket, so a QR code cannot be produced without it. Absent in tests and + * before the user has ever switched the web interface on, in which case + * pairing reports that there is nothing to pair to rather than inventing a + * port. + */ + getWebServer?: () => { getSecurityToken: () => string; getPort: () => number } | null; +} + +const handlerOpts = (operation: string): Pick => ({ + context: LOG_CONTEXT, + operation, +}); + +/** + * The provider selection the live service was built from. A change here means + * the next start rebuilds the service, which is how a Voice Providers change + * takes effect without an app restart. + */ +let activeProviderKey: string | null = null; +let activeSubstitutions: VoiceProviderSubstitution[] = []; + +/** + * The live pipeline. + * + * Held here rather than inside the session service because it outlives one + * session and because it owns real resources - a Whisper model, an ONNX session, + * a llama context, a realtime socket. Dropping the reference without calling + * `dispose()` would leak every one of them. + */ +let activePipeline: VoiceProviderResolution | null = null; + +/** + * The audio bridge for the live service. Module state for the same reason the + * service is: the frame and status listeners are registered once, for the life of + * the app, while the thing behind them is rebuilt whenever the provider trio + * changes. + */ +let audioBridge: VoiceAudioBridge | null = null; + +/** + * The microphones the audio host last reported. + * + * Cached because only the host can enumerate them - `enumerateDevices` is a DOM + * API - and a settings panel that had to open a hidden window and wait for a + * round trip just to draw a list would either block or render empty. The host + * republishes on boot, on device change, and after each capture starts (which is + * when Chromium stops redacting the labels). + */ +let inputDevices: AudioDeviceInfo[] = []; + +/** + * The two global voice hotkeys. + * + * Installed once, for the life of the app, and NOT rebuilt with the pipeline: a + * system-wide combo that is released and re-registered every time the user + * changes a voice would be a combo another app can steal in the gap. + */ +let voiceHotkeys: VoiceHotkeyInstallation | null = null; + +/** + * The wake-word tuning run behind the Test button in Settings. + * + * Its own detector rather than the session's, because the point is to run the + * wake word with NO session: a user tuning sensitivity is asking "would this + * have fired", not "please start listening to me". + */ +let wakeTestDetector: WakeDetector | null = null; + +/** + * The tap on dispatched agent output. + * + * Module state alongside the service and rebuilt with it, because it holds + * listeners on the process manager: a tap dropped without `dispose()` would keep + * filtering output for a session that no longer exists, and a second one built + * over it would speak every chunk twice. + */ +let agentOutputTap: AgentOutputTap | null = null; + +/** + * The paired-device transport. Module state alongside the hotkeys and for the + * same reason: it holds live signaling sessions and a Bonjour advert, both of + * which outlive any one voice session. + */ +let transport: ACappellaTransport | null = null; + +/** Push one command to the hidden audio host. A window that is not open is a no-op. */ +function sendAudioHostCommand(command: AudioHostCommand): void { + const win = getAcappellaAudioHostWindow(); + if (!win || win.webContents.isDestroyed()) return; + win.webContents.send(ACAPPELLA_AUDIO_COMMAND_CHANNEL, command); +} + +/** + * Shape guard for an inbound frame. + * + * The sender check is the real security boundary; this is a crash guard. Frames + * arrive fifty times a second, so a malformed one must not become fifty identical + * unhandled exceptions a second inside an `ipcMain` listener. + */ +function isAudioFrame(value: unknown): value is AudioFrame { + const frame = value as AudioFrame | null; + return ( + !!frame && + typeof frame.seq === 'number' && + typeof frame.rms === 'number' && + frame.pcm instanceof ArrayBuffer + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** The Encore gate. See `src/shared/acappella/feature-flag.ts`. */ +const requireEnabled = requireACappellaEnabled; + +/** An unrecognised scope is conductor scope; a malformed AGENT scope is not. */ +function parseScope(raw: unknown): VoiceScope { + if (!isRecord(raw)) return { kind: 'conductor' }; + if (raw.kind !== 'agent') return { kind: 'conductor' }; + // Falling back to the conductor here would send a spoken instruction to + // whichever agent happens to be active, which is the one outcome worse than + // an error. + if (typeof raw.sessionId !== 'string' || !raw.sessionId) { + throw new Error('InvalidVoiceScope'); + } + return { kind: 'agent', sessionId: raw.sessionId }; +} + +/** + * Teach the permission tracker what the device just did. + * + * Windows and Linux have no usable permission query, so a failed capture is the + * only evidence there is, and a successful one is the only proof of a grant that + * exists on any platform: Chromium hands over a live track only after the user + * agrees. Kept here, at the one place host statuses arrive, so the tracker + * cannot fall out of step with the microphone the user is actually looking at. + */ +function notePermissionFromStatus(status: AudioHostStatus): void { + if (status.kind === 'capture-start') { + noteCaptureStarted(); + return; + } + if (status.kind === 'mic-error') noteCaptureFailure(status.code); +} + +/** + * Cache the host's device list and tell every client it changed. + * + * Broadcast rather than request-only because the list changes for reasons no + * renderer initiated: a headset is unplugged, or a first capture finally reveals + * the labels Chromium had redacted. A picker that only ever pulled would keep + * showing a device that is now gone. + */ +function noteInputDevices(status: AudioHostStatus, safeSend: SafeSendFn): void { + if (status.kind !== 'input-devices') return; + inputDevices = status.devices; + safeSend(ACAPPELLA_INPUT_DEVICES_CHANNEL, inputDevices); +} + +/** An IPC caller is a client by definition, so a bare request is a button press. */ +function parseInterruptSource(raw: unknown): InterruptSource { + return raw === 'voice' ? 'voice' : 'client-button'; +} + +/** + * Validate a credential message. + * + * The service name is checked against the known set rather than passed through: + * it becomes a keychain account name, and an arbitrary string from the renderer + * would let a caller write entries into the user's credential store under any + * name it liked. + */ +function parseCredentialPayload(raw: unknown): { service: VoiceCredentialService; key: string } { + if (!isRecord(raw)) throw new Error('InvalidCredential'); + const service = raw.service; + if (typeof service !== 'string' || !VOICE_CREDENTIAL_SERVICES.includes(service as never)) { + throw new Error('InvalidCredential'); + } + return { + service: service as VoiceCredentialService, + key: typeof raw.key === 'string' ? raw.key : '', + }; +} + +/** + * Attach audio wiring to a service that has none. + * + * Audio is wired only when there is a host window to wire it to. Without one + * (tests, and any path that did not pass `audioHostDeps`) the session still + * runs: it is simply text-in, which is exactly the mock tier's contract. + * + * Called from BOTH paths in `ensureService`, because the two lifetimes differ: + * switching the Encore Feature off disposes the bridge and deliberately leaves + * the session service alone, so the next start reuses a live service that has + * no audio at all. That failure is silent and total - the host window opens, the + * device is captured, and every frame lands on a null bridge - which is why it + * cannot be left to the fresh-service path. + */ +function ensureAudioBridge(service: VoiceSessionService, deps: ACappellaHandlerDependencies): void { + if (audioBridge || !deps.audioHostDeps) return; + audioBridge = createVoiceAudioBridge({ + session: service, + sendCommand: sendAudioHostCommand, + // Read at each capture start rather than captured here, so picking a + // different microphone takes effect on the next session without rebuilding + // the provider pipeline. + getInputDeviceId: () => readVoiceProviderSettings(deps.settingsStore).inputDeviceId, + }); + // The user's volume applies from the FIRST sentence, not from the first time + // they touch the slider. A fresh bridge that defaulted to full output would + // undo a quiet setting (or a mute) every time the Encore Feature was toggled. + audioBridge.setPlaybackVolume( + readVoiceProviderSettings(deps.settingsStore).volume ?? DEFAULT_TTS_VOLUME + ); +} + +/** + * The live service, built on first use and rebuilt whenever the provider + * selection changes. + * + * `initVoiceSessionService` disposes the old instance, which drops its + * subscribers, so the fan-out is re-registered here rather than accumulating a + * second copy. + */ +async function ensureService(deps: ACappellaHandlerDependencies): Promise<{ + service: VoiceSessionService; + substitutions: VoiceProviderSubstitution[]; +}> { + const settings = readVoiceProviderSettings(deps.settingsStore); + const key = pipelineKey(settings); + + const existing = getVoiceSessionService(); + if (existing && key === activeProviderKey) { + // The service survives an Encore toggle but the bridge does not, so a reused + // service can arrive here with no audio wiring at all. + ensureAudioBridge(existing, deps); + return { service: existing, substitutions: activeSubstitutions }; + } + + // The old pipeline is torn down BEFORE the new one is built, so two loaded + // models are never resident at once. + await activePipeline?.pipeline.dispose(); + const resolution = resolveVoicePipeline({ settings }); + audioBridge?.dispose(); + audioBridge = null; + + const service = await buildService(resolution, deps); + + activePipeline = resolution; + activeProviderKey = key; + activeSubstitutions = resolution.substitutions; + return { service, substitutions: resolution.substitutions }; +} + +/** + * Build the tap for a fresh service, or null when there is no process manager to + * listen to. + * + * The sink reads the service through the module getter rather than closing over + * the instance: the tap outlives no service, but a chunk can be in flight while + * one is being replaced, and delivering it into the old instance would speak + * into a session nobody is subscribed to. + */ +function buildAgentOutputTap(deps: ACappellaHandlerDependencies): AgentOutputTap | null { + agentOutputTap?.dispose(); + agentOutputTap = null; + + const source = deps.getProcessManager?.(); + if (!source) return null; + + agentOutputTap = createAgentOutputTap({ + source, + getAgentType: deps.getAgentType, + onChunk: (chunk) => getVoiceSessionService()?.pushAgentOutput(chunk), + }); + return agentOutputTap; +} + +/** Construct the session service around a resolved pipeline and wire its fan-out. */ +async function buildService( + resolution: VoiceProviderResolution, + deps: ACappellaHandlerDependencies +): Promise { + const tap = buildAgentOutputTap(deps); + const service = await initVoiceSessionService({ + providers: { + ...resolution.providers, + // The Conductor router wraps whichever Brain the registry resolved. It + // keeps that provider's id and tier, so `provider-state` still names the + // engine that is really running; what it adds is the bounded context, the + // recall shortlist, roster validation, one constrained retry, and the + // refusal to guess below the confidence threshold. + brain: createConductorRouter({ brain: resolution.providers.brain }), + }, + pipelineShape: resolution.shape, + getRoster: readAgentRoster, + // The capability gate. The service refuses to start when a required slot is + // unsatisfied and names the missing piece; it never asks for, and cannot be + // handed, a replacement provider. + checkReadiness: () => readVoiceReadiness(deps.settingsStore), + // What is actually running, including any slot that could not be built. The + // service cannot derive this: it is handed a trio and never learns what was + // requested. + getProviderState: () => buildProviderState(resolution), + // How long a sentence is held before it counts as a finished thought. Read + // through the getter so tuning it lands on the next thought rather than the + // next session. + getUtteranceComposerConfig: () => { + const settings = readVoiceProviderSettings(deps.settingsStore); + return { + // Hold mode swaps the short "did they stop talking" guess for a long + // backstop, because in that mode the send phrase is how a request + // finishes and the timer only catches the times you forget to say it. + settleMs: settings.holdUntilSend ? settings.sendHoldMs : settings.turnSettleMs, + sendPhrases: settings.sendPhrases, + }; + }, + // Read per turn, so switching modes applies to the next thing said rather + // than to the next session. + getConversationalMode: () => + readVoiceProviderSettings(deps.settingsStore).conversationalMode === true, + executeRoute: createVoiceRouteExecutor({ + bridge: createRendererVoiceBridge(deps.getMainWindow, deps.getWindowForSession), + }), + // Read through the module variable rather than captured: the bridge cannot + // exist yet (it takes the service), and the two are replaced together. + onSpeechChunk: (chunk) => audioBridge?.handleSpeechChunk(chunk), + // The live tap. With it, a reply is spoken as the agent writes it; without + // it (no process manager yet, and in tests) the session waits for a whole + // reply through `submitAgentReply`. + agentReplyStream: tap ?? undefined, + // The pipeline ducks on a candidate frame long before this fires. These are + // the other door: a client button, and the Phase 10 phone. + duckPlayback: (gain, ms) => audioBridge?.duckPlayback(gain, ms), + flushPlayback: () => audioBridge?.flushPlayback(), + // Read per call rather than captured, so switching it in Settings takes + // effect on the next completion instead of on the next app start. Through + // the one settings reader, which already knows where the key lives. + getBackgroundAnnouncementSetting: () => + readVoiceProviderSettings(deps.settingsStore).speakBackgroundCompletions, + // Same reasoning, one turn finer: read per SENTENCE, so a speed slider + // dragged mid-reply is heard on the next sentence rather than the next + // session. This is the seam that makes "applies live" true. + getSpeechOptions: () => { + const current = readVoiceProviderSettings(deps.settingsStore); + return { voiceId: current.voiceId, rate: current.rate }; + }, + }); + service.subscribe((event) => deps.safeSend(ACAPPELLA_EVENT_CHANNEL, event)); + service.subscribe(recordRoutingOutcome); + ensureAudioBridge(service, deps); + return service; +} + +/** + * Close the routing log's loop from the event stream. + * + * The router records what it DECIDED; only the session knows what became of it, + * and the difference between those two is the entire value of the log. Doing it + * here rather than inside the service keeps the service free of the router's + * storage, and doing it from events rather than from call sites means a new + * failure path cannot forget to report itself. + */ +function recordRoutingOutcome(event: VoiceEvent): void { + const turnId = lastRoutingTurn()?.id; + if (!turnId) return; + + if (event.type === 'dispatch') { + noteRoutingOutcome(turnId, 'dispatched', `${event.agentName} / ${event.action}`); + return; + } + if (event.type === 'route-correction') { + // The turn being corrected is the one BEFORE this correction's own entry, + // which is why the correction is matched on the dispatch it replaced. + const corrected = readRoutingLog() + .reverse() + .find((entry) => entry.outcome === 'dispatched'); + if (corrected) { + noteRoutingOutcome(corrected.id, 'corrected', `moved to ${event.agentName}`); + } + return; + } + if (event.type === 'session-error' && event.code === 'dispatch-failed') { + noteRoutingOutcome(turnId, 'failed', event.message); + } +} + +/** + * Apply a provider change to the running app. + * + * Called by the settings panel after it writes a selection. A swap while a turn + * is in flight is REFUSED rather than queued: splicing two engines into one + * exchange would transcribe with one model, route with another, and answer in a + * third voice, and the user would have no idea why. + */ +export async function applyACappellaProviders( + deps: ACappellaHandlerDependencies +): Promise<{ status: 'swapped' | 'unchanged' | 'refused'; reason?: string }> { + const settings = readVoiceProviderSettings(deps.settingsStore); + const service = getVoiceSessionService(); + + const result = await swapVoicePipeline({ + settings, + current: activePipeline + ? { pipeline: activePipeline.pipeline, key: activeProviderKey ?? '' } + : null, + isBusy: isTurnInFlight(service), + }); + + if (result.status !== 'swapped' || !result.resolution) { + return { status: result.status, reason: result.reason }; + } + + audioBridge?.dispose(); + audioBridge = null; + + const rebuilt = await buildService(result.resolution, deps); + activePipeline = result.resolution; + activeProviderKey = pipelineKey(settings); + activeSubstitutions = result.resolution.substitutions; + // Announce the new engines even though no session is open: a client showing + // "you are on Whisper" has to stop saying so the moment that stops being true. + rebuilt.publishProviderState(); + + return { status: 'swapped' }; +} + +/** A provider that can enumerate its voices. Duck-typed: not every one can. */ +interface VoiceListingProvider { + listVoices?: () => + | Promise> + | Array<{ id: string; name: string }>; +} + +/** + * The voices the current TTS provider offers. + * + * Empty for a provider with one voice or none, which the picker renders as + * "Provider default" rather than as an error: a mock has no voices and that is + * not a failure. + */ +async function listVoiceOptions( + deps: ACappellaHandlerDependencies +): Promise> { + const resolution = + activePipeline ?? + resolveVoicePipeline({ + settings: readVoiceProviderSettings(deps.settingsStore), + }); + // Not cached into `activePipeline`: this can run before any session has ever + // been started, and building the live pipeline as a side effect of drawing a + // settings panel would load models nobody asked for. + const provider = resolution.providers.tts as VoiceListingProvider; + if (typeof provider.listVoices !== 'function') return []; + return provider.listVoices(); +} + +/** + * Speak one fixed line through the configured voice. + * + * Refused while a session is live: the preview and the assistant would be + * talking over each other through the same output device, and the user would + * have no way to tell which voice they were hearing. + * + * @returns false when nothing could be spoken (no audio host, or a provider with + * no audio behind it). + */ +async function previewVoiceLine( + deps: ACappellaHandlerDependencies, + text: string, + voiceId?: string +): Promise { + const live = getVoiceSessionService(); + if (live && live.getState() !== 'idle') return false; + + if (deps.audioHostDeps) ensureAcappellaAudioHostWindow(deps.audioHostDeps); + await ensureService(deps); + if (!activePipeline || !audioBridge) return false; + + const settings = readVoiceProviderSettings(deps.settingsStore); + let spoke = false; + for await (const chunk of activePipeline.providers.tts.speak(text, { + utteranceId: `preview-${Date.now()}`, + // The caller may name a voice it has NOT selected. That is the point of a + // per-voice preview: hearing a voice before committing to it beats + // selecting each one in turn and undoing the ones you did not want. + voiceId: voiceId ?? settings.voiceId, + rate: settings.rate, + })) { + audioBridge.handleSpeechChunk(chunk); + spoke = spoke || Boolean(chunk.audio?.byteLength); + } + return spoke; +} + +/** + * Whether a turn is mid-flight. + * + * `idle` and `listening` are the two safe moments: nothing has been said yet, or + * everything said has been answered. Every other state has a turn in it. + */ +function isTurnInFlight(service: VoiceSessionService | null): boolean { + if (!service) return false; + const state = service.getState(); + return state !== 'idle' && state !== 'listening' && state !== 'error'; +} + +/** + * Register the A Cappella IPC handlers. + * + * Wired from `setupIpcHandlers()` (src/main/ipc/bootstrap/index.ts), which is + * what the running app calls. A handler registered only through + * `registerAllHandlers()` in handlers/index.ts would be dead. + */ +export function registerACappellaHandlers(deps: ACappellaHandlerDependencies): void { + const { settingsStore } = deps; + + watchRosterChanges(); + + const wrappedStart = withIpcErrorLogging( + handlerOpts('startSession'), + async (rawScope: unknown, windowId: VoiceWindowId): Promise => { + const scope = parseScope(rawScope); + // The microphone is asked for HERE and nowhere earlier. Not at app + // launch, not when the Encore Feature is switched on: a first run that + // prompts for the microphone for a feature nobody turned on spends trust + // the app has not earned. This is the first moment the user has asked for + // something that genuinely needs a device. + // + // The result is not branched on. A refusal belongs to the capability + // gate, which names the microphone as its own blocking slot with its own + // recovery; throwing a second, differently-worded error from here would + // give the same problem two voices. + await requestMicPermission(); + // First start is what pays for the audio host: enabling the Encore + // Feature opens no device and builds no second renderer. + if (deps.audioHostDeps) ensureAcappellaAudioHostWindow(deps.audioHostDeps); + const { service, substitutions } = await ensureService(deps); + const snapshot = await service.startSession({ + scope, + source: 'client-button', + windowId, + }); + return { snapshot, substitutions }; + } + ); + + const wrappedStop = withIpcErrorLogging(handlerOpts('stopSession'), async (): Promise => { + await getVoiceSessionService()?.stopSession('user'); + }); + + const wrappedSubmitUtterance = withIpcErrorLogging( + handlerOpts('submitUtterance'), + // Returns false when the session cannot take an utterance right now, so a + // stray Send in the dev harness is a no-op rather than a thrown error. + async (text: unknown): Promise => { + if (typeof text !== 'string') throw new Error('InvalidUtterance'); + return getVoiceSessionService()?.submitUtterance(text) ?? false; + } + ); + + const wrappedInterrupt = withIpcErrorLogging( + handlerOpts('interrupt'), + // Barge-in: cancels speech and KEEPS the floor. Distinct from the stop word + // on purpose - talking over the assistant must not hang up on it. + async (source: unknown): Promise => + getVoiceSessionService()?.interrupt(parseInterruptSource(source)) ?? false + ); + + const wrappedStopWord = withIpcErrorLogging( + handlerOpts('stopWord'), + async (payload: unknown): Promise => { + const body = isRecord(payload) ? payload : {}; + const phrase = typeof body.phrase === 'string' ? body.phrase : undefined; + await getVoiceSessionService()?.hardStop(parseInterruptSource(body.source), phrase); + } + ); + + const wrappedSubmitAgentReply = withIpcErrorLogging( + handlerOpts('submitAgentReply'), + // The reply seam. Phase 05 wires real agent output straight into the + // service in-process; until then this is how anything outside main gets a + // session past `dispatching`, which is what makes the dev harness able to + // demonstrate speech, barge-in, and the difference between the two. + async (payload: unknown): Promise => { + if (!isRecord(payload)) throw new Error('InvalidAgentReply'); + const { agentSessionId, tabId, text } = payload; + if ( + typeof agentSessionId !== 'string' || + !agentSessionId || + typeof tabId !== 'string' || + !tabId || + typeof text !== 'string' + ) { + throw new Error('InvalidAgentReply'); + } + return ( + (await getVoiceSessionService()?.submitAgentReply({ agentSessionId, tabId, text })) ?? false + ); + } + ); + + const wrappedOpenMicSettings = withIpcErrorLogging( + handlerOpts('openMicSettings'), + // Its own channel rather than `shell:openExternal`, which allows only + // http/https/mailto. Widening that allowlist so one button can open one + // hard-coded URL would trade a real security property for nothing; here the + // URL is a constant the caller cannot influence. + async (): Promise => openMicSystemSettings() + ); + + const wrappedMicPermission = withIpcErrorLogging( + handlerOpts('micPermission'), + // A pure query. It never prompts, which is what lets the HUD and Settings + // call it on render without the app asking for the microphone behind a user + // who has not asked for voice. + async (): Promise => getMicPermission() + ); + + const wrappedGetRoster = withIpcErrorLogging( + handlerOpts('getRoster'), + async (): Promise => readAgentRoster() + ); + + const wrappedListCredentials = withIpcErrorLogging( + handlerOpts('listCredentials'), + // Configured-or-not, never the key. Nothing in the renderer needs to read a + // credential back, and a channel that returned one would put it in a + // renderer heap and in every crash dump taken afterwards. + async (): Promise => listCredentialStates() + ); + + const wrappedSetCredential = withIpcErrorLogging( + handlerOpts('setCredential'), + async (payload: unknown) => { + const { service, key } = parseCredentialPayload(payload); + return key ? setCredential(service, key) : clearCredential(service); + } + ); + + const wrappedValidateCredential = withIpcErrorLogging( + handlerOpts('validateCredential'), + // An optional key so Test works before Save: a user should be able to find + // out a key is wrong without storing it first. + async (payload: unknown): Promise => { + const { service, key } = parseCredentialPayload(payload); + return validateCredential(service, key || undefined); + } + ); + + const wrappedApplyProviders = withIpcErrorLogging(handlerOpts('applyProviders'), async () => + applyACappellaProviders(deps) + ); + + const wrappedListVoices = withIpcErrorLogging(handlerOpts('listVoices'), async () => + listVoiceOptions(deps) + ); + + const wrappedPreviewVoice = withIpcErrorLogging( + handlerOpts('previewVoice'), + async (text: unknown, voiceId: unknown): Promise => { + if (typeof text !== 'string' || !text.trim()) throw new Error('InvalidPreviewText'); + if (voiceId !== undefined && typeof voiceId !== 'string') throw new Error('InvalidVoiceId'); + return previewVoiceLine(deps, text, voiceId || undefined); + } + ); + + /** + * Apply an output volume to whatever is playing RIGHT NOW. + * + * Deliberately does NOT persist: the caller has already written the setting + * (or is muting, which is session-scoped and must not survive a restart), and + * a channel that both saved and applied would make a mute permanent the first + * time somebody used it. + * + * Resolves false when there is no audio host to apply it to, which the HUD + * treats as "nothing is playing" rather than as a failure. + */ + const wrappedSetVolume = withIpcErrorLogging( + handlerOpts('setVolume'), + async (volume: unknown): Promise => { + if (typeof volume !== 'number' || !Number.isFinite(volume)) { + throw new Error('InvalidVolume'); + } + if (!audioBridge) return false; + // Zero is legal here and only here: mute is a real state the HUD owns, + // while the SLIDER floors above zero so it cannot become a silent mute. + audioBridge.setPlaybackVolume(Math.min(1, Math.max(0, volume))); + return true; + } + ); + + /** + * The microphones this machine offers, plus which one is selected. + * + * Serves the cache and asks the host to refresh in the background rather than + * awaiting it: only a renderer can enumerate devices, so a caller that waited + * would hang whenever the audio host is not open - which is most of the time, + * since nothing opens it until a session starts. + */ + const wrappedInputDevices = withIpcErrorLogging( + handlerOpts('inputDevices'), + async (): Promise<{ devices: AudioDeviceInfo[]; selectedId: string }> => { + if (getAcappellaAudioHostWindow()) sendAudioHostCommand({ kind: 'list-input-devices' }); + return { + devices: inputDevices, + selectedId: + readVoiceProviderSettings(deps.settingsStore).inputDeviceId ?? + ACAPPELLA_SYSTEM_DEFAULT_INPUT, + }; + } + ); + + /** + * Choose the microphone. + * + * Takes effect on the NEXT capture rather than mid-utterance: swapping the + * device under a live recogniser would splice two rooms into one sentence, and + * the pre-roll buffer in front of it belongs to the old one. + */ + const wrappedSetInputDevice = withIpcErrorLogging( + handlerOpts('setInputDevice'), + async (rawId: unknown): Promise => { + if (typeof rawId !== 'string' || !rawId) throw new Error('InvalidInputDevice'); + const stored = (deps.settingsStore.get(ACAPPELLA_SETTINGS_KEY, {}) ?? {}) as Record< + string, + unknown + >; + const audio = (stored.audio ?? {}) as Record; + if (!deps.settingsStore.set) throw new Error('SettingsNotWritable'); + deps.settingsStore.set(ACAPPELLA_SETTINGS_KEY, { + ...stored, + audio: { + ...audio, + // The sentinel is stored as "no preference" so the setting keeps + // following the OS rather than pinning today's default device. + inputDeviceId: rawId === ACAPPELLA_SYSTEM_DEFAULT_INPUT ? undefined : rawId, + }, + }); + return true; + } + ); + + const wrappedLastTurn = withIpcErrorLogging( + handlerOpts('lastTurn'), + async (): Promise => lastTurn() + ); + + // The floor's view of the session. Narrow by construction: the hotkeys can + // open, close, and interrupt, and nothing else. + const floorSession: FloorControlSession = { + getState: () => getVoiceSessionService()?.getState() ?? 'idle', + startSession: async ({ scope, source, origin }) => { + // A remote origin does NOT skip this: the desktop still opens its audio + // host, because that window is where the peer connection and the playback + // live. What it skips is nothing at all, which is the point - a remote + // session takes the same path as a local one. + await requestMicPermission(); + if (deps.audioHostDeps) ensureAcappellaAudioHostWindow(deps.audioHostDeps); + const { service } = await ensureService(deps); + return service.startSession({ + scope, + source, + origin, + // No IPC sender to resolve: a global hotkey, a wake word, and a paired + // phone all arrive with no window behind them, so the HUD belongs to + // the window the user is looking at. + windowId: deps.resolveVoiceWindowId?.() ?? null, + }); + }, + stopSession: async (reason) => { + await getVoiceSessionService()?.stopSession(reason); + }, + interrupt: (source) => getVoiceSessionService()?.interrupt(source) ?? false, + }; + + const hotkeys = (voiceHotkeys ??= installVoiceHotkeys({ + settingsStore, + session: floorSession, + getMainWindow: deps.getMainWindow, + getFocusedAgentSessionId: deps.getFocusedAgentSessionId ?? (() => null), + // Two halves of one gesture, and both are needed. The bridge owns the + // recogniser handle, so the flush goes through it; the session owns the + // composer, which would otherwise buffer the resulting final and sit out a + // settle window the user already answered by letting go of the key. + endUtterance: () => { + audioBridge?.endUtterance(); + getVoiceSessionService()?.endUtteranceNow(); + }, + // Broadcast rather than logged: a hotkey that did nothing has to say why, + // or the user concludes the key is broken. + onRefused: (info) => deps.safeSend(ACAPPELLA_EVENT_CHANNEL + ':hotkey-refused', info), + })); + + /** + * The paired-device transport. + * + * Built here, next to the floor, because it presses the SAME controller a + * hotkey does: a phone's talk button and a keyboard chord are two surfaces + * over one state machine, not two ways to open a microphone. Constructed + * eagerly and cheaply - it opens no socket and advertises nothing until a user + * asks it to. + */ + transport ??= initACappellaTransport({ + settingsStore, + userDataPath: app.getPath('userData'), + sendToAudioHost: (command) => { + const win = getAcappellaAudioHostWindow(); + if (!win || win.webContents.isDestroyed()) return; + win.webContents.send(ACAPPELLA_WEBRTC_COMMAND_CHANNEL, command); + }, + acquireFloor: (scope, origin) => hotkeys.acquireFloor(scope, origin), + getSession: () => getVoiceSessionService(), + getServerToken: () => deps.getWebServer?.()?.getSecurityToken() ?? null, + getServerPort: () => deps.getWebServer?.()?.getPort() ?? null, + getAppVersion: () => app.getVersion(), + getMachineName: () => hostname(), + onDevicesChanged: () => deps.safeSend(ACAPPELLA_DEVICES_CHANNEL, null), + onPairingRequest: (request) => deps.safeSend(ACAPPELLA_PAIRING_REQUEST_CHANNEL, request), + }); + + registerACappellaDeviceHandlers({ settingsStore }); + + const wrappedHotkeyStatus = withIpcErrorLogging( + handlerOpts('hotkeyStatus'), + async (): Promise<{ statuses: GlobalHotkeyStatus[]; capability: string; note: string }> => ({ + statuses: voiceHotkeys?.statuses() ?? [], + capability: voiceHotkeys?.controller.capability ?? 'tap-only', + note: describePressHoldCapability(voiceHotkeys?.controller.capability ?? 'tap-only'), + }) + ); + + const wrappedWakeTest = withIpcErrorLogging( + handlerOpts('wakeTest'), + async (payload: unknown): Promise => startWakeTest(deps, payload) + ); + + const wrappedWakeTestStop = withIpcErrorLogging( + handlerOpts('wakeTestStop'), + async (): Promise => stopWakeTest() + ); + + const wrappedCorrectRoute = withIpcErrorLogging( + handlerOpts('correctRoute'), + // The HUD's "wrong tab" control. Returns false when there is nothing to + // move, so a stray click is a no-op rather than an error. + async (agentSessionId: unknown): Promise => { + if (typeof agentSessionId !== 'string' || !agentSessionId) { + throw new Error('InvalidCorrectionTarget'); + } + return ( + (await getVoiceSessionService()?.correctLastDispatch(agentSessionId, 'client-button')) ?? + false + ); + } + ); + + const wrappedRoutingLog = withIpcErrorLogging( + handlerOpts('routingLog'), + async (): Promise<{ entries: RoutingLogEntry[]; quality: RoutingQuality }> => { + await loadRoutingLog(); + return { entries: readRoutingLog(), quality: routingQuality() }; + } + ); + + const wrappedGetState = withIpcErrorLogging( + handlerOpts('getState'), + // Null means the service has never been built, so no provider is resolved + // yet. Synthesising an idle snapshot here would have to name provider ids + // that nothing has resolved, and reporting a requested-but-unavailable + // provider as running is the substitution lie in a different costume. + async (): Promise => + getVoiceSessionService()?.getSnapshot() ?? null + ); + + ipcMain.handle( + 'acappella:start-session', + async (event, scope: unknown): Promise => { + requireEnabled(settingsStore); + // Resolved HERE rather than inside the wrapped handler, because + // `withIpcErrorLogging` strips the event and the sender is the only + // evidence of which window the user actually clicked in. + return wrappedStart(event, scope, deps.resolveVoiceWindowId?.(event?.sender) ?? null); + } + ); + + // Deliberately ungated: turning the Encore Feature off while a session is + // live must still be able to release the floor. + ipcMain.handle('acappella:stop-session', wrappedStop); + + ipcMain.handle('acappella:submit-utterance', async (event, text: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedSubmitUtterance(event, text); + }); + + ipcMain.handle('acappella:interrupt', async (event, source: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedInterrupt(event, source); + }); + + ipcMain.handle('acappella:stop-word', async (event, payload: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedStopWord(event, payload); + }); + + ipcMain.handle( + 'acappella:submit-agent-reply', + async (event, payload: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedSubmitAgentReply(event, payload); + } + ); + + // Ungated, like `stop-session`: a user whose microphone was denied has to be + // able to reach the OS setting, and a session that could not open a device is + // exactly the situation in which the feature may already have been turned off. + ipcMain.handle('acappella:open-mic-settings', wrappedOpenMicSettings); + + // Ungated for the same reason: a client showing "microphone access denied" + // must still be able to read the state after the feature was switched off. + ipcMain.handle('acappella:mic-permission', wrappedMicPermission); + + ipcMain.handle('acappella:get-roster', async (event): Promise => { + requireEnabled(settingsStore); + return wrappedGetRoster(event); + }); + + ipcMain.handle('acappella:get-state', async (event): Promise => { + requireEnabled(settingsStore); + return wrappedGetState(event); + }); + + ipcMain.handle('acappella:correct-route', async (event, agentSessionId: unknown) => { + requireEnabled(settingsStore); + return wrappedCorrectRoute(event, agentSessionId); + }); + + // Ungated: the routing log is how somebody works out why yesterday's dispatch + // went where it did, and switching the feature off is a thing people do + // BECAUSE of a misroute. + ipcMain.handle('acappella:routing-log', wrappedRoutingLog); + + // The credential channels are ungated, like the microphone ones: a user has to + // be able to add or remove a key while the feature is off, and removing one is + // exactly what somebody switching the feature off may want to do. + ipcMain.handle('acappella:list-credentials', wrappedListCredentials); + ipcMain.handle('acappella:set-credential', wrappedSetCredential); + ipcMain.handle('acappella:validate-credential', wrappedValidateCredential); + + ipcMain.handle('acappella:apply-providers', async (event) => { + requireEnabled(settingsStore); + return wrappedApplyProviders(event); + }); + + ipcMain.handle('acappella:list-voices', async (event) => { + requireEnabled(settingsStore); + return wrappedListVoices(event); + }); + + ipcMain.handle( + 'acappella:preview-voice', + async (event, text: unknown, voiceId: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedPreviewVoice(event, text, voiceId); + } + ); + + ipcMain.handle('acappella:set-volume', async (event, volume: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedSetVolume(event, volume); + }); + + // Ungated deliberately: Voice Setup draws the microphone picker while the user + // is deciding whether to switch the feature on, and a picker that throws until + // then would make the panel look broken. + ipcMain.handle('acappella:input-devices', wrappedInputDevices); + ipcMain.handle('acappella:set-input-device', wrappedSetInputDevice); + + ipcMain.handle('acappella:last-turn', async (event): Promise => { + requireEnabled(settingsStore); + return wrappedLastTurn(event); + }); + + // Ungated: the Voice Controls rows show a hotkey's registration state, and the + // most interesting time to read it is right after the feature was switched + // off and both combos were released. + ipcMain.handle('acappella:hotkey-status', wrappedHotkeyStatus); + + ipcMain.handle('acappella:wake-test', async (event, payload: unknown): Promise => { + requireEnabled(settingsStore); + return wrappedWakeTest(event, payload); + }); + + // Ungated, like `stop-session`: a tuning run has an open microphone, and + // switching the feature off must still be able to close it. + ipcMain.handle('acappella:wake-test-stop', wrappedWakeTestStop); + + // The audio host's own control link. `on`, not `handle`: frames arrive fifty + // times a second and nothing about them needs a reply, so a promise round trip + // per 20 ms of audio would be pure overhead. + // + // Both listeners check the sender. The preload exposes `voiceAudioHost` to every + // window because it is one shared preload, so "only the audio host may speak + // here" has to be enforced at the receiving end - a browser tab that found the + // channel must not be able to inject PCM into a live voice session. + ipcMain.on(ACAPPELLA_AUDIO_FRAME_CHANNEL, (event, frame: unknown) => { + if (!isAcappellaAudioHostContents(event.sender)) return; + if (!isAudioFrame(frame)) return; + audioBridge?.handleFrame(frame); + // The tuning run taps the same frames rather than opening a second capture: + // there is one microphone, and two consumers of it is one device conflict. + if (wakeTestDetector) wakeTestDetector.pushFrame(new Int16Array(frame.pcm)); + }); + + ipcMain.on(ACAPPELLA_AUDIO_STATUS_CHANNEL, (event, status: unknown) => { + if (!isAcappellaAudioHostContents(event.sender)) return; + if (!status || typeof (status as AudioHostStatus).kind !== 'string') return; + notePermissionFromStatus(status as AudioHostStatus); + noteInputDevices(status as AudioHostStatus, deps.safeSend); + audioBridge?.handleStatus(status as AudioHostStatus); + }); + + // The peer-connection control plane, from the same window and with the same + // sender check: an answer or a data-channel message forged by any other + // renderer would be a paired device's traffic with no pairing behind it. + ipcMain.on(ACAPPELLA_WEBRTC_EVENT_CHANNEL, (event, hostEvent: unknown) => { + if (!isAcappellaAudioHostContents(event.sender)) return; + if (!hostEvent || typeof (hostEvent as WebRtcHostEvent).kind !== 'string') return; + getACappellaTransport()?.handleHostEvent(hostEvent as WebRtcHostEvent); + }); + + // Release the floor on the way out, and with it the microphone: the audio host + // window holds a real capture device, and a session left running would keep it + // open past the last app window. Fire-and-forget: `will-quit` is synchronous, + // and this is the last thing the session will ever do. + app.on('will-quit', () => { + void disposeVoiceSessionService(); + closeAcappellaAudioHostWindow(); + // The log is written on a debounce, so the last few turns of a session are + // still in memory when the app is told to quit. + void flushRoutingLog(); + resetACappellaHandlerState(); + }); +} + +/** + * Drop the cached routing context whenever an agent or a tab changes. + * + * The cache has a short TTL as a backstop, but a TTL alone is not good enough + * here: within those seconds the roster can lose the very tab a decision is + * about to name, and a routing turn is exactly the moment a user has just + * finished rearranging their workspace. + * + * Best-effort by design. A store with no change feed (tests, an older + * electron-store) simply falls back to the TTL rather than failing registration. + */ +function watchRosterChanges(): void { + try { + const store = getSessionsStore() as unknown as { + onDidChange?: (key: string, callback: () => void) => void; + }; + store.onDidChange?.('sessions', invalidateRoutingContext); + } catch { + /* no store to watch: the context cache falls back to its TTL */ + } +} + +/** + * Stop capture and drop the audio wiring, leaving the session service alone. + * + * Called when the Encore Feature is switched off: the audio host window goes + * away with it, so a bridge still holding a running pipeline would be counting + * frames from a device nobody owns. + */ +export function disposeACappellaAudioBridge(): void { + audioBridge?.dispose(); + audioBridge = null; +} + +/** + * Wind everything down because the Encore Feature was switched off. + * + * The switch has to mean what it says. Before this existed, turning A Cappella + * off released the microphone and closed the audio host, and left three things + * running that a user would reasonably believe were gone: a live voice session, a + * loaded inference pipeline holding native runtimes and model files open, and the + * transport with its Bonjour advert and its connected phones. + * + * The pipeline is the one with a second consequence. The reclaim-disk button + * lives on the Models page precisely so it can be pressed after the feature is + * off, and on Windows `fs.rm` of a directory whose files are still mapped by a + * loaded runtime fails outright. Dropping the pipeline here is what makes + * reclaiming disk work rather than error. + * + * What it deliberately does NOT do is dispose the hotkey installation or the + * transport object. Both are constructed once per process at handler + * registration, so tearing them down would make switching the feature back on a + * no-op until the next restart. The hotkeys release their combos through their + * own settings watcher; the transport stands down and stays reusable. + */ +/** + * End the session if it belonged to a window that just closed. + * + * A session is shown by exactly one window, so closing that window would + * otherwise leave an open microphone with no surface anywhere - the same + * "something is listening and nobody can see it" failure the HUD's close button + * exists to prevent, arrived at by a different route. Closing the window that + * holds the floor is a clear enough intent to end the session. + * + * A session with no window (null) is deliberately left alone: it is not this + * window's to end, and the primary window still shows it. + */ +export async function stopVoiceSessionForClosedWindow(windowId: string): Promise { + const service = getVoiceSessionService(); + if (!service || service.getSnapshot().windowId !== windowId) return; + logger.info(`Voice session ended: its window (${windowId}) closed`, LOG_CONTEXT); + await service.stopSession('shutdown'); +} + +export async function shutdownACappellaForDisable(): Promise { + // The advert and the phones go first. A device that still holds the floor + // would otherwise be able to reopen the microphone during the teardown below. + getACappellaTransport()?.standDown(); + await stopWakeTest(); + // `shutdown` rather than `user`: nobody pressed stop, and the distinction is + // what the session's own telemetry and the History entry read. + await getVoiceSessionService()?.stopSession('shutdown'); + // Order matters, and matches the audio host teardown in main/index.ts: the + // bridge stops capture through a window it is about to lose. + disposeACappellaAudioBridge(); + await activePipeline?.pipeline.dispose(); + activePipeline = null; + // Cleared together. `activeProviderKey` is the memo of what `activePipeline` + // was built from, so leaving it set would make the next start believe a + // disposed pipeline still matches settings and reuse it. + activeProviderKey = null; + activeSubstitutions = []; +} + +/** + * Run the wake word with no session behind it, so a user can tune sensitivity by + * saying the phrase and watching it light up rather than by guessing. + * + * Refused while a session is live: the detector would be scoring the same frames + * the session is already using, and a hit would light the Test dot for a phrase + * that also just woke the assistant. + * + * @returns false when there is no audio host to capture through. + */ +async function startWakeTest( + deps: ACappellaHandlerDependencies, + payload: unknown +): Promise { + await stopWakeTest(); + if (getVoiceSessionService()?.getState() !== undefined) { + const state = getVoiceSessionService()?.getState(); + if (state && state !== 'idle') return false; + } + if (!deps.audioHostDeps) return false; + + const body = isRecord(payload) ? payload : {}; + const phrase = + typeof body.phrase === 'string' && body.phrase.trim() ? body.phrase.trim() : undefined; + const sensitivity = typeof body.sensitivity === 'number' ? body.sensitivity : undefined; + const phrases: WakePhrase[] = [globalWakePhrase(phrase, sensitivity)]; + + await requestMicPermission(); + ensureAcappellaAudioHostWindow(deps.audioHostDeps); + + const detector = createWakeDetector({ + getPhrases: () => phrases, + onWake: (detection: WakeDetection) => { + const event: WakeTestEvent = { + phraseId: detection.phraseId, + phrase: detection.phrase, + score: detection.score, + at: detection.at, + }; + deps.safeSend(ACAPPELLA_WAKE_TEST_CHANNEL, event); + }, + }); + await detector.start(); + wakeTestDetector = detector; + sendAudioHostCommand({ kind: 'start-capture' }); + return true; +} + +/** End a tuning run and close the microphone it opened. Safe when none is running. */ +async function stopWakeTest(): Promise { + const detector = wakeTestDetector; + if (!detector) return; + wakeTestDetector = null; + await detector.stop(); + // Only when no session owns the device: a tuning run that ended while a + // session was starting must not close that session's microphone. + const state = getVoiceSessionService()?.getState() ?? 'idle'; + if (state === 'idle') sendAudioHostCommand({ kind: 'stop-capture' }); +} + +/** + * Drop the cached provider selection. Test-only seam: the service singleton is + * module state in `src/main/acappella/index.ts`, and this file's memo of what it + * was built from has to be cleared alongside it. + */ +export function resetACappellaHandlerState(): void { + activeProviderKey = null; + activeSubstitutions = []; + agentOutputTap?.dispose(); + agentOutputTap = null; + voiceHotkeys?.dispose(); + voiceHotkeys = null; + disposeACappellaTransport(); + transport = null; + void stopWakeTest(); + // Fire and forget: this runs from `will-quit`, which is synchronous, and from + // tests, which do not care how long a model file takes to close. + void activePipeline?.pipeline.dispose(); + activePipeline = null; + audioBridge?.dispose(); + audioBridge = null; +} diff --git a/src/main/ipc/handlers/debug.ts b/src/main/ipc/handlers/debug.ts index 52b9d75aeb..5eceed5c1e 100644 --- a/src/main/ipc/handlers/debug.ts +++ b/src/main/ipc/handlers/debug.ts @@ -26,6 +26,7 @@ import { getProfilingStatus, finalizeCapture, } from '../../profiling'; +import { runSelfTest, type RuntimeSelfTestReport } from '../../acappella/runtime/runtime-selftest'; import { AgentDetector } from '../../agents'; import { ProcessManager } from '../../process-manager'; import { WebServer } from '../../web-server'; @@ -143,6 +144,25 @@ export function registerDebugHandlers(deps: DebugHandlerDependencies): void { }) ); + // A Cappella voice self-test. Lives on the debug surface rather than the voice + // one because it is a diagnostic, and because it must stay runnable when the + // voice feature is exactly what is not working. It loads each native runtime + // and no model, so running it is cheap and safe at any time. + ipcMain.handle( + 'debug:voiceSelfTest', + createIpcHandler( + handlerOpts('voiceSelfTest'), + async (): Promise<{ report: RuntimeSelfTestReport }> => { + const report = await runSelfTest(); + logger.info(`Voice self-test: ${report.passed ? 'pass' : 'fail'}`, LOG_CONTEXT); + // Nested rather than spread: `createIpcHandler` merges the result into + // its `{ success }` envelope, and a report field named `passed` sitting + // beside `success` is two words for two different things. + return { report }; + } + ) + ); + // Snapshot of runtime memory / process info for the Debug: View Application Stats modal ipcMain.handle( 'debug:getAppStats', diff --git a/src/main/ipc/handlers/index.ts b/src/main/ipc/handlers/index.ts index 7f65c61f91..4707eff9bf 100644 --- a/src/main/ipc/handlers/index.ts +++ b/src/main/ipc/handlers/index.ts @@ -66,6 +66,15 @@ import { registerCrossAgentHandlers } from './cross-agent'; import { registerCueHandlers, CueHandlerDependencies } from './cue'; import { registerCueBackupHandlers } from './cue-backup'; import { registerPianolaHandlers, PianolaHandlerDependencies } from './pianola'; +import { + registerACappellaHandlers, + stopVoiceSessionForClosedWindow, + ACappellaHandlerDependencies, +} from './acappella'; +import { + registerACappellaModelsHandlers, + ACappellaModelsHandlerDependencies, +} from './acappella-models'; import { registerPluginsHandlers, PluginsHandlerDependencies } from './plugins'; import { registerWakatimeHandlers } from './wakatime'; import { registerCoworkingHandlers } from './coworking'; @@ -146,6 +155,10 @@ export type { CueHandlerDependencies }; export { registerCueBackupHandlers }; export { registerPianolaHandlers }; export type { PianolaHandlerDependencies }; +export { registerACappellaHandlers, stopVoiceSessionForClosedWindow }; +export type { ACappellaHandlerDependencies }; +export { registerACappellaModelsHandlers }; +export type { ACappellaModelsHandlerDependencies }; export { registerPluginsHandlers }; export type { PluginsHandlerDependencies }; export { registerWakatimeHandlers }; diff --git a/src/main/ipc/handlers/windows.ts b/src/main/ipc/handlers/windows.ts index c15729a961..c87c2d136c 100644 --- a/src/main/ipc/handlers/windows.ts +++ b/src/main/ipc/handlers/windows.ts @@ -12,7 +12,7 @@ * manager's `createSecondaryWindow`, which registers the new window itself. */ -import { BrowserWindow, ipcMain } from 'electron'; +import { ipcMain } from 'electron'; import type { WindowBounds, WindowHighlightDropZonePayload, @@ -95,6 +95,10 @@ function isBridgeEvent(event: Electron.IpcMainInvokeEvent | undefined): boolean * claim the agent into a window the remote user never chose, and a remote panel * collapse would silently rewrite a desktop window's persisted state. Reads * answering for the primary are defensible; writes landing on it are not. + * + * The sender -> window resolution itself lives on `registry.findBySender`, so + * A Cappella (which scopes a voice session to the window that opened it) does + * not need a second copy of it. Only the bridge carve-out is local. */ function resolveCallingWindow( event: Electron.IpcMainInvokeEvent, @@ -112,9 +116,7 @@ function resolveCallingWindow( // letting BrowserWindow.fromWebContents throw on the missing WebContents. return undefined; } - const browserWindow = BrowserWindow.fromWebContents(event.sender); - if (!browserWindow) return undefined; - return registry.getAll().find((entry) => entry.browserWindow === browserWindow); + return registry.findBySender(event.sender); } /** diff --git a/src/main/plugins/authorization-ledger.ts b/src/main/plugins/authorization-ledger.ts index f276796357..6894ab7f52 100644 --- a/src/main/plugins/authorization-ledger.ts +++ b/src/main/plugins/authorization-ledger.ts @@ -33,6 +33,12 @@ import * as fs from 'fs'; import * as path from 'path'; import { randomBytes } from 'crypto'; +import { + createKeyringEntry, + loadKeyringModule, + type KeyringEntry, + type KeyringModuleLoader, +} from '../utils/keyring'; import { isAllowlistScoped, isValidAllowlistMember } from '../../shared/plugins/permissions'; import type { PermissionGrant, PluginCapability } from '../../shared/plugins/permissions'; import type { SignatureStatus } from '../../shared/plugins/signing'; @@ -495,12 +501,12 @@ export function safeStorageSeal(safeStorage: SafeStorageLike): SealProvider { * Production `AnchorStore` over a named OS credential entry. `entryFactory` * lazily constructs the keyring entry so a missing/unavailable native module * degrades to `available() === false` (→ session-only) instead of throwing. + * + * The entry shape and the lazy module load live in `src/main/utils/keyring.ts`, + * shared with A Cappella's credential store; they are re-exported here so this + * module's existing importers keep their one import site. */ -export interface KeyringEntry { - getPassword(): string | null; - setPassword(password: string): void; - deletePassword(): boolean; -} +export type { KeyringEntry, KeyringModule } from '../utils/keyring'; export function keyringAnchor(entryFactory: () => KeyringEntry | null): AnchorStore { let entry: KeyringEntry | null | undefined; @@ -565,28 +571,13 @@ export function noAnchor(): AnchorStore { }; } -export interface KeyringModule { - Entry: new (service: string, account: string) => KeyringEntry; -} - /** Lazily adapt `@napi-rs/keyring` without making app startup depend on it. */ export function createKeyringAnchor( service: string, account: string, - loadModule: () => KeyringModule | null = () => { - try { - const mod = require('@napi-rs/keyring') as Partial; - return typeof mod.Entry === 'function' ? (mod as KeyringModule) : null; - } catch { - return null; - } - } + loadModule: KeyringModuleLoader = loadKeyringModule ): AnchorStore { - return keyringAnchor(() => { - const mod = loadModule(); - if (!mod) return null; - return new mod.Entry(service, account); - }); + return keyringAnchor(() => createKeyringEntry(service, account, loadModule)); } export function createAuthorizationStore(opts: { safeStorage: SafeStorageLike; diff --git a/src/main/preload/acappella.ts b/src/main/preload/acappella.ts new file mode 100644 index 0000000000..00e30b0cf2 --- /dev/null +++ b/src/main/preload/acappella.ts @@ -0,0 +1,528 @@ +/** + * Preload API for A Cappella (voice sessions). + * + * Provides the `window.maestro.voice` namespace. Every channel is gated in the + * main process on the `aCappella` Encore flag; when it is off they reject with + * 'ACappellaDisabled', which callers treat as "feature off" rather than "no + * session". `stop()` is the one exception and always works, so a client can + * release the floor even after the flag is turned off mid-session. + * + * `onEvent` is the whole protocol: state, transcript, routing, dispatch, and + * speech all arrive as `VoiceEvent`s on one ordered stream, so a client renders + * from the stream rather than from the return values of these calls. + */ + +import { ipcRenderer } from 'electron'; +import type { RosterAgent, VoiceEvent, VoiceScope } from '../../shared/acappella/protocol'; +import type { VoiceReadiness } from '../../shared/acappella/readiness'; +import type { VoiceSessionSnapshot } from '../acappella'; +import type { VoiceStartSessionResult, WakeTestEvent } from '../ipc/handlers/acappella'; +import type { GlobalHotkeyStatus } from '../../shared/global-hotkeys'; +import type { VoiceHotkeyRefusalInfo } from '../acappella/hotkeys/voice-hotkeys'; +import type { VoiceModelListing } from '../ipc/handlers/acappella-models'; +import type { DownloadProgress, DownloadResult } from '../acappella/models/model-downloader'; +import type { ModelFootprint, VerifyResult } from '../acappella/models/model-store'; +import type { MicPermissionInfo } from '../acappella/permissions/mic-permission'; +import type { RoutingLogEntry, RoutingQuality } from '../acappella/router/routing-log'; +import type { CredentialState, CredentialValidation } from '../acappella/providers/credentials'; +import type { VoiceCredentialService } from '../../shared/acappella/provider-catalog'; +import type { TurnBreakdown } from '../acappella/telemetry/turn-metrics'; +import type { DeviceStatus, PairingPayload } from '../acappella/transport'; +import type { IceTransportSettings } from '../acappella/transport/ice-config'; +import type { DiscoveryStatus } from '../acappella/pairing/discovery'; +import type { PairingRequest } from '../acappella/pairing/pairing-service'; +import type { IceProbeResult } from '../../shared/acappella/webrtc-host'; + +/** + * `window.maestro.voice.models.*` - the model manager. + * + * Nothing in here touches the network except {@link download} and + * {@link resume}. `list()` is a disk read against a frozen local catalog, which + * is what lets Voice Setup show a full bill of materials (name, size, license, + * install path) before the user has agreed to fetch a single byte. + * + * `remove`, `removeAll`, and `footprint` deliberately keep working when the + * Encore Feature is off, because that is exactly when someone wants their disk + * back. + */ +function createVoiceModelsApi() { + return { + /** Every catalog model joined to its on-disk status and install paths. */ + list: (): Promise => ipcRenderer.invoke('models:list'), + + /** + * Start or resume a download. The ONLY call in this namespace that opens a + * connection. Resolves when the model is installed, paused, cancelled, or + * has failed; watch {@link onProgress} for the intervening detail. + */ + download: (modelId: string): Promise => + ipcRenderer.invoke('models:download', modelId), + + /** Abort the transfer, keep the partial file. */ + pause: (modelId: string): Promise => ipcRenderer.invoke('models:pause', modelId), + + /** Continue from the partial file rather than starting over. */ + resume: (modelId: string): Promise => + ipcRenderer.invoke('models:resume', modelId), + + /** Abort and delete everything the download wrote. */ + cancel: (modelId: string): Promise => ipcRenderer.invoke('models:cancel', modelId), + + /** + * Re-hash an installed model. A mismatch is REPORTED, never repaired: the + * result carries both hashes so the UI can offer Re-verify or Re-download + * rather than silently spending a gigabyte. + */ + verify: (modelId: string): Promise => + ipcRenderer.invoke('models:verify', modelId), + + /** Delete one model. Resolves to the bytes reclaimed. */ + remove: (modelId: string): Promise => ipcRenderer.invoke('models:remove', modelId), + + /** Delete every A Cappella model. Resolves to the bytes reclaimed. */ + removeAll: (): Promise => ipcRenderer.invoke('models:remove-all'), + + /** Disk used by A Cappella models, including directories no longer in the catalog. */ + footprint: (): Promise => ipcRenderer.invoke('models:footprint'), + + /** + * The capability gate's verdict. One source of truth for the HUD, the + * hotkeys, and Settings, so a disabled microphone button and the Voice Setup + * panel can never disagree about why. + */ + readiness: (): Promise => ipcRenderer.invoke('models:readiness'), + + /** + * Download progress, throttled at the source. Broadcast, so every window + * sees the same transfer. + * + * @returns Cleanup function to unsubscribe. + */ + onProgress: (handler: (progress: DownloadProgress) => void): (() => void) => { + const wrappedHandler = (_event: Electron.IpcRendererEvent, progress: DownloadProgress) => + handler(progress); + ipcRenderer.on('models:progress', wrappedHandler); + return () => ipcRenderer.removeListener('models:progress', wrappedHandler); + }, + }; +} + +/** + * `window.maestro.voice.credentials.*` - API keys for the hosted tier. + * + * There is deliberately no `get`. Keys live in the OS keychain and are read only + * in the main process; a channel that handed one back would put it in a renderer + * heap, in a devtools frame, and in any crash dump taken afterwards, for no + * capability the panel actually needs. + */ +function createVoiceCredentialsApi() { + return { + /** Which services have a key stored, and whether this machine has a keychain. */ + list: (): Promise => ipcRenderer.invoke('acappella:list-credentials'), + + /** Store a key, or clear it when `key` is empty. */ + set: (service: VoiceCredentialService, key: string): Promise<{ ok: boolean; error?: string }> => + ipcRenderer.invoke('acappella:set-credential', { service, key }), + + /** + * Check a key against the service. Pass one to test before saving; omit it to + * test the stored one. Rate limiting comes back as its own status, because a + * throttled account is not a bad key. + */ + validate: (service: VoiceCredentialService, key?: string): Promise => + ipcRenderer.invoke('acappella:validate-credential', { service, key }), + }; +} + +/** + * `window.maestro.voice.devices.*` - paired phones and the transport they use. + * + * The pairing payload returned by {@link startPairing} contains the server + * token, because that is what a device needs to reach the signaling socket at + * all. It exists to be rendered as a QR code on a screen the user is looking at, + * and nothing in the renderer should persist it. + * + * `list`, `revoke`, and `revokeAll` deliberately keep working when the Encore + * Feature is off, for the same reason model removal does: the moment somebody + * switches voice off is exactly when they may want to cut a phone loose. + */ +function createVoiceDevicesApi() { + return { + /** + * Open a pairing window and get the QR payload. Null when the web server is + * not running, which the panel shows as "start the web interface first". + */ + startPairing: (): Promise => + ipcRenderer.invoke('acappella:start-pairing'), + + /** The open window, any device waiting for approval, and the advert's state. */ + pairingStatus: (): Promise<{ + payload: PairingPayload | null; + request: PairingRequest | null; + discovery: DiscoveryStatus | null; + manualHint: string; + }> => ipcRenderer.invoke('acappella:pairing-status'), + + /** Close the pairing window without pairing anything. */ + cancelPairing: (): Promise => ipcRenderer.invoke('acappella:cancel-pairing'), + + /** + * Approve a waiting device. THE affirmative action: without it, knowing a + * six-character code would be enough to hold somebody's microphone. + */ + approve: (requestId: string, name?: string): Promise => + ipcRenderer.invoke('acappella:approve-device', { requestId, name }), + + deny: (requestId: string): Promise => + ipcRenderer.invoke('acappella:deny-device', requestId), + + /** Every paired device, joined onto its live connection state and quality. */ + list: (): Promise => ipcRenderer.invoke('acappella:list-devices'), + + rename: (deviceId: string, name: string): Promise => + ipcRenderer.invoke('acappella:rename-device', { deviceId, name }), + + /** + * End a pairing. Takes effect on a LIVE connection, immediately: the peer is + * torn down and any session that device was holding ends. + */ + revoke: (deviceId: string): Promise => + ipcRenderer.invoke('acappella:revoke-device', deviceId), + + /** Remove a revoked device from the list entirely. */ + forget: (deviceId: string): Promise => + ipcRenderer.invoke('acappella:forget-device', deviceId), + + revokeAll: (): Promise => ipcRenderer.invoke('acappella:revoke-all-devices'), + + /** Drop every live connection WITHOUT revoking anything. They can reconnect. */ + disconnectAll: (): Promise => ipcRenderer.invoke('acappella:disconnect-all-devices'), + + /** ICE configuration, what it can reach, and the Cloudflare tunnel caveat. */ + iceSettings: (): Promise<{ + settings: IceTransportSettings; + reach: string; + tunnelNote: string; + discovery: DiscoveryStatus | null; + }> => ipcRenderer.invoke('acappella:ice-settings'), + + /** + * Gather ICE candidates against the configured servers and report which + * types actually came back. A relay candidate is proof the TURN credentials + * work, rather than a claim that they should. + */ + testConnection: (): Promise => ipcRenderer.invoke('acappella:test-connection'), + + /** Start or stop the Bonjour advert. Returns what the advert is actually doing. */ + setDiscovery: (enabled: boolean): Promise => + ipcRenderer.invoke('acappella:set-discovery', enabled), + + /** + * The device list or a connection state changed. Broadcast, so every window + * repaints. + * + * @returns Cleanup function to unsubscribe. + */ + onChanged: (handler: () => void): (() => void) => { + const wrappedHandler = () => handler(); + ipcRenderer.on('acappella:devices-changed', wrappedHandler); + return () => ipcRenderer.removeListener('acappella:devices-changed', wrappedHandler); + }, + + /** + * A device is asking to pair, or stopped asking (`null`). + * + * @returns Cleanup function to unsubscribe. + */ + onPairingRequest: (handler: (request: PairingRequest | null) => void): (() => void) => { + const wrappedHandler = (_event: Electron.IpcRendererEvent, request: PairingRequest | null) => + handler(request); + ipcRenderer.on('acappella:pairing-request', wrappedHandler); + return () => ipcRenderer.removeListener('acappella:pairing-request', wrappedHandler); + }, + }; +} + +/** + * Creates the A Cappella voice API object for contextBridge exposure. + */ +export function createVoiceApi() { + return { + /** The model manager: catalog, downloads, verification, and disk. */ + models: createVoiceModelsApi(), + + /** API keys for the hosted tier, stored in the OS keychain. */ + credentials: createVoiceCredentialsApi(), + + /** Paired phones, the pairing flow, and the WebRTC transport's settings. */ + devices: createVoiceDevicesApi(), + + /** + * Apply a provider change to the running app. + * + * Called after the settings panel writes a selection. Refused while a turn + * is in flight rather than queued, so two engines can never be spliced into + * one exchange. + */ + applyProviders: (): Promise<{ + status: 'swapped' | 'unchanged' | 'refused'; + reason?: string; + }> => ipcRenderer.invoke('acappella:apply-providers'), + + /** + * The last turn's per-hop timings, or null before any turn has completed. + * What turns "voice feels slow" into a specific hop. + */ + lastTurn: (): Promise => ipcRenderer.invoke('acappella:last-turn'), + + /** + * Move the last dispatch to a different agent: the HUD's "wrong tab" + * control, and what a spoken "no, the other one" does. + * + * @returns false when there is nothing to correct, so a stray click is a + * no-op rather than an error. + */ + correctRoute: (agentSessionId: string): Promise => + ipcRenderer.invoke('acappella:correct-route', agentSessionId), + + /** + * Every routing decision this install has made, with what became of it. + * + * The point of the aggregate is that a decision the user immediately + * corrected is a miss even though nothing errored, so routing quality is + * measurable rather than anecdotal. + */ + routingLog: (): Promise<{ entries: RoutingLogEntry[]; quality: RoutingQuality }> => + ipcRenderer.invoke('acappella:routing-log'), + + /** + * Voices the configured TTS provider offers. Empty for a provider with one + * voice or none, which the picker shows as "Provider default". + */ + listVoices: (): Promise> => + ipcRenderer.invoke('acappella:list-voices'), + + /** + * Speak one line through a voice. + * + * `voiceId` overrides the selection for this preview only, which is what + * makes a per-voice Preview button possible: you hear a voice before + * choosing it, rather than selecting each one in turn and undoing the ones + * you did not want. Omit it to preview whatever is configured. + * + * @returns false when nothing could be spoken: no audio host, a silent + * provider, or a live session that the preview must not talk over. + */ + previewVoice: (text: string, voiceId?: string): Promise => + ipcRenderer.invoke('acappella:preview-voice', text, voiceId), + + /** + * Every microphone this machine offers, plus the selected one. + * + * `selectedId` is {@link ACAPPELLA_SYSTEM_DEFAULT_INPUT} when the user has + * expressed no preference, which is a real choice ("follow the OS") rather + * than an absent value. + * + * Labels can be empty until a capture has been granted once - Chromium + * redacts them - so a picker should fall back to the id and re-read when + * `onInputDevices` fires. + */ + inputDevices: (): Promise<{ + devices: Array<{ deviceId: string; label: string }>; + selectedId: string; + }> => ipcRenderer.invoke('acappella:input-devices'), + + /** + * Choose the microphone, persistently. + * + * Takes effect on the next capture, not mid-utterance: swapping the device + * under a live recogniser splices two rooms into one sentence. + */ + setInputDevice: (deviceId: string): Promise => + ipcRenderer.invoke('acappella:set-input-device', deviceId), + + /** + * The device list changed: one was plugged in or pulled out, or a first + * capture just revealed the labels Chromium had redacted. + * + * @returns Cleanup function to unsubscribe. + */ + onInputDevices: ( + handler: (devices: Array<{ deviceId: string; label: string }>) => void + ): (() => void) => { + const wrapped = ( + _event: Electron.IpcRendererEvent, + devices: Array<{ deviceId: string; label: string }> + ) => handler(devices); + ipcRenderer.on('acappella:input-devices', wrapped); + return () => ipcRenderer.removeListener('acappella:input-devices', wrapped); + }, + + /** + * Apply an output volume (0 to 1) to the assistant's voice, immediately. + * + * Applies only; it does NOT persist. The volume slider saves the value + * itself and then calls this so the change is audible on the sentence + * currently playing, and the HUD's mute button calls it WITHOUT saving, + * because a mute that survived a restart is a voice assistant that has + * silently stopped talking. + * + * @returns false when there is no audio host to apply it to. + */ + setVolume: (volume: number): Promise => + ipcRenderer.invoke('acappella:set-volume', volume), + + /** + * Open a voice session. Omit the scope for conductor scope. Any live + * session is replaced rather than stacked. + * + * Returns the snapshot plus any provider substitutions: a role that fell + * back to the mock tier is reported here, never applied silently. + */ + start: (scope?: VoiceScope): Promise => + ipcRenderer.invoke('acappella:start-session', scope), + + /** End the session and return to idle. Safe to call when already idle. */ + stop: (): Promise => ipcRenderer.invoke('acappella:stop-session'), + + /** + * Hand the service a settled utterance. This is the same seam a real STT + * final transcript lands on, so the dev harness and a microphone are + * indistinguishable downstream. + * + * @returns false when the session cannot take an utterance right now. + */ + submitUtterance: (text: string): Promise => + ipcRenderer.invoke('acappella:submit-utterance', text), + + /** + * Barge-in: cancel speech and KEEP the floor. Distinct from {@link stopWord} + * on purpose - talking over the assistant must not hang up on it. + * + * @returns false when nothing was speaking. + */ + interrupt: (source: 'voice' | 'client-button' = 'client-button'): Promise => + ipcRenderer.invoke('acappella:interrupt', source), + + /** The stop word: end the session from any state. */ + stopWord: (payload?: { phrase?: string; source?: 'voice' | 'client-button' }): Promise => + ipcRenderer.invoke('acappella:stop-word', payload), + + /** + * Hand the service an agent's answer, which it reshapes for the ear and + * speaks. Phase 05 feeds real agent output to the same seam in-process; + * this channel is what lets a client drive a turn past `dispatching`. + * + * @returns false when the session was not waiting on a reply. + */ + submitAgentReply: (params: { + agentSessionId: string; + tabId: string; + text: string; + }): Promise => ipcRenderer.invoke('acappella:submit-agent-reply', params), + + /** Current agents and their AI tabs, as the Brain sees them. */ + getRoster: (): Promise => ipcRenderer.invoke('acappella:get-roster'), + + /** + * Open the OS microphone permission settings, the one recovery for a denied + * microphone that the app itself cannot perform. + * + * @returns false on a platform with no such link (Linux), so a caller can + * offer words instead of a button that would do nothing. + */ + openMicSettings: (): Promise => ipcRenderer.invoke('acappella:open-mic-settings'), + + /** + * The microphone permission as the OS reports it, plus whether asking would + * actually show a prompt and where the privacy pane is. + * + * A pure query: calling this NEVER prompts. The prompt happens once, at the + * first session start, because asking for the microphone before the user has + * asked for voice is a trust problem rather than a convenience. + * + * Kept separate from `models.readiness()` on purpose. A denied microphone + * and a missing model are different failures with different recoveries, and + * a UI that can only say "voice unavailable" has already lost the user. + */ + micPermission: (): Promise => ipcRenderer.invoke('acappella:mic-permission'), + + /** + * Catch-up snapshot. Null when no session service exists yet (nothing is + * constructed until the first `start`), which a client reads as idle. + */ + getState: (): Promise => ipcRenderer.invoke('acappella:get-state'), + + /** + * Subscribe to the protocol event stream. Events carry a monotonic `seq` + * per voice session, so a gap means events were lost. + * + * @returns Cleanup function to unsubscribe. + */ + onEvent: (handler: (event: VoiceEvent) => void): (() => void) => { + const wrappedHandler = (_event: Electron.IpcRendererEvent, voiceEvent: VoiceEvent) => + handler(voiceEvent); + ipcRenderer.on('acappella:event', wrappedHandler); + return () => ipcRenderer.removeListener('acappella:event', wrappedHandler); + }, + + /** + * Whether each voice hotkey is actually bound, and what a press can do on + * this platform. + * + * The settings rows render this inline rather than assuming success: a combo + * the OS already owns is the single most common way a global hotkey silently + * does nothing, and "registered" is the only honest thing to show next to a + * key the user just recorded. + */ + hotkeyStatus: (): Promise<{ + statuses: GlobalHotkeyStatus[]; + capability: 'hold-and-tap' | 'tap-only'; + note: string; + }> => ipcRenderer.invoke('acappella:hotkey-status'), + + /** + * Start a wake-word tuning run: the local detector, no session, so a user can + * say the phrase and watch it fire while moving the sensitivity slider. + * + * @returns false when a session is already running or there is no audio host. + */ + wakeTest: (payload?: { phrase?: string; sensitivity?: number }): Promise => + ipcRenderer.invoke('acappella:wake-test', payload), + + /** End a tuning run and close the microphone it opened. */ + wakeTestStop: (): Promise => ipcRenderer.invoke('acappella:wake-test-stop'), + + /** + * Hits from a tuning run. Its own channel rather than a protocol event: a + * run has no session, and synthesising one so a settings panel can light a + * dot would put a fake session in every client's stream. + * + * @returns Cleanup function to unsubscribe. + */ + onWakeTest: (handler: (event: WakeTestEvent) => void): (() => void) => { + const wrappedHandler = (_event: Electron.IpcRendererEvent, hit: WakeTestEvent) => + handler(hit); + ipcRenderer.on('acappella:wake-test', wrappedHandler); + return () => ipcRenderer.removeListener('acappella:wake-test', wrappedHandler); + }, + + /** + * A voice hotkey was pressed and did nothing, with the reason. Subscribed by + * the HUD so a refused press says why rather than looking like a dead key. + * + * @returns Cleanup function to unsubscribe. + */ + onHotkeyRefused: (handler: (info: VoiceHotkeyRefusalInfo) => void): (() => void) => { + const wrappedHandler = (_event: Electron.IpcRendererEvent, info: VoiceHotkeyRefusalInfo) => + handler(info); + ipcRenderer.on('acappella:event:hotkey-refused', wrappedHandler); + return () => ipcRenderer.removeListener('acappella:event:hotkey-refused', wrappedHandler); + }, + }; +} + +/** + * TypeScript type for the A Cappella voice API. + */ +export type VoiceApi = ReturnType; diff --git a/src/main/preload/acappellaAudio.ts b/src/main/preload/acappellaAudio.ts new file mode 100644 index 0000000000..a5ed33c38d --- /dev/null +++ b/src/main/preload/acappellaAudio.ts @@ -0,0 +1,92 @@ +/** + * Preload API for A Cappella's hidden audio host window. + * + * Provides `window.maestro.voiceAudioHost`, the only bridge the audio host + * renderer needs: it ships PCM frames and device/playback status up to main, + * receives capture and playback commands back, and carries the WebRTC control + * plane for the paired-device peers that terminate in the same window. + * + * This is deliberately NOT part of `window.maestro.voice`. That namespace is the + * client-facing protocol (any window, and later the phone, may call it); this + * one is a device driver's control link. Main answers frames only from the audio + * host's own webContents, so exposing it on the shared preload is a convenience + * for the audio host rather than an API surface for the app window. + * + * `send`, not `invoke`: frames arrive 50 times a second and nothing about them + * needs a reply, so paying for a promise round trip per 20 ms of audio would be + * pure overhead. A dropped frame is a counted drop, not an exception. + */ + +import { ipcRenderer } from 'electron'; +import { + ACAPPELLA_AUDIO_COMMAND_CHANNEL, + ACAPPELLA_AUDIO_FRAME_CHANNEL, + ACAPPELLA_AUDIO_STATUS_CHANNEL, + type AudioFrame, + type AudioHostCommand, + type AudioHostStatus, +} from '../../shared/acappella/audio-host'; +import { + ACAPPELLA_WEBRTC_COMMAND_CHANNEL, + ACAPPELLA_WEBRTC_EVENT_CHANNEL, + type WebRtcHostCommand, + type WebRtcHostEvent, +} from '../../shared/acappella/webrtc-host'; + +/** + * Creates the A Cappella audio host API object for contextBridge exposure. + */ +export function createVoiceAudioHostApi() { + return { + /** Ship one 20 ms frame of 16 kHz mono PCM to the main process. */ + sendFrame: (frame: AudioFrame): void => { + ipcRenderer.send(ACAPPELLA_AUDIO_FRAME_CHANNEL, frame); + }, + + /** Report readiness, a device change, a capture failure, or playback state. */ + sendStatus: (status: AudioHostStatus): void => { + ipcRenderer.send(ACAPPELLA_AUDIO_STATUS_CHANNEL, status); + }, + + /** + * Subscribe to capture and playback commands from the main process. + * + * @returns Cleanup function to unsubscribe. + */ + onCommand: (handler: (command: AudioHostCommand) => void): (() => void) => { + const wrappedHandler = (_event: Electron.IpcRendererEvent, command: AudioHostCommand) => + handler(command); + ipcRenderer.on(ACAPPELLA_AUDIO_COMMAND_CHANNEL, wrappedHandler); + return () => ipcRenderer.removeListener(ACAPPELLA_AUDIO_COMMAND_CHANNEL, wrappedHandler); + }, + + /** + * Report a peer event: an answer, a trickled candidate, a connection state, + * a stats reading, or one inbound data-channel message. + * + * The same `send` reasoning as frames. Stats arrive on a timer and inbound + * levels arrive twenty times a second, and neither needs a reply. + */ + sendWebRtcEvent: (event: WebRtcHostEvent): void => { + ipcRenderer.send(ACAPPELLA_WEBRTC_EVENT_CHANNEL, event); + }, + + /** + * Subscribe to peer commands from the main process: accept an offer, trickle + * a candidate in, close a peer, send a protocol message. + * + * @returns Cleanup function to unsubscribe. + */ + onWebRtcCommand: (handler: (command: WebRtcHostCommand) => void): (() => void) => { + const wrappedHandler = (_event: Electron.IpcRendererEvent, command: WebRtcHostCommand) => + handler(command); + ipcRenderer.on(ACAPPELLA_WEBRTC_COMMAND_CHANNEL, wrappedHandler); + return () => ipcRenderer.removeListener(ACAPPELLA_WEBRTC_COMMAND_CHANNEL, wrappedHandler); + }, + }; +} + +/** + * TypeScript type for the A Cappella audio host API. + */ +export type VoiceAudioHostApi = ReturnType; diff --git a/src/main/preload/debug.ts b/src/main/preload/debug.ts index 718b4adb5f..c99067e5f3 100644 --- a/src/main/preload/debug.ts +++ b/src/main/preload/debug.ts @@ -9,8 +9,27 @@ import { ipcRenderer } from 'electron'; import type { DebugPackageOptions } from '../../shared/debugPackage'; +import type { RuntimeSelfTestReport } from '../acappella/runtime/runtime-selftest'; + +// Re-exported rather than re-declared: rc moved the canonical definition to +// `shared/debugPackage`, and a second local copy of the same shape is exactly +// how the two drift apart. export type { DebugPackageOptions }; +/** + * The voice self-test response. + * + * `success` is the IPC envelope (did the handler run), `report.passed` is the + * diagnosis (did the runtimes work). They are different questions and a caller + * has to be able to tell "the self-test could not run" from "the self-test says + * ONNX is broken". + */ +export interface VoiceSelfTestResponse { + success: boolean; + report?: RuntimeSelfTestReport; + error?: string; +} + /** * Document graph file change event */ @@ -122,6 +141,11 @@ export function createDebugApi() { getAppStats: (): Promise => ipcRenderer.invoke('debug:getAppStats'), + // A Cappella voice self-test: loads each native runtime, runs a trivial + // operation, and reports per-runtime pass/fail with timings. Loads no model, + // so it is safe to run at any time, including before anything is downloaded. + voiceSelfTest: (): Promise => ipcRenderer.invoke('debug:voiceSelfTest'), + // Performance profiling (Chromium contentTracing). Off by default with no // steady-state cost; see src/main/profiling. getProfilingStatus: (): Promise => diff --git a/src/main/preload/index.ts b/src/main/preload/index.ts index e4568065df..eaf7f2dd6d 100644 --- a/src/main/preload/index.ts +++ b/src/main/preload/index.ts @@ -68,6 +68,8 @@ import { createCoworkingApi } from './coworking'; import { createBrowserSessionApi } from './browserSession'; import { createWindowsApi } from './windows'; import { createImagesApi } from './images'; +import { createVoiceApi } from './acappella'; +import { createVoiceAudioHostApi } from './acappellaAudio'; import { MAESTRO_CLI_PATH_ARG_PREFIX } from '../../shared/maestro-cli'; /** @@ -274,6 +276,12 @@ contextBridge.exposeInMainWorld('maestro', { // Session Images API (resolve maestro-image:// refs back to data URLs) images: createImagesApi(), + + // A Cappella Voice API (headless voice session in main; this is one client) + voice: createVoiceApi(), + + // A Cappella audio host control link (only the hidden audio window uses it) + voiceAudioHost: createVoiceAudioHostApi(), }); // Re-export factory functions for external consumers (e.g., tests) @@ -378,6 +386,9 @@ export { createWindowsApi, // Session Images createImagesApi, + // A Cappella Voice + createVoiceApi, + createVoiceAudioHostApi, }; // Re-export types for TypeScript consumers @@ -666,3 +677,11 @@ export type { // From images ImagesApi, } from './images'; +export type { + // From acappella + VoiceApi, +} from './acappella'; +export type { + // From acappellaAudio + VoiceAudioHostApi, +} from './acappellaAudio'; diff --git a/src/main/preload/process/tabRemote.ts b/src/main/preload/process/tabRemote.ts index 538d5477e9..03f78be254 100644 --- a/src/main/preload/process/tabRemote.ts +++ b/src/main/preload/process/tabRemote.ts @@ -30,6 +30,38 @@ export function createTabRemoteApi() { ipcRenderer.send(responseChannel, result); }, + /** + * Subscribe to a request to land on one specific AI tab. + * + * Separate from `onRemoteSelectTab` because this one ANSWERS: the caller + * (A Cappella's dispatch executor) has to know whether the tab was simply + * focused, woken out of a snooze, or reopened from the closed-tab history, + * and only the renderer can tell it apart. + */ + onRemoteFocusAiTab: ( + callback: (sessionId: string, tabId: string, responseChannel: string) => void + ): (() => void) => { + const handler = (_: unknown, sessionId: string, tabId: string, responseChannel: string) => + callback(sessionId, tabId, responseChannel); + ipcRenderer.on('remote:focusAiTab', handler); + return () => ipcRenderer.removeListener('remote:focusAiTab', handler); + }, + + /** + * Send response for a remote AI tab focus + */ + sendRemoteFocusAiTabResponse: ( + responseChannel: string, + result: { + ok: boolean; + tabId?: string; + action?: 'focused' | 'woke' | 'reopened'; + reason?: string; + } + ): void => { + ipcRenderer.send(responseChannel, result); + }, + /** * Subscribe to remote close tab from web interface */ diff --git a/src/main/preload/system.ts b/src/main/preload/system.ts index 8c37ceae28..265c6ab173 100644 --- a/src/main/preload/system.ts +++ b/src/main/preload/system.ts @@ -6,6 +6,7 @@ import { ipcRenderer } from 'electron'; import type { IpcRendererEvent } from 'electron'; +import type { GlobalHotkeyStatus } from '../../shared/global-hotkeys'; import type { ParsedDeepLink, ShellInfo, UpdateStatus } from '../../shared/types'; export type { ShellInfo, UpdateStatus } from '../../shared/types'; @@ -225,10 +226,13 @@ export function createAppApi() { /** * Listen for global hotkey registration failures (e.g. another app already * owns the combo). Renderer should surface this to the user so they pick a - * different key. + * different key. The payload names the failing hotkey id, because Maestro + * registers several and "a global hotkey failed" is not actionable. */ - onGlobalHotkeyRegistrationFailed: (callback: (keys: string[]) => void): (() => void) => { - const handler = (_: unknown, keys: string[]) => callback(keys); + onGlobalHotkeyRegistrationFailed: ( + callback: (status: GlobalHotkeyStatus) => void + ): (() => void) => { + const handler = (_: unknown, status: GlobalHotkeyStatus) => callback(status); ipcRenderer.on('globalHotkey:registrationFailed', handler); return () => ipcRenderer.removeListener('globalHotkey:registrationFailed', handler); }, diff --git a/src/main/utils/keyring.ts b/src/main/utils/keyring.ts new file mode 100644 index 0000000000..2a888ad44c --- /dev/null +++ b/src/main/utils/keyring.ts @@ -0,0 +1,64 @@ +/** + * The one adapter over `@napi-rs/keyring`. + * + * Two callers need an OS credential entry for completely different reasons - the + * plugin authorization ledger's freshness anchor and A Cappella's API keys - and + * both need the same three properties, which is why the loader lives here rather + * than being written twice: + * + * 1. **Lazy.** The native module is `require`d on first use, not at import, so + * app startup does not depend on a keyring being present. + * 2. **Never throws.** A machine with no keyring daemon (headless Linux, a + * locked login keychain) gets `null` back and the caller degrades. A missing + * credential store is a capability the machine lacks, not a crash. + * 3. **One module id.** A second `require('@napi-rs/keyring')` elsewhere would + * be a second place to keep the packaging config honest. + */ + +/** The slice of `@napi-rs/keyring`'s `Entry` this codebase uses. */ +export interface KeyringEntry { + getPassword(): string | null; + setPassword(password: string): void; + deletePassword(): boolean; +} + +export interface KeyringModule { + Entry: new (service: string, account: string) => KeyringEntry; +} + +/** Loads the native module, or null when it is absent or will not load. */ +export type KeyringModuleLoader = () => KeyringModule | null; + +/** + * The production loader. Uses `require` rather than a dynamic import because the + * main bundle is CommonJS and this must stay synchronous: a caller asking whether + * a credential exists cannot be made async by the loader's module system. + */ +export const loadKeyringModule: KeyringModuleLoader = () => { + try { + const mod = require('@napi-rs/keyring') as Partial; + return typeof mod.Entry === 'function' ? (mod as KeyringModule) : null; + } catch { + return null; + } +}; + +/** + * One credential entry, or null when this machine has no usable keyring. + * + * `loadModule` is injectable for tests; production always uses + * {@link loadKeyringModule}. + */ +export function createKeyringEntry( + service: string, + account: string, + loadModule: KeyringModuleLoader = loadKeyringModule +): KeyringEntry | null { + try { + const mod = loadModule(); + if (!mod) return null; + return new mod.Entry(service, account); + } catch { + return null; + } +} diff --git a/src/main/utils/networkUtils.ts b/src/main/utils/networkUtils.ts index a35343f009..d0c720023e 100644 --- a/src/main/utils/networkUtils.ts +++ b/src/main/utils/networkUtils.ts @@ -88,6 +88,19 @@ function getIpViaUdp(): Promise { * Prefers interfaces that look like they connect to the internet. */ function getIpFromInterfaces(): string { + const candidates = rankedIpv4Candidates(); + if (candidates.length === 0) { + return 'localhost'; + } + return candidates[0].ip; +} + +/** + * Every usable IPv4 address, sorted best-routing first. The one scoring pass + * behind both `getIpFromInterfaces` (takes the head) and + * `listLocalIpv4Addresses` (takes all of it). + */ +function rankedIpv4Candidates(): Array<{ ip: string; priority: number }> { const interfaces = networkInterfaces(); const candidates: Array<{ ip: string; priority: number }> = []; @@ -132,13 +145,9 @@ function getIpFromInterfaces(): string { } } - if (candidates.length === 0) { - return 'localhost'; - } - - // Sort by priority (highest first) and return the best + // Sort by priority (highest first) so the head is the best route out. candidates.sort((a, b) => b.priority - a.priority); - return candidates[0].ip; + return candidates; } /** @@ -167,3 +176,20 @@ function isPrivateIp(ip: string): boolean { export function getLocalIpAddressSync(): string { return getIpFromInterfaces(); } + +/** + * Every non-internal IPv4 address on this machine, best-routing first. + * + * `getLocalIpAddress` answers "which one address should I advertise"; this + * answers "which addresses could someone reach me on", which is a different + * question with a different right answer. A machine on WiFi and a Tailscale-style + * overlay at the same time has two working addresses, and a pairing QR code that + * offered only the highest-priority one would send a phone that is on the overlay + * but not the WiFi to a relay for a connection it could have had directly. + * + * Ordering reuses the same interface-priority scoring as the single-address + * picker, so the two can never disagree about which address is the primary one. + */ +export function listLocalIpv4Addresses(): string[] { + return rankedIpv4Candidates().map((candidate) => candidate.ip); +} diff --git a/src/main/web-server/WebServer.ts b/src/main/web-server/WebServer.ts index de7ebf6db7..0d73be27f7 100644 --- a/src/main/web-server/WebServer.ts +++ b/src/main/web-server/WebServer.ts @@ -33,6 +33,7 @@ import { logger } from '../utils/logger'; import { getLocalIpAddress } from '../utils/networkUtils'; import { captureException } from '../utils/sentry'; import { WebSocketMessageHandler } from './handlers'; +import { handleACappellaSignalDisconnect } from './handlers/messageHandlers/acappellaSignal'; import { BroadcastService } from './services'; import { ApiRoutes, StaticRoutes, WsRoute } from './routes'; import { LiveSessionManager, CallbackRegistry } from './managers'; @@ -909,6 +910,10 @@ export class WebServer { ); } } + // Tears down any A Cappella peer connection and closes the voice + // session if this socket's device was holding the microphone. A + // dropped socket must not leave a hot mic behind. + handleACappellaSignalDisconnect(clientId); this.webClients.delete(clientId); logger.info( `Client disconnected: ${clientId} (total: ${this.webClients.size})`, @@ -916,6 +921,7 @@ export class WebServer { ); }, onClientError: (clientId) => { + handleACappellaSignalDisconnect(clientId); this.webClients.delete(clientId); }, handleMessage: (clientId, message) => { diff --git a/src/main/web-server/handlers/messageHandlers/WebSocketMessageHandler.ts b/src/main/web-server/handlers/messageHandlers/WebSocketMessageHandler.ts index 117845669f..d1d55f677f 100644 --- a/src/main/web-server/handlers/messageHandlers/WebSocketMessageHandler.ts +++ b/src/main/web-server/handlers/messageHandlers/WebSocketMessageHandler.ts @@ -21,6 +21,7 @@ * - get_file_tree: Read directory tree from filesystem for web file explorer * - get_settings: Fetch current web settings * - set_setting: Modify a single setting (allowlisted keys only) + * - acappella_signal: WebRTC signaling for a paired A Cappella voice device * - list_desktop_sessions: Enumerate open AI tabs across all agents (CLI: `session list`) * - get_session_history: Return tab conversation history with --since/--tail filters (CLI: `session show`) * @@ -41,6 +42,7 @@ import type { MessageHandlerContext, } from './types'; import { handleSendCommand, handleSwitchMode, handleSelectSession } from './commands'; +import { ACAPPELLA_SIGNAL_MESSAGE, handleACappellaSignal } from './acappellaSignal'; import { handleGetSessions, handleCreateSession, @@ -637,6 +639,10 @@ export class WebSocketMessageHandler { handleGetSessionHistory(this.ctx, client, message); break; + case ACAPPELLA_SIGNAL_MESSAGE: + handleACappellaSignal(this.ctx, client, message); + break; + case 'bridge.invoke': void this.handleBridgeInvoke(client, message); break; diff --git a/src/main/web-server/handlers/messageHandlers/acappellaSignal.ts b/src/main/web-server/handlers/messageHandlers/acappellaSignal.ts new file mode 100644 index 0000000000..f658438ed5 --- /dev/null +++ b/src/main/web-server/handlers/messageHandlers/acappellaSignal.ts @@ -0,0 +1,66 @@ +/** + * WebRTC signaling over the existing authenticated WebSocket. + * + * One message type, `acappella_signal`, whose `payload` is a + * `SignalingClientMessage`. It rides `/$TOKEN/ws` rather than a port of its own + * so a paired device inherits the token check, the client registry, and the + * connection lifecycle that already exist - see + * `docs/architecture/acappella/decisions/adr-001-webrtc-transport.md`. + * + * Everything about who may say what lives in + * `src/main/acappella/transport/signaling.ts`. This file is the adapter: it + * finds the transport, gives it a way to write back to this socket, and hands + * over the payload. + */ + +import { getACappellaTransport } from '../../../acappella'; +import type { SignalingServerMessage } from '../../../acappella/transport/signaling'; +import { ACAPPELLA_SIGNAL_MESSAGE } from '../../../acappella/transport/signaling'; +import type { WebClient, WebClientMessage, MessageHandlerContext } from './types'; + +export { ACAPPELLA_SIGNAL_MESSAGE }; + +/** + * Handle one signaling frame. + * + * Registration is lazy and idempotent: a browser client that never speaks A + * Cappella should not cost a signaling session, and a device that does gets one + * on its first message. + */ +export function handleACappellaSignal( + ctx: MessageHandlerContext, + client: WebClient, + message: WebClientMessage +): void { + const transport = getACappellaTransport(); + // Both halves of one condition. The transport is built once at boot and kept + // so the feature can be switched back on without a restart, so its existence + // does NOT mean the feature is on: a device that was connected when the user + // unticked the box would otherwise keep signaling into a live transport. + if (!transport || !transport.featureEnabled()) { + // A stated refusal rather than silence: a phone that gets no answer cannot + // tell "the feature is off" from "the network ate it", and only one of those + // is worth retrying. + ctx.send(client, { + type: ACAPPELLA_SIGNAL_MESSAGE, + payload: { + op: 'error', + code: 'not-authenticated', + message: 'A Cappella is not running on this desktop. Turn it on in Encore Features.', + }, + }); + return; + } + + transport.registerClient({ + clientId: client.id, + send: (payload: SignalingServerMessage) => + ctx.send(client, { type: ACAPPELLA_SIGNAL_MESSAGE, payload }), + }); + void transport.handleSignalMessage(client.id, (message as { payload?: unknown }).payload); +} + +/** A socket went away. Tears down its peer and any floor it was holding. */ +export function handleACappellaSignalDisconnect(clientId: string): void { + getACappellaTransport()?.handleClientDisconnect(clientId); +} diff --git a/src/main/web-server/routes/staticRoutes.ts b/src/main/web-server/routes/staticRoutes.ts index 9f89e03ef8..0790a54753 100644 --- a/src/main/web-server/routes/staticRoutes.ts +++ b/src/main/web-server/routes/staticRoutes.ts @@ -11,6 +11,7 @@ * - /$TOKEN/sw.js - PWA service worker * - /$TOKEN - Web-desktop interface (the default UI) * - /$TOKEN/desktop - Legacy alias for the web-desktop interface + * - /$TOKEN/acappella - A Cappella reference client (the device half of the voice protocol) * - /$TOKEN/session/:sessionId - Deprecated deep link, serves the desktop interface * - /:token - Invalid token catch-all, redirect to GitHub */ @@ -164,6 +165,48 @@ export class StaticRoutes { } } + /** + * Serve the A Cappella reference client. + * + * A second page in the same bundle, and deliberately NOT the desktop SPA: it + * is the device half of the voice protocol, the independent endpoint the + * WebRTC transport is tested against, and what a phone developer reads to see + * the wire behaviour. It pairs with a code like any other device, so it gets + * no config injection at all - handing it the token would let it skip the one + * flow it exists to exercise. + */ + private serveACappellaClient(reply: FastifyReply): void { + const indexPath = this.webDesktopPath + ? path.join(this.webDesktopPath, 'acappella-client', 'index.html') + : null; + if (!indexPath || !existsSync(indexPath)) { + reply.code(503).send({ + error: 'Service Unavailable', + message: 'A Cappella reference client not built. Run "npm run build:web-desktop".', + }); + return; + } + + try { + // Read fresh so rebuilt asset hashes are reflected immediately, and point + // both relative forms at the one asset mount: the page sits a directory + // down from the bundle root, so Vite emits `../assets/`. + let html = readFileSync(indexPath, 'utf-8'); + const token = this.securityToken; + html = html.replace(/\.\.\/assets\//g, `/${token}/desktop/assets/`); + html = html.replace(/\.\/assets\//g, `/${token}/desktop/assets/`); + html = html.replace(/="\/assets\//g, `="/${token}/desktop/assets/`); + reply.type('text/html').send(html); + } catch (err) { + void captureException(err); + logger.error('Error serving the A Cappella reference client', LOG_CONTEXT, err); + reply.code(500).send({ + error: 'Internal Server Error', + message: 'Failed to serve the A Cappella reference client.', + }); + } + } + /** * Register all static routes on the Fastify server */ @@ -225,6 +268,15 @@ export class StaticRoutes { this.serveDesktopIndex(reply); }); + // The A Cappella reference client. Registered before the `/:token` + // catch-all, which would otherwise swallow it and serve the desktop SPA. + server.get(`/${token}/acappella`, async (_request, reply) => { + this.serveACappellaClient(reply); + }); + server.get(`/${token}/acappella/`, async (_request, reply) => { + this.serveACappellaClient(reply); + }); + // Deprecated single-session deep link. The desktop app manages its own // session selection, so this just serves the full interface. server.get(`/${token}/session/:sessionId`, async (_request, reply) => { diff --git a/src/main/web-server/services/index.ts b/src/main/web-server/services/index.ts index dfd23d59f2..c3fceb52ae 100644 --- a/src/main/web-server/services/index.ts +++ b/src/main/web-server/services/index.ts @@ -4,8 +4,11 @@ * Re-exports all service modules for the web server. */ -export { - BroadcastService, +export { BroadcastService } from './broadcastService'; +// Split out because everything below is a type: `isolatedModules` compiles each +// file alone, so a type re-exported through a value `export` has no runtime +// binding to emit and is a hard error. +export type { WebClientInfo, CustomAICommand, AITabData, diff --git a/src/main/window-registry.ts b/src/main/window-registry.ts index 37128cb65e..d7fb2c8cb6 100644 --- a/src/main/window-registry.ts +++ b/src/main/window-registry.ts @@ -1,17 +1,19 @@ // src/main/window-registry.ts import { EventEmitter } from 'events'; -import type { BrowserWindow } from 'electron'; +import { BrowserWindow } from 'electron'; +import type { WebContents } from 'electron'; import { isPointInWindowBounds, type WindowPanelState } from '../shared/window-types'; import { generateUUID } from '../shared/uuid'; /** * The kind of a registered window. `app` windows own agents (sessions) and take * part in all the multi-window machinery (session moves, persistence, the "Move - * to Window" menu, telemetry). Special kinds like `cadenza-hud` are host-owned - * feature windows that own no sessions; the multi-window consumers skip them. + * to Window" menu, telemetry). Special kinds like `cadenza-hud` and + * `acappella-audio` are host-owned feature windows that own no sessions; the + * multi-window consumers skip them. */ -export type WindowKind = 'app' | 'cadenza-hud'; +export type WindowKind = 'app' | 'cadenza-hud' | 'acappella-audio'; /** * A single window tracked by the registry. `sessionIds` are agent IDs (what @@ -159,6 +161,45 @@ export class WindowRegistry extends EventEmitter { return undefined; } + /** + * The window an IPC message came from, or undefined when there is no window + * behind it. + * + * "No window" is a real, expected answer, not a failure: the web-desktop + * bridge invokes handlers with a synthetic event that has no `sender` at all + * (`FAKE_EVENT` in `web-server/handlers/bridgeHandlers.ts`), and a web client + * is not a window. Passing that straight to `BrowserWindow.fromWebContents` + * throws, so the check is here rather than at each call site. + */ + findBySender(sender: WebContents | null | undefined): RegisteredWindow | undefined { + if (!sender) return undefined; + const browserWindow = BrowserWindow.fromWebContents(sender); + if (!browserWindow) return undefined; + for (const entry of this.windows.values()) { + if (entry.browserWindow === browserWindow) return entry; + } + return undefined; + } + + /** + * The window that should own a surface started without one: the focused + * window, else the primary. + * + * For triggers that have no IPC sender to resolve - a global hotkey, a wake + * word, a paired phone. The focused window is what "the window the user is + * looking at" means, and falling back to the primary means such a trigger + * always lands somewhere rather than nowhere. + */ + getFocusedAppWindow(): RegisteredWindow | undefined { + const focused = BrowserWindow.getFocusedWindow(); + if (focused) { + for (const entry of this.windows.values()) { + if (entry.kind === 'app' && entry.browserWindow === focused) return entry; + } + } + return this.getPrimary(); + } + /** Stop tracking a window (e.g. after it is closed). */ remove(windowId: string): void { if (this.windows.delete(windowId)) { diff --git a/src/prompts/acappella-router.md b/src/prompts/acappella-router.md new file mode 100644 index 0000000000..240d4ef212 --- /dev/null +++ b/src/prompts/acappella-router.md @@ -0,0 +1,116 @@ +You are the Conductor. You route spoken instructions inside Maestro, a desktop app that runs several AI coding agents at once. + +You are given one utterance, the list of running agents with their open tabs, and the last few things the user said. Decide which agent the utterance is for, what to do with that agent's tabs, and what prompt to actually send. + +## Output + +Answer with ONE JSON object and nothing else. No prose, no code fence. + +| Field | Meaning | +| ------------ | ----------------------------------------------------------------------------------------------------------- | +| `target` | Either the string `"conductor"` or `{"sessionId": ""}`. Never invent an id. | +| `tabAction` | `"current"`, `"new"`, or `"recall"`. | +| `tabId` | Required by `recall`. Must be one of that agent's tab ids. | +| `tabName` | A short name for a `new` tab, three words at most. | +| `prompt` | What the agent should receive: the request itself, with the routing words removed. Keep the user's wording. | +| `confidence` | 0 to 1. Be honest: a guess is 0.4, hearing an agent named out loud is 0.9. | +| `clarify` | One short spoken question. Set it INSTEAD of guessing. Leave it out otherwise. | + +## Choosing a target + +- Name an agent when the utterance names it, describes its project, or continues work only that agent has been doing. +- Use `"conductor"` when the utterance is about Maestro itself ("how many agents are running", "turn on dark mode") or about the fleet as a whole rather than about one repository. +- Never invent a session id. If nothing in the roster fits, the conductor takes it. + +## Choosing a tab action + +- `current` - the utterance continues the topic of the agent's active tab. This is the common case; prefer it when in doubt between `current` and `new`. +- `new` - a clearly different topic from what the active tab is about. Give it a `tabName`. +- `recall` - the utterance points at prior work: "back to", "the auth thing", "what we discussed yesterday", "that migration conversation". Match it against the tab topics and set `tabId`. + +A tab marked `snoozed` or `closed` is still a valid `recall` target. It will be woken or reopened. + +## When you are not sure + +Do not guess between two plausible agents or two plausible tabs. Set `clarify` to one short question naming the alternatives ("the backend agent or the API agent?") and leave `confidence` low. A question costs the user two seconds; a misroute costs them a prompt in the wrong repository. + +## Examples + +```json +{ + "target": { "sessionId": "a1" }, + "tabAction": "current", + "prompt": "run the tests", + "confidence": 0.9 +} +``` + +```json +{ + "target": { "sessionId": "a2" }, + "tabAction": "new", + "tabName": "Rate Limiting", + "prompt": "add a rate limiter to the public API", + "confidence": 0.8 +} +``` + +```json +{ + "target": { "sessionId": "a1" }, + "tabAction": "recall", + "tabId": "t7", + "prompt": "did we ever land that fix?", + "confidence": 0.7 +} +``` + +```json +{ + "target": "conductor", + "tabAction": "current", + "prompt": "", + "confidence": 0.3, + "clarify": "the backend agent or the API agent?" +} +``` + +## Talking instead of sending + +These rules apply only when the prompt you are given includes a conversation +section saying you may reply. In command mode they do not exist and every +utterance is routed. + +- `reply` is one short spoken line back to the user. Setting it means you are + TALKING: no agent is contacted and the floor stays with the user. +- Reply while the user is still thinking out loud, describing a problem, or has + said something that is not yet a doable task. +- Do NOT reply once one concrete, doable thing has been stated. Send it instead. + An agent can work out the details; your job is to notice that there is a job. +- When you send after a conversation, `prompt` is the distilled request - a + sentence or two in the user's own words, not a transcript of the discussion. +- Keep a reply to one or two sentences. It is spoken aloud, not read. + +Still thinking out loud, so talk back: + +```json +{ + "target": "conductor", + "tabAction": "current", + "prompt": "", + "confidence": 0.3, + "reply": "The refresh failing only on the second load sounds like the token cache. Want me to have someone look?" +} +``` + +A doable thing has been stated, so send it: + +```json +{ + "target": { "sessionId": "agent-backend" }, + "tabAction": "new", + "tabName": "Token refresh", + "prompt": "Find out why the token refresh fails on the second load and fix it.", + "confidence": 0.8 +} +``` diff --git a/src/prompts/acappella-translator.md b/src/prompts/acappella-translator.md new file mode 100644 index 0000000000..2cd118c04a --- /dev/null +++ b/src/prompts/acappella-translator.md @@ -0,0 +1,45 @@ +You turn an AI coding agent's written answer into something worth hearing out loud. + +The person listening has no screen. They asked for something by voice, they are waiting, and they will interrupt you the moment you stop being useful. Your job is to tell them what happened in the time it takes to say one or two sentences, and to offer the detail rather than deliver it. + +## Rules + +- Speak the outcome, never the transcript. "Done, the auth bug was a stale token check" beats a summary of the steps taken to find it. +- Use contractions. Say it the way a colleague would say it across a desk. +- No markdown. No headings, no bullet lists, no bold, no code fences, no tables. If the answer was a list, say how many things there were and name the interesting one. +- Never read code, diffs, command output, URLs, or file paths aloud. Say "the router file" rather than spelling out a path. +- Numbers and identifiers get spoken naturally: "version one two three", "about forty lines", "three tests". +- Offer the detail instead of giving it. End with a short offer when there is more to say: "want the details?", "want me to read the error?". Do not offer when there is nothing behind the offer. +- If the agent asked the user a question, ask that question directly and drop everything else. +- If the agent failed, say so plainly and say what it failed at. Never soften it into an ambiguous answer, and never go silent. +- Answer with the spoken text only. No preamble, no quotes around it, no stage directions. + +## Length + +Two sentences is the target and the cap unless the caller asks for more. A third sentence is only ever the offer of detail. + +## Continuity + +You are given the last few things you said out loud. Refer back to them the way a person would ("like I said about the token check") instead of repeating yourself. Never re-explain something you already said this conversation. + +## Examples + +Agent wrote four hundred lines ending in a summary of a fixed authentication bug: + +> Done. The auth bug was a stale token check in the middle of the refresh path. Want the details? + +Agent wrote a diff touching six files: + +> That's six files changed, mostly the session store. Want me to walk you through them? + +Agent wrote "Yes, the tests pass.": + +> Yes, the tests pass. + +Agent errored out against a rate limit: + +> It stopped, the API rate limited us. I can retry whenever you want. + +Agent asked which database to migrate first: + +> It's asking which database to migrate first, staging or production. diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index 36dea4675f..fcdd3429d9 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -3014,6 +3014,7 @@ function MaestroConsoleInner() { useNativeTitleBar={useNativeTitleBar} isMdDownViewport={isMdDownViewport} concertoEnabled={encoreFeatures.concerto === true} + aCappellaEnabled={encoreFeatures.aCappella === true} activeGroupChatId={activeGroupChatId} groupChats={groupChats} groups={groups} diff --git a/src/renderer/acappella-audio/AudioHostRoot.tsx b/src/renderer/acappella-audio/AudioHostRoot.tsx new file mode 100644 index 0000000000..fb37c3789c --- /dev/null +++ b/src/renderer/acappella-audio/AudioHostRoot.tsx @@ -0,0 +1,255 @@ +/** + * AudioHostRoot - render entry for A Cappella's hidden audio window. + * + * The main process loads the ordinary renderer bundle with `?acappellaAudio` + * (see `src/main/acappella/audio-host-window.ts`) and `main.tsx` mounts this + * instead of the app. It renders nothing: the window exists only because + * `AudioContext`, `getUserMedia`, and `AudioWorklet` live in a renderer and the + * main process has none of them. + * + * The controller is a plain object rather than a hook so the audio lifecycle is + * not tangled with React's. A microphone and an `AudioContext` are OS resources; + * they need one owner with an explicit start and stop, not something that + * re-runs when a dependency array changes. + */ + +import { useEffect } from 'react'; + +import type { AudioHostCommand, AudioHostStatus } from '../../shared/acappella/audio-host'; +import type { WebRtcHostCommand, WebRtcHostEvent } from '../../shared/acappella/webrtc-host'; +import { logger } from '../utils/logger'; +import { createAudioHostBridge, type AudioHostBridge } from './bridge'; +import { MicCapture, listInputDevices } from './capture'; +import { PeerRegistry, applyWebRtcCommand, type PeerAudioBinding } from './peer-connection'; +import { TtsPlayback } from './playback'; +import { pcmWorkletUrl } from './worklet-url'; + +const LOG_CONTEXT = 'ACappellaAudioHost'; + +export interface AudioHostController { + handleCommand(command: AudioHostCommand): void; + /** Peer lifecycle and signaling, for the paired-device leg. */ + handleWebRtcCommand(command: WebRtcHostCommand): void; + dispose(): void; +} + +export interface CreateAudioHostControllerOptions { + bridge?: AudioHostBridge; + workletUrl?: string; + /** Seam for tests; production always wants a real `AudioContext`. */ + createContext?: () => AudioContext; + /** Seam for tests; production constructs a real `RTCPeerConnection`. */ + createPeerConnection?: (config: RTCConfiguration) => RTCPeerConnection; +} + +/** + * Wire the bridge to a capture and a playback over one shared `AudioContext`. + * + * The context is created on the first command that needs it, not here: the + * window is built when a session starts, but a session that only speaks (or one + * the user abandons) should never open an audio device. + */ +export function createAudioHostController( + options: CreateAudioHostControllerOptions = {} +): AudioHostController { + const bridge = options.bridge ?? createAudioHostBridge(); + const workletUrl = options.workletUrl ?? pcmWorkletUrl; + const createContext = options.createContext ?? (() => new AudioContext()); + + let context: AudioContext | null = null; + let capture: MicCapture | null = null; + let playback: TtsPlayback | null = null; + let disposed = false; + + /** The device whose microphone is currently feeding the capture pipeline. */ + let remoteCaptureDeviceId: string | null = null; + /** The tap that turns the shared playback output into a sendable track. */ + let outboundDestination: MediaStreamAudioDestinationNode | null = null; + + const onStatus = (status: AudioHostStatus) => bridge.sendStatus(status); + + /** + * Publish the input list. + * + * Called on boot, on every device change, and after capture starts. The last + * one matters: Chromium redacts device LABELS until a capture has been granted + * once, so the boot-time list is often a set of unnamed entries, and a picker + * built from it alone would offer "Microphone 1 / Microphone 2" forever. + */ + const publishInputDevices = async (): Promise => { + if (disposed) return; + try { + onStatus({ kind: 'input-devices', devices: await listInputDevices() }); + } catch { + // Enumeration failing is not a session failure - the picker just has + // nothing to add beyond the system default. + } + }; + + const ensureContext = (): AudioContext => { + if (!context) context = createContext(); + return context; + }; + + const ensureCapture = (): MicCapture => { + if (!capture) { + capture = new MicCapture({ + context: ensureContext(), + workletUrl, + onFrame: (frame) => bridge.sendFrame(frame), + onStatus, + }); + } + return capture; + }; + + const ensurePlayback = (): TtsPlayback => { + if (!playback) playback = new TtsPlayback({ context: ensureContext(), onStatus }); + return playback; + }; + + const handleCommand = (command: AudioHostCommand): void => { + if (disposed) return; + switch (command.kind) { + case 'start-capture': + void ensureCapture() + .start(command.deviceId) + // Labels are readable once a capture has been granted, so this is the + // moment the picker can finally show real device names. + .then(() => publishInputDevices()); + break; + case 'stop-capture': + capture?.stop('requested'); + break; + case 'list-input-devices': + void publishInputDevices(); + break; + case 'play': + void ensurePlayback().enqueue({ + utteranceId: command.utteranceId, + format: command.format, + sampleRate: command.sampleRate, + data: command.data, + }); + break; + case 'end-utterance': + playback?.endUtterance(command.utteranceId); + break; + case 'flush': + // Barge-in. Nothing to flush when playback was never built, and + // building one here just to empty it would open an audio device. + playback?.flush(); + break; + case 'duck': + playback?.duck(command.gain, command.ms); + break; + case 'set-volume': + // Built on demand, unlike `flush` and `duck`: the volume has to be in + // place BEFORE the first chunk arrives, or the opening sentence of a + // session comes out at the previous level. + ensurePlayback().setVolume(command.volume); + break; + } + }; + + /** + * How a peer reaches the audio graph. + * + * Both directions are TAPS on the existing pipeline rather than new ones: the + * remote microphone goes into the same worklet a local microphone does, and + * the outbound voice comes off the same node the speakers hear. That is what + * makes a remote turn byte-for-byte identical to a local one downstream. + */ + const peerAudio: PeerAudioBinding = { + attachRemoteStream: (stream, deviceId) => { + remoteCaptureDeviceId = deviceId; + void ensureCapture().startWithStream(stream, { deviceId }); + }, + detachRemoteStream: (deviceId) => { + // Only the device that actually holds the capture may close it: a + // detach from a device that already lost the floor would shut the + // microphone of the one that just took it. + if (remoteCaptureDeviceId !== deviceId) return; + remoteCaptureDeviceId = null; + capture?.stop('requested'); + }, + getOutboundTrack: () => { + const ctx = ensureContext(); + if (!outboundDestination) { + outboundDestination = ctx.createMediaStreamDestination(); + ensurePlayback().outputNode.connect(outboundDestination); + } + return outboundDestination.stream.getAudioTracks()[0] ?? null; + }, + }; + + const peers = new PeerRegistry({ + audio: peerAudio, + createPeerConnection: options.createPeerConnection, + callbacks: { + onAnswer: (deviceId, answer) => sendPeerEvent({ kind: 'answer', deviceId, answer }), + onIceCandidate: (deviceId, candidate) => + sendPeerEvent({ kind: 'ice-candidate', deviceId, candidate }), + onConnectionState: (deviceId, state) => + sendPeerEvent({ kind: 'connection-state', deviceId, state }), + onStats: (stats) => sendPeerEvent({ kind: 'stats', stats }), + onMessage: (deviceId, message) => sendPeerEvent({ kind: 'message', deviceId, message }), + onError: (deviceId, message) => sendPeerEvent({ kind: 'peer-error', deviceId, message }), + }, + }); + + function sendPeerEvent(event: WebRtcHostEvent): void { + if (disposed) return; + bridge.sendWebRtcEvent(event); + } + + // The switch itself lives next to the registry it drives, so the conformance + // harness can apply the identical commands with no DOM around it. + const handleWebRtcCommand = (command: WebRtcHostCommand): void => { + if (disposed) return; + applyWebRtcCommand(peers, command, sendPeerEvent); + }; + + const unsubscribe = bridge.onCommand(handleCommand); + const unsubscribeWebRtc = bridge.onWebRtcCommand(handleWebRtcCommand); + bridge.sendStatus({ kind: 'ready' }); + // So a picker has something to show before the first session. Labels may be + // redacted at this point; `start-capture` republishes once they are not. + void publishInputDevices(); + // A microphone plugged in or pulled out changes what is selectable, and the + // picker has to follow rather than showing a device that is no longer there. + navigator.mediaDevices?.addEventListener?.('devicechange', () => { + void publishInputDevices(); + }); + logger.info('A Cappella audio host ready', LOG_CONTEXT); + + return { + handleCommand, + handleWebRtcCommand, + dispose: () => { + if (disposed) return; + disposed = true; + unsubscribe(); + unsubscribeWebRtc(); + peers.closeAll('the desktop closed the audio host'); + capture?.dispose(); + playback?.dispose(); + // Closing the context releases the output device too; a hidden window + // holding one open is invisible and therefore never noticed. + void context?.close(); + capture = null; + playback = null; + outboundDestination = null; + context = null; + }, + }; +} + +export function AudioHostRoot(): null { + useEffect(() => { + const controller = createAudioHostController(); + return () => controller.dispose(); + }, []); + + return null; +} diff --git a/src/renderer/acappella-audio/bridge.ts b/src/renderer/acappella-audio/bridge.ts new file mode 100644 index 0000000000..1e6443fc3f --- /dev/null +++ b/src/renderer/acappella-audio/bridge.ts @@ -0,0 +1,65 @@ +/** + * The audio host's link to the main process. + * + * Thin by design: it adds nothing to `window.maestro.voiceAudioHost` except a + * safe no-op fallback for the case where the API is missing. That case is real - + * the module is exercised in jsdom, and the audio host would otherwise throw + * during boot in any environment without the preload bridge - and a throwing + * bridge would take the whole audio host down with it. + * + * Everything about pacing, buffering, and drop accounting lives on the main side + * (Phase 02 `audio-pipeline.ts`). A frame handed to the bridge is on its way. + */ + +import type { + AudioFrame, + AudioHostCommand, + AudioHostStatus, +} from '../../shared/acappella/audio-host'; +import type { WebRtcHostCommand, WebRtcHostEvent } from '../../shared/acappella/webrtc-host'; +import { logger } from '../utils/logger'; + +const LOG_CONTEXT = 'ACappellaAudioHost'; + +export interface AudioHostBridge { + sendFrame(frame: AudioFrame): void; + sendStatus(status: AudioHostStatus): void; + /** @returns Cleanup function to unsubscribe. */ + onCommand(handler: (command: AudioHostCommand) => void): () => void; + /** Peer answers, candidates, connection state, stats, inbound device messages. */ + sendWebRtcEvent(event: WebRtcHostEvent): void; + /** @returns Cleanup function to unsubscribe. */ + onWebRtcCommand(handler: (command: WebRtcHostCommand) => void): () => void; +} + +type VoiceAudioHostApi = NonNullable['voiceAudioHost'] | undefined; + +/** + * Wrap the preload API, or return a bridge that quietly drops everything when + * there is no preload to wrap. + */ +export function createAudioHostBridge( + api: VoiceAudioHostApi = window.maestro?.voiceAudioHost +): AudioHostBridge { + if (!api) { + logger.warn( + 'A Cappella audio host started without its IPC bridge; audio will not reach the session.', + LOG_CONTEXT + ); + return { + sendFrame: () => {}, + sendStatus: () => {}, + onCommand: () => () => {}, + sendWebRtcEvent: () => {}, + onWebRtcCommand: () => () => {}, + }; + } + + return { + sendFrame: (frame) => api.sendFrame(frame), + sendStatus: (status) => api.sendStatus(status), + onCommand: (handler) => api.onCommand(handler), + sendWebRtcEvent: (event) => api.sendWebRtcEvent(event), + onWebRtcCommand: (handler) => api.onWebRtcCommand(handler), + }; +} diff --git a/src/renderer/acappella-audio/capture.ts b/src/renderer/acappella-audio/capture.ts new file mode 100644 index 0000000000..3fa6cac747 --- /dev/null +++ b/src/renderer/acappella-audio/capture.ts @@ -0,0 +1,390 @@ +/** + * Microphone capture for the A Cappella audio host. + * + * Opens `getUserMedia` with Chromium's own audio processing module switched on - + * acoustic echo cancellation, noise suppression, auto gain - which is the whole + * reason the capture lives in a renderer rather than in a native addon. AEC is + * what makes full duplex possible: the mic can stay open while TTS is playing + * because the canceller subtracts our own output from the input, so the system + * does not hear itself and barge-in detection is not triggered by the assistant's + * own voice. + * + * Failure is classified, never silent. A denied permission, a missing device, + * and a device yanked mid-sentence are three different problems with three + * different fixes, and all three present identically ("nothing is happening") if + * they are swallowed. Each becomes a `mic-error` status, which main turns into a + * protocol `session-error` (see `audioHostErrorToSessionError`). + */ + +import { + ACAPPELLA_AUDIO_FRAME_SAMPLES, + ACAPPELLA_AUDIO_SAMPLE_RATE, + ACAPPELLA_PCM_WORKLET_NAME, + ACAPPELLA_SYSTEM_DEFAULT_INPUT, + type AudioDeviceInfo, + type AudioFrame, + type AudioHostErrorCode, + type AudioHostStatus, + type CaptureStopReason, +} from '../../shared/acappella/audio-host'; +import type { PcmWorkletFrameMessage } from './pcm-worklet'; + +/** + * Chromium's libwebrtc audio processing chain. `channelCount: 1` is not just a + * bandwidth saving: the AEC and the noise suppressor are specified for mono + * capture, and asking for stereo can silently disable them on some devices. + */ +export const ACAPPELLA_MIC_CONSTRAINTS: MediaStreamConstraints = { + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + channelCount: 1, + }, + video: false, +}; + +/** + * The same constraints, aimed at one device. + * + * `exact` on purpose. The forgiving form lets Chromium quietly hand back a + * different microphone when the requested one is busy or gone, which produces + * the single most confusing outcome this feature has: a picker that says one + * device while another is being recorded. `exact` fails instead, and a failure + * is classified into a `mic-error` the user can read. + */ +export function micConstraintsForDevice(deviceId?: string): MediaStreamConstraints { + const audio = ACAPPELLA_MIC_CONSTRAINTS.audio as MediaTrackConstraints; + if (!deviceId || deviceId === ACAPPELLA_SYSTEM_DEFAULT_INPUT) return ACAPPELLA_MIC_CONSTRAINTS; + return { audio: { ...audio, deviceId: { exact: deviceId } }, video: false }; +} + +/** + * Every microphone this machine offers. + * + * Labels are redacted by Chromium until a capture has been granted at least + * once, so an early call legitimately returns entries with empty labels; the + * host re-publishes after `capture-start` when they are populated. + */ +export async function listInputDevices(): Promise { + if (!navigator.mediaDevices?.enumerateDevices) return []; + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices + .filter((device) => device.kind === 'audioinput') + .map((device) => ({ deviceId: device.deviceId, label: device.label })); +} + +export interface MicCaptureOptions { + /** Shared with playback, so the echo canceller has a real reference signal. */ + context: AudioContext; + /** URL of the bundled PCM worklet chunk. */ + workletUrl: string; + onFrame: (frame: AudioFrame) => void; + onStatus: (status: AudioHostStatus) => void; +} + +/** + * Map a `getUserMedia` rejection onto a code the user can act on. + * + * The DOM spec's exception names are the only reliable signal here - the + * messages are Chromium-internal and change between versions. + */ +export function classifyCaptureError(error: unknown): AudioHostErrorCode { + const name = error instanceof Error ? error.name : ''; + switch (name) { + // The user said no, or the OS has not granted the app microphone access. + case 'NotAllowedError': + case 'SecurityError': + return 'permission-denied'; + // Nothing matched the constraints: no input device at all. + case 'NotFoundError': + case 'OverconstrainedError': + return 'no-device'; + // The device exists but the OS would not hand it over (in use, unplugged + // during the open, driver asleep). + case 'NotReadableError': + case 'AbortError': + return 'device-lost'; + default: + return 'audio-init-failed'; + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Owns the capture graph: mic stream -> worklet -> frames. + * + * Safe to start and stop repeatedly; the AudioWorklet module is added once per + * context, and the microphone is only held while capture is active. + */ +export class MicCapture { + private readonly options: MicCaptureOptions; + private stream: MediaStream | null = null; + private source: MediaStreamAudioSourceNode | null = null; + private node: AudioWorkletNode | null = null; + private sink: GainNode | null = null; + private moduleAdded = false; + /** True when the live stream came from a peer connection rather than a device. */ + private externalStream = false; + private starting: Promise | null = null; + private disposed = false; + private seq = 0; + /** + * Epoch ms corresponding to `context.currentTime === 0`. Frames carry audio + * clock timestamps converted through this, so their spacing stays exactly + * 20 ms even when the main thread is busy - `Date.now()` at receipt would + * jitter by however long the event loop was blocked. + */ + private epochAtContextZero = 0; + + constructor(options: MicCaptureOptions) { + this.options = options; + navigator.mediaDevices?.addEventListener?.('devicechange', this.handleDeviceChange); + } + + get active(): boolean { + return this.stream !== null; + } + + /** + * Open the microphone. + * + * @param deviceId The user's chosen input, or undefined / the system-default + * sentinel to follow the OS. + * @returns true once frames are flowing, false when capture could not start. + */ + start(deviceId?: string): Promise { + if (this.disposed) return Promise.resolve(false); + if (this.active) return Promise.resolve(true); + if (!this.starting) { + this.starting = this.startInternal(deviceId).finally(() => { + this.starting = null; + }); + } + return this.starting; + } + + /** + * Capture from a stream we did not open: the remote audio track of a paired + * device's peer connection. + * + * The SAME graph as a local microphone - worklet, 16 kHz mono downsample, + * 20 ms frames, identical `AudioFrame` on the identical channel - because + * downstream there is one recogniser, one VAD, one wake detector and one + * router, and a second capture path would be a second place for them to + * disagree. The phone is a microphone, not a second brain. + * + * Echo cancellation for this path runs on the DEVICE, not here: the echo + * happens in the room the phone is in, and cancelling it needs the phone's own + * speaker output as the reference signal. Nothing on this side has that + * signal, so anything this end did would be guesswork. The offer asks the + * device to enable it (`RemoteAudioConfig.requestRemoteEchoCancellation`), + * which is the honest extent of the desktop's influence over it. + * + * @returns true once frames are flowing. + */ + async startWithStream( + stream: MediaStream, + info: { deviceId?: string; label?: string } = {} + ): Promise { + if (this.disposed) return false; + // A remote stream displaces a local one rather than mixing: two microphones + // summed into one utterance transcribe as neither. + if (this.active) this.stop('requested'); + + const { onStatus } = this.options; + try { + await this.ensureWorklet(); + } catch (error) { + onStatus({ kind: 'mic-error', code: 'audio-init-failed', message: errorMessage(error) }); + return false; + } + if (this.disposed) return false; + + this.externalStream = true; + this.attachStream(stream, { + deviceId: info.deviceId ?? '', + label: info.label ?? 'Remote device', + }); + return true; + } + + private async startInternal(deviceId?: string): Promise { + const { onStatus } = this.options; + + if (!navigator.mediaDevices?.getUserMedia) { + onStatus({ + kind: 'mic-error', + code: 'unsupported', + message: 'This build has no microphone API available.', + }); + return false; + } + + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia(micConstraintsForDevice(deviceId)); + } catch (error) { + onStatus({ + kind: 'mic-error', + code: classifyCaptureError(error), + message: errorMessage(error), + }); + return false; + } + + try { + await this.ensureWorklet(); + } catch (error) { + stream.getTracks().forEach((track) => track.stop()); + onStatus({ + kind: 'mic-error', + code: 'audio-init-failed', + message: errorMessage(error), + }); + return false; + } + + // Disposed while we were awaiting: drop the device rather than leaking it. + if (this.disposed) { + stream.getTracks().forEach((track) => track.stop()); + return false; + } + + this.externalStream = false; + this.attachStream(stream); + return true; + } + + /** Add the worklet module once per context and wake a suspended context. */ + private async ensureWorklet(): Promise { + const { context, workletUrl } = this.options; + if (!this.moduleAdded) { + await context.audioWorklet.addModule(workletUrl); + this.moduleAdded = true; + } + // A hidden window never gets a user gesture, so a context that started + // suspended would stay suspended forever. + if (context.state === 'suspended') await context.resume(); + } + + /** + * Build the capture graph over `stream`: source -> worklet -> muted sink. + * + * Shared by the local microphone and by a paired device's remote track, which + * is the whole point - one graph means one frame format, one sequence counter, + * and one place where the downsample can be wrong. + */ + private attachStream( + stream: MediaStream, + deviceOverride?: { deviceId: string; label: string } + ): void { + const { context, onStatus } = this.options; + + this.stream = stream; + this.seq = 0; + this.epochAtContextZero = Date.now() - context.currentTime * 1000; + + this.source = context.createMediaStreamSource(stream); + this.node = new AudioWorkletNode(context, ACAPPELLA_PCM_WORKLET_NAME, { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: { + targetSampleRate: ACAPPELLA_AUDIO_SAMPLE_RATE, + frameSamples: ACAPPELLA_AUDIO_FRAME_SAMPLES, + }, + }); + this.node.port.onmessage = this.handleWorkletMessage; + + // A node with nothing downstream is not guaranteed to be pulled by the + // renderer, so the worklet is terminated into a muted gain node wired to the + // destination. `gain = 0` matters: this path exists to keep the graph alive, + // and routing the microphone to the speakers at any audible level would + // create the feedback loop the AEC is here to prevent. + this.sink = context.createGain(); + this.sink.gain.value = 0; + this.source.connect(this.node); + this.node.connect(this.sink); + this.sink.connect(context.destination); + + const track = stream.getAudioTracks()[0]; + track?.addEventListener('ended', this.handleTrackEnded); + + onStatus({ + kind: 'capture-start', + device: deviceOverride ?? { + deviceId: track?.getSettings?.().deviceId ?? '', + label: track?.label ?? '', + }, + contextSampleRate: context.sampleRate, + }); + } + + /** Release the microphone. Idempotent. */ + stop(reason: CaptureStopReason = 'requested'): void { + if (!this.stream) return; + + this.stream.getAudioTracks().forEach((track) => { + track.removeEventListener('ended', this.handleTrackEnded); + }); + // A remote stream belongs to its peer connection, not to us. Stopping its + // tracks here would kill the receiver, so the device would have to + // renegotiate to be heard again after a single floor handover. + if (!this.externalStream) this.stream.getTracks().forEach((track) => track.stop()); + this.stream = null; + this.externalStream = false; + + if (this.node) { + this.node.port.onmessage = null; + this.node.disconnect(); + this.node = null; + } + this.source?.disconnect(); + this.source = null; + this.sink?.disconnect(); + this.sink = null; + + this.options.onStatus({ kind: 'capture-stop', reason }); + } + + /** Stop capture and detach from the device list. The capture is unusable after this. */ + dispose(): void { + if (this.disposed) return; + this.disposed = true; + navigator.mediaDevices?.removeEventListener?.('devicechange', this.handleDeviceChange); + this.stop('requested'); + } + + private readonly handleWorkletMessage = (event: MessageEvent) => { + const { pcm, rms, t } = event.data; + this.seq += 1; + this.options.onFrame({ + seq: this.seq, + capturedAt: this.epochAtContextZero + t * 1000, + rms, + pcm, + }); + }; + + /** + * The OS took the device away (unplugged headset, switched output profile). + * Recoverable: the caller can start again once a device is back, which is why + * it is reported rather than thrown. + */ + private readonly handleTrackEnded = () => { + this.options.onStatus({ + kind: 'mic-error', + code: 'device-lost', + message: 'The microphone was disconnected.', + }); + this.stop('device-lost'); + }; + + private readonly handleDeviceChange = () => { + this.options.onStatus({ kind: 'device-change' }); + }; +} diff --git a/src/renderer/acappella-audio/index.ts b/src/renderer/acappella-audio/index.ts new file mode 100644 index 0000000000..7e4466586f --- /dev/null +++ b/src/renderer/acappella-audio/index.ts @@ -0,0 +1,16 @@ +/** + * A Cappella audio host - the renderer half of the audio pipeline. + * + * Loaded only by the hidden `?acappellaAudio` window. See + * `src/main/acappella/audio-host-window.ts` for why it exists and + * `src/shared/acappella/audio-host.ts` for the wire contract. + */ + +export { AudioHostRoot, createAudioHostController } from './AudioHostRoot'; +export type { AudioHostController, CreateAudioHostControllerOptions } from './AudioHostRoot'; +export { createAudioHostBridge } from './bridge'; +export type { AudioHostBridge } from './bridge'; +export { classifyCaptureError, MicCapture, ACAPPELLA_MIC_CONSTRAINTS } from './capture'; +export type { MicCaptureOptions } from './capture'; +export { pcm16ToFloat32, TtsPlayback } from './playback'; +export type { PlaybackChunk, TtsPlaybackOptions } from './playback'; diff --git a/src/renderer/acappella-audio/pcm-worklet.ts b/src/renderer/acappella-audio/pcm-worklet.ts new file mode 100644 index 0000000000..9a5b534c7d --- /dev/null +++ b/src/renderer/acappella-audio/pcm-worklet.ts @@ -0,0 +1,159 @@ +/** + * A Cappella PCM worklet - downmix, resample, quantise, emit. + * + * Runs on the audio rendering thread inside `AudioWorkletGlobalScope`. It takes + * whatever the microphone gives us (48 kHz stereo on most machines, 44.1 kHz on + * some, 16 kHz on a headset that already speaks our language) and posts fixed + * 20 ms frames of 16 kHz signed 16-bit mono to the main thread. + * + * **The resampling happens here, not on the main thread, on purpose.** The audio + * thread is real-time scheduled and cannot be blocked by React rendering, a + * garbage collection pause, or a busy IPC queue. Doing the same arithmetic in a + * `message` handler would make every dropout in the renderer an audible hole in + * the transcript, and STT accuracy falls off a cliff with missing audio. + * + * Loaded as a URL rather than imported: `capture.ts` hands + * `AudioWorklet.addModule()` the bundled chunk that Vite emits for this file + * (`?worker&url`). Importing it directly would link it into the main renderer + * chunk, where `AudioWorkletProcessor` does not exist. + */ + +import { + ACAPPELLA_AUDIO_FRAME_SAMPLES, + ACAPPELLA_AUDIO_SAMPLE_RATE, + ACAPPELLA_PCM_WORKLET_NAME, +} from '../../shared/acappella/audio-host'; + +/** + * The handful of `AudioWorkletGlobalScope` globals we use. They are not in + * TypeScript's DOM lib, and declaring them with `declare` would leak them into + * every renderer file, so they are read off `globalThis` through a local type + * instead. + */ +interface AudioWorkletScope { + /** The AudioContext's rate, fixed for the life of the processor. */ + readonly sampleRate: number; + /** Start of the current render quantum, in the context's clock. */ + readonly currentTime: number; + registerProcessor(name: string, processorCtor: unknown): void; + readonly AudioWorkletProcessor: { + new (options?: unknown): { readonly port: MessagePort }; + }; +} + +const scope = globalThis as unknown as AudioWorkletScope; +const { AudioWorkletProcessor, registerProcessor, sampleRate } = scope; + +/** What the worklet posts per frame. `capture.ts` stamps seq and wall time. */ +export interface PcmWorkletFrameMessage { + /** Signed 16-bit little-endian mono samples, transferred (not copied). */ + pcm: ArrayBuffer; + /** Root mean square over the frame, 0 to 1. */ + rms: number; + /** Context time at emit, for deriving a wall clock without per-frame `Date.now()`. */ + t: number; +} + +interface PcmProcessorOptions { + processorOptions?: { + targetSampleRate?: number; + frameSamples?: number; + }; +} + +class PcmDownsampleProcessor extends AudioWorkletProcessor { + /** Input samples consumed per output sample. 3 for 48 kHz -> 16 kHz. */ + private readonly ratio: number; + private readonly frameSamples: number; + private readonly frame: Int16Array; + private frameFill = 0; + private sumSquares = 0; + + /** + * Fractional read position into the CURRENT block. Carried across blocks (it + * goes negative, meaning "between the last sample of the previous block and + * the first of this one") so the output has no periodic seam at the 128-sample + * render quantum boundary. A seam every 2.6 ms is audible as a buzz and is + * exactly the artefact naive per-block resamplers produce. + */ + private readPos = 0; + /** Last sample of the previous block, the left neighbour when `readPos` is negative. */ + private tail = 0; + + private mono: Float32Array = new Float32Array(0); + + constructor(options?: PcmProcessorOptions) { + super(); + const targetRate = options?.processorOptions?.targetSampleRate ?? ACAPPELLA_AUDIO_SAMPLE_RATE; + this.frameSamples = options?.processorOptions?.frameSamples ?? ACAPPELLA_AUDIO_FRAME_SAMPLES; + this.ratio = sampleRate / targetRate; + this.frame = new Int16Array(this.frameSamples); + } + + process(inputs: Float32Array[][]): boolean { + const channels = inputs[0]; + // No input yet (the graph is still connecting) or the track ended. Staying + // alive is right either way: returning false would retire the processor and + // the node would have to be rebuilt to resume. + if (!channels || channels.length === 0) return true; + + const blockLength = channels[0].length; + if (blockLength === 0) return true; + + const mono = this.downmix(channels, blockLength); + const last = blockLength - 1; + + while (this.readPos <= last) { + const index = Math.floor(this.readPos); + const frac = this.readPos - index; + const left = index < 0 ? this.tail : mono[index]; + // When `index === last` the right neighbour lives in the next block, but + // `frac` is 0 there, so `left` is the exact answer. + const right = index + 1 <= last ? mono[index + 1] : left; + this.push(left + (right - left) * frac); + this.readPos += this.ratio; + } + + this.tail = mono[last]; + this.readPos -= blockLength; + return true; + } + + /** Average the channels into a reusable scratch buffer. */ + private downmix(channels: Float32Array[], blockLength: number): Float32Array { + if (channels.length === 1) return channels[0]; + + if (this.mono.length !== blockLength) this.mono = new Float32Array(blockLength); + const mono = this.mono; + mono.set(channels[0]); + for (let c = 1; c < channels.length; c++) { + const channel = channels[c]; + for (let i = 0; i < blockLength; i++) mono[i] += channel[i]; + } + const scale = 1 / channels.length; + for (let i = 0; i < blockLength; i++) mono[i] *= scale; + return mono; + } + + private push(sample: number): void { + const clamped = sample > 1 ? 1 : sample < -1 ? -1 : sample; + this.sumSquares += clamped * clamped; + // Asymmetric scaling: two's complement reaches -32768 but only +32767, so + // scaling both directions by 32768 would clip every full-scale positive peak. + this.frame[this.frameFill++] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + if (this.frameFill === this.frameSamples) this.emit(); + } + + private emit(): void { + const rms = Math.sqrt(this.sumSquares / this.frameSamples); + // Copy, then transfer the copy: the processor keeps reusing `this.frame`, + // and transferring it would detach the buffer we are about to write into. + const pcm = this.frame.slice(); + const message: PcmWorkletFrameMessage = { pcm: pcm.buffer, rms, t: scope.currentTime }; + this.port.postMessage(message, [pcm.buffer]); + this.frameFill = 0; + this.sumSquares = 0; + } +} + +registerProcessor(ACAPPELLA_PCM_WORKLET_NAME, PcmDownsampleProcessor); diff --git a/src/renderer/acappella-audio/peer-connection.ts b/src/renderer/acappella-audio/peer-connection.ts new file mode 100644 index 0000000000..a094d35037 --- /dev/null +++ b/src/renderer/acappella-audio/peer-connection.ts @@ -0,0 +1,509 @@ +/** + * The WebRTC media leg, terminated in the hidden audio window. + * + * This is the desktop half of "the phone is a remote microphone and speaker". + * A paired device offers, this answers, and from that point: + * + * - the device's microphone arrives as a remote track and is spliced into the + * SAME capture pipeline the local microphone feeds (`capture.ts`), so there + * is one recogniser, one VAD, one router; + * - the assistant's voice leaves as an outgoing track tapped off the SAME + * playback node the speakers hear (`playback.ts`), so the phone hears the + * ElevenLabs voice the user configured rather than a second synthesis; + * - two data channels carry the protocol (`shared/acappella/device-protocol.ts`). + * + * It lives in a renderer because `RTCPeerConnection` is a DOM object and because + * an `AudioContext` is the only place a remote track and a local microphone can + * meet. Electron ships Chromium's libwebrtc, so this needs no native dependency + * at all. + * + * **Exactly one device holds the floor.** Every peer's audio is received, but + * only the holder's track is connected to the capture pipeline; the rest are + * parked. Mixing two microphones into one utterance produces a transcript of + * neither, and picking one silently produces a user who is talking to nothing. + * The takeover rule itself lives in `main/acappella/transport/remote-session.ts` + * - this file only obeys `set-floor-holder`. + */ + +import { + RELIABLE_CHANNEL_INIT, + RELIABLE_CHANNEL_LABEL, + UNRELIABLE_CHANNEL_INIT, + UNRELIABLE_CHANNEL_LABEL, + decodeDeviceMessage, + deviceChannelForMessage, + encodeDeviceMessage, + type DeviceMessage, +} from '../../shared/acappella/device-protocol'; +import { + PEER_STATS_INTERVAL_MS, + applyOpusPreferences, + summarizeStats, +} from '../../shared/acappella/peer-tuning'; +import type { + IceCandidatePayload, + IceProbeResult, + IceServerConfig, + PeerConnectionState, + PeerQualityStats, + RemoteAudioConfig, + SessionDescriptionPayload, + WebRtcHostCommand, + WebRtcHostEvent, +} from '../../shared/acappella/webrtc-host'; +import { logger } from '../utils/logger'; + +const LOG_CONTEXT = 'ACappellaAudioHost'; + +export interface PeerConnectionCallbacks { + onAnswer(deviceId: string, answer: SessionDescriptionPayload): void; + onIceCandidate(deviceId: string, candidate: IceCandidatePayload): void; + onConnectionState(deviceId: string, state: PeerConnectionState): void; + onStats(stats: PeerQualityStats): void; + onMessage(deviceId: string, message: DeviceMessage): void; + onError(deviceId: string, message: string): void; +} + +/** The capture and playback seams a peer needs. Injected so tests need no audio. */ +export interface PeerAudioBinding { + /** Route this device's microphone into the shared capture pipeline. */ + attachRemoteStream(stream: MediaStream, deviceId: string): void; + /** Stop consuming a device's microphone. */ + detachRemoteStream(deviceId: string): void; + /** + * A `MediaStreamTrack` carrying the assistant's voice, tapped off playback. + * + * One track shared by every peer: the output is the same audio, and creating + * a destination node per device would mean N copies of one synthesis running + * through N resamplers for no benefit. + */ + getOutboundTrack(): MediaStreamTrack | null; +} + +type RTCFactory = (config: RTCConfiguration) => RTCPeerConnection; + +export interface PeerRegistryOptions { + callbacks: PeerConnectionCallbacks; + audio: PeerAudioBinding; + /** Test seam. Production constructs a real `RTCPeerConnection`. */ + createPeerConnection?: RTCFactory; + statsIntervalMs?: number; +} + +// --------------------------------------------------------------------------- +// One peer +// --------------------------------------------------------------------------- + +class DevicePeer { + readonly deviceId: string; + private readonly pc: RTCPeerConnection; + private readonly callbacks: PeerConnectionCallbacks; + private readonly audio: PeerAudioBinding; + private readonly audioConfig: RemoteAudioConfig; + private reliable: RTCDataChannel | null = null; + private unreliable: RTCDataChannel | null = null; + private remoteStream: MediaStream | null = null; + private statsTimer: ReturnType | null = null; + private lastBytes: { bytesReceived: number; at: number } | undefined; + private holdsFloor = false; + private closed = false; + + constructor(params: { + deviceId: string; + pc: RTCPeerConnection; + callbacks: PeerConnectionCallbacks; + audio: PeerAudioBinding; + audioConfig: RemoteAudioConfig; + statsIntervalMs: number; + }) { + this.deviceId = params.deviceId; + this.pc = params.pc; + this.callbacks = params.callbacks; + this.audio = params.audio; + this.audioConfig = params.audioConfig; + + this.pc.onicecandidate = (event) => { + if (!event.candidate) return; + this.callbacks.onIceCandidate(this.deviceId, { + candidate: event.candidate.candidate, + sdpMid: event.candidate.sdpMid, + sdpMLineIndex: event.candidate.sdpMLineIndex, + usernameFragment: event.candidate.usernameFragment, + }); + }; + this.pc.onconnectionstatechange = () => { + this.callbacks.onConnectionState( + this.deviceId, + this.pc.connectionState as PeerConnectionState + ); + }; + this.pc.ontrack = (event) => { + this.remoteStream = event.streams[0] ?? new MediaStream([event.track]); + // Only routed into the pipeline when this device holds the floor. The + // track is received either way, so a takeover is a graph reconnection + // rather than a renegotiation. + if (this.holdsFloor) this.audio.attachRemoteStream(this.remoteStream, this.deviceId); + }; + this.pc.ondatachannel = (event) => this.bindChannel(event.channel); + + this.statsTimer = setInterval(() => void this.pollStats(), params.statsIntervalMs); + this.statsTimer.unref?.(); + } + + /** + * Apply an offer and produce an answer. + * + * Also the renegotiation path: a phone changing network re-offers, and + * applying it to the existing peer is what makes a WiFi-to-LTE handover a + * hiccup rather than a dropped call. + */ + async acceptOffer(offer: SessionDescriptionPayload): Promise { + await this.pc.setRemoteDescription({ + type: 'offer', + sdp: offer.sdp ? applyOpusPreferences(offer.sdp, this.audioConfig) : offer.sdp, + }); + + // The outbound voice track is added once and reused across renegotiations. + const outbound = this.audio.getOutboundTrack(); + if (outbound && this.pc.getSenders().every((sender) => sender.track !== outbound)) { + this.pc.addTrack(outbound); + } + + const answer = await this.pc.createAnswer(); + const sdp = answer.sdp ? applyOpusPreferences(answer.sdp, this.audioConfig) : answer.sdp; + await this.pc.setLocalDescription({ type: 'answer', sdp }); + this.applySenderBitrate(); + this.callbacks.onAnswer(this.deviceId, { type: 'answer', sdp }); + } + + async addIceCandidate(candidate: IceCandidatePayload): Promise { + try { + await this.pc.addIceCandidate(candidate as RTCIceCandidateInit); + } catch (error) { + // A candidate that arrives before the remote description, or one for a + // bundled m-line ICE has already given up on, is normal trickle traffic. + // It is not worth failing a connection that is otherwise negotiating. + logger.debug( + `Discarded ICE candidate for ${this.deviceId}: ${errorText(error)}`, + LOG_CONTEXT + ); + } + } + + /** Connect or park this device's microphone. */ + setFloor(holdsFloor: boolean): void { + if (this.holdsFloor === holdsFloor) return; + this.holdsFloor = holdsFloor; + if (holdsFloor) { + if (this.remoteStream) this.audio.attachRemoteStream(this.remoteStream, this.deviceId); + } else { + this.audio.detachRemoteStream(this.deviceId); + } + } + + send(message: DeviceMessage): void { + const kind = deviceChannelForMessage(message); + const channel = kind === 'reliable' ? this.reliable : this.unreliable; + // The reliable channel is the fallback for a lossy message whose channel is + // not open yet: late is better than never for the FIRST floor-state a device + // sees, and by the time the meter is running both channels exist. + const target = channel ?? this.reliable; + if (!target || target.readyState !== 'open') return; + try { + target.send(encodeDeviceMessage(message)); + } catch (error) { + logger.warn(`Data channel send failed: ${errorText(error)}`, LOG_CONTEXT); + } + } + + close(reason: string): void { + if (this.closed) return; + this.closed = true; + this.send({ type: 'revoked', message: reason }); + if (this.statsTimer) clearInterval(this.statsTimer); + this.statsTimer = null; + this.audio.detachRemoteStream(this.deviceId); + this.reliable?.close(); + this.unreliable?.close(); + this.pc.onicecandidate = null; + this.pc.onconnectionstatechange = null; + this.pc.ontrack = null; + this.pc.ondatachannel = null; + this.pc.close(); + } + + private bindChannel(channel: RTCDataChannel): void { + if (channel.label === RELIABLE_CHANNEL_LABEL) this.reliable = channel; + else if (channel.label === UNRELIABLE_CHANNEL_LABEL) this.unreliable = channel; + else { + // A channel we did not name is a client bug or an attack surface; either + // way there is nothing correct to do with its traffic. + channel.close(); + return; + } + channel.onmessage = (event) => { + const message = decodeDeviceMessage( + typeof event.data === 'string' ? event.data : String(event.data) + ); + if (!message) return; + this.callbacks.onMessage(this.deviceId, message); + }; + } + + /** + * Cap the outgoing bitrate. + * + * `maxaveragebitrate` in the SDP is what the ENCODER aims for; this is what + * the sender is allowed to use. Both, because either one alone is routinely + * ignored depending on which end negotiated what. + */ + private applySenderBitrate(): void { + for (const sender of this.pc.getSenders()) { + if (sender.track?.kind !== 'audio') continue; + const parameters = sender.getParameters(); + if (!parameters.encodings || parameters.encodings.length === 0) { + parameters.encodings = [{}]; + } + for (const encoding of parameters.encodings) { + encoding.maxBitrate = this.audioConfig.maxAverageBitrate; + encoding.networkPriority = 'high'; + } + void sender.setParameters(parameters).catch((error: unknown) => { + logger.debug(`Could not set sender parameters: ${errorText(error)}`, LOG_CONTEXT); + }); + } + } + + private async pollStats(): Promise { + if (this.closed) return; + try { + const report = await this.pc.getStats(); + const reports: Array> = []; + report.forEach((value) => reports.push(value as unknown as Record)); + const { bytesReceived, ...stats } = summarizeStats(this.deviceId, reports, this.lastBytes); + this.lastBytes = { bytesReceived, at: performance.now() }; + this.callbacks.onStats(stats); + // Both ends draw the same bar from the same numbers rather than each + // measuring its own half of the link and disagreeing about it. + this.send({ + type: 'link-quality', + rttMs: stats.rttMs, + jitterMs: stats.jitterMs, + packetLoss: stats.packetLoss, + candidateType: stats.candidateType, + }); + } catch (error) { + logger.debug(`Stats poll failed: ${errorText(error)}`, LOG_CONTEXT); + } + } +} + +// --------------------------------------------------------------------------- +// Registry +// --------------------------------------------------------------------------- + +/** Every live peer, and the single-floor rule over them. */ +export class PeerRegistry { + private readonly peers = new Map(); + private readonly options: PeerRegistryOptions; + private readonly createPeerConnection: RTCFactory; + private readonly statsIntervalMs: number; + private floorHolder: string | null = null; + + constructor(options: PeerRegistryOptions) { + this.options = options; + this.createPeerConnection = + options.createPeerConnection ?? ((config) => new RTCPeerConnection(config)); + this.statsIntervalMs = options.statsIntervalMs ?? PEER_STATS_INTERVAL_MS; + } + + async acceptOffer(params: { + deviceId: string; + offer: SessionDescriptionPayload; + iceServers: IceServerConfig[]; + audio: RemoteAudioConfig; + }): Promise { + let peer = this.peers.get(params.deviceId); + if (!peer) { + peer = new DevicePeer({ + deviceId: params.deviceId, + pc: this.createPeerConnection({ + iceServers: params.iceServers as RTCIceServer[], + // A small pool so the first candidates exist before the offer is + // answered, which takes a visible chunk off time-to-first-audio. + iceCandidatePoolSize: 2, + }), + callbacks: this.options.callbacks, + audio: this.options.audio, + audioConfig: params.audio, + statsIntervalMs: this.statsIntervalMs, + }); + this.peers.set(params.deviceId, peer); + peer.setFloor(this.floorHolder === params.deviceId); + } + try { + await peer.acceptOffer(params.offer); + } catch (error) { + this.options.callbacks.onError(params.deviceId, errorText(error)); + this.close(params.deviceId, 'the connection could not be negotiated'); + } + } + + addIceCandidate(deviceId: string, candidate: IceCandidatePayload): void { + void this.peers.get(deviceId)?.addIceCandidate(candidate); + } + + send(deviceId: string, message: DeviceMessage): void { + this.peers.get(deviceId)?.send(message); + } + + broadcast(message: DeviceMessage): void { + for (const peer of this.peers.values()) peer.send(message); + } + + /** Exactly one device consumes the microphone. Everyone else is parked. */ + setFloorHolder(deviceId: string | null): void { + this.floorHolder = deviceId; + for (const [id, peer] of this.peers) peer.setFloor(id === deviceId); + } + + close(deviceId: string, reason: string): void { + const peer = this.peers.get(deviceId); + if (!peer) return; + this.peers.delete(deviceId); + if (this.floorHolder === deviceId) this.floorHolder = null; + peer.close(reason); + } + + closeAll(reason: string): void { + for (const deviceId of [...this.peers.keys()]) this.close(deviceId, reason); + } + + get size(): number { + return this.peers.size; + } + + /** + * Gather candidates against a configuration and report what came back: the + * Test Connection button. + * + * A real gather, not a reachability guess. A `relay` candidate can only exist + * if the TURN server accepted the credentials, so its presence is proof the + * configuration works rather than a claim that it should. + */ + async probeIce(iceServers: IceServerConfig[], timeoutMs: number): Promise { + const result: IceProbeResult = { host: false, stun: false, relay: false, best: 'unknown' }; + let pc: RTCPeerConnection; + try { + pc = this.createPeerConnection({ iceServers: iceServers as RTCIceServer[] }); + } catch (error) { + return { ...result, error: errorText(error) }; + } + + try { + await new Promise((resolve) => { + const timer = setTimeout(resolve, timeoutMs); + pc.onicecandidate = (event) => { + if (!event.candidate) { + clearTimeout(timer); + resolve(); + return; + } + const type = / typ (\w+)/.exec(event.candidate.candidate)?.[1]; + if (type === 'host') result.host = true; + else if (type === 'srflx' || type === 'prflx') result.stun = true; + else if (type === 'relay') { + result.relay = true; + // A relay is the last thing that will be gathered and the only + // thing this test cannot infer any other way. Stop there rather + // than waiting out the full timeout for candidates nobody reads. + clearTimeout(timer); + resolve(); + } + }; + // A data channel is enough to make ICE gather; no media, no permission + // prompt, and nothing that could open a microphone during a test. + pc.createDataChannel('probe'); + void pc + .createOffer() + .then((offer) => pc.setLocalDescription(offer)) + .catch(() => { + clearTimeout(timer); + resolve(); + }); + }); + } finally { + pc.onicecandidate = null; + pc.close(); + } + + result.best = result.relay ? 'relay' : result.stun ? 'stun' : result.host ? 'lan' : 'unknown'; + return result; + } +} + +// --------------------------------------------------------------------------- +// The command surface +// --------------------------------------------------------------------------- + +/** + * Apply one {@link WebRtcHostCommand} to a registry, emitting whatever it + * produces. + * + * The audio host window is one caller; the protocol conformance harness at + * `src/__tests__/acappella/conformance/` is the other, and it exists to drive + * the real desktop stack against an independent client. Keeping the switch here + * rather than inside `AudioHostRoot.tsx` is what makes that possible without a + * DOM, and it means a new command kind is covered by conformance the moment it + * is handled rather than the next time somebody remembers to mirror it. + */ +export function applyWebRtcCommand( + peers: PeerRegistry, + command: WebRtcHostCommand, + emit: (event: WebRtcHostEvent) => void +): void { + switch (command.kind) { + case 'accept-offer': + void peers.acceptOffer({ + deviceId: command.deviceId, + offer: command.offer, + iceServers: command.iceServers, + audio: command.audio, + }); + return; + case 'add-ice-candidate': + peers.addIceCandidate(command.deviceId, command.candidate); + return; + case 'close-peer': + peers.close(command.deviceId, command.reason); + return; + case 'send': + peers.send(command.deviceId, command.message); + return; + case 'broadcast': + peers.broadcast(command.message); + return; + case 'set-floor-holder': + peers.setFloorHolder(command.deviceId); + return; + case 'probe-ice': + void peers + .probeIce(command.iceServers, command.timeoutMs) + .then((result) => emit({ kind: 'ice-probe-result', probeId: command.probeId, result })); + return; + } +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export { RELIABLE_CHANNEL_INIT, UNRELIABLE_CHANNEL_INIT }; + +/** + * Re-exported from `shared/acappella/peer-tuning.ts`, where they moved when the + * browser reference client needed the identical SDP shaping and stats + * reduction from the other side of the wire. Kept exported here because this is + * the module every desktop-side caller and test already imports them from. + */ +export { PEER_STATS_INTERVAL_MS, applyOpusPreferences, summarizeStats }; diff --git a/src/renderer/acappella-audio/playback.ts b/src/renderer/acappella-audio/playback.ts new file mode 100644 index 0000000000..61a96f1aa9 --- /dev/null +++ b/src/renderer/acappella-audio/playback.ts @@ -0,0 +1,261 @@ +/** + * TTS playback for the A Cappella audio host. + * + * One `AudioContext` output chain shared with capture, which is what gives + * Chromium's echo canceller a reference signal to subtract: the assistant's own + * voice is removed from the microphone input, so the mic can stay open while it + * speaks and barge-in detection never fires on our own audio. + * + * Two operations carry the barge-in guarantee: + * + * - {@link TtsPlayback.duck} ramps the output down so the user's voice wins + * the room while the pipeline decides whether they meant to interrupt. + * - {@link TtsPlayback.flush} stops every scheduled source immediately and + * drops everything queued. Not "stop scheduling new audio" - already + * scheduled buffers keep playing, and the felt latency of an interruption is + * exactly how long the last one runs. + */ + +import { + ACAPPELLA_AUDIO_SAMPLE_RATE, + type AudioHostStatus, + type PlaybackFormat, +} from '../../shared/acappella/audio-host'; + +export interface PlaybackChunk { + utteranceId: string; + format: PlaybackFormat; + /** Required for `pcm16`. Ignored for `encoded`, where the container decides. */ + sampleRate?: number; + data: ArrayBuffer; +} + +export interface TtsPlaybackOptions { + context: AudioContext; + onStatus: (status: AudioHostStatus) => void; +} + +/** Signed 16-bit little-endian mono -> the float samples an AudioBuffer wants. */ +export function pcm16ToFloat32(data: ArrayBuffer): Float32Array { + const source = new Int16Array(data); + const out = new Float32Array(source.length); + for (let i = 0; i < source.length; i++) { + // Mirrors the worklet's asymmetric quantisation, so a round trip is lossless. + out[i] = source[i] < 0 ? source[i] / 0x8000 : source[i] / 0x7fff; + } + return out; +} + +/** Long enough that a volume change is a fade rather than a click. */ +const VOLUME_RAMP_MS = 40; + +export class TtsPlayback { + private readonly options: TtsPlaybackOptions; + private readonly gain: GainNode; + /** The user's output volume. Ducking multiplies this; a flush restores it. */ + private volume = 1; + private readonly sources = new Set(); + /** Context time the next chunk starts at, so consecutive chunks are gapless. */ + private nextStartTime = 0; + private currentUtteranceId: string | null = null; + /** Utterances main has said it is done sending chunks for. */ + private readonly endedUtterances = new Set(); + private pendingDecodes = 0; + /** + * Bumped by {@link flush}. A decode that started before a flush resolves after + * it, and scheduling that buffer would resurrect audio the user just talked + * over - the single most jarring failure this class can have. + */ + private generation = 0; + private disposed = false; + + constructor(options: TtsPlaybackOptions) { + this.options = options; + this.gain = options.context.createGain(); + this.gain.connect(options.context.destination); + } + + /** + * The node every scheduled chunk passes through on its way to the speakers. + * + * Exposed so the WebRTC leg can tap the SAME output the local speakers hear + * (`peer-connection.ts` wires it into a `MediaStreamAudioDestinationNode`). + * Tapping here rather than re-synthesising per device is what guarantees a + * phone hears the voice the user configured, at the volume they set, with the + * same ducking - one TTS run, two places it comes out. + */ + get outputNode(): AudioNode { + return this.gain; + } + + /** Audio scheduled but not yet heard. Bounds how late a barge-in can land. */ + get queuedMs(): number { + return Math.max(0, (this.nextStartTime - this.options.context.currentTime) * 1000); + } + + get playing(): boolean { + return this.sources.size > 0 || this.pendingDecodes > 0; + } + + /** Decode a chunk and schedule it after everything already queued. */ + async enqueue(chunk: PlaybackChunk): Promise { + if (this.disposed) return; + const generation = this.generation; + this.currentUtteranceId = chunk.utteranceId; + this.endedUtterances.delete(chunk.utteranceId); + this.pendingDecodes += 1; + this.emitState(); + + let buffer: AudioBuffer; + try { + buffer = await this.decode(chunk); + } finally { + this.pendingDecodes -= 1; + } + + // Flushed (or disposed) while decoding: this audio belongs to a run the user + // already interrupted. + if (this.disposed || generation !== this.generation) { + this.emitState(); + return; + } + this.schedule(buffer); + } + + /** + * No more chunks are coming for `utteranceId`. Playback keeps running; this + * only lets the drain report an honest idle instead of a maybe-more-coming + * pause. + */ + endUtterance(utteranceId: string): void { + this.endedUtterances.add(utteranceId); + if (this.playing) return; + // Already drained before the end marker arrived (a short final sentence): + // close it out here, or nothing else ever will. + if (this.currentUtteranceId === utteranceId) { + this.endedUtterances.delete(utteranceId); + this.currentUtteranceId = null; + } + this.emitState(); + } + + /** + * Stop now and discard the queue. + * + * Gain is restored to the USER'S volume here, not to 1. Ducking only has + * meaning while something is playing, so leaving a barge-in's duck in place + * would make the next utterance come out inaudible with nothing on screen to + * explain it - and restoring to full would just as silently undo a mute. + */ + flush(): void { + this.generation += 1; + for (const source of this.sources) { + source.onended = null; + try { + source.stop(); + } catch { + // Already stopped or never started; disconnecting is all that is left. + } + source.disconnect(); + } + this.sources.clear(); + this.nextStartTime = 0; + this.currentUtteranceId = null; + this.endedUtterances.clear(); + this.setGain(this.volume, 0); + this.emitState(); + } + + /** + * Ramp output gain to `gain` (0 to 1) over `ms`, RELATIVE to the user's volume. + * + * Relative rather than absolute because ducking to 0.2 means "a fifth as loud + * as whatever we were", and a duck that jumped to an absolute 0.2 would make + * a barge-in LOUDER for anyone running the assistant quietly. + */ + duck(gain: number, ms: number): void { + this.setGain(this.volume * Math.min(1, Math.max(0, gain)), Math.max(0, ms)); + } + + /** + * Set the base output volume (0 to 1). + * + * Applied with a short ramp rather than instantly: a step change in gain on a + * playing buffer is an audible click, and the one thing a volume control must + * not do is make a noise of its own. + */ + setVolume(volume: number): void { + this.volume = Math.min(1, Math.max(0, Number.isFinite(volume) ? volume : 1)); + this.setGain(this.volume, VOLUME_RAMP_MS); + } + + dispose(): void { + if (this.disposed) return; + this.flush(); + this.disposed = true; + this.gain.disconnect(); + } + + private setGain(value: number, ms: number): void { + const now = this.options.context.currentTime; + const param = this.gain.gain; + param.cancelScheduledValues(now); + // Pin the current value first: without it the ramp starts from whatever was + // last *scheduled*, which jumps when a duck interrupts a duck. + param.setValueAtTime(param.value, now); + if (ms <= 0) param.setValueAtTime(value, now); + else param.linearRampToValueAtTime(value, now + ms / 1000); + } + + private async decode(chunk: PlaybackChunk): Promise { + const { context } = this.options; + if (chunk.format === 'encoded') return context.decodeAudioData(chunk.data); + + const samples = pcm16ToFloat32(chunk.data); + const rate = chunk.sampleRate ?? ACAPPELLA_AUDIO_SAMPLE_RATE; + // A buffer whose rate differs from the context's is resampled by the source + // node on playback, so a 22 kHz local voice needs no work here. + const buffer = context.createBuffer(1, samples.length, rate); + buffer.getChannelData(0).set(samples); + return buffer; + } + + private schedule(buffer: AudioBuffer): void { + const { context } = this.options; + const source = context.createBufferSource(); + source.buffer = buffer; + source.connect(this.gain); + + const startAt = Math.max(context.currentTime, this.nextStartTime); + source.start(startAt); + this.nextStartTime = startAt + buffer.duration; + this.sources.add(source); + + source.onended = () => { + source.disconnect(); + this.sources.delete(source); + if (!this.playing) { + this.nextStartTime = 0; + // Only forget the utterance once main has said no more chunks are + // coming. A streaming TTS run drains between sentences, and reporting + // "nothing is speaking" in that gap would let the pipeline close a + // speech run that is still mid-sentence. + if (this.currentUtteranceId && this.endedUtterances.has(this.currentUtteranceId)) { + this.endedUtterances.delete(this.currentUtteranceId); + this.currentUtteranceId = null; + } + } + this.emitState(); + }; + this.emitState(); + } + + private emitState(): void { + this.options.onStatus({ + kind: 'playback-state', + playing: this.playing, + utteranceId: this.currentUtteranceId, + queuedMs: this.queuedMs, + }); + } +} diff --git a/src/renderer/acappella-audio/worklet-url.ts b/src/renderer/acappella-audio/worklet-url.ts new file mode 100644 index 0000000000..5571a6b524 --- /dev/null +++ b/src/renderer/acappella-audio/worklet-url.ts @@ -0,0 +1,20 @@ +/** + * The bundled URL of the PCM worklet. + * + * `?worker&url` makes Vite emit `pcm-worklet.ts` as its own self-contained chunk + * and hand back its URL instead of linking it into the renderer bundle - which + * is exactly what `AudioWorklet.addModule()` needs, and exactly what a plain + * import would get wrong (`AudioWorkletProcessor` does not exist on the main + * thread). + * + * Deliberately its own one-line module. A `Blob` URL would sidestep the bundler + * entirely, but worklet module fetches are checked against `script-src`, which + * `src/renderer/index.html` pins to `'self'` - so an inline blob would be + * blocked at load, and widening the app's CSP to `blob:` for one worklet is not + * a trade worth making. Isolating the import also keeps `capture.ts` free of + * bundler-specific syntax, so it can be unit tested without a Vite pipeline. + */ + +import pcmWorkletUrl from './pcm-worklet.ts?worker&url'; + +export { pcmWorkletUrl }; diff --git a/src/renderer/assets.d.ts b/src/renderer/assets.d.ts index 3f466c5e01..e74f53f2ce 100644 --- a/src/renderer/assets.d.ts +++ b/src/renderer/assets.d.ts @@ -28,6 +28,14 @@ declare module '*.webp' { export default src; } +// Vite emits the referenced module as its own self-contained chunk and resolves +// the import to that chunk's URL. Used for AudioWorklet modules, which have to +// be fetched by URL rather than linked into the renderer bundle. +declare module '*?worker&url' { + const src: string; + export default src; +} + // Vite-injected build-time constants declare const __APP_VERSION__: string; declare const __COMMIT_HASH__: string; diff --git a/src/renderer/components/ACappella/VoiceDevHarness.tsx b/src/renderer/components/ACappella/VoiceDevHarness.tsx new file mode 100644 index 0000000000..4d272f0ef3 --- /dev/null +++ b/src/renderer/components/ACappella/VoiceDevHarness.tsx @@ -0,0 +1,190 @@ +/** + * VoiceDevHarness - type an utterance, watch the whole pipeline run. + * + * Development-only, and only with the A Cappella Encore flag on. Until a real + * microphone lands (Phase 05) this is the only way to drive a session, and it + * is deliberately the SAME seam: `submitUtterance()` goes through the STT + * provider's text-in hook, so nothing downstream can tell a typed utterance + * from a spoken one. + * + * Four controls, because the prototype has to prove four distinct behaviours: + * - **Send** runs a turn: partials, a route decision, a real tab dispatch. + * - **Reply** feeds the session an agent answer. Nothing in Phase 01 produces + * one on its own, so without this the demo stops at `dispatch` and speech + * is never exercised. + * - **Interrupt** is barge-in: it cuts speech and KEEPS the floor. + * - **Stop** is the stop word: it ends the session. + * The last two look alike and are not, which is exactly why both are here. + */ + +import { useCallback, useState } from 'react'; +import { MessageSquare, Send, Square, Zap } from 'lucide-react'; +import type { Theme } from '../../types'; +import { readableTextOn } from '../../../shared/colorContrast'; +import { isVoiceSessionActive } from '../../../shared/acappella/session-state'; +import { useVoiceSessionStore } from '../../stores/voiceSessionStore'; +import type { VoiceSessionActions } from './useVoiceSession'; + +export interface VoiceDevHarnessProps { + theme: Theme; + actions: VoiceSessionActions; +} + +/** What a reply would look like, so the demo does not need an agent to answer. */ +const SAMPLE_REPLY = + 'I refactored the auth middleware to verify the refresh token before issuing a new access token. The two failing tests now pass.'; + +export function VoiceDevHarness({ theme, actions }: VoiceDevHarnessProps) { + const state = useVoiceSessionStore((s) => s.state); + const lastDispatch = useVoiceSessionStore((s) => s.lastDispatch); + const [text, setText] = useState(''); + const [busy, setBusy] = useState(false); + + const active = isVoiceSessionActive(state); + const onAccent = readableTextOn(theme.colors.accentForeground, [theme.colors.accent]); + + const run = useCallback(async (fn: () => Promise) => { + setBusy(true); + try { + await fn(); + } finally { + setBusy(false); + } + }, []); + + const handleSend = useCallback(() => { + const utterance = text.trim(); + if (!utterance) return; + // Starting on demand keeps the "enabling the feature opens no device" + // promise: nothing runs until the user asks for a turn. + void run(async () => { + if (!isVoiceSessionActive(useVoiceSessionStore.getState().state)) { + await actions.start(); + } + const accepted = await actions.submitUtterance(utterance); + if (accepted) setText(''); + }); + }, [actions, run, text]); + + const handleReply = useCallback(() => { + if (!lastDispatch) return; + void run(() => + actions.submitAgentReply({ + agentSessionId: lastDispatch.agentSessionId, + tabId: lastDispatch.tabId, + text: SAMPLE_REPLY, + }) + ); + }, [actions, lastDispatch, run]); + + return ( +
+
+ setText(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') handleSend(); + }} + placeholder="Type an utterance..." + className="flex-1 min-w-0 px-2 py-1 text-[11px] rounded outline-none border" + style={{ + backgroundColor: theme.colors.bgMain, + borderColor: theme.colors.border, + color: theme.colors.textMain, + }} + /> + } + fill={theme.colors.accent} + fg={onAccent} + /> +
+ +
+ } + /> + void run(actions.interrupt)} + icon={} + /> + void run(actions.stop)} + icon={} + /> +
+
+ ); +} + +function HarnessButton({ + theme, + testId, + label, + title, + icon, + disabled, + onClick, + fill, + fg, +}: { + theme: Theme; + testId: string; + label: string; + title: string; + icon: React.ReactNode; + disabled: boolean; + onClick: () => void; + fill?: string; + fg?: string; +}) { + return ( + + ); +} + +export default VoiceDevHarness; diff --git a/src/renderer/components/ACappella/VoiceHud.tsx b/src/renderer/components/ACappella/VoiceHud.tsx new file mode 100644 index 0000000000..cf818f6232 --- /dev/null +++ b/src/renderer/components/ACappella/VoiceHud.tsx @@ -0,0 +1,606 @@ +/** + * VoiceHud - the one on-screen surface for an A Cappella voice session. + * + * Mounted once, app-wide (next to the other single-instance hosts in AppShell) + * and gated on the `aCappella` Encore flag. It owns the `acappella:event` + * subscription, so a second mount would project every protocol event twice. It + * renders nothing until there is something to show: a live session, an error + * worth reading, or the dev harness in a development build. + * + * **Minimize and close are different actions, and must stay that way.** + * Minimize hides the widget and leaves the session running, handing the + * indicator to `VoiceStatusIndicator` in the Left Bar header; close ENDS the + * session. That pairing is the opposite of the media player's, and deliberately + * so: there, sound is the evidence that something is still running, so hiding + * the widget is safe. Here, silence is - a microphone with no visible surface is + * one the user cannot see, so the button that hides the widget must leave an + * indicator behind, and the button that looks like an exit must actually close + * the floor. + * + * The widget is draggable and remembers where it was put, through + * `usePointerDrag` (the same gesture the Concerto surfaces use) and the `ui` + * section of the A Cappella settings blob. + */ + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import type { PointerEvent as ReactPointerEvent } from 'react'; +import { Minus, MicOff } from 'lucide-react'; +import type { Theme } from '../../types'; +import { readableTextOn } from '../../../shared/colorContrast'; +import { MODAL_PRIORITIES } from '../../constants/modalPriorities'; +import { useModalLayer } from '../../hooks/ui/useModalLayer'; +import { usePointerDrag } from '../../hooks/utils/usePointerDrag'; +import { useEventListener } from '../../hooks/utils/useEventListener'; +import { isVoiceSessionActive } from '../../../shared/acappella/session-state'; +import { + VOICE_HUD_STATE_DESCRIPTIONS, + VOICE_HUD_STATE_LABELS, + voiceHudVisualState, +} from '../../../shared/acappella/hud-state'; +import { + clampVoiceHudPosition, + defaultVoiceHudPosition, + type VoiceHudPosition, +} from '../../../shared/acappella/ui-prefs'; +import { DEFAULT_TTS_VOLUME } from '../../../shared/acappella/voice-controls'; +import type { MicIssue } from '../../../shared/acappella/protocol'; +import { micSettingsLabel, micSettingsUrl } from '../../../shared/acappella/mic-settings'; +import { getPlatform } from '../../utils/platformUtils'; +import { selectVoiceRemoteDevice, useVoiceSessionStore } from '../../stores/voiceSessionStore'; +import { useVoiceUiStore } from '../../stores/voiceUiStore'; +import { EscCloseButton } from '../ui/EscCloseButton'; +import { VoiceDevHarness } from './VoiceDevHarness'; +import { VoiceHudControls } from './VoiceHudControls'; +import { VoiceIndicator } from './VoiceIndicator'; +import { VoiceTranscript } from './VoiceTranscript'; +import { useVoiceScope } from './useVoiceScope'; +import { useOwnsVoiceSession } from './useOwnsVoiceSession'; +import { VoiceInputPicker } from './VoiceInputPicker'; +import { useVoiceInputDevices } from './useVoiceInputDevices'; +import { useVoiceSession } from './useVoiceSession'; + +export interface VoiceHudProps { + theme: Theme; + /** The A Cappella Encore flag. False renders nothing and subscribes to nothing. */ + enabled: boolean; + /** + * Show the dev harness - the type-an-utterance box that drives a session + * without a microphone. + * + * Defaults to OFF, including in development, and opts in through + * {@link DEV_HARNESS_STORAGE_KEY}. It used to default to the development + * build, and because the harness is also a reason for the widget to render, + * every dev build opened a voice panel at startup that nobody had asked for - + * the one thing A Cappella must never do. A debugging tool is not a reason to + * put a microphone widget on screen. + */ + showDevHarness?: boolean; +} + +/** + * Opt in to the dev harness: `localStorage.setItem(key, 'true')`, then reload. + * + * localStorage rather than a build flag so it can be switched on in the window + * that is misbehaving, and read once at mount so toggling it mid-session cannot + * make a widget appear underneath the user. + */ +export const DEV_HARNESS_STORAGE_KEY = 'maestro.acappella.devHarness'; + +function devHarnessOptedIn(): boolean { + try { + return globalThis.localStorage?.getItem(DEV_HARNESS_STORAGE_KEY) === 'true'; + } catch { + // A window with storage denied has not opted in to anything. + return false; + } +} + +/** Widget width. Fixed: this is a status readout, not a document. */ +const HUD_WIDTH = 340; + +/** Height used for clamping before the widget has been measured. */ +const HUD_FALLBACK_HEIGHT = 160; + +/** + * What a broken microphone says. Plain, specific, and free of alarm language: a + * denied permission is a setting the user has not turned on yet, not a crash, + * and dressing it in error red teaches people to ignore the colour that matters. + */ +const MIC_ISSUE_MESSAGES: Record = { + 'permission-denied': 'Maestro does not have microphone access yet.', + 'no-device': 'No microphone was found.', + 'device-lost': 'The microphone was disconnected.', + unavailable: 'Audio capture is unavailable on this system.', +}; + +function viewport() { + return { width: window.innerWidth, height: window.innerHeight }; +} + +export function VoiceHud({ theme, enabled, showDevHarness }: VoiceHudProps) { + const actions = useVoiceSession(enabled); + + const state = useVoiceSessionStore((s) => s.state); + const partial = useVoiceSessionStore((s) => s.partialTranscript); + const speech = useVoiceSessionStore((s) => s.speech); + const mic = useVoiceSessionStore((s) => s.mic); + const error = useVoiceSessionStore((s) => s.error); + const substitutions = useVoiceSessionStore((s) => s.substitutions); + const sttHearsAudio = useVoiceSessionStore((s) => s.sttHearsAudio); + const lostEvents = useVoiceSessionStore((s) => s.lostEvents); + const dismissed = useVoiceSessionStore((s) => s.dismissed); + /** + * The paired device holding the floor, or null for this machine's own + * microphone. Rendered next to the state so a Mac at home visibly reflects + * that a phone is the thing listening - a listening indicator over a shut + * local microphone is the one lie this widget must never tell. + */ + const remoteDevice = useVoiceSessionStore(selectVoiceRemoteDevice); + const setDismissed = useVoiceSessionStore((s) => s.setDismissed); + + const loadPrefs = useVoiceUiStore((s) => s.load); + const storedPosition = useVoiceUiStore((s) => s.hudPosition); + const setHudPosition = useVoiceUiStore((s) => s.setHudPosition); + const transcriptVisible = useVoiceUiStore((s) => s.transcriptVisible); + const toggleTranscript = useVoiceUiStore((s) => s.toggleTranscript); + const minimized = useVoiceUiStore((s) => s.minimized); + const setMinimized = useVoiceUiStore((s) => s.setMinimized); + const minimizeBehavior = useVoiceUiStore((s) => s.minimizeBehavior); + const muted = useVoiceUiStore((s) => s.muted); + const setMuted = useVoiceUiStore((s) => s.setMuted); + const holdThresholdMs = useVoiceUiStore((s) => s.holdThresholdMs); + + const scope = useVoiceScope(theme); + // `HTMLElement`, not `HTMLDivElement`: the collapsed form is a button, and the + // clamp measures whichever one is currently mounted. + const rootRef = useRef(null); + // A callback ref, because the same ref is attached to a
in the expanded + // form and a + +
+ + {/* Anything the user is running that they did not ask for. */} + {substitutions.length > 0 && ( +
+ {substitutions.map((sub) => ( +
+ {sub.message} +
+ ))} +
+ )} + + {lostEvents && ( +
+ Some voice events were lost; this transcript may be incomplete. +
+ )} + + {micIssue && } + + {showError && error && ( +
+ {error.message} +
+ )} + + {/* The last thing heard, always visible. The full scrollback is behind + the transcript toggle; this one line is what stops the collapsed HUD + from being a widget with no content at all. */} + {!transcriptVisible && (partial || spoken > 0) && ( +
+ {partial || speech?.sentences[speech.sentences.length - 1]} +
+ )} + + {transcriptVisible && } + + {/* The one thing "Listening" cannot say for itself. The floor really is + open here - the state machine is not lying - but a text-in recogniser + opens no capture device, so nothing spoken can ever arrive. Said + plainly, and only when it is true. */} + {active && sttHearsAudio === false && ( +
+ This recogniser does not listen to the microphone - it takes typed input only, so + speaking will not produce a transcript. Pick a speech-to-text provider in Settings > + Plugins > A Cappella > Voice Setup. +
+ )} + + {/* The microphone in use, and the way to change it. In the HUD because + "nothing is being heard" is discovered HERE, mid-session, and sending + someone to Settings to find out which device is open is the gap that + made a silent session indistinguishable from a wrong input. Writes + the same persisted setting Voice Setup does. */} +
+ +
+ + void toggleTranscript()} + onToggleMute={handleToggleMute} + /> + + {devHarness && } + + + ); +} + +/** + * The microphone is not going to work, said calmly. + * + * Only `permission-denied` gets the button, because it is the only issue the OS + * settings pane can fix: sending someone to a privacy checkbox to solve an + * unplugged microphone wastes their time and their trust in the next button. On + * a platform with no deep link (Linux) the sentence carries the instruction + * instead, since a button that opens nothing is worse than no button at all. + */ +function MicIssueNotice({ theme, issue, color }: { theme: Theme; issue: MicIssue; color: string }) { + const platform = getPlatform(); + const settingsUrl = issue === 'permission-denied' ? micSettingsUrl(platform) : null; + + const openSettings = useCallback(() => { + void window.maestro.voice.openMicSettings(); + }, []); + + return ( +
+
+ ); +} + +export default VoiceHud; diff --git a/src/renderer/components/ACappella/VoiceHudControls.tsx b/src/renderer/components/ACappella/VoiceHudControls.tsx new file mode 100644 index 0000000000..fd1b4cd619 --- /dev/null +++ b/src/renderer/components/ACappella/VoiceHudControls.tsx @@ -0,0 +1,249 @@ +/** + * The HUD's control row: talk, interrupt, stop, transcript, mute. + * + * The talk button is the only interesting one. It has to be BOTH push-to-talk + * and tap-to-toggle, because those are two different habits and a voice UI that + * picks one has half its users fighting it: someone dictating a paragraph holds + * the button like a walkie-talkie, and someone having a conversation taps it + * once and forgets about it. The classifier is the same one the global hotkey + * uses, off the same `holdThresholdMs`, so the button and the key cannot decide + * "hold" at different moments. + * + * Unlike the global hotkey, this surface HAS a real release event (see the note + * at the top of `main/acappella/hotkeys/press-hold.ts`), so it drives start/stop + * directly instead of going through the polling detector. + * + * Every control is a real ` + + + + + +
+ + + + +
+ ); +} + +function IconControl({ + theme, + testId, + icon: Icon, + label, + disabled, + pressed, + onClick, +}: { + theme: Theme; + testId: string; + icon: LucideIcon; + label: string; + disabled?: boolean; + pressed?: boolean; + onClick: () => void; +}) { + return ( + + ); +} + +export default VoiceHudControls; diff --git a/src/renderer/components/ACappella/VoiceIndicator.tsx b/src/renderer/components/ACappella/VoiceIndicator.tsx new file mode 100644 index 0000000000..8c69ffd2d5 --- /dev/null +++ b/src/renderer/components/ACappella/VoiceIndicator.tsx @@ -0,0 +1,212 @@ +/** + * The one thing in the HUD you are meant to read from across the room. + * + * Five states have to be distinguishable, and they have to be distinguishable by + * more than hue: they are all drawn from the same theme accent, and a + * colour-only difference is no difference at all to a large minority of users. + * So each state differs in SHAPE and in MOTION as well: + * + * idle-armed an outlined ring around a radio glyph, still + * listening an outlined ring around a microphone, filled by a disc that + * tracks the real input level + * thinking an outlined ring around a spinner + * speaking a FILLED accent disc around a speaker, pulsing with the outgoing + * audio + * error an outlined ring in the error colour around a warning glyph + * + * The level meter is why this component reads the store itself instead of taking + * the level as a prop. `audio-level` lands ~20 times a second, and a + * subscription in the HUD body would re-render the transcript, the controls, and + * the harness at meter rate to move a disc a few pixels. The subscription + * belongs in the smallest component that draws the number. + * + * Under `prefers-reduced-motion` every one of those animations is replaced by a + * static indicator. That is not a nicety: this widget is designed to be left on + * screen all day, and a permanently animating element is a genuine accessibility + * problem for people with vestibular disorders. + */ + +import { memo } from 'react'; +import { AlertTriangle, Loader2, Mic, Radio, Volume2 } from 'lucide-react'; +import type { Theme } from '../../types'; +import { readableTextOn } from '../../../shared/colorContrast'; +import { + VOICE_HUD_STATE_DESCRIPTIONS, + VOICE_HUD_STATE_LABELS, + type VoiceHudVisualState, +} from '../../../shared/acappella/hud-state'; +import { usePrefersReducedMotion } from '../../hooks/utils/usePrefersReducedMotion'; +import { selectVoiceAudioLevel, useVoiceSessionStore } from '../../stores/voiceSessionStore'; + +/** + * Where the level sits when the meter is full. Speech from a laptop mic at arm's + * length lands around 0.05 to 0.2 RMS, so a bar scaled linearly to 1.0 would + * barely move; the square root spends the range where the voice actually is. + */ +const METER_FULL_SCALE = 0.25; + +export function meterFill(level: number): number { + if (!Number.isFinite(level) || level <= 0) return 0; + return Math.min(1, Math.sqrt(level / METER_FULL_SCALE)); +} + +export interface VoiceIndicatorProps { + theme: Theme; + state: VoiceHudVisualState; + /** The microphone in use, shown on hover. Null when nothing is being captured. */ + deviceLabel?: string | null; + /** Diameter in px. The HUD header uses 28; the collapsed pill uses 20. */ + size?: number; +} + +export const VoiceIndicator = memo(function VoiceIndicator({ + theme, + state, + deviceLabel = null, + size = 28, +}: VoiceIndicatorProps) { + // Read here rather than in the HUD body: this is the only thing that moves at + // meter rate, so it is the only thing that should re-render at meter rate. + const level = useVoiceSessionStore(selectVoiceAudioLevel); + const reducedMotion = usePrefersReducedMotion(); + + const label = VOICE_HUD_STATE_LABELS[state]; + const title = deviceLabel + ? `${VOICE_HUD_STATE_DESCRIPTIONS[state]} (${deviceLabel})` + : VOICE_HUD_STATE_DESCRIPTIONS[state]; + const glyph = Math.round(size * 0.5); + const box = { width: size, height: size } as const; + + if (state === 'speaking') { + const onAccent = readableTextOn(theme.colors.accentForeground, [theme.colors.accent]); + return ( +
+ +
+ ); + } + + if (state === 'listening') { + const fill = meterFill(level); + return ( +
+ {/* + * The level itself, behind the icon. A real signal is worth more than a + * canned pulse: a ring that moves with the room proves the microphone is + * live, which is the single question a user has while looking at this + * thing. It floors at a visible sliver so an open floor in a silent room + * still reads as open rather than as switched off. + * + * With reduced motion the disc stops tracking and sits at a fixed size: + * the ring alone still says the floor is open, and the transcript says + * whether anything is being heard. + */} +
+ ); + } + + if (state === 'thinking') { + return ( +
+ +
+ ); + } + + if (state === 'error') { + return ( +
+ +
+ ); + } + + return ( +
+ +
+ ); +}); + +export default VoiceIndicator; diff --git a/src/renderer/components/ACappella/VoiceInputPicker.tsx b/src/renderer/components/ACappella/VoiceInputPicker.tsx new file mode 100644 index 0000000000..52217ce65e --- /dev/null +++ b/src/renderer/components/ACappella/VoiceInputPicker.tsx @@ -0,0 +1,84 @@ +/** + * VoiceInputPicker - choose which microphone A Cappella listens to. + * + * One component, two placements: `compact` for the HUD's control row, where it + * sits beside the transcript and mute buttons, and the default for Voice Setup, + * where it is a labelled row. Both write the same persisted setting, so the + * "quick" picker is not a separate, temporary choice - changing it in the HUD IS + * changing the default. Two selectors that disagreed about which microphone is + * chosen would be worse than having only one. + * + * A native ` void devices.select(event.target.value)} + aria-label="Microphone" + className={`rounded border bg-transparent outline-none ${ + compact ? 'text-[10px] px-1 py-0.5 w-full' : 'text-xs px-2 py-1 w-full' + }`} + style={{ + borderColor: theme.colors.border, + color: theme.colors.textMain, + backgroundColor: theme.colors.bgMain, + }} + > + {/* Always first, and always present even with no devices enumerated: it is + the only option that is guaranteed to resolve to something. */} + + {devices.devices + // The OS's own "default" entry is what the sentinel above already + // means, so listing it again offers the same choice twice under two + // names - and picking the wrong one pins the device that happens to be + // default today. + .filter((device) => device.deviceId && device.deviceId !== 'default') + .map((device, index) => ( + + ))} + + ); + + if (compact) return select; + + return ( + + ); +} diff --git a/src/renderer/components/ACappella/VoiceStatusIndicator.tsx b/src/renderer/components/ACappella/VoiceStatusIndicator.tsx new file mode 100644 index 0000000000..f3c206e293 --- /dev/null +++ b/src/renderer/components/ACappella/VoiceStatusIndicator.tsx @@ -0,0 +1,87 @@ +/** + * The minimized voice HUD: a live state glyph in the Left Bar header. + * + * Minimizing the floating widget parks it here rather than ending the session, + * so this is both the "Maestro is listening" indicator and the way back to the + * controls. It sits in the same header row as `NowPlayingIndicator`, for the + * same reason: a background capability with no visible surface is one the user + * can neither find nor stop. + * + * **This is why minimize is allowed to hide the widget at all.** A microphone is + * not like audio playback - silence is not evidence that it stopped - so the + * HUD's minimize button is only honest while something on screen keeps saying + * the floor is open. That something is this. Do not let the HUD's minimized + * state render nothing on the grounds that the session is still in the store: a + * state nobody can see is a microphone nobody knows about. + * + * One button, not the media indicator's two. There, the halves are play/pause + * and restore, which are genuinely different actions; here the only thing a + * header icon can do is bring the widget back, because talking is done by voice, + * by the hotkey, or by the composer button. + */ + +import { memo } from 'react'; + +import { isVoiceSessionActive } from '../../../shared/acappella/session-state'; +import { VOICE_HUD_STATE_LABELS, voiceHudVisualState } from '../../../shared/acappella/hud-state'; +import { selectACappellaEnabled, useSettingsStore } from '../../stores/settingsStore'; +import { useVoiceSessionStore } from '../../stores/voiceSessionStore'; +import { useOwnsVoiceSession } from './useOwnsVoiceSession'; +import { useVoiceUiStore } from '../../stores/voiceUiStore'; +import type { Theme } from '../../types'; +import { VoiceIndicator } from './VoiceIndicator'; +import { useVoiceScope } from './useVoiceScope'; + +interface VoiceStatusIndicatorProps { + theme: Theme; + /** + * Drop the scope label, for a Left Bar with no room for it - the collapsed + * rail, or a sidebar too narrow to take one. The glyph always stays: it is + * the entire point of the control. + */ + compact?: boolean; +} + +export const VoiceStatusIndicator = memo(function VoiceStatusIndicator({ + theme, + compact = false, +}: VoiceStatusIndicatorProps) { + const enabled = useSettingsStore(selectACappellaEnabled); + const state = useVoiceSessionStore((s) => s.state); + const minimized = useVoiceUiStore((s) => s.minimized); + const setMinimized = useVoiceUiStore((s) => s.setMinimized); + const scope = useVoiceScope(theme); + // The minimized HUD is still the session's surface, so it follows the HUD's + // window: a session opened in another window must not leave a live microphone + // indicator in this one's Left Bar. + const ownsSession = useOwnsVoiceSession(); + + // `minimized` alone, deliberately not `minimized || dismissed`. Dismissed is + // the close button, which ends the session, so an indicator that also + // respected it would sit there claiming an open microphone for the moment + // between the click and the service confirming the session is gone. + if (!enabled || !ownsSession || !isVoiceSessionActive(state) || !minimized) return null; + + const visualState = voiceHudVisualState(state); + const stateLabel = VOICE_HUD_STATE_LABELS[visualState]; + + // Bordered pill, matching the now-playing indicator beside it, so the two read + // as the same class of thing: something running that you can get back to. + // Never shrinks - the wordmark is this row's shrink target. + return ( + + ); +}); diff --git a/src/renderer/components/ACappella/VoiceTranscript.tsx b/src/renderer/components/ACappella/VoiceTranscript.tsx new file mode 100644 index 0000000000..985bd15c5b --- /dev/null +++ b/src/renderer/components/ACappella/VoiceTranscript.tsx @@ -0,0 +1,289 @@ +/** + * The live transcript: what was said, where it went, and what is being said back. + * + * Three things happen here that a plain message list does not do: + * + * - **Partials settle rather than jump.** An interim hypothesis renders as a + * dimmed, italic line at the bottom; when it settles it is replaced by the + * final in the feed. Rendering partials as ordinary lines would leave the + * scrollback full of half-heard sentences that were never actually said. + * - **Route chips are addresses, not decoration.** The line narrating a + * dispatch carries where it went, and the chip is clickable: "I said that + * out loud, where did it land" is the question this panel exists to answer, + * and answering it with text the user then has to go and find by hand + * answers only half of it. + * - **The sentence being spoken is highlighted.** The scheduler emits + * `speak-sentence` just BEFORE the audio reaches the sink, so the last + * sentence in the run is the one currently coming out of the speakers. + * + * Virtualized through `@tanstack/react-virtual` with `measureElement`, the same + * pattern the group chat and history panels use: rows here are markdown and + * genuinely variable in height, so a fixed row estimate would misplace the + * scroll position on any reply longer than a line. + */ + +import { useCallback, useEffect, useMemo, useRef } from 'react'; +import { useVirtualizer } from '@tanstack/react-virtual'; +import { CornerDownRight } from 'lucide-react'; +import type { Theme } from '../../types'; +import { readableTextOn } from '../../../shared/colorContrast'; +import type { DispatchAction } from '../../../shared/acappella/protocol'; +import { jumpToVoiceTab } from '../../hooks/voice/useVoiceAgentActions'; +import { + useVoiceSessionStore, + type VoiceFeedEntry, + type VoiceFeedRoute, + type VoiceSpeechRun, +} from '../../stores/voiceSessionStore'; +import { Markdown } from '../Markdown'; + +export interface VoiceTranscriptProps { + theme: Theme; + /** Max height of the scroll area, in px. */ + maxHeight?: number; +} + +/** Who said a line. `Maestro` is the session narrating itself. */ +const KIND_LABELS: Record = { + you: 'You', + assistant: 'Agent', + system: 'Maestro', +}; + +/** What the dispatch did to the tab, in the words the chip shows. */ +const ACTION_LABELS: Record = { + created: 'new tab', + recalled: 'back to tab', + focused: 'current tab', +}; + +/** + * A row's height before it has been measured. + * + * Deliberately near the short end: an underestimate is corrected upward on the + * first measure pass, while an overestimate leaves a gap under the last row that + * the user sees as the list failing to reach the bottom. + */ +const ESTIMATED_ROW_HEIGHT = 34; + +export function VoiceTranscript({ theme, maxHeight = 220 }: VoiceTranscriptProps) { + const feed = useVoiceSessionStore((s) => s.feed); + const partial = useVoiceSessionStore((s) => s.partialTranscript); + const speech = useVoiceSessionStore((s) => s.speech); + + const containerRef = useRef(null); + + const virtualizer = useVirtualizer({ + count: feed.length, + getScrollElement: () => containerRef.current, + estimateSize: () => ESTIMATED_ROW_HEIGHT, + getItemKey: (index) => feed[index]?.id ?? index, + overscan: 6, + // jsdom has no layout, so without a seed rect the virtualizer reports a zero + // viewport and renders nothing at all - which would make every test here + // pass against an empty list. + initialRect: { width: 320, height: maxHeight }, + }); + + // Follow the conversation. A voice transcript is read at the bottom by + // definition: the interesting line is the one being said right now. + useEffect(() => { + const container = containerRef.current; + if (!container || feed.length === 0) return; + container.scrollTop = container.scrollHeight; + }, [feed.length, partial, speech]); + + const accentText = useMemo( + () => readableTextOn(theme.colors.accent, [theme.colors.bgSidebar]), + [theme.colors.accent, theme.colors.bgSidebar] + ); + + const items = virtualizer.getVirtualItems(); + + return ( +
+
+ {feed.length === 0 && !partial && ( +
+ Nothing said yet. +
+ )} + + {feed.length > 0 && ( +
+ {items.map((item) => { + const entry = feed[item.index]; + if (!entry) return null; + return ( +
+ +
+ ); + })} +
+ )} + + {/* + * The live hypothesis, outside the virtual list. It has no id, it is + * replaced wholesale on every update, and it is always last - giving it + * a row in a measured list would remeasure the whole tail several times + * a second for a line that is about to be deleted. + */} + {partial && ( +
+ {partial} +
+ )} + + {speech && speech.sentences.length > 0 && ( + + )} +
+
+ ); +} + +function TranscriptLine({ + theme, + entry, + accentText, +}: { + theme: Theme; + entry: VoiceFeedEntry; + accentText: string; +}) { + return ( +
+ + {KIND_LABELS[entry.kind]} + + {entry.kind === 'assistant' ? ( + // Agents write markdown even when they are being read out loud, and the + // shared chat preset is what already knows how to render it (and how to + // make its file paths clickable). A hand-rolled ReactMarkdown here would + // be a second, drifting copy of that map. + + + + ) : ( + {entry.text} + )} + {entry.route && } +
+ ); +} + +/** + * "Backend / Auth Refactor (new tab)", clickable. + * + * A real ` + ); +} + +/** + * The reply as it is spoken, one sentence at a time. + * + * Outside the virtual list for the same reason the partial is: it grows by one + * sentence every couple of seconds and is always at the bottom, so it is cheaper + * and steadier to render it whole than to keep remeasuring a growing tail row. + */ +function SpokenRun({ + theme, + speech, + accentText, +}: { + theme: Theme; + speech: VoiceSpeechRun; + accentText: string; +}) { + // `speak-sentence` is emitted just before the audio reaches the sink, so the + // newest sentence is the one being heard - until the run ends, at which point + // nothing is being spoken and nothing should be highlighted. + const speakingIndex = speech.endedReason === null ? speech.sentences.length - 1 : -1; + + return ( +
+ {speech.sentences.map((sentence, index) => { + const current = index === speakingIndex; + return ( + + {sentence}{' '} + + ); + })} + {speech.endedReason === 'cancelled' && ( + (cut off) + )} +
+ ); +} + +export default VoiceTranscript; diff --git a/src/renderer/components/ACappella/index.ts b/src/renderer/components/ACappella/index.ts new file mode 100644 index 0000000000..f0cbe3a683 --- /dev/null +++ b/src/renderer/components/ACappella/index.ts @@ -0,0 +1,25 @@ +/** + * A Cappella renderer surfaces. + * + * The HUD is the only thing outside this folder should mount that TALKS to the + * service: it owns the event subscription (one subscriber, or every event + * applies twice) and renders the controls, the transcript, and the dev harness + * itself. `VoiceStatusIndicator` is the one exception, and only reads store + * state - it is the minimized HUD's home in the Left Bar header. + */ + +export { VoiceHud, DEV_HARNESS_STORAGE_KEY, type VoiceHudProps } from './VoiceHud'; +export { VoiceStatusIndicator } from './VoiceStatusIndicator'; +export { VoiceHudControls, type VoiceHudControlsProps } from './VoiceHudControls'; +export { VoiceIndicator, type VoiceIndicatorProps, meterFill } from './VoiceIndicator'; +export { VoiceTranscript, type VoiceTranscriptProps } from './VoiceTranscript'; +export { VoiceDevHarness, type VoiceDevHarnessProps } from './VoiceDevHarness'; +export { VoiceInputPicker, type VoiceInputPickerProps } from './VoiceInputPicker'; +export { + useVoiceInputDevices, + deviceLabel, + type VoiceInputDevice, + type VoiceInputDevicesState, +} from './useVoiceInputDevices'; +export { useVoiceScope, type VoiceScopeDisplay } from './useVoiceScope'; +export { useVoiceSession, type VoiceSessionActions } from './useVoiceSession'; diff --git a/src/renderer/components/ACappella/useOwnsVoiceSession.ts b/src/renderer/components/ACappella/useOwnsVoiceSession.ts new file mode 100644 index 0000000000..212ebd7b60 --- /dev/null +++ b/src/renderer/components/ACappella/useOwnsVoiceSession.ts @@ -0,0 +1,37 @@ +/** + * useOwnsVoiceSession - does THIS window get to show the voice session? + * + * There is one voice session for the whole app, and every window receives its + * whole event stream: `acappella:event` is broadcast like every other + * main -> renderer push (see the multi-window invariant in + * `src/main/utils/safe-send.ts`), so each window mirrors the same session and + * decides for itself whether to draw a surface. Without that decision, opening + * voice in one window drew an identical HUD in every window, and one microphone + * looked like several. + * + * The rule, in one place because two surfaces need it (the HUD and the Left Bar + * indicator) and a second copy would drift into showing one without the other: + * + * - The session names a window, and it is this one -> yes. + * - The session names a window that is NOT this one -> no. + * - The session names no window -> the primary window shows it, so a session + * started by something with no window behind it always has exactly one + * surface rather than none. + * - Web-desktop is not one of several Electron windows. It mirrors the whole + * app, so it shows everything, matching `WindowContext`'s own permit-all. + * - Outside a `WindowProvider` (a single-window host, an isolation test) there + * is no window to be the wrong one, so it shows everything. + */ + +import { useWindowContextOptional } from '../../contexts/WindowContext'; +import { useVoiceSessionStore } from '../../stores/voiceSessionStore'; +import { isWebDesktop } from '../../utils/runtimeContext'; + +export function useOwnsVoiceSession(): boolean { + const sessionWindowId = useVoiceSessionStore((s) => s.windowId); + const windowContext = useWindowContextOptional(); + + if (isWebDesktop() || !windowContext) return true; + if (sessionWindowId === null) return windowContext.isMainWindow; + return sessionWindowId === windowContext.windowId; +} diff --git a/src/renderer/components/ACappella/useVoiceInputDevices.ts b/src/renderer/components/ACappella/useVoiceInputDevices.ts new file mode 100644 index 0000000000..08aacd6fcc --- /dev/null +++ b/src/renderer/components/ACappella/useVoiceInputDevices.ts @@ -0,0 +1,107 @@ +/** + * useVoiceInputDevices - the microphones this machine offers, and which one is + * chosen. + * + * One hook for both pickers (the HUD's quick selector and Voice Setup's + * persistent one) for the reason `useGitAgentActions` is one hook for three + * menus: two copies of "read the list, subscribe to changes, write the setting" + * drift, and the one that drifts is the one the user reaches for mid-session. + * + * Two facts about the list are worth knowing before rendering it: + * + * - **Labels are redacted until a capture has been granted.** Chromium reports + * `deviceId` but an empty `label` until the user has allowed the microphone + * at least once, so entries legitimately arrive nameless. `deviceLabel()` + * supplies a stable fallback rather than rendering a blank row. + * - **The list changes without anyone asking.** A headset is unplugged, or a + * first capture reveals the labels. The audio host pushes on + * `onInputDevices`, so this subscribes rather than reading once. + */ + +import { useCallback, useEffect, useState } from 'react'; +import { ACAPPELLA_SYSTEM_DEFAULT_INPUT } from '../../../shared/acappella/audio-host'; + +export interface VoiceInputDevice { + deviceId: string; + label: string; +} + +export interface VoiceInputDevicesState { + /** Every microphone, in the order the OS reported them. */ + devices: VoiceInputDevice[]; + /** The chosen id, or {@link ACAPPELLA_SYSTEM_DEFAULT_INPUT} for "follow the OS". */ + selectedId: string; + /** Persist a choice. Takes effect on the next capture, never mid-utterance. */ + select: (deviceId: string) => Promise; + /** True until the first read resolves, so a picker can avoid flashing "none". */ + loading: boolean; +} + +/** + * What to show for a device. + * + * Never blank: an unlabelled entry is a real device the user may need to pick, + * and a dropdown row with no text is unclickable in practice. + */ +export function deviceLabel(device: VoiceInputDevice, index: number): string { + if (device.label) return device.label; + if (device.deviceId === 'default') return 'System default'; + return `Microphone ${index + 1}`; +} + +export function useVoiceInputDevices(enabled: boolean): VoiceInputDevicesState { + const [devices, setDevices] = useState([]); + const [selectedId, setSelectedId] = useState(ACAPPELLA_SYSTEM_DEFAULT_INPUT); + const [loading, setLoading] = useState(true); + + useEffect(() => { + if (!enabled) { + setLoading(false); + return; + } + let cancelled = false; + + void window.maestro.voice + .inputDevices() + .then((result) => { + if (cancelled) return; + setDevices(result.devices); + setSelectedId(result.selectedId); + }) + // A failed read leaves the system default selected, which is the same + // thing the session would open anyway. + .catch(() => undefined) + .finally(() => { + if (!cancelled) setLoading(false); + }); + + // Pushed, not polled: the list changes when hardware does, and when a first + // capture finally un-redacts the labels. + const unsubscribe = window.maestro.voice.onInputDevices((next) => { + if (!cancelled) setDevices(next); + }); + + return () => { + cancelled = true; + unsubscribe(); + }; + }, [enabled]); + + const select = useCallback(async (deviceId: string) => { + // Optimistic, with the rollback target read from a ref rather than captured: + // a callback that closed over `selectedId` would either be rebuilt on every + // render or restore a device the user had already moved on from. + let previous = ACAPPELLA_SYSTEM_DEFAULT_INPUT; + setSelectedId((current) => { + previous = current; + return deviceId; + }); + try { + await window.maestro.voice.setInputDevice(deviceId); + } catch { + setSelectedId(previous); + } + }, []); + + return { devices, selectedId, select, loading }; +} diff --git a/src/renderer/components/ACappella/useVoiceScope.ts b/src/renderer/components/ACappella/useVoiceScope.ts new file mode 100644 index 0000000000..35b2669c35 --- /dev/null +++ b/src/renderer/components/ACappella/useVoiceScope.ts @@ -0,0 +1,82 @@ +/** + * What the voice session is bound to, in the words and colour the HUD shows. + * + * This is the single most important line in the widget. Everything else on + * screen is feedback about a turn that already happened; this says where the + * NEXT sentence is going to land. Speaking a refactor instruction into the wrong + * repository is the failure mode the whole HUD exists to prevent, so the scope + * gets the agent's own colour, its name, and the tab the last dispatch actually + * used - not a generic "connected" pill. + * + * The tab comes from `lastDispatch` rather than from the scope, because the + * scope only names an agent. Which TAB a prompt landed in is decided per turn by + * the router, and it is the part a user is most likely to have lost track of. + */ + +import { useMemo } from 'react'; +import type { Theme } from '../../types'; +import { readableTextOn } from '../../../shared/colorContrast'; +import { generateParticipantColor } from '../../utils/participantColors'; +import { useVoiceSessionStore } from '../../stores/voiceSessionStore'; + +export interface VoiceScopeDisplay { + /** "Conductor", or the agent's name. Never empty. */ + label: string; + /** The tab the last dispatch used, or null when nothing has been dispatched. */ + tabLabel: string | null; + /** The agent's colour, already contrast-corrected for the HUD background. */ + color: string; + /** The bound agent, or null for the conductor. */ + agentSessionId: string | null; + /** The tab the last dispatch used, so the HUD can offer to jump to it. */ + tabId: string | null; +} + +/** + * The conductor's colour index. + * + * Zero is the group-chat moderator's reserved blue, which is the same idea + * wearing a different hat: the one participant that coordinates the others. Two + * features using one palette slot for the same role is consistency, not a + * collision. + */ +const CONDUCTOR_COLOR_INDEX = 0; + +export function useVoiceScope(theme: Theme): VoiceScopeDisplay { + const scope = useVoiceSessionStore((s) => s.scope); + const roster = useVoiceSessionStore((s) => s.roster); + const lastDispatch = useVoiceSessionStore((s) => s.lastDispatch); + + return useMemo(() => { + const agentSessionId = scope && scope.kind === 'agent' ? scope.sessionId : null; + const index = agentSessionId + ? roster.findIndex((agent) => agent.sessionId === agentSessionId) + : -1; + const agent = index >= 0 ? roster[index] : null; + + // The colour index is the agent's position in the roster, which is the only + // stable per-agent ordinal this side of IPC. `+ 1` keeps every agent off the + // conductor's reserved slot. + const raw = generateParticipantColor(agentSessionId ? index + 1 : CONDUCTOR_COLOR_INDEX, theme); + + // The HUD sits on the sidebar background and the colour comes from a + // palette, so a theme whose background lands near one of the hues would + // otherwise paint the scope label invisibly on top of itself. + const color = readableTextOn(raw, [theme.colors.bgSidebar, theme.colors.bgMain]); + + // The dispatch tab only describes this scope when it went to this agent. A + // conductor session dispatches all over the fleet, so its last tab is not + // "the tab you are talking to" and showing it would claim a binding that + // does not exist. + const dispatchMatches = + !!lastDispatch && (!agentSessionId || lastDispatch.agentSessionId === agentSessionId); + + return { + label: agentSessionId ? (agent?.name ?? 'Agent') : 'Conductor', + tabLabel: dispatchMatches ? (lastDispatch.tabName ?? null) : null, + color, + agentSessionId, + tabId: dispatchMatches ? lastDispatch.tabId : null, + }; + }, [lastDispatch, roster, scope, theme]); +} diff --git a/src/renderer/components/ACappella/useVoiceSession.ts b/src/renderer/components/ACappella/useVoiceSession.ts new file mode 100644 index 0000000000..c5d0dc0e53 --- /dev/null +++ b/src/renderer/components/ACappella/useVoiceSession.ts @@ -0,0 +1,93 @@ +/** + * useVoiceSession - the renderer's one connection to the voice session. + * + * Subscribes to the `acappella:event` push stream, projects every event into + * `voiceSessionStore`, and hands back the actions. Mounted once (by the HUD): + * the stream is a broadcast, so a second subscriber would apply every event + * twice. + * + * On the IPC subscription: `window.maestro.voice.onEvent()` already returns its + * own unsubscribe, so this is a plain `useEffect` that returns it rather than a + * `useEventListener()` call. That hook wraps `addEventListener`/ + * `removeEventListener` on a DOM `EventTarget`, which the preload bridge is + * not; using it here would mean inventing a DOM event just to hop through it. + * The rule it enforces - never hand-pair add/remove inside an effect - is + * satisfied: nothing is paired by hand, the bridge owns the teardown. + */ + +import { useCallback, useEffect } from 'react'; +import type { VoiceScope } from '../../../shared/acappella/protocol'; +import { beginVoiceSession, useVoiceSessionStore } from '../../stores/voiceSessionStore'; + +export interface VoiceSessionActions { + /** Open a session. Omit the scope for conductor scope. */ + start: (scope?: VoiceScope) => Promise; + /** End the session and return to idle. */ + stop: () => Promise; + submitUtterance: (text: string) => Promise; + /** Barge-in: cancel speech, keep the floor. */ + interrupt: () => Promise; + /** Feed the session an agent answer so it has something to speak. */ + submitAgentReply: (params: { + agentSessionId: string; + tabId: string; + text: string; + }) => Promise; +} + +/** + * Subscribe to the voice event stream and return the session actions. + * + * @param enabled Mirror of the A Cappella Encore flag. False unsubscribes and + * clears the mirrored state, so turning the feature off cannot + * leave a stale session on screen. + */ +export function useVoiceSession(enabled: boolean): VoiceSessionActions { + useEffect(() => { + if (!enabled) { + useVoiceSessionStore.getState().reset(); + return; + } + + const unsubscribe = window.maestro.voice.onEvent((event) => { + useVoiceSessionStore.getState().applyEvent(event); + }); + + // Catch-up: a window opened mid-session (or reloaded) has missed every + // event so far, and the stream alone would leave it claiming idle. + void window.maestro.voice + .getState() + .then((snapshot) => useVoiceSessionStore.getState().applySnapshot(snapshot)) + .catch(() => { + // The one expected rejection is 'ACappellaDisabled' from a flag that + // flipped off between the guard above and this call. There is no + // session to show either way. + }); + + return unsubscribe; + }, [enabled]); + + // A role that fell back to the mock tier has to reach the user, and every + // other trigger needs the same guarantee, so the projection lives in + // `beginVoiceSession` rather than here. + const start = useCallback((scope?: VoiceScope) => beginVoiceSession(scope), []); + + const stop = useCallback(async () => { + await window.maestro.voice.stop(); + }, []); + + const submitUtterance = useCallback( + (text: string) => window.maestro.voice.submitUtterance(text), + [] + ); + + const interrupt = useCallback(() => window.maestro.voice.interrupt('client-button'), []); + + const submitAgentReply = useCallback( + (params: { agentSessionId: string; tabId: string; text: string }) => + window.maestro.voice.submitAgentReply(params), + [] + ); + + return { start, stop, submitUtterance, interrupt, submitAgentReply }; +} diff --git a/src/renderer/components/AppShell.tsx b/src/renderer/components/AppShell.tsx index 6449d93de6..a6c796d19c 100644 --- a/src/renderer/components/AppShell.tsx +++ b/src/renderer/components/AppShell.tsx @@ -25,6 +25,7 @@ import { ContextTimelinePanel } from './ContextTimelinePanel'; import { PermissionPrompt } from './PermissionPrompt'; import { CadenzaLayer } from './Cadenza'; import { ConcertoStageModal } from './Concerto/ConcertoStageModal'; +import { VoiceHud } from './ACappella'; import { useCadenzaStore } from '../stores/cadenzaStore'; import { useMovementStore } from '../stores/movementStore'; import { selectActiveSession, useSessionStore } from '../stores/sessionStore'; @@ -44,6 +45,7 @@ export interface AppShellProps { useNativeTitleBar: boolean; isMdDownViewport: boolean; concertoEnabled: boolean; + aCappellaEnabled: boolean; activeGroupChatId: string | null; groupChats: GroupChat[]; @@ -86,6 +88,7 @@ export function AppShell({ useNativeTitleBar, isMdDownViewport, concertoEnabled, + aCappellaEnabled, activeGroupChatId, groupChats, groups, @@ -285,6 +288,12 @@ export function AppShell({ {/* --- PERMISSION PROMPT (Claude Code standard mode; portal) --- */} + {/* --- A CAPPELLA VOICE HUD (single, app-wide; Encore-gated) --- + Owns the one `acappella:event` subscription, so it is mounted + unconditionally and gates itself: a second mount would project every + protocol event twice. Renders nothing, and subscribes to nothing, + while the Encore Feature is off. */} + {/* --- CONCERTO --- Cadenzas float over the app; the movement stage lives in its own resizable window (Alt+C / command palette / hamburger menu). Both stay diff --git a/src/renderer/components/InputArea/InputArea.tsx b/src/renderer/components/InputArea/InputArea.tsx index 15351dc85e..bf5aad80a9 100644 --- a/src/renderer/components/InputArea/InputArea.tsx +++ b/src/renderer/components/InputArea/InputArea.tsx @@ -17,7 +17,8 @@ import { ContextWarningSash } from '../ContextWarningSash'; import { SummarizeProgressOverlay } from '../SummarizeProgressOverlay'; import { WizardInputPanel } from '../InlineWizard'; import { useImageAnnotatorStore } from '../ImageAnnotator/imageAnnotatorStore'; -import { useAgentCapabilities, useScrollIntoView, useVoiceInput } from '../../hooks'; +import { useAgentCapabilities, useScrollIntoView } from '../../hooks'; +import { useComposerVoice } from '../../hooks/voice/useComposerVoice'; import { useThinkingItems } from '../../hooks/session/useThinkingItems'; import { useWindowContextOptional } from '../../contexts/WindowContext'; import { filterSlashCommands } from '../../utils/search'; @@ -364,22 +365,23 @@ export const InputArea = React.memo(function InputArea(props: InputAreaProps) { setAtMentionCategory, }); - // Voice dictation (Web Speech API). Interim results live-update the draft via - // setInputValue; the final transcript is appended to the value captured when - // listening began. Disabled in terminal mode (the button only renders in AI - // mode anyway). The hook is a no-op where the Web Speech API is unavailable. - const voice = useVoiceInput({ + // The composer microphone. With the A Cappella Encore Feature on it opens a + // voice session bound to this agent; with it off it is the Web Speech + // dictation this button has always been, unchanged. See `useComposerVoice`. + // Disabled in terminal mode (the button only renders in AI mode anyway). + const voice = useComposerVoice({ + session, currentValue: inputValue, onTranscriptionChange: setInputValue, focusRef: inputRef, disabled: isTerminalMode, }); - // toggleVoiceInput's identity changes on every keystroke (it closes over the - // live draft value), so wrap it in a stable callback. This keeps the memoized - // ToolbarControls from re-rendering on each keystroke - it only re-renders - // when isListening actually flips. - const voiceToggleRef = useRef(voice.toggleVoiceInput); - voiceToggleRef.current = voice.toggleVoiceInput; + // The Web Speech toggle's identity changes on every keystroke (it closes over + // the live draft value), so wrap it in a stable callback. This keeps the + // memoized ToolbarControls from re-rendering on each keystroke - it only + // re-renders when isListening actually flips. + const voiceToggleRef = useRef(voice.toggle); + voiceToggleRef.current = voice.toggle; const handleToggleVoiceInput = useCallback(() => voiceToggleRef.current(), []); // Show summarization progress overlay when active for this tab @@ -645,6 +647,7 @@ export const InputArea = React.memo(function InputArea(props: InputAreaProps) { voiceSupported={voice.voiceSupported} isVoiceListening={voice.isListening} onToggleVoiceInput={handleToggleVoiceInput} + voiceHandledElsewhere={voice.usesACappella} onOpenPromptComposer={onOpenPromptComposer} shortcuts={shortcuts} showFlashNotification={showFlashNotification} @@ -685,6 +688,12 @@ export const InputArea = React.memo(function InputArea(props: InputAreaProps) { theme={theme} isTerminalMode={isTerminalMode} processInput={processInput} + // Only when A Cappella owns the microphone. With it off, the button + // stays the Web Speech dictation one in the toolbar row, which is + // touch-only - so there is never a second microphone on screen. + showVoiceButton={!isTerminalMode && voice.usesACappella} + isVoiceListening={voice.isListening} + onToggleVoice={handleToggleVoiceInput} /> diff --git a/src/renderer/components/InputArea/components/NotificationSendControls.tsx b/src/renderer/components/InputArea/components/NotificationSendControls.tsx index 4fb27cc7ef..dbca159615 100644 --- a/src/renderer/components/InputArea/components/NotificationSendControls.tsx +++ b/src/renderer/components/InputArea/components/NotificationSendControls.tsx @@ -1,5 +1,5 @@ import { memo, useRef, useState } from 'react'; -import { ArrowUp, Bell } from 'lucide-react'; +import { ArrowUp, Bell, Mic } from 'lucide-react'; import type { Theme } from '../../../types'; import { NotificationPopover } from '../../NotificationPopover'; @@ -7,12 +7,27 @@ interface NotificationSendControlsProps { theme: Theme; isTerminalMode: boolean; processInput: () => void; + /** + * Show the A Cappella microphone under Send. + * + * True only when the Encore Feature owns the composer's microphone. The Web + * Speech dictation button stays where it has always been, in the toolbar row + * and on touch pointers only, so exactly one microphone is ever on screen. + */ + showVoiceButton?: boolean; + /** True while this agent holds the voice floor. */ + isVoiceListening?: boolean; + /** Start or end the voice session. Stable identity (see InputArea). */ + onToggleVoice?: () => void; } export const NotificationSendControls = memo(function NotificationSendControls({ theme, isTerminalMode, processInput, + showVoiceButton = false, + isVoiceListening = false, + onToggleVoice, }: NotificationSendControlsProps) { const [notificationPopoverOpen, setNotificationPopoverOpen] = useState(false); const notificationBtnRef = useRef(null); @@ -52,6 +67,31 @@ export const NotificationSendControls = memo(function NotificationSendControls({ > + {/* Under Send rather than in the toolbar row: speaking is a way of + submitting a message, so it belongs with the other submit control + instead of among the per-tab toggles. Accented while the floor is open, + so the one button that can leave a microphone running never looks the + same open as shut. */} + {showVoiceButton && onToggleVoice && ( + + )} ); }); diff --git a/src/renderer/components/InputArea/components/ToolbarControls.tsx b/src/renderer/components/InputArea/components/ToolbarControls.tsx index 58be873812..69c1dc7d4d 100644 --- a/src/renderer/components/InputArea/components/ToolbarControls.tsx +++ b/src/renderer/components/InputArea/components/ToolbarControls.tsx @@ -47,6 +47,14 @@ interface ToolbarControlsProps { isVoiceListening?: boolean; /** Toggle voice dictation on/off. Stable identity (see InputArea). */ onToggleVoiceInput?: () => void; + /** + * A Cappella owns the composer's microphone, so this row must not draw one. + * + * The A Cappella button lives under Send in `NotificationSendControls`, where + * it is visible on every pointer type. Without this flag a touch device with + * the Encore Feature on would show two microphones wired to the same toggle. + */ + voiceHandledElsewhere?: boolean; onOpenPromptComposer?: () => void; shortcuts?: Record; showFlashNotification?: (message: string) => void; @@ -82,6 +90,7 @@ export const ToolbarControls = memo(function ToolbarControls({ voiceSupported, isVoiceListening, onToggleVoiceInput, + voiceHandledElsewhere, onOpenPromptComposer, shortcuts, showFlashNotification, @@ -114,7 +123,12 @@ export const ToolbarControls = memo(function ToolbarControls({ // targets would defeat the point. Shown only when the Web Speech API is // supported AND the primary pointer is coarse (touch), so mouse/keyboard // desktop users never see it. - const showVoiceButton = isAiMode && !!voiceSupported && !!onToggleVoiceInput && isCoarsePointer(); + const showVoiceButton = + isAiMode && + !!voiceSupported && + !!onToggleVoiceInput && + !voiceHandledElsewhere && + isCoarsePointer(); const activeTab = session.aiTabs?.find((t) => t.id === session.activeTabId); const rawPermissionMode: 'full' | 'standard' | 'readonly' = resolveTabPermissionMode(activeTab); @@ -124,8 +138,12 @@ export const ToolbarControls = memo(function ToolbarControls({ const currentPermissionMode: 'full' | 'standard' | 'readonly' = rawPermissionMode === 'standard' && !hasStandardCapability ? 'full' : rawPermissionMode; + // mt-auto pins the row to the bottom of the composer box. The A Cappella + // microphone makes the Send column taller than the textarea, and the box + // stretches to match it - without this the pills float in the middle with + // dead space under them. return ( -
+
{isTerminalMode && (
- + +
{/* Git Status Widget - compact mode handled via CSS container queries */} diff --git a/src/renderer/components/NewInstanceModal/EditAgentModal.tsx b/src/renderer/components/NewInstanceModal/EditAgentModal.tsx index 09ff2303bf..ff1799b95c 100644 --- a/src/renderer/components/NewInstanceModal/EditAgentModal.tsx +++ b/src/renderer/components/NewInstanceModal/EditAgentModal.tsx @@ -8,6 +8,8 @@ import { normalizeAdditionalDirectories } from '../../../shared/additionalDirect import { formatTokensCompact } from '../../../shared/formatters'; import { getActiveTab } from '../../utils/tabHelpers'; import { useSessionStore, selectSessionById } from '../../stores/sessionStore'; +import { useSettingsStore, selectACappellaEnabled } from '../../stores/settingsStore'; +import { AgentWakePhraseSection } from '../Settings/ACappella/AgentWakePhraseSection'; import { resolveContextWindow, isStoredContextWindowOverridden, @@ -104,6 +106,9 @@ export function EditAgentModal({ // they never write (or erase) the per-session model/contextWindow/effort. const globalConfigRef = useRef>({}); + // One owner for the voice gate: the same selector the HUD and Voice Setup read. + const aCappellaEnabled = useSettingsStore(selectACappellaEnabled); + // Clear copy timeout and reset copied state on unmount, close, or session change useEffect(() => { if (!isOpen) { @@ -699,6 +704,13 @@ export function EditAgentModal({ {/* Provider Failover: backup Anthropic-compatible endpoints for this agent. */} + {/* Voice wake phrase. Renders nothing unless A Cappella is switched on. */} + + {/* Working Directory (read-only) */}
s.transcriptVisible); + const toggleVoiceTranscript = useVoiceUiStore((s) => s.toggleTranscript); // Output search is scoped per agent+AI-tab; open the active window's slot so // the Find bar doesn't follow the user to other agents/tabs. const openActiveOutputSearch = useCallback( @@ -767,6 +774,13 @@ export const QuickActionsModal = memo(function QuickActionsModal(props: QuickAct openUrl, logger, }), + ...buildVoiceCommands({ + activeSession, + voiceActions, + transcriptVisible: voiceTranscriptVisible, + toggleTranscript: toggleVoiceTranscript, + setQuickActionOpen, + }), ...buildRightPanelCommands({ autoRunDisabled: useSettingsStore.getState().autoRunDisabled, autoRunSelectedDocument, diff --git a/src/renderer/components/QuickActionsModal/commands/voiceCommands.ts b/src/renderer/components/QuickActionsModal/commands/voiceCommands.ts new file mode 100644 index 0000000000..e8561e5bcf --- /dev/null +++ b/src/renderer/components/QuickActionsModal/commands/voiceCommands.ts @@ -0,0 +1,124 @@ +/** + * A Cappella entries for the command palette. + * + * One surface of `useVoiceAgentActions`, alongside the composer's microphone and + * the Left Bar right-click menu. It takes the action set rather than re-deriving + * it, exactly as `buildGitWorktreeCommands` takes `GitAgentActions`: the palette + * IS a surface, not a reimplementation of the others. + * + * It is also the ONLY surface for some of these. The composer microphone talks + * to the agent on screen and the Left Bar menu to the agent under the cursor, so + * "Talk to the Conductor", "Show Voice HUD", and the transcript toggle are + * reachable here and nowhere else. + * + * Returns nothing when the Encore Feature is off, so the palette has no + * A Cappella entries at all for users who never turned it on. + */ + +import type { VoiceAgentActions } from '../../../hooks/voice/useVoiceAgentActions'; +import type { Session } from '../../../types'; +import type { QuickAction } from '../types'; + +interface BuildVoiceCommandsArgs { + activeSession: Session | undefined; + /** The same action set the header pill and the Left Bar menu use. */ + voiceActions: VoiceAgentActions; + /** Whether the transcript panel is currently open. */ + transcriptVisible: boolean; + toggleTranscript: () => Promise; + setQuickActionOpen: (open: boolean) => void; +} + +/** + * What a user types when they want this. + * + * "Talk to Backend" is the clearest label for starting a voice session and the + * least findable one: palette search is over labels, so before these existed, + * typing "voice" surfaced the transcript toggle and nothing that could actually + * start talking. Both the feature's name and the thing it is are here, because + * nobody searches for "A Cappella" and everybody searches for "voice". + */ +const VOICE_KEYWORDS = ['voice', 'acappella', 'a cappella', 'talk', 'speak', 'microphone', 'mic']; + +export function buildVoiceCommands({ + activeSession, + voiceActions, + transcriptVisible, + toggleTranscript, + setQuickActionOpen, +}: BuildVoiceCommandsArgs): QuickAction[] { + if (!voiceActions.enabled) return []; + + const commands: QuickAction[] = []; + + if (activeSession) { + commands.push({ + id: 'voiceTalkToAgent', + keywords: VOICE_KEYWORDS, + label: `Talk to ${activeSession.name}`, + // The wake phrase is surfaced here too: the palette is where people go + // looking for a capability, and it is the cheapest place to teach them + // they never needed to open it. + subtext: voiceActions.wakePhrase + ? `Or say "${voiceActions.wakePhrase}"` + : 'Open a voice session bound to this agent', + action: () => { + void voiceActions.talkToAgent(); + setQuickActionOpen(false); + }, + }); + } + + commands.push({ + id: 'voiceTalkToConductor', + keywords: VOICE_KEYWORDS, + label: 'Talk to the Conductor', + subtext: 'Open a voice session that can route to any agent', + action: () => { + void voiceActions.talkToConductor(); + setQuickActionOpen(false); + }, + }); + + // Recovery for a hidden HUD, the same entry the media player has for its + // hidden widget: minimizing leaves the session running, so there has to be a + // way back to the controls that does not depend on finding the Left Bar pill. + if (voiceActions.hudHidden) { + commands.push({ + id: 'voiceShowHud', + keywords: VOICE_KEYWORDS, + label: 'Show Voice HUD', + subtext: 'Bring back the minimized voice controls', + action: () => { + voiceActions.showHud(); + setQuickActionOpen(false); + }, + }); + } + + commands.push({ + id: 'voiceToggleTranscript', + keywords: VOICE_KEYWORDS, + label: transcriptVisible ? 'Hide Voice Transcript' : 'Show Voice Transcript', + action: () => { + void toggleTranscript(); + setQuickActionOpen(false); + }, + }); + + // Only offered when there is a session to end, for the same reason the header + // menu hides it: an entry that does nothing teaches people the palette lies. + if (voiceActions.hasVoiceFloor) { + commands.push({ + id: 'voiceEndSession', + keywords: VOICE_KEYWORDS, + label: 'End Voice Session', + action: () => { + void voiceActions.endVoiceSession(); + setQuickActionOpen(false); + }, + }); + } + + return commands; +} diff --git a/src/renderer/components/QuickActionsModal/types.ts b/src/renderer/components/QuickActionsModal/types.ts index 024a8fb594..9af7f4c502 100644 --- a/src/renderer/components/QuickActionsModal/types.ts +++ b/src/renderer/components/QuickActionsModal/types.ts @@ -20,6 +20,20 @@ export interface QuickAction { label: string; action: () => void | Promise; subtext?: string; + /** + * Extra words this command should match, for when the label is not what a + * user would type. + * + * Search is over the label, which is right for almost everything: the label + * is the command's name. It breaks down when a feature's name and its verb + * are different words - "Talk to Backend" is the clearest possible label for + * starting a voice session, and it is invisible to someone typing "voice". + * + * Opt-in, so a command with no keywords matches exactly as it always did. + * Keep them to words a user would actually type; this is a search hint, not a + * place to stuff synonyms until everything matches everything. + */ + keywords?: string[]; shortcut?: Shortcut; // Agents-mode only: marks an agent whose state is not 'idle' so we can // bucket "active" agents at the top with a divider beneath them. Also true diff --git a/src/renderer/components/QuickActionsModal/utils/quickActionSorting.ts b/src/renderer/components/QuickActionsModal/utils/quickActionSorting.ts index 3b813889d2..e773aef01b 100644 --- a/src/renderer/components/QuickActionsModal/utils/quickActionSorting.ts +++ b/src/renderer/components/QuickActionsModal/utils/quickActionSorting.ts @@ -29,7 +29,11 @@ export function filterAndSortQuickActions( if (isDebugCommand && !showDebugCommands) { return false; } - return a.label.toLowerCase().includes(searchLower); + if (a.label.toLowerCase().includes(searchLower)) return true; + // Keywords are opt-in and almost nothing declares them, so this is a + // second chance for the handful of commands whose name is not the word + // a user would search for - not a broadening of the match rule. + return !!a.keywords?.some((keyword) => keyword.toLowerCase().includes(searchLower)); }) .sort((a, b) => { const sameAgent = diff --git a/src/renderer/components/SessionItem.tsx b/src/renderer/components/SessionItem.tsx index 03838a4cb5..456223bfa4 100644 --- a/src/renderer/components/SessionItem.tsx +++ b/src/renderer/components/SessionItem.tsx @@ -16,6 +16,7 @@ import { WorktreePill } from './ui/WorktreePill'; import { CueIndicator } from './SessionList/CueIndicator'; import { StartupCommandIndicator } from './SessionList/StartupCommandIndicator'; import { WizardIndicator } from './SessionList/WizardIndicator'; +import { AgentVoiceIndicator } from './SessionList/AgentVoiceIndicator'; import { WindowBadge } from './SessionList/WindowBadge'; import { PluginUiItemsSlot } from './plugins/PluginUiItemsSlot'; import { useSettingsStore } from '../stores/settingsStore'; @@ -399,6 +400,10 @@ export const SessionItem = memo(function SessionItem({ /> {/* Inline wizard indicator: shown while /wizard is in dialog or doc-gen phase. */} + {/* A Cappella: holds the voice floor, is being spoken, or has a wake + phrase. Reads the voice stores itself and renders null when the + Encore Feature is off, so nothing needs threading through here. */} + {/* Worktree badge to visually mark worktree children */} {variant === 'worktree' && showWorktreePill && }
diff --git a/src/renderer/components/SessionList/AgentVoiceIndicator.tsx b/src/renderer/components/SessionList/AgentVoiceIndicator.tsx new file mode 100644 index 0000000000..7a30c9997f --- /dev/null +++ b/src/renderer/components/SessionList/AgentVoiceIndicator.tsx @@ -0,0 +1,104 @@ +/** + * Per-agent voice indicators for the Left Bar. + * + * Three separate facts, three separate glyphs, all next to the agent's name: + * + * - **Holding the floor.** This agent is what the live voice session is bound + * to, so anything said next lands here. A filled microphone. + * - **Being spoken.** This agent's reply is coming out of the speakers right + * now, which is not the same thing - a Conductor session speaks replies from + * whichever agent it routed to, and that agent may not hold the floor at all. + * - **Has a wake phrase.** Saying the phrase jumps straight into this agent. + * Without a badge that mapping is invisible, and an invisible mapping is one + * nobody uses. + * + * These COMPOSE with the status dot rather than replacing it. Green/yellow/red + * still means ready/busy/error; the voice glyphs sit beside them and say + * something the status colour cannot. Overloading the dot would mean losing the + * agent's state for as long as it was being talked to, which is exactly when the + * user most wants to know whether it is working. + * + * The component reads the stores itself instead of taking props, for the same + * reason `VoiceIndicator` does: SessionItem is memoized on primitive props and + * renders once per row, and threading four voice fields through SessionList + * would re-render every row in the Left Bar on every voice event. + */ + +import { memo } from 'react'; +import { Mic, Volume2, Waves } from 'lucide-react'; +import type { Theme } from '../../types'; +import { readableTextOn } from '../../../shared/colorContrast'; +import { usePrefersReducedMotion } from '../../hooks/utils/usePrefersReducedMotion'; +import { selectACappellaEnabled, useSettingsStore } from '../../stores/settingsStore'; +import { useVoiceSessionStore } from '../../stores/voiceSessionStore'; +import { selectWakePhraseFor, useVoiceUiStore } from '../../stores/voiceUiStore'; +import { isVoiceSessionActive } from '../../../shared/acappella/session-state'; + +export interface AgentVoiceIndicatorProps { + /** The agent this row is for. */ + sessionId: string; + theme: Theme; +} + +export const AgentVoiceIndicator = memo(function AgentVoiceIndicator({ + sessionId, + theme, +}: AgentVoiceIndicatorProps) { + const enabled = useSettingsStore(selectACappellaEnabled); + const state = useVoiceSessionStore((s) => s.state); + const scope = useVoiceSessionStore((s) => s.scope); + const lastDispatch = useVoiceSessionStore((s) => s.lastDispatch); + const wakePhrase = useVoiceUiStore(selectWakePhraseFor(sessionId)); + const reducedMotion = usePrefersReducedMotion(); + + if (!enabled) return null; + + const hasFloor = + isVoiceSessionActive(state) && scope?.kind === 'agent' && scope.sessionId === sessionId; + const speaking = state === 'speaking' && lastDispatch?.agentSessionId === sessionId; + + if (!hasFloor && !speaking && !wakePhrase) return null; + + // The accent is the theme's own colour and the Left Bar row background is + // too, so a theme whose accent sits near its sidebar would otherwise paint + // these glyphs invisibly. + const accent = readableTextOn(theme.colors.accent, [theme.colors.bgSidebar, theme.colors.bgMain]); + + return ( + <> + {hasFloor && ( + + + + )} + {speaking && ( + + + + )} + {wakePhrase && ( + + + + )} + + ); +}); + +export default AgentVoiceIndicator; diff --git a/src/renderer/components/SessionList/SessionContextMenu.tsx b/src/renderer/components/SessionList/SessionContextMenu.tsx index bc702534ab..99432d9c80 100644 --- a/src/renderer/components/SessionList/SessionContextMenu.tsx +++ b/src/renderer/components/SessionList/SessionContextMenu.tsx @@ -20,6 +20,7 @@ import { Zap, Fingerprint, AppWindow, + Mic, Plus, Pencil, Check, @@ -28,6 +29,7 @@ import type { Group, Session, Theme } from '../../types'; import { useClickOutside, useContextMenuPosition } from '../../hooks'; import { compareNamesIgnoringEmojis } from '../../../shared/emojiUtils'; import { useGitAgentActions } from '../../hooks/git/useGitAgentActions'; +import { useVoiceAgentActions } from '../../hooks/voice/useVoiceAgentActions'; import { GitChangeCounts } from '../ui/GitChangeCounts'; import { GitRunningBadge } from '../ui/GitRunningBadge'; import { formatGitChangeSummary } from '../../../shared/gitUtils'; @@ -226,6 +228,7 @@ export function SessionContextMenu({ // branch pill's dropdown uses, so both entry points behave identically - // here they act on the right-clicked agent rather than the active one. const gitActions = useGitAgentActions(session); + const voiceActions = useVoiceAgentActions(session); // `onCreatePR` is the worktree-child path that App already wires up; for any // other git agent the shared action opens the same modal for this session. @@ -579,6 +582,36 @@ export function SessionContextMenu({
)} + {/* A Cappella - the same "Talk to this agent" entry the header and the + command palette offer, off the one hook, so the three cannot drift. */} + {voiceActions.enabled && ( + <> +
+ + + )} + {/* Git actions - mirrors the header branch pill's dropdown so the same operations are reachable from either place. */} {gitActions.isGitRepo && ( diff --git a/src/renderer/components/SessionList/SessionList.tsx b/src/renderer/components/SessionList/SessionList.tsx index 72a70935eb..34ab2b46e0 100644 --- a/src/renderer/components/SessionList/SessionList.tsx +++ b/src/renderer/components/SessionList/SessionList.tsx @@ -25,6 +25,7 @@ import { import { GhostIconButton } from '../ui/GhostIconButton'; import { HamburgerDropdown } from './HamburgerDropdown'; import { NowPlayingIndicator } from '../MediaPlayback/NowPlayingIndicator'; +import { VoiceStatusIndicator } from '../ACappella/VoiceStatusIndicator'; import type { Session, Group, Theme } from '../../types'; import { isWorktreeGroup } from '../../../shared/types'; import { canSetGroupParent, removeGroupAndPromoteChildren } from '../../../shared/groupHierarchy'; @@ -1349,6 +1350,13 @@ function SessionListInner(props: SessionListProps) { theme={theme} compact={leftSidebarWidthState < NOW_PLAYING_LABEL_MIN_WIDTH + headerBadgeWidth} /> + {/* Voice session - the minimized HUD's home, and the same bargain + as the now-playing pill above it: something that is running + while its widget is away has to stay visible somewhere. */} + {/* Global LIVE Toggle - hidden in the web-desktop bundle, where toggling it would kill the webserver the user's browser is currently connected to. */} @@ -1445,7 +1453,15 @@ function SessionListInner(props: SessionListProps) { // strip, and a media control there competes with the agent pills for // the one thing the rail is for. Expand the sidebar, or run "Show // Floating Media Player" from the Command Palette. + // + // The voice indicator is the deliberate exception, and for the reason that + // decided the media pill rather than in spite of it: audio evidences + // itself, so a hidden media control still announces what it is doing, + // while a microphone's only tell is this glyph. Dropping it here would + // make the collapsed rail the one place a live microphone is invisible - + // see the minimize/close note in `VoiceHud.tsx`.
+ setMenuOpen(!menuOpen)} padding="p-2" title="Menu"> + {enabled && } + {enabled && } + {enabled && } + {enabled && } + {enabled && } + {enabled && } + +
+ ); +} diff --git a/src/renderer/components/Settings/ACappella/AgentWakePhraseSection.tsx b/src/renderer/components/Settings/ACappella/AgentWakePhraseSection.tsx new file mode 100644 index 0000000000..ac9f075182 --- /dev/null +++ b/src/renderer/components/Settings/ACappella/AgentWakePhraseSection.tsx @@ -0,0 +1,66 @@ +/** + * One agent's wake phrase, surfaced where the agent itself is configured. + * + * The same value the Voice Controls panel edits, read and written through the + * same hook and stored in the same `acappella` settings blob. Deliberately NOT a + * field on the `Session` record: an agent's wake phrase has to be resolvable by + * the always-on detector in the main process, and a second copy on the session + * would be a second answer to "what does this agent respond to". + * + * Renders nothing when A Cappella is off. A wake phrase field on a machine with + * no voice feature is a question the user cannot act on. + */ + +import type { Theme } from '../../../types'; +import { useVoiceControls } from './useVoiceControls'; + +export interface AgentWakePhraseSectionProps { + theme: Theme; + /** The agent this phrase binds to. */ + agentSessionId: string; + /** Mirror of the A Cappella Encore flag. */ + enabled: boolean; +} + +export function AgentWakePhraseSection({ + theme, + agentSessionId, + enabled, +}: AgentWakePhraseSectionProps) { + const controls = useVoiceControls(enabled); + if (!enabled) return null; + + const phrase = + controls.agentPhrases.find((entry) => entry.agentSessionId === agentSessionId)?.phrase ?? ''; + + return ( + // No `data-setting-id`: this renders in the agent modal, not the Settings + // modal, and a registry entry would send Settings search to a tab that does + // not contain it. The Voice Controls panel carries the searchable copy. +
+
+ Voice Wake Phrase +
+ void controls.setAgentPhrase(agentSessionId, event.target.value)} + className="w-full p-2 rounded border text-sm" + style={{ + borderColor: theme.colors.border, + backgroundColor: theme.colors.bgMain, + color: theme.colors.textMain, + }} + /> +

+ Saying this opens a voice session bound straight to this agent, with no routing step. Leave + blank to use the global wake phrase and let the Conductor decide. +

+
+ ); +} diff --git a/src/renderer/components/Settings/ACappella/PairedDevicesPanel.tsx b/src/renderer/components/Settings/ACappella/PairedDevicesPanel.tsx new file mode 100644 index 0000000000..df0b361476 --- /dev/null +++ b/src/renderer/components/Settings/ACappella/PairedDevicesPanel.tsx @@ -0,0 +1,469 @@ +/** + * Paired Devices: which phones may hold this computer's microphone, and how they + * get here. + * + * The panel is arranged around the four questions a person actually has: + * + * - **How do I add one?** A QR code with a live pairing status, and an + * approval step on THIS screen. The approval is not a formality: a pairing + * code is short enough to be read over a shoulder, so knowing it buys a row + * in a dialog and nothing else. + * - **What is connected, and how?** Each device shows its name, platform, last + * connection, live state, and the ICE candidate type that actually won, in + * plain words: LAN, through NAT, or relayed. "Relayed" is the one that costs + * latency and somebody's bandwidth, so it is said rather than hidden. + * - **How do I stop one?** Revoke, per device, effective on a LIVE connection. + * Plus one control that drops everything at once. + * - **Why will this not connect from outside?** The reach line and the tunnel + * note, stated as facts. The Cloudflare quick tunnel that serves the browser + * interface cannot carry this audio, and a user who does not know that will + * blame the wrong thing every time. + */ + +import { useCallback, useState } from 'react'; +import { Check, Radio, Smartphone, Trash2, Wifi, X } from 'lucide-react'; +import { QRCodeSVG } from 'qrcode.react'; + +import { CANDIDATE_TYPE_LABELS } from '../../../../shared/acappella/device-protocol'; +import { formatRelativeTime } from '../../../../shared/formatters'; +import type { Theme } from '../../../types'; +import { ToggleSwitch } from '../../ui/ToggleSwitch'; +import { SettingsSectionHeading } from '../SettingsSectionHeading'; +import { SectionCard } from '../tabs/DisplayTab/components/SectionCard'; +import { usePairedDevices, type DeviceStatus } from './usePairedDevices'; + +export interface PairedDevicesPanelProps { + theme: Theme; + /** Mirror of the A Cappella Encore flag. */ + enabled: boolean; +} + +/** + * The one sentence about what a paired device can do. + * + * Written out rather than implied, because "paired" is a word that hides a + * capability: this is a device that can open a microphone on this machine and + * put words in front of your agents. + */ +const CAPABILITY_STATEMENT = + 'A paired device can hold this computer’s microphone, hear replies in your configured voice, ' + + 'and dispatch spoken prompts to your agents. It cannot read your files or change your settings.'; + +export function PairedDevicesPanel({ theme, enabled }: PairedDevicesPanelProps) { + const devices = usePairedDevices(enabled); + const [renaming, setRenaming] = useState<{ id: string; value: string } | null>(null); + + const commitRename = useCallback(async () => { + if (!renaming) return; + const { id, value } = renaming; + setRenaming(null); + await devices.rename(id, value); + }, [devices, renaming]); + + const ice = devices.ice; + + return ( + <> + Paired Devices + + {/* -- Add a device ------------------------------------------------ */} +
+ +
+
+ Add a device +
+

{CAPABILITY_STATEMENT}

+
+ + {!devices.pairing && ( + + )} + + {devices.pairing && ( +
+
+ +
+
+
+ {devices.pairing.code} +
+

+ Fingerprint {devices.pairing.fingerprint}. The device shows the same one - if it + does not match, do not approve it. +

+

+ Reachable at {devices.pairing.hosts.join(', ') || 'this computer'} on port{' '} + {devices.pairing.port}. +

+ +
+
+ )} + + {devices.request && ( +
+
+ {devices.request.name} wants to pair +
+

+ {devices.request.platform} + {devices.request.remoteAddress ? ` from ${devices.request.remoteAddress}` : ''}. + Only approve this if it is the device in your hand. +

+
+ + +
+
+ )} + +
+
+
+ Advertise on the local network +
+

+ {describeDiscovery(devices)} Turning this off does not stop pairing: the QR code and + manual entry still work. +

+
+ void devices.setDiscovery(checked)} + /> +
+ +
+ + {/* -- The device list --------------------------------------------- */} +
+ +
+
+ Devices +
+ {devices.devices.length > 0 && ( +
+ + +
+ )} +
+ + {devices.devices.length === 0 && ( +

+ {devices.loaded ? 'No devices are paired with this computer.' : 'Loading devices...'} +

+ )} + + {devices.devices.map((device) => ( +
+
+ {renaming?.id === device.id ? ( + setRenaming({ id: device.id, value: event.target.value })} + onBlur={() => void commitRename()} + onKeyDown={(event) => { + if (event.key === 'Enter') void commitRename(); + if (event.key === 'Escape') setRenaming(null); + }} + className="px-2 py-1 rounded border text-sm" + style={{ + borderColor: theme.colors.border, + backgroundColor: theme.colors.bgMain, + color: theme.colors.textMain, + }} + /> + ) : ( +
+ {device.name} + {device.holdsFloor && ( + + holding the microphone + + )} +
+ )} +

{describeDevice(device)}

+
+
+ + {device.revokedAt === null ? ( + + ) : ( + + )} +
+
+ ))} + +
+ + {/* -- Connection ---------------------------------------------------- */} +
+ +
+
+ Connection +
+

{devices.reach}

+

{devices.tunnelNote}

+
+ + {ice && ( + <> + + + +
+ + + + + +
+
+ + + +
+ + +
+
+ idle + Not connected + +
+

+

+

+ +
+
+ +
+

Project wheel

+
+
+ +
+ +
+ + +
+
+ +
+

Transcript

+
+
+ +
+

Protocol log

+
+
+ + + + + + + + diff --git a/src/web-desktop/acappella-client/main.ts b/src/web-desktop/acappella-client/main.ts new file mode 100644 index 0000000000..69c2c75b32 --- /dev/null +++ b/src/web-desktop/acappella-client/main.ts @@ -0,0 +1,384 @@ +/** + * The browser reference client, wired to the browser. + * + * Everything DOM lives here: the WebSocket, the `RTCPeerConnection`, the + * microphone, the level meter, playback, and the push-to-talk gesture. The + * protocol itself is in `client.ts` and knows about none of it, which is what + * lets the conformance suite drive the same code with fakes. + * + * Served by the desktop at `//acappella` and buildable on its own with + * `npm run dev:web-desktop` (then open `/acappella-client/`). Point it at a + * desktop by pasting the JSON behind the pairing QR code. + */ + +import { DEFAULT_HOLD_THRESHOLD_MS } from '../../shared/acappella/voice-controls'; +import type { RosterAgent, VoiceScope } from '../../shared/acappella/protocol'; +import { + ACappellaReferenceClient, + type ClientState, + type PairingStore, + type PairingTarget, + type SignalingSocket, + type SignalingSocketHandlers, + type StoredPairing, +} from './client'; +import { appendLog, createTranscript, renderStatus, renderWheel, type StatusElements } from './ui'; + +/** What the app calls itself in the desktop's approval sheet and device list. */ +const CLIENT_NAME = 'Browser reference client'; +const CLIENT_PLATFORM = 'browser'; + +/** Where the pairing is kept. `localStorage` is this platform's Keychain. */ +const STORAGE_KEY = 'maestro.acappella.pairing'; + +/** + * How loud counts as speech, as a linear RMS. + * + * Crude on purpose. A real client runs a VAD; this one exists to prove the + * ORDER of the barge-in path (duck locally, then send) rather than to be a good + * speech detector. + */ +const VAD_RMS_THRESHOLD = 0.06; + +// --------------------------------------------------------------------------- +// Element lookup +// --------------------------------------------------------------------------- + +function el(id: string): T { + const node = document.getElementById(id); + if (!node) throw new Error(`Missing #${id} in the reference client page.`); + return node as T; +} + +const fields = { + payload: el('qr-payload'), + host: el('host'), + port: el('port'), + token: el('token'), + code: el('code'), + secure: el('secure'), +}; +const buttons = { + connect: el('connect'), + disconnect: el('disconnect'), + forget: el('forget'), + talk: el('talk'), + barge: el('barge'), + stop: el('stop'), +}; +const status: StatusElements = { + phase: el('phase'), + message: el('message'), + mic: el('mic-pill'), + floor: el('floor-line'), + quality: el('quality-line'), + suspect: el('suspect'), + version: el('version-line'), +}; +const wheel = el('wheel'); +const transcriptNode = el('transcript'); +const logNode = el('log'); +const meter = el('meter-fill'); +const playback = el('playback'); + +const transcript = createTranscript(transcriptNode); + +// --------------------------------------------------------------------------- +// Seams, implemented for a browser +// --------------------------------------------------------------------------- + +const store: PairingStore = { + read(): StoredPairing | null { + try { + const raw = localStorage.getItem(STORAGE_KEY); + return raw ? (JSON.parse(raw) as StoredPairing) : null; + } catch { + return null; + } + }, + write(pairing: StoredPairing): void { + localStorage.setItem(STORAGE_KEY, JSON.stringify(pairing)); + }, + clear(): void { + localStorage.removeItem(STORAGE_KEY); + }, +}; + +function openSocket(url: string, handlers: SignalingSocketHandlers): SignalingSocket { + const socket = new WebSocket(url); + socket.onopen = () => handlers.onOpen(); + socket.onclose = () => handlers.onClose(); + socket.onerror = () => appendLog(logNode, 'error', 'The WebSocket reported an error.'); + socket.onmessage = (event) => { + let parsed: unknown; + try { + parsed = JSON.parse(typeof event.data === 'string' ? event.data : String(event.data)); + } catch { + return; + } + // The app's ordinary envelope. Anything that is not an A Cappella frame on + // this socket belongs to some other feature and is not ours to read. + const frame = parsed as { type?: string; payload?: unknown }; + if (frame.type !== 'acappella_signal' || !frame.payload) return; + handlers.onMessage(frame.payload as Parameters[0]); + }; + return { + send: (message) => socket.send(JSON.stringify({ type: 'acappella_signal', payload: message })), + close: () => socket.close(), + }; +} + +const client = new ACappellaReferenceClient({ + identity: { name: CLIENT_NAME, platform: CLIENT_PLATFORM, appVersion: __APP_VERSION__ }, + store, + openSocket, + createPeerConnection: (config) => new RTCPeerConnection(config), + openMicrophone: () => + navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }), +}); + +// --------------------------------------------------------------------------- +// Level meter and the local VAD +// --------------------------------------------------------------------------- + +let audioContext: AudioContext | null = null; +let analyser: AnalyserNode | null = null; +let meterFrame = 0; +let speaking = false; + +function startMeter(stream: MediaStream): void { + stopMeter(); + audioContext = new AudioContext(); + analyser = audioContext.createAnalyser(); + analyser.fftSize = 512; + audioContext.createMediaStreamSource(stream).connect(analyser); + const buffer = new Float32Array(analyser.fftSize); + + const tick = (): void => { + if (!analyser) return; + analyser.getFloatTimeDomainData(buffer); + let sum = 0; + for (const sample of buffer) sum += sample * sample; + const rms = Math.sqrt(sum / buffer.length); + meter.style.width = `${Math.min(100, Math.round(rms * 400))}%`; + const isSpeech = rms > VAD_RMS_THRESHOLD; + client.reportAudioLevel(rms, isSpeech); + // Talking over the reply. The duck happens inside `requestBargeIn` before + // the frame goes out, which is the whole point of doing this locally. + if (isSpeech && speaking) client.requestBargeIn(); + meterFrame = requestAnimationFrame(tick); + }; + meterFrame = requestAnimationFrame(tick); +} + +function stopMeter(): void { + if (meterFrame) cancelAnimationFrame(meterFrame); + meterFrame = 0; + analyser = null; + void audioContext?.close(); + audioContext = null; + meter.style.width = '0%'; +} + +// --------------------------------------------------------------------------- +// Push to talk +// --------------------------------------------------------------------------- + +/** + * Tap to toggle, or press and hold to release. + * + * `press` goes out on pointer-down, before the gesture has been classified, + * because the desktop's press is idempotent and waiting out the threshold would + * put it in front of every utterance. Only the RELEASE depends on which gesture + * this turned out to be. The threshold is the desktop's own + * `DEFAULT_HOLD_THRESHOLD_MS` rather than a second copy of the number. + */ +let pressedAt = 0; +let latched = false; + +buttons.talk.addEventListener('pointerdown', (event) => { + event.preventDefault(); + buttons.talk.setPointerCapture(event.pointerId); + if (latched) { + // A tap while latched is the second half of the toggle. + latched = false; + client.releaseFloor(); + pressedAt = 0; + return; + } + pressedAt = performance.now(); + client.pressFloor(selectedScope); +}); + +buttons.talk.addEventListener('pointerup', () => { + if (!pressedAt) return; + const held = performance.now() - pressedAt; + pressedAt = 0; + if (held >= DEFAULT_HOLD_THRESHOLD_MS) { + client.releaseFloor(); + latched = false; + return; + } + // A tap latches the floor open until the next tap. + latched = true; + buttons.talk.classList.add('is-latched'); +}); + +buttons.barge.addEventListener('click', () => client.requestBargeIn()); +buttons.stop.addEventListener('click', () => { + latched = false; + client.requestStop(); +}); + +// --------------------------------------------------------------------------- +// Connection form +// --------------------------------------------------------------------------- + +/** + * Read the JSON behind the desktop's pairing QR code. + * + * The desktop encodes `JSON.stringify(PairingPayload)` into the QR, so pasting + * it is exactly what a camera scan produces. Tolerant of a raw code being typed + * into the field instead. + */ +export function parsePairingPayload(text: string): Partial | null { + const trimmed = text.trim(); + if (!trimmed) return null; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return null; + } + if (!parsed || typeof parsed !== 'object') return null; + const payload = parsed as { + kind?: string; + hosts?: unknown; + port?: unknown; + token?: unknown; + code?: unknown; + }; + // A scanner has to be able to reject an unrelated QR code, which is the only + // reason `kind` is in the payload at all. + if (payload.kind !== 'maestro-acappella') return null; + const hosts = Array.isArray(payload.hosts) ? payload.hosts.filter(isString) : []; + return { + host: hosts[0], + port: typeof payload.port === 'number' ? payload.port : undefined, + token: isString(payload.token) ? payload.token : undefined, + code: isString(payload.code) ? payload.code : undefined, + }; +} + +function isString(value: unknown): value is string { + return typeof value === 'string'; +} + +fields.payload.addEventListener('input', () => { + const parsed = parsePairingPayload(fields.payload.value); + if (!parsed) return; + if (parsed.host) fields.host.value = parsed.host; + if (parsed.port) fields.port.value = String(parsed.port); + if (parsed.token) fields.token.value = parsed.token; + if (parsed.code) fields.code.value = parsed.code; + appendLog(logNode, 'info', 'Read the pairing payload. Press Connect.'); +}); + +buttons.connect.addEventListener('click', () => { + const target: PairingTarget = { + host: fields.host.value.trim(), + port: Number(fields.port.value), + token: fields.token.value.trim(), + code: fields.code.value.trim() || undefined, + secure: fields.secure.checked, + }; + if (!target.host || !target.port || !target.token) { + appendLog(logNode, 'error', 'Host, port, and token are all required.'); + return; + } + transcript.clear(); + client.connect(target); +}); + +buttons.disconnect.addEventListener('click', () => client.disconnect()); +buttons.forget.addEventListener('click', () => client.forget()); + +// --------------------------------------------------------------------------- +// The wheel +// --------------------------------------------------------------------------- + +let agents: RosterAgent[] = []; +let selectedScope: VoiceScope = { kind: 'conductor' }; + +function paintWheel(): void { + renderWheel(wheel, agents, selectedScope, (scope) => { + selectedScope = scope; + paintWheel(); + }); +} +paintWheel(); + +// --------------------------------------------------------------------------- +// Client events +// --------------------------------------------------------------------------- + +let lastState: ClientState | null = null; + +client.subscribe((event) => { + switch (event.type) { + case 'state': { + const state = event.state; + renderStatus(status, state); + buttons.talk.dataset.self = String(state.floor.isSelf); + buttons.talk.textContent = state.floor.isSelf ? 'Talking' : 'Hold to talk'; + if (!state.floor.isSelf) buttons.talk.classList.remove('is-latched'); + buttons.disconnect.disabled = state.phase === 'idle'; + // The meter follows the microphone, which follows the floor. + if (state.sending && !lastState?.sending) { + const stream = client.microphone; + if (stream) startMeter(stream); + } + if (!state.sending && lastState?.sending) stopMeter(); + lastState = state; + return; + } + + case 'voice-event': + if (event.event.type === 'agent-roster') { + // A snapshot. Replaced wholesale, never merged. + agents = event.event.agents; + paintWheel(); + } + if (event.event.type === 'speak-start') speaking = true; + if (event.event.type === 'speak-end') speaking = false; + transcript.apply(event.event); + return; + + case 'remote-track': + playback.srcObject = event.stream; + void playback.play().catch(() => { + appendLog(logNode, 'warn', 'Playback needs a click. Press Connect again to allow audio.'); + }); + return; + + case 'duck': + // One property, applied immediately. A duck is a level change, not a + // pause: the reply is still being spoken and may be resumed. + playback.volume = event.ducked ? 0.15 : 1; + return; + + case 'log': + appendLog(logNode, event.level, event.text); + return; + } +}); + +// A deliberate teardown on the way out, so the desktop is not left waiting for +// ICE to notice. C-14. +window.addEventListener('beforeunload', () => client.disconnect('the browser page closed')); diff --git a/src/web-desktop/acappella-client/styles.css b/src/web-desktop/acappella-client/styles.css new file mode 100644 index 0000000000..77eb9022f7 --- /dev/null +++ b/src/web-desktop/acappella-client/styles.css @@ -0,0 +1,318 @@ +/* + * Deliberately plain. No framework, no theme system, no build-time CSS: this + * page is reference material, and a Swift developer reading it should not have + * to work out which of six abstractions painted a button. + */ + +:root { + color-scheme: dark; + --bg: #101215; + --panel: #181b20; + --border: #2a2f37; + --text: #e6e8eb; + --muted: #8b939f; + --accent: #6ea8fe; + --warn: #f0b429; + --live: #4ade80; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font: + 14px/1.5 ui-sans-serif, + system-ui, + -apple-system, + sans-serif; +} + +.app { + max-width: 880px; + margin: 0 auto; + padding: 24px 16px 64px; + display: flex; + flex-direction: column; + gap: 16px; +} + +h1 { + font-size: 20px; + margin: 0 0 4px; +} + +h2 { + font-size: 13px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--muted); + margin: 0 0 8px; +} + +.sub { + margin: 0; + color: var(--muted); +} + +.panel { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 10px; + padding: 14px; +} + +.field { + display: flex; + flex-direction: column; + gap: 4px; + flex: 1 1 160px; + min-width: 0; +} + +.field > span { + font-size: 12px; + color: var(--muted); +} + +.field.short { + flex: 0 0 110px; +} + +.field.check { + flex: 0 0 auto; + flex-direction: row; + align-items: center; + gap: 6px; + align-self: end; + padding-bottom: 6px; +} + +.field.wide { + margin-bottom: 10px; +} + +input, +textarea { + background: #0d0f12; + border: 1px solid var(--border); + border-radius: 6px; + color: var(--text); + padding: 6px 8px; + font: inherit; + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 12px; +} + +.row-fields { + display: flex; + flex-wrap: wrap; + gap: 10px; +} + +.buttons { + display: flex; + gap: 8px; + margin-top: 12px; + flex-wrap: wrap; +} + +button { + background: #22262d; + border: 1px solid var(--border); + color: var(--text); + border-radius: 6px; + padding: 6px 12px; + font: inherit; + cursor: pointer; +} + +button:hover:not(:disabled) { + border-color: var(--accent); +} + +button:disabled { + opacity: 0.45; + cursor: default; +} + +button.primary { + border-color: var(--accent); + color: var(--accent); +} + +.status-top { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +.pill { + border: 1px solid var(--border); + border-radius: 999px; + padding: 2px 10px; + font-size: 12px; +} + +.pill[data-phase='connected'] { + border-color: var(--live); + color: var(--live); +} + +.pill[data-phase='terminal'] { + border-color: var(--warn); + color: var(--warn); +} + +.pill.mic[data-on='true'] { + border-color: var(--live); + color: var(--live); +} + +.message { + margin: 8px 0 0; +} + +.muted { + color: var(--muted); + margin: 2px 0 0; + font-size: 12px; +} + +.warn { + color: var(--warn); + margin: 8px 0 0; + font-size: 12px; +} + +.meter { + margin-top: 10px; + height: 6px; + border-radius: 3px; + background: #0d0f12; + overflow: hidden; +} + +.meter-fill { + height: 100%; + width: 0; + background: var(--live); + transition: width 60ms linear; +} + +/* -- Project wheel ------------------------------------------------------- */ + +.wheel { + display: flex; + gap: 10px; + overflow-x: auto; + padding-bottom: 6px; + scroll-snap-type: x mandatory; +} + +.wheel-item { + flex: 0 0 160px; + scroll-snap-align: start; + text-align: left; + display: flex; + flex-direction: column; + gap: 2px; + padding: 10px; +} + +.wheel-item.is-selected { + border-color: var(--accent); +} + +.wheel-name { + font-weight: 600; +} + +.wheel-detail { + color: var(--muted); + font-size: 12px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* -- Talk ---------------------------------------------------------------- */ + +.talk-panel { + display: flex; + flex-direction: column; + align-items: center; +} + +.talk { + width: 160px; + height: 160px; + border-radius: 50%; + font-size: 15px; + touch-action: none; + user-select: none; +} + +.talk[data-self='true'] { + border-color: var(--live); + color: var(--live); + box-shadow: 0 0 0 6px rgba(74, 222, 128, 0.12); +} + +.talk.is-latched::after { + content: ' (latched)'; + color: var(--muted); +} + +/* -- Transcript and log --------------------------------------------------- */ + +.transcript, +.log { + max-height: 320px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 6px; +} + +.row { + border-left: 2px solid var(--border); + padding-left: 8px; +} + +.row-user { + border-color: var(--accent); +} + +.row-assistant { + border-color: var(--live); +} + +.row-system { + border-color: var(--border); + color: var(--muted); + font-size: 12px; +} + +.row-caption { + color: var(--muted); + font-size: 11px; +} + +.log { + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 11px; + gap: 2px; +} + +.log-warn { + color: var(--warn); +} + +.log-error { + color: #f87171; +} diff --git a/src/web-desktop/acappella-client/ui.ts b/src/web-desktop/acappella-client/ui.ts new file mode 100644 index 0000000000..31fbd5bbc0 --- /dev/null +++ b/src/web-desktop/acappella-client/ui.ts @@ -0,0 +1,337 @@ +/** + * The reference client's screen: a project wheel, a talk button, a transcript, + * and a status strip. + * + * Plain DOM on purpose. The main renderer is React with a theme system, stores, + * and several hundred components, and none of that would help a Swift developer + * work out what a `route-correction` does to a row. Everything here is one + * function per surface, in the order the events arrive. + * + * The one rule this file exists to demonstrate: **the button is drawn from + * `floor-state`, never from the gesture.** The gesture asks; the desktop + * answers; the answer is what paints. + */ + +import type { RosterAgent, VoiceEvent, VoiceScope } from '../../shared/acappella/protocol'; +import type { ClientState } from './client'; + +/** One line of conversation, as the transcript holds it. */ +interface TranscriptRow { + id: string; + kind: 'user' | 'assistant' | 'system'; + text: string; + /** The routing caption under a user row. Rewritten in place, never appended. */ + caption?: string; + pending?: boolean; + element: HTMLElement; +} + +export interface TranscriptHandle { + apply(event: VoiceEvent): void; + clear(): void; +} + +// --------------------------------------------------------------------------- +// Project wheel +// --------------------------------------------------------------------------- + +/** + * Draw the roster. + * + * `agent-roster` is a snapshot and this replaces the wheel wholesale. Merging + * would accumulate agents the desktop has already closed, which is how a phone + * ends up offering to talk to something that no longer exists. C-24. + */ +export function renderWheel( + container: HTMLElement, + agents: RosterAgent[], + selected: VoiceScope, + onSelect: (scope: VoiceScope) => void +): void { + container.replaceChildren(); + const entries: Array<{ scope: VoiceScope; name: string; detail: string }> = [ + { + scope: { kind: 'conductor' }, + name: 'Conductor', + detail: 'Routed by what you say', + }, + ...agents.map((agent) => ({ + scope: { kind: 'agent' as const, sessionId: agent.sessionId }, + name: agent.name, + detail: agent.recentWork || `${agent.tabs.length} tab${agent.tabs.length === 1 ? '' : 's'}`, + })), + ]; + + for (const entry of entries) { + const button = document.createElement('button'); + button.type = 'button'; + button.className = 'wheel-item'; + button.setAttribute('aria-pressed', String(sameScope(entry.scope, selected))); + if (sameScope(entry.scope, selected)) button.classList.add('is-selected'); + const name = document.createElement('span'); + name.className = 'wheel-name'; + name.textContent = entry.name; + const detail = document.createElement('span'); + detail.className = 'wheel-detail'; + detail.textContent = entry.detail; + button.append(name, detail); + button.addEventListener('click', () => onSelect(entry.scope)); + container.append(button); + } +} + +export function sameScope(a: VoiceScope, b: VoiceScope): boolean { + if (a.kind !== b.kind) return false; + return a.kind === 'agent' && b.kind === 'agent' ? a.sessionId === b.sessionId : true; +} + +// --------------------------------------------------------------------------- +// Status strip +// --------------------------------------------------------------------------- + +/** + * What the microphone is doing, in the client's own words. + * + * Three states, not two, and the reason is the same one that gives the iOS pill + * three: the browser lights its recording indicator for any open microphone, so + * a client that only says "on" or "off" is describing something other than what + * the user can see. C-50. + */ +export function micPillText(state: ClientState): string { + if (state.sending) return 'Sending'; + if (state.phase === 'connected') return 'Mic off'; + return 'Not connected'; +} + +export function renderStatus(elements: StatusElements, state: ClientState): void { + elements.phase.textContent = state.phase; + elements.phase.dataset.phase = state.phase; + elements.message.textContent = state.message; + elements.mic.textContent = micPillText(state); + elements.mic.dataset.on = String(state.sending); + + const holder = state.floor.isSelf + ? 'You hold the floor' + : state.floor.holder === 'local' + ? 'The desktop holds the floor' + : state.floor.holder + ? `${state.floor.takenOverBy ?? 'Another device'} holds the floor` + : 'Nobody holds the floor'; + elements.floor.textContent = holder; + + elements.quality.textContent = state.quality + ? `${state.quality.candidateType}` + + (state.quality.rttMs === null ? '' : ` - ${state.quality.rttMs} ms`) + + ` - ${(state.quality.packetLoss * 100).toFixed(1)}% loss` + : 'no link stats yet'; + + elements.suspect.hidden = !state.transcriptSuspect; + elements.version.textContent = `protocol v${state.protocolVersion}${ + state.desktopVersion ? ` - desktop ${state.desktopVersion}` : '' + }`; +} + +export interface StatusElements { + phase: HTMLElement; + message: HTMLElement; + mic: HTMLElement; + floor: HTMLElement; + quality: HTMLElement; + suspect: HTMLElement; + version: HTMLElement; +} + +// --------------------------------------------------------------------------- +// Transcript +// --------------------------------------------------------------------------- + +/** + * The transcript, driven straight off the session-event catalogue. + * + * Two behaviours here are the ones a first client usually gets wrong, so they + * are written plainly: + * + * - a `partial-transcript` REPLACES the in-flight user row rather than + * appending, because `text` is the whole hypothesis rather than a delta; + * - a `route-correction` REWRITES the caption of the row it corrects. The user + * said one sentence, so there is one row. Appending a second is how a + * transcript starts disagreeing with the conversation. C-25. + */ +export function createTranscript(container: HTMLElement): TranscriptHandle { + let rows: TranscriptRow[] = []; + /** The utterance whose sentences are currently being spoken. */ + let speakingUtteranceId: string | null = null; + + function addRow(kind: TranscriptRow['kind'], text: string, pending = false): TranscriptRow { + const element = document.createElement('div'); + element.className = `row row-${kind}`; + const body = document.createElement('div'); + body.className = 'row-text'; + body.textContent = text; + element.append(body); + container.append(element); + container.scrollTop = container.scrollHeight; + const row: TranscriptRow = { id: `${kind}-${rows.length}`, kind, text, pending, element }; + rows.push(row); + return row; + } + + function setText(row: TranscriptRow, text: string): void { + row.text = text; + const body = row.element.querySelector('.row-text'); + if (body) body.textContent = text; + } + + function setCaption(row: TranscriptRow, caption: string): void { + row.caption = caption; + let node = row.element.querySelector('.row-caption'); + if (!node) { + node = document.createElement('div'); + node.className = 'row-caption'; + row.element.append(node); + } + node.textContent = caption; + } + + function pendingRow(kind: TranscriptRow['kind']): TranscriptRow | undefined { + return [...rows].reverse().find((row) => row.kind === kind && row.pending); + } + + function lastRow(kind: TranscriptRow['kind']): TranscriptRow | undefined { + return [...rows].reverse().find((row) => row.kind === kind); + } + + return { + clear(): void { + rows = []; + speakingUtteranceId = null; + container.replaceChildren(); + }, + + apply(event: VoiceEvent): void { + switch (event.type) { + case 'listen-start': + addRow('system', `Listening (${event.sttProviderId}).`); + return; + + case 'partial-transcript': { + const row = pendingRow('user') ?? addRow('user', '', true); + setText(row, event.text); + return; + } + + case 'final-transcript': { + const row = pendingRow('user') ?? addRow('user', '', true); + setText(row, event.text); + row.pending = false; + row.element.classList.remove('is-pending'); + return; + } + + case 'route-decision': { + const row = lastRow('user'); + if (row) { + setCaption(row, `Routing (${event.brainProviderId}, ${event.latencyMs} ms)`); + } + return; + } + + case 'dispatch': { + const row = lastRow('user'); + if (row) { + setCaption( + row, + `${event.action} ${event.agentName} / ${event.tabName ?? event.tabId}` + + (event.promptSent ? '' : ' (prompt not sent)') + ); + } + return; + } + + case 'route-correction': { + // In place. One sentence, one row. + const row = lastRow('user'); + if (row) { + setCaption( + row, + `corrected to ${event.agentName} / ${event.tabName ?? event.tabId}` + + (event.promptSent ? '' : ' (prompt not sent)') + ); + } + return; + } + + case 'agent-reply': + addRow('assistant', event.text, true); + return; + + case 'speak-start': + speakingUtteranceId = event.utteranceId; + addRow( + 'system', + `Speaking ${event.sentenceCount}${event.streaming ? '+' : ''} sentence(s) via ${ + event.ttsProviderId + }.` + ); + return; + + case 'speak-sentence': { + // A sentence from a cancelled run arriving late is dropped, and the + // index is never clamped to `sentenceCount`: while `streaming` is true + // that count is a lower bound. C-26, C-27. + if (event.utteranceId !== speakingUtteranceId) return; + const row = pendingRow('assistant') ?? addRow('assistant', '', true); + setCaption(row, `sentence ${event.index + 1}`); + return; + } + + case 'speak-end': { + speakingUtteranceId = null; + const row = pendingRow('assistant'); + if (row) { + row.pending = false; + setCaption(row, `spoken: ${event.reason}`); + } + return; + } + + case 'barge-in': + addRow('system', `Interrupted (${event.source}). The floor is kept.`); + return; + + case 'stop-word': + addRow('system', `Stopped${event.phrase ? ` on "${event.phrase}"` : ''}.`); + return; + + case 'session-error': + // The message is written for a human and is shown as written. + addRow('system', `${event.code}: ${event.message}`); + return; + + case 'provider-state': + // Verbatim, wherever the app answers "where does my audio go". C-30. + addRow('system', event.egressStatement); + return; + + default: + // Every other event either updates a surface other than the + // transcript or is deliberately not shown. An unrecognised type lands + // here too, and doing nothing with it is the correct behaviour. C-23. + return; + } + }, + }; +} + +// --------------------------------------------------------------------------- +// Log +// --------------------------------------------------------------------------- + +export function appendLog(container: HTMLElement, level: string, text: string): void { + const line = document.createElement('div'); + line.className = `log-line log-${level}`; + line.textContent = text; + container.append(line); + while (container.childElementCount > 200) container.firstElementChild?.remove(); + container.scrollTop = container.scrollHeight; +} diff --git a/src/web-desktop/electron-shim.ts b/src/web-desktop/electron-shim.ts index f985143dba..892ab40022 100644 --- a/src/web-desktop/electron-shim.ts +++ b/src/web-desktop/electron-shim.ts @@ -13,6 +13,12 @@ * window.maestro with the same factory output the desktop gets. */ +// Type-only, and it is what declares `window.__MAESTRO_CONFIG__` for this file: +// the server injects one config object and there is one declaration of its +// shape. A second, narrower copy here disagreed with it the moment the bundle +// was type-checked alongside the renderer. +import type {} from '../web/utils/config'; + import { captureException } from './sentry-shim'; type Listener = (event: { senderFrame: null }, ...args: unknown[]) => void; @@ -26,12 +32,6 @@ interface BridgeConfig { wsUrl: string; } -declare global { - interface Window { - __MAESTRO_CONFIG__?: { wsUrl: string; apiBase: string; securityToken: string }; - } -} - function getWsUrl(): string { const cfg = window.__MAESTRO_CONFIG__; if (cfg && typeof cfg.wsUrl === 'string') { diff --git a/tsconfig.json b/tsconfig.json index 3d1d2991c6..4123331561 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,5 +15,5 @@ "noUnusedParameters": true, "noFallthroughCasesInSwitch": true }, - "include": ["src/renderer", "src/web", "src/shared", "src/types"] + "include": ["src/renderer", "src/web", "src/web-desktop", "src/shared", "src/types"] } diff --git a/tsconfig.lint.json b/tsconfig.lint.json index bb96e111b1..9801dd419a 100644 --- a/tsconfig.lint.json +++ b/tsconfig.lint.json @@ -4,5 +4,5 @@ "noUnusedLocals": false, "noUnusedParameters": false }, - "include": ["src/renderer", "src/shared", "src/web", "src/types"] + "include": ["src/renderer", "src/shared", "src/web", "src/web-desktop", "src/types"] } diff --git a/tsconfig.scripts.json b/tsconfig.scripts.json new file mode 100644 index 0000000000..b0e8018dcd --- /dev/null +++ b/tsconfig.scripts.json @@ -0,0 +1,15 @@ +{ + // Type-check the TypeScript under `scripts/`. + // + // These are developer tools that import main-process modules, so they need the + // main config's options - but not its `rootDir`/`outDir`, which exist to emit + // the app. Checked only, and kept out of tsconfig.main.json so a harness can + // never end up in a build. + "extends": "./tsconfig.main.json", + "compilerOptions": { + "noEmit": true, + "rootDir": "." + }, + "include": ["scripts/**/*.ts"], + "exclude": ["node_modules", "dist", "release"] +} diff --git a/vite.config.web-desktop.mts b/vite.config.web-desktop.mts index fc4fa4ab64..1321bb1d7b 100644 --- a/vite.config.web-desktop.mts +++ b/vite.config.web-desktop.mts @@ -74,6 +74,11 @@ export default defineConfig(({ mode }) => ({ rollupOptions: { input: { main: path.join(__dirname, 'src/web-desktop/index.html'), + // The A Cappella reference client: a second, self-contained page in the + // same bundle. It shares nothing with the renderer beyond the shared + // protocol modules, which is the point - it is the independent endpoint + // the desktop's WebRTC transport is tested against. + 'acappella-client': path.join(__dirname, 'src/web-desktop/acappella-client/index.html'), }, output: { manualChunks: (id) => {