diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 000000000..1be6f1a8b --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,4 @@ +bunx lint-staged +bun run typecheck +python3 -m pytest backend/tests/test_fixture_contract.py -q +ruff check backend/tests/conftest.py backend/tests/test_generation_unload.py backend/tests/test_mcp_mount_slashes.py backend/tests/test_fixture_contract.py backend/tests/test_tts_vram_churn.py backend/tests/test_whisper_long_audio_e2e.py diff --git a/.lintstagedrc b/.lintstagedrc new file mode 100644 index 000000000..d0bf80e1e --- /dev/null +++ b/.lintstagedrc @@ -0,0 +1,3 @@ +{ + "*": "prettier --ignore-unknown --write" +} diff --git a/.mcp.json b/.mcp.json index de6d6caf2..2df463879 100644 --- a/.mcp.json +++ b/.mcp.json @@ -2,10 +2,10 @@ "mcpServers": { "voicebox": { "type": "http", - "url": "http://127.0.0.1:17493/mcp", + "url": "http://127.0.0.1:17493/mcp/", "headers": { "X-Voicebox-Client-Id": "claude-code" } } } -} \ No newline at end of file +} diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 000000000..0bd86d915 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,9 @@ +{ + "useTabs": false, + "tabWidth": 2, + "printWidth": 80, + "singleQuote": false, + "trailingComma": "es5", + "semi": true, + "arrowParens": "always" +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b03f5e46..a97b4475f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ ### Linux +- **Evaluation and regression coverage for Whisper/TTS/MCP.** Added CI-safe + unload and MCP slash tests, opt-in Common Voice long-audio fixture generation, + backend/dictation contract harnesses, and documented GPU evaluation commands. - **ROCm setup works on Linux AMD systems.** Docker ROCm builds now keep PyTorch on the ROCm wheel index during dependency installation, so later installs do not replace it with CUDA wheels. The ROCm compose overlay no longer assumes @@ -137,6 +140,7 @@ A patch focused on two user-impacting reliability fixes: macOS DMG notarization This release localizes the entire app. English, Simplified Chinese (zh-CN), Traditional Chinese (zh-TW), and Japanese (ja) are wired up end-to-end across every tab, modal, dialog, and toast — 559 translation keys per locale, parity verified. Plus a batch of reliability fixes: offline-mode now actually stays offline, Chatterbox accepts reference samples it used to reject, MLX Qwen 0.6B points at the right repo, and macOS system audio survives backgrounding. ### Internationalization ([#508](https://github.com/jamiepine/voicebox/pull/508)) + - **i18next foundation** with an in-app language switcher that re-renders the tree on change — lazy-loaded components were holding stale strings without an explicit key-bump on the React root. - **Four locales** at full coverage: English, Simplified Chinese, Traditional Chinese, Japanese. No partial/English-fallback surfaces. - **Every user-visible surface translated**: Stories (list, content editor, dialogs, toasts), Effects (list, detail, chain editor, built-in preset names), Voices (table, search, inspector, Create/Edit modal, audio sample panels), Audio Channels (list, dialogs, device picker), history + story dropdown menus, ProfileCard / ProfileList / HistoryTable, and the unsupported-model note. @@ -145,6 +149,7 @@ This release localizes the entire app. English, Simplified Chinese (zh-CN), Trad - **559 translation keys** across all four locales. ### Reliability + - **`HF_HUB_OFFLINE` now guards every inference path** ([#503](https://github.com/jamiepine/voicebox/pull/503)) — some engines were still attempting a HuggingFace metadata roundtrip on first load when offline mode was enabled, causing hangs on airgapped or flaky networks. - **Chatterbox reference samples are preprocessed instead of rejected** ([#502](https://github.com/jamiepine/voicebox/pull/502)) — samples outside the expected sample rate or channel layout are resampled to match, rather than failing with an opaque error. - **MLX Qwen 0.6B repo path fixed** ([#501](https://github.com/jamiepine/voicebox/pull/501)) — now points at the published `mlx-community` repo so the model actually downloads on Apple Silicon. @@ -152,15 +157,18 @@ This release localizes the entire app. English, Simplified Chinese (zh-CN), Trad - **MLX backend `miniaudio` dependency pinned** ([#506](https://github.com/jamiepine/voicebox/pull/506)) — `mlx_audio.stt` needs it at runtime and nothing else transitively pulled it in, so `--no-deps` installs were breaking on first use. ### Landing / Docs + - **New `/download` page** ([#487](https://github.com/jamiepine/voicebox/pull/487)) — no more dumping first-time visitors onto the GitHub releases list. The API example snippet on the landing page also got an accuracy pass. - **Download redirects work behind reverse proxies** ([#498](https://github.com/jamiepine/voicebox/pull/498)) — uses the public origin instead of `localhost` when resolving platform-specific installer URLs. - **MDX docs audited against the multi-engine backend** ([#484](https://github.com/jamiepine/voicebox/pull/484)) — stale single-engine assumptions removed. - **Three more tutorials + mobile navbar / hero CTA fixes** ([#483](https://github.com/jamiepine/voicebox/pull/483)). ### Linux + - **Still not shipping.** The re-enable attempt ([#488](https://github.com/jamiepine/voicebox/pull/488)) landed on `main` but CI still hangs in the `tauri-action` bundler step on `ubuntu-22.04` — no output for 25+ minutes after `rpm` bundling, even with `createUpdaterArtifacts: false` and `--bundles deb,rpm`. The matrix entry is disabled again for 0.4.2; the ubuntu-specific setup steps stay in the workflow so re-enabling is a one-line change once we identify the hang. Next release will take another pass. ### New Contributors + - [@shekharyv](https://github.com/shekharyv) — download redirects behind reverse proxies ([#498](https://github.com/jamiepine/voicebox/pull/498)) ## [0.4.1] - 2026-04-18 @@ -170,33 +178,40 @@ A fast follow-up to 0.4.0 focused on making the new engines actually load in the 0.4.0 introduced three new TTS engines, but the frozen PyInstaller binary tripped over several Python-ecosystem quirks that don't show up in the dev venv: `transformers` opening `.py` sources at runtime, `scipy.stats._distn_infrastructure` hitting a frozen-importer `NameError`, and `chatterbox-multilingual` failing to find its Chinese segmenter dictionary. This release patches all of those in one sweep. ### Frozen-Binary Reliability ([#438](https://github.com/jamiepine/voicebox/pull/438)) + - **Kokoro** now bundles `.py` sources alongside `.pyc` via `--collect-all kokoro` so `transformers`' `_can_set_attn_implementation` regex scan can read them — previously `FileNotFoundError: kokoro/modules.py` killed Kokoro loading in production builds - **Chatterbox Multilingual** now bundles `spacy_pkuseg/dicts/default.pkl` and the package's native `.so` extensions via `--collect-all spacy_pkuseg` — previously the Chinese word segmenter crashed with `FileNotFoundError` on first load -- **scipy.stats._distn_infrastructure** — new runtime hook source-patches the trailing `del obj` (which raises `NameError` under PyInstaller's frozen importer because the preceding list comprehension evaluates empty) to `globals().pop('obj', None)`, unblocking `librosa` → `scipy.signal` → `scipy.stats` for every TTS engine that depends on librosa +- **scipy.stats.\_distn_infrastructure** — new runtime hook source-patches the trailing `del obj` (which raises `NameError` under PyInstaller's frozen importer because the preceding list comprehension evaluates empty) to `globals().pop('obj', None)`, unblocking `librosa` → `scipy.signal` → `scipy.stats` for every TTS engine that depends on librosa - **transformers.masking_utils** — same runtime hook forces `_is_torch_greater_or_equal_than_2_6 = False` so the older `sdpa_mask_older_torch` path is selected; the 2.6+ path uses `TransformGetItemToIndex()`, a real `torch._dynamo` graph transform our permissive stub can't reproduce -- **torch._dynamo** — no-op stub replaces the real module before `transformers` imports it, preventing the `torch._numpy._ufuncs` import crash (`NameError: name 'name' is not defined`) that blocked Kokoro and every engine pulling in `flex_attention` +- **torch.\_dynamo** — no-op stub replaces the real module before `transformers` imports it, preventing the `torch._numpy._ufuncs` import crash (`NameError: name 'name' is not defined`) that blocked Kokoro and every engine pulling in `flex_attention` - `.spec` paths are now repo-relative instead of absolute, so the generated spec is portable across machines and CI ### Generation + - **Cancel queued or running generations** ([#444](https://github.com/jamiepine/voicebox/pull/444)) — new `/generate/{id}/cancel` endpoint and a Stop button on the history row while generating. The serial queue now tracks per-ID state (queued / running / cancelled) so queued jobs are skipped before the worker picks them up and running jobs are `.cancel()`-ed mid-flight; `run_generation` catches `CancelledError` and marks the row `failed` with a "cancelled" error. - **Legacy `data/` path prefix resolution** ([#440](https://github.com/jamiepine/voicebox/pull/440)) — generations stored with the old `data/` prefix under pre-0.4 installs now resolve correctly after the storage root moved, fixing 404s for historical audio. ### Model Migration + - Migration dialog no longer hangs when the cache is empty ([#439](https://github.com/jamiepine/voicebox/pull/439)) — the backend now emits a completion SSE event even when zero models are moved. - Storage-change flow surfaces a toast when there's nothing to migrate ([#433](https://github.com/jamiepine/voicebox/pull/433)) instead of proceeding with a no-op move and restarting the server. - Deleting all generations from a voice profile now deletes the associated version files and DB rows too ([#447](https://github.com/jamiepine/voicebox/pull/447)) — previously orphaned versions accumulated in storage. ### Platform + - **Linux system audio capture** ([#457](https://github.com/jamiepine/voicebox/pull/457)) — `cpal`'s ALSA backend doesn't expose PulseAudio/PipeWire monitor sources by name, so the previous device-name search never matched and silently fell back to the microphone. Detection now uses `pactl get-default-sink` + `pactl list short sources` and routes via `PULSE_SOURCE`, with the name-based search retained as a fallback when `pactl` is absent. ### Frontend CI + - First PR-time quality gate ([#418](https://github.com/jamiepine/voicebox/pull/418)) — new `.github/workflows/ci.yml` runs `bun run typecheck` + `bun run build:web` on every PR. Fixed pre-existing type issues that were being suppressed with `@ts-expect-error`, cleaned up a dep-array typo (`[platform.metadata.isTauricheckOnMountcheckForUpdates]`) in `useAutoUpdater`, and removed 100+ lines of dead `ModelItem` code from `ModelManagement.tsx`. - Follow-up: widened `apiClient.migrateModels()` return type to include `moved` and `errors` so the storage-change handler typechecks against the real backend response ([#470](https://github.com/jamiepine/voicebox/pull/470)). ### Docs + - Clarified in the Quick Start + README that paralinguistic tags (`[laugh]`, `[sigh]`) only work with Chatterbox Turbo; other engines read them as literal text ([#450](https://github.com/jamiepine/voicebox/pull/450)). ### New Contributors + - [@Bortlesboat](https://github.com/Bortlesboat) — generation cancellation (#444) - [@gaojulong](https://github.com/gaojulong) — migration dialog hang fix (#439) - [@fuleinist](https://github.com/fuleinist) — migration no-op toast (#433) @@ -212,6 +227,7 @@ This release also marks a big community moment: **13 new contributors** shipped ### New TTS Engines #### HumeAI TADA — Expressive English & Multilingual ([#296](https://github.com/jamiepine/voicebox/pull/296)) + - Added `tada-1b` (English) and `tada-3b-ml` (multilingual) backends - Replaced `descript-audio-codec` with a lightweight DAC shim to cut dependencies - Switched audio decoding to `soundfile` to sidestep `torchcodec` bundling issues @@ -220,11 +236,13 @@ This release also marks a big community moment: **13 new contributors** shipped - Fixed TorchScript error in frozen builds #### Kokoro 82M — Fast Lightweight TTS ([#325](https://github.com/jamiepine/voicebox/pull/325)) + - Added Kokoro 82M engine with a new voice profile type system that distinguishes preset voices from cloned profiles - Profile grid now handles engine compatibility directly — removed redundant dropdown filtering - Tightened Kokoro profile handling so preset voices can't be edited like cloned profiles #### Qwen CustomVoice ([#328](https://github.com/jamiepine/voicebox/pull/328)) + - Added `qwen-custom-voice` preset engine backed by Qwen3-TTS - Enforced preset/profile engine compatibility across the generation flow - Floating generator now shows all engines instead of silently filtering @@ -246,22 +264,26 @@ This release ships the resolution before it ever reaches a tagged version: ### GPU & Platform #### Intel Arc (XPU) Support ([#320](https://github.com/jamiepine/voicebox/pull/320)) + - First-class Intel Arc support across all PyTorch-based backends - Device-aware seeding, XPU detection in the GPU status panel, and setup flow detection - Reports correct device name and VRAM in settings #### Blackwell / RTX 50-series Support ([#316](https://github.com/jamiepine/voicebox/pull/316), [#401](https://github.com/jamiepine/voicebox/pull/401)) + - Upgraded the CUDA backend from cu126 → cu128 for RTX 50-series support - Added `sm_120+PTX` to the CUDA build via `TORCH_CUDA_ARCH_LIST` for forward-compatibility with Blackwell architectures (closes 5 open reports: #386, #395, #396, #399, #400) - GPU settings UI fixes around install/uninstall state #### GPU Compatibility Diagnostics ([#367](https://github.com/jamiepine/voicebox/pull/367), adapted) + - New `check_cuda_compatibility()` compares the current device's compute capability against the bundled PyTorch's architecture list - Health endpoint exposes a `gpu_compatibility_warning` field so the UI can surface mismatches - Startup logs a `WARN` when the installed PyTorch build doesn't support the detected GPU - GPU status label shows `[UNSUPPORTED - see logs]` — no more silent "no kernel image" failures #### Split CUDA Backend ([#298](https://github.com/jamiepine/voicebox/pull/298)) + - CUDA backend now ships as two independently versioned archives: a small server binary and a large libs archive (the ~4 GB of PyTorch/CUDA DLLs) - Upgrading Voicebox no longer redownloads the libs archive when only the server binary changed - Added `asyncio.Lock` around `download_cuda_binary()` so auto-update and manual download can't race on the same temp file ([#428](https://github.com/jamiepine/voicebox/pull/428)) @@ -271,9 +293,11 @@ This release ships the resolution before it ever reaches a tagged version: ### Bug Fixes #### Critical: TTS Generation + - **numpy 2.x `torch.from_numpy` crash** ([#361](https://github.com/jamiepine/voicebox/pull/361)) — torch compiled against numpy 1.x ABI fails silently when paired with numpy 2.x, causing `RuntimeError: Numpy is not available` / `Unable to create tensor` on every TTS request in bundled macOS Intel / Rosetta builds. Pinned `numpy<2.0` in requirements and added a PyInstaller runtime hook with a `ctypes.memmove` fallback as belt-and-suspenders. Hardened afterward to raise on unknown dtypes instead of silently reinterpreting bytes as float32. #### Platform Reliability + - **Windows background server** ([#402](https://github.com/jamiepine/voicebox/pull/402)) — "keep server running after close" now actually keeps the server running. The HTTP `/watchdog/disable` request could lose the race against process exit on Windows; added a `.keep-running` sentinel file as a synchronous fallback, with stale-sentinel cleanup on startup to avoid orphan server processes - **macOS 11 launch crash** ([#424](https://github.com/jamiepine/voicebox/pull/424)) — weak-linked ScreenCaptureKit so the app can launch on macOS < 12.3 instead of crashing at dyld resolution. Gated system audio capture behind a real `sw_vers` version check so unsupported systems cleanly advertise "not available" rather than crashing at runtime - **macOS Intel (x86_64) setup** ([#416](https://github.com/jamiepine/voicebox/pull/416)) — relaxed `torch>=2.7.0` → `torch>=2.2.0`. PyTorch dropped pre-built x86_64 wheels after 2.2.2, so Intel Mac devs could no longer `pip install`. Now resolves to the latest compatible torch per platform @@ -285,18 +309,22 @@ This release ships the resolution before it ever reaches a tagged version: - **Effects service import** ([#384](https://github.com/jamiepine/voicebox/pull/384)) — fixed `ModuleNotFoundError` on preset create/update by switching to relative imports (#349) #### Audio & Playback + - **cpal stream silent playback** ([#405](https://github.com/jamiepine/voicebox/pull/405)) — `cpal::Stream` was dropped on function return immediately after `play()`, causing every playback to fall silent. Now holds the stream until either the buffer drains or the stop flag fires (#404) #### Stories & History + - **Clip-splitting race** ([#403](https://github.com/jamiepine/voicebox/pull/403)) — rapid double-clicks on split could race through `split_story_item` with inconsistent state. Added `with_for_update()` row locking on the backend and an `isPending` guard on the frontend (#366) - **History `status` staleness** ([#394](https://github.com/jamiepine/voicebox/pull/394)) — `GET /history/{id}` was hardcoding `status="completed"` regardless of the DB row, breaking any client polling for job completion. Now returns `status`, `error`, `engine`, `model_size`, and `is_favorited` from the actual row - **"Clear failed" bulk button** ([#412](https://github.com/jamiepine/voicebox/pull/412)) — new `DELETE /history/failed` endpoint and a header strip showing `"N failed generations"` with a Clear button, complementing the per-row trash icon added in #321 (#410) - **Delete failed generations** ([#321](https://github.com/jamiepine/voicebox/pull/321)) — added a trash icon next to the retry button so failed entries can be cleaned up without having to retry first #### Security & Safety + - **Voice prompt cache hardening** ([#429](https://github.com/jamiepine/voicebox/pull/429)) — `torch.load(weights_only=True)` on cached voice prompts per PyTorch 2.6 recommendation; replaced string-based SPA path guard with `Path.is_relative_to()` for more robust path-traversal protection #### Infrastructure & Docker + - **Docker web build** ([#344](https://github.com/jamiepine/voicebox/pull/344)) — include `CHANGELOG.md` in the Docker web build so the in-app changelog page works in Docker deployments - **Docker numba cache** ([#425](https://github.com/jamiepine/voicebox/pull/425)) — set `NUMBA_CACHE_DIR` in docker-compose so numba can write its JIT cache in container runtime (#308) - **Relative media paths** ([#332](https://github.com/jamiepine/voicebox/pull/332)) — media paths now stored relative to the configured data dir rather than resolved against CWD, so the data directory is portable between installs @@ -321,6 +349,7 @@ This release rewrites the backend into a modular architecture, overhauls the set The backend's 3,000-line monolith `main.py` has been decomposed into domain routers, a services layer, and a proper database package. A style guide and ruff configuration now enforce consistency. On the frontend, settings have been split into dedicated routed pages with server logs, a changelog viewer, and an about page. The audio player no longer freezes mid-playback, and model loading status is now visible in the UI. Seven user-reported bugs have been fixed, including server crashes during sample uploads, generation list staleness, cryptic error messages, and CUDA support for RTX 50-series GPUs. ### Settings Overhaul ([#294](https://github.com/jamiepine/voicebox/pull/294)) + - Split settings into routed sub-tabs: General, Generation, GPU, Logs, Changelog, About - Added live server log viewer with auto-scroll - Added in-app changelog page that parses `CHANGELOG.md` at build time @@ -328,6 +357,7 @@ The backend's 3,000-line monolith `main.py` has been decomposed into domain rout - Extracted reusable `SettingRow` component for consistent setting layouts ### Audio Player Fix ([#293](https://github.com/jamiepine/voicebox/pull/293)) + - Fixed audio player freezing during playback - Improved playback UX with better state management and listener cleanup - Fixed restart race condition during regeneration @@ -335,6 +365,7 @@ The backend's 3,000-line monolith `main.py` has been decomposed into domain rout - Improved accessibility across player controls ### Backend Refactor ([#285](https://github.com/jamiepine/voicebox/pull/285)) + - Extracted all routes from `main.py` into 13 domain routers under `backend/routes/` — `main.py` dropped from ~3,100 lines to ~10 - Moved CRUD and service modules into `backend/services/`, platform detection into `backend/utils/` - Split monolithic `database.py` into a `database/` package with separate `models`, `session`, `migrations`, and `seed` modules @@ -348,6 +379,7 @@ The backend's 3,000-line monolith `main.py` has been decomposed into domain rout - Reject model migration when target is a subdirectory of source cache ### Documentation Rewrite ([#288](https://github.com/jamiepine/voicebox/pull/288)) + - Migrated docs site from Mintlify to Fumadocs (Next.js-based) - Rewrote introduction and root page with content from README - Added "Edit on GitHub" links and last-updated timestamps on all pages @@ -357,6 +389,7 @@ The backend's 3,000-line monolith `main.py` has been decomposed into domain rout - Added OG image metadata and `/og` preview page ### UI & Frontend + - Added model loading status indicator and effects preset dropdown ([3187344](https://github.com/jamiepine/voicebox/commit/3187344)) - Fixed take-label race condition during regeneration - Added accessible focus styling to select component @@ -364,6 +397,7 @@ The backend's 3,000-line monolith `main.py` has been decomposed into domain rout - Addressed 4 critical and 12 major issues from CodeRabbit review ### Bug Fixes ([#295](https://github.com/jamiepine/voicebox/pull/295)) + - Fixed sample uploads crashing the server — audio decoding now runs in a thread pool instead of blocking the async event loop ([#278](https://github.com/jamiepine/voicebox/issues/278)) - Fixed generation list not updating when a generation completes — switched to `refetchQueries` for reliable cache busting, added SSE error fallback, and page reset on completion ([#231](https://github.com/jamiepine/voicebox/issues/231)) - Fixed error toasts showing `[object Object]` instead of the actual error message ([#290](https://github.com/jamiepine/voicebox/issues/290)) @@ -375,12 +409,14 @@ The backend's 3,000-line monolith `main.py` has been decomposed into domain rout - Eliminated redundant double audio decode in sample processing pipeline ### Platform Fixes + - Replaced `netstat` with `TcpStream` + PowerShell for Windows port detection ([#277](https://github.com/jamiepine/voicebox/pull/277)) - Fixed Docker frontend build and cleaned up Docker docs - Fixed macOS download links to use `.dmg` instead of `.app.tar.gz` - Added dynamic download redirect routes to landing site ### Release Tooling + - Added `draft-release-notes` and `release-bump` agent skills - Wired CI release workflow to extract notes from `CHANGELOG.md` for GitHub Releases - Backfilled changelog with all historical releases diff --git a/backend/app.py b/backend/app.py index ea8411cbb..99be37799 100644 --- a/backend/app.py +++ b/backend/app.py @@ -129,7 +129,7 @@ def safe_content_disposition(disposition_type: str, filename: str) -> str: def create_app() -> FastAPI: """Create and configure the FastAPI application.""" - from .mcp_server.server import build_mcp_server, compose_lifespan + from .mcp_server.server import MountRootSlashRewrite, build_mcp_server, compose_lifespan from .mcp_server.context import ClientIdMiddleware # Build the MCP app up-front so we can wire its lifespan into FastAPI's — @@ -167,7 +167,7 @@ async def voicebox_lifespan(app: FastAPI): _configure_cors(application) application.add_middleware(ClientIdMiddleware) register_routers(application) - application.mount("/mcp", mcp_app) + application.mount("/mcp", MountRootSlashRewrite(mcp_app)) logger.info("MCP: mounted at /mcp") _mount_frontend(application) diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index 142c430c7..65d5e727a 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -12,12 +12,15 @@ # HF_HUB_OFFLINE=1 and on network failures. from ..utils import hf_offline_patch # noqa: F401 +import logging import threading from dataclasses import dataclass, field from typing import Protocol, Optional, Tuple, List from typing_extensions import runtime_checkable import numpy as np +logger = logging.getLogger(__name__) + DEFAULT_LLM_MAX_TOKENS = 512 DEFAULT_LLM_TEMPERATURE = 0.7 @@ -794,3 +797,45 @@ def reset_backends(): _tts_backends.clear() _stt_backend = None _llm_backends.clear() + + +def unload_all_models() -> None: + """Unload every registered backend and release accelerator caches. + + Generation workers call this after each job so chained TTS, STT, and LLM + work cannot retain model weights indefinitely. Backend instances are + removed from the registry afterwards and recreated lazily on the next + request. + """ + registered = [*_tts_backends.values(), *_llm_backends.values()] + if _stt_backend is not None: + registered.append(_stt_backend) + if _tts_backend is not None: + registered.append(_tts_backend) + + # Keep the cleanup idempotent even if a legacy singleton points at an + # instance already present in one of the engine registries. + backends = list({id(backend): backend for backend in registered}.values()) + for backend in backends: + try: + backend.unload_model() + except Exception: + logger.warning("Failed to unload backend %r", backend, exc_info=True) + + reset_backends() + + # Return cached blocks to the driver. torch is optional in MLX/CPU builds. + try: + import torch + except ImportError: + return + try: + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + logger.warning("Failed to empty CUDA cache", exc_info=True) + try: + if hasattr(torch, "mps") and torch.mps.is_available(): + torch.mps.empty_cache() + except Exception: + logger.warning("Failed to empty MPS cache", exc_info=True) diff --git a/backend/backends/pytorch_backend.py b/backend/backends/pytorch_backend.py index f8ae79b86..520ebc822 100644 --- a/backend/backends/pytorch_backend.py +++ b/backend/backends/pytorch_backend.py @@ -296,7 +296,13 @@ def _load_model_sync(self, model_size: str): logger.info("Loading Whisper model %s on %s...", model_size, self.device) self.processor = WhisperProcessor.from_pretrained(model_name) - self.model = WhisperForConditionalGeneration.from_pretrained(model_name) + # Usar bfloat16 en lugar de float32 para ahorrar VRAM en GPUs + # modernas (RTX 40/50, Ada/Blackwell) sin incurrir en el bug de + # dtype de float16 (c10::Half) que se da en esta arquitectura. + self.model = WhisperForConditionalGeneration.from_pretrained( + model_name, + torch_dtype=torch.bfloat16, + ) self.model.to(self.device) self.model_size = model_size @@ -342,17 +348,24 @@ def _transcribe_sync(): # state — forcing offline here (issue #462) broke online users # whose `get_decoder_prompt_ids` / tokenizer calls issue # legitimate metadata lookups. - # Process audio - inputs = self.processor( - audio, - sampling_rate=16000, - return_tensors="pt", - ) - inputs = inputs.to(self.device) + # Whisper's encoder only accepts 30s of audio per pass (3000 mel + # frames); feeding it a longer clip silently transcribes just the + # first 30 seconds. Chunk into 30s windows and join the results. + chunk_samples = 30 * 16000 + chunks = [ + audio[i : i + chunk_samples] + for i in range(0, len(audio), chunk_samples) + ] or [audio] # Generate transcription # If language is provided, force it; otherwise let Whisper auto-detect - generate_kwargs = {} + generate_kwargs = { + # Sin timestamps, Whisper a veces emite <|endoftext|> tras la + # primera frase de una ventana y descarta el resto del audio. + # Forzar timestamps obliga al modelo a recorrer la ventana + # completa en cada chunk. + "return_timestamps": True, + } if language: forced_decoder_ids = self.processor.get_decoder_prompt_ids( language=language, @@ -360,19 +373,34 @@ def _transcribe_sync(): ) generate_kwargs["forced_decoder_ids"] = forced_decoder_ids - with torch.no_grad(): - predicted_ids = self.model.generate( - inputs["input_features"], - **generate_kwargs, + transcriptions = [] + for chunk in chunks: + # Process audio + inputs = self.processor( + chunk, + sampling_rate=16000, + return_tensors="pt", + ) + # Mover al device y convertir al mismo dtype que el modelo para + # evitar el error "Input type (float) and bias type (...) should be + # the same" en GPUs como la RTX 5060. + inputs = inputs.to(device=self.device, dtype=self.model.dtype) + + with torch.no_grad(): + predicted_ids = self.model.generate( + inputs["input_features"], + **generate_kwargs, + ) + + # Decode + transcriptions.append( + self.processor.batch_decode( + predicted_ids, + skip_special_tokens=True, + )[0] ) - # Decode - transcription = self.processor.batch_decode( - predicted_ids, - skip_special_tokens=True, - )[0] - - return transcription.strip() + return " ".join(t.strip() for t in transcriptions).strip() # Run blocking transcription in thread pool return await asyncio.to_thread(_transcribe_sync) diff --git a/backend/mcp_server/README.md b/backend/mcp_server/README.md index 4c9b426f2..b4c96641f 100644 --- a/backend/mcp_server/README.md +++ b/backend/mcp_server/README.md @@ -15,7 +15,7 @@ Preferred — direct HTTP: { "mcpServers": { "voicebox": { - "url": "http://127.0.0.1:17493/mcp", + "url": "http://127.0.0.1:17493/mcp/", "headers": { "X-Voicebox-Client-Id": "claude-code" } } } @@ -41,18 +41,18 @@ Claude Code one-liner: ``` claude mcp add voicebox \ --transport http \ - --url http://127.0.0.1:17493/mcp \ + --url http://127.0.0.1:17493/mcp/ \ --header "X-Voicebox-Client-Id: claude-code" ``` ## Tools -| Name | Purpose | -|---|---| -| `voicebox.speak` | Speak text in a voice profile. Returns a generation id you can poll. | -| `voicebox.transcribe` | Whisper transcription of a base64 blob or an absolute local path. | -| `voicebox.list_captures` | Recent captures (dictation / recording / file) with transcripts. | -| `voicebox.list_profiles` | Available voice profiles (cloned + preset). | +| Name | Purpose | +| ------------------------ | -------------------------------------------------------------------- | +| `voicebox.speak` | Speak text in a voice profile. Returns a generation id you can poll. | +| `voicebox.transcribe` | Whisper transcription of a base64 blob or an absolute local path. | +| `voicebox.list_captures` | Recent captures (dictation / recording / file) with transcripts. | +| `voicebox.list_profiles` | Available voice profiles (cloned + preset). | All tools resolve voice profiles in this precedence: @@ -66,7 +66,7 @@ Settings → MCP. ## Debug with MCP Inspector ``` -npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp +npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp/ ``` Point it at the URL, hit "List tools," call `voicebox.list_profiles` diff --git a/backend/mcp_server/server.py b/backend/mcp_server/server.py index 3a408df9d..1f6eaa6be 100644 --- a/backend/mcp_server/server.py +++ b/backend/mcp_server/server.py @@ -9,11 +9,12 @@ from __future__ import annotations import logging -from contextlib import AsyncExitStack, asynccontextmanager from collections.abc import Callable +from contextlib import AsyncExitStack, asynccontextmanager from fastapi import FastAPI from fastmcp import FastMCP +from starlette.types import ASGIApp, Receive, Scope, Send from .context import ClientIdMiddleware from .tools import register_tools @@ -22,6 +23,31 @@ logger = logging.getLogger(__name__) +class MountRootSlashRewrite: + """ASGI wrapper that maps the bare mount root onto FastMCP's ``/`` route. + + Starlette's ``Mount`` strips the ``/mcp`` prefix, so ``POST /mcp`` (no + trailing slash) arrives inside the sub-application with ``path == ""`` + and FastMCP's router — which only knows ``/`` — answers 405. Most MCP + clients (Claude Code, Cursor, …) point at the bare ``/mcp`` URL and do + not follow 307 redirects on POST, so we rewrite the empty path to ``/`` + instead of redirecting. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] == "http" and scope.get("path") == "": + scope = dict(scope) + scope["path"] = "/" + # Keep path and raw_path coherent for ASGI routers that inspect + # both fields (Starlette normally supplies raw_path from the + # original request, e.g. b"/mcp"). + scope["raw_path"] = b"/" + await self.app(scope, receive, send) + + def build_mcp_server() -> FastMCP: """Create the FastMCP instance with Voicebox tools registered.""" mcp = FastMCP( @@ -55,7 +81,7 @@ def mount_into( # by the time tool handlers execute. Starlette composes middlewares # outermost-first, so adding here on the parent app is correct. app.add_middleware(ClientIdMiddleware) - app.mount("/mcp", mcp_app) + app.mount("/mcp", MountRootSlashRewrite(mcp_app)) app.state.mcp_lifespan = mcp_app.router.lifespan_context logger.info("MCP: mounted at /mcp (FastMCP %s)", getattr(mcp, "version", "")) diff --git a/backend/services/generation.py b/backend/services/generation.py index a4b2e8a3f..5f675aaf5 100644 --- a/backend/services/generation.py +++ b/backend/services/generation.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import logging import traceback from typing import Literal, Optional @@ -25,6 +26,26 @@ from ..database import get_db from ..utils.tasks import get_task_manager +logger = logging.getLogger(__name__) + +#: tada degenerates on long inputs (produces ~1s of audio or hallucinated +#: text), so its chunks are capped well below the global 800-char default. +TADA_MAX_CHUNK_CHARS = 250 + + +def effective_max_chunk_chars(engine: str, requested: Optional[int]) -> Optional[int]: + """Resolve the chunk size used for long-text TTS splitting. + + ``GenerationRequest.max_chunk_chars`` defaults to 800, so callers almost + never pass ``None``. For tada we force :data:`TADA_MAX_CHUNK_CHARS` + unless the caller explicitly asked for an even smaller chunk. + + Returns the requested value unchanged for other engines. + """ + if engine == "tada" and (requested is None or requested > TADA_MAX_CHUNK_CHARS): + return TADA_MAX_CHUNK_CHARS + return requested + async def run_generation( *, @@ -55,6 +76,8 @@ async def run_generation( load_engine_model, ) from ..utils.chunked_tts import generate_chunked + + max_chunk_chars = effective_max_chunk_chars(engine, max_chunk_chars) from ..utils.audio import has_tts_runaway, normalize_audio, save_audio, trim_tts_output task_manager = get_task_manager() @@ -156,6 +179,14 @@ async def run_generation( finally: task_manager.complete_generation(generation_id) bg_db.close() + # Liberar TODA la VRAM tras cada generación (TTS + Whisper + LLM). + # Liberar solo el motor usado dejaba Whisper cargado y la GPU de 8 GB + # acababa en CUDA OOM al encadenar varias generaciones. + try: + from ..backends import unload_all_models + unload_all_models() + except Exception: + logger.warning("Failed to unload models after generation", exc_info=True) def _notify_speak_end(generation_id: str, *, status: str) -> None: @@ -281,6 +312,8 @@ async def generate_audio_sync( load_engine_model, ) from ..utils.chunked_tts import generate_chunked + + max_chunk_chars = effective_max_chunk_chars(engine, max_chunk_chars) from ..utils.audio import has_tts_runaway, normalize_audio, trim_tts_output from . import tts @@ -297,6 +330,13 @@ async def generate_audio_sync( ) finally: bg_db.close() + # Unload TTS model to free VRAM after speak + try: + tts_backend = get_tts_backend_for_engine(engine) + if tts_backend.is_loaded(): + tts_backend.unload_model() + except Exception: + logger.warning("Failed to unload TTS model after speak", exc_info=True) trim_fn = trim_tts_output if engine_needs_trim(engine) else None runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else None diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 000000000..0b6560fd1 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,138 @@ +"""Shared pytest fixtures and markers for the voicebox backend suite. + +Markers: + gpu — needs a local CUDA GPU; excluded from CI runs. + e2e — spawns the real backend process; excluded from CI runs. + slow — long-running checks (e.g. test_rocm_build.py's full build). + +CI-safe selection: ``python -m pytest -m "not gpu and not e2e"``. +""" + +from __future__ import annotations + +import contextlib +import os +import socket +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +HEALTH_TIMEOUT = 120 + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "gpu: requires a local CUDA GPU (not run in CI)") + config.addinivalue_line("markers", "e2e: spawns the real backend process (not run in CI)") + config.addinivalue_line("markers", "slow: long-running test (deselect with -m 'not slow')") + + +@pytest.fixture(scope="session") +def gpu_or_skip() -> bool: + """Require a CUDA-capable GPU; skip the test otherwise.""" + try: + import torch + except ImportError: + pytest.skip("torch is not installed") + if not torch.cuda.is_available(): + pytest.skip("no CUDA GPU available") + return True + + +@pytest.fixture(scope="session") +def vram_mb(): + """Return a callable reading GPU 0 used VRAM (MiB) via nvidia-smi.""" + + def _read() -> int: + out = subprocess.run( + ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], + capture_output=True, + text=True, + check=True, + ) + return int(out.stdout.strip().splitlines()[0]) + + return _read + + +def _pick_free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _wait_for_health(base_url: str, proc: subprocess.Popen, timeout: int) -> None: + import httpx + + deadline = time.time() + timeout + with httpx.Client(timeout=5.0) as client: + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"backend exited early with code {proc.returncode}") + try: + response = client.get(f"{base_url}/health") + if response.status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(1.0) + raise TimeoutError(f"backend did not become healthy within {timeout}s") + + +@pytest.fixture(scope="session") +def live_backend(tmp_path_factory): + """Spawn the real backend on a free port with an isolated data directory.""" + data_dir = tmp_path_factory.mktemp("voicebox-data") + port = _pick_free_port() + base_url = f"http://127.0.0.1:{port}" + log_path = data_dir / "server.log" + log_fh = open(log_path, "w", encoding="utf-8", errors="replace") # noqa: SIM115 + + try: + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "backend.server", + "--host", + "127.0.0.1", + "--port", + str(port), + "--data-dir", + str(data_dir), + "--parent-pid", + str(os.getpid()), + ], + cwd=str(REPO_ROOT), + stdout=log_fh, + stderr=subprocess.STDOUT, + ) + except Exception as exc: + log_fh.close() + pytest.skip(f"could not spawn backend process: {exc}") + + try: + _wait_for_health(base_url, proc, timeout=HEALTH_TIMEOUT) + except Exception as exc: + proc.kill() + proc.wait(timeout=10) + log_fh.close() + with contextlib.suppress(OSError): + tail = "\n".join(log_path.read_text(errors="replace").splitlines()[-40:]) + if "tail" not in locals(): + tail = "" + pytest.skip(f"backend did not become healthy: {exc}\n--- server log tail ---\n{tail}") + + try: + yield base_url + finally: + proc.terminate() + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(timeout=10) + if proc.poll() is None: + proc.kill() + proc.wait(timeout=5) + log_fh.close() diff --git a/backend/tests/fixtures/README.md b/backend/tests/fixtures/README.md index 341b98acb..f6ca0f515 100644 --- a/backend/tests/fixtures/README.md +++ b/backend/tests/fixtures/README.md @@ -1,16 +1,43 @@ # E2E Test Fixtures +## TTS reference voice + Place two files here before running `test_all_models_e2e.py`: - `reference_voice.wav` — a clean speech sample, mono, 16–24 kHz, ~5–15 seconds. - `reference_voice.txt` — the **exact** transcription of the WAV (single line, no trailing newline required). -These are used to create a cloned voice profile for every cloning-capable engine (qwen, luxtts, chatterbox, chatterbox_turbo, tada). Keep them out of version control if they contain personal audio — this directory is not gitignored by default, so add them to `.gitignore` locally if needed. +These are used to create a cloned voice profile for every cloning-capable engine (qwen, luxtts, chatterbox, chatterbox_turbo, tada). Keep personal audio out of version control; the directory is not gitignored by default, so add local files to `.git/info/exclude`. You can point the test at different files with: -``` +```bash python backend/tests/test_all_models_e2e.py \ --reference-wav /path/to/your.wav \ --reference-text "exact transcription here" ``` + +## Whisper long-audio fixtures + +`generate_fixtures.sh` creates optional, untracked `generated/` media from a +local Mozilla Common Voice archive. Common Voice is used only as a public +source; do not commit downloaded audio or derived media. + +```bash +bash backend/tests/fixtures/generate_fixtures.sh /path/to/common-voice/en +``` + +The generator requires `ffmpeg` and a TSV containing `path` and `sentence` +columns (defaults to `validated.tsv`). It writes `cv_10s.wav`, `cv_30s.wav`, +`cv_6m43s.{wav,flac,mp3}`, `references.tsv`, and `SHA256SUMS` under +`generated/` for local GPU evaluation. + +The long-form GPU tests are intentionally not enabled until the generated +fixtures exist. This keeps CI deterministic and prevents private/user voice +material from entering the repository. + +## Licensing + +Verify the current Common Voice dataset terms before each download. Record +the source release and checksums in local evaluation results; no dataset +archive or generated media belongs in this repository. diff --git a/backend/tests/fixtures/generate_fixtures.sh b/backend/tests/fixtures/generate_fixtures.sh new file mode 100755 index 000000000..711fda601 --- /dev/null +++ b/backend/tests/fixtures/generate_fixtures.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Generate local, untracked Whisper evaluation fixtures from Common Voice. +# +# Pass a directory containing Common Voice clips and a TSV with +# client-provided transcriptions. The source archive and derived media stay +# outside version control. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="${SCRIPT_DIR}/generated" +SOURCE_DIR="${1:-}" +TSV="${2:-${SOURCE_DIR:+${SOURCE_DIR}/validated.tsv}}" + +if [[ -z "$SOURCE_DIR" || -z "$TSV" ]]; then + cat >&2 <<'USAGE' +Usage: generate_fixtures.sh COMMON_VOICE_DIR [validated.tsv] + +COMMON_VOICE_DIR must contain Common Voice audio clips and a TSV with at least +path and sentence columns. Obtain the source archive from Mozilla Common Voice +under its current dataset terms. +USAGE + exit 2 +fi + +command -v ffmpeg >/dev/null 2>&1 || { echo "ffmpeg is required" >&2; exit 2; } +[[ -d "$SOURCE_DIR" ]] || { echo "source directory not found: $SOURCE_DIR" >&2; exit 2; } +[[ -f "$TSV" ]] || { echo "TSV not found: $TSV" >&2; exit 2; } + +mkdir -p "$OUT_DIR" + +python3 - "$TSV" "$SOURCE_DIR" "$OUT_DIR/selected.tsv" <<'PY' +import csv +import sys +from pathlib import Path + +source_tsv, source_dir, output = sys.argv[1:] +rows = [] +with open(source_tsv, newline="", encoding="utf-8") as f: + for row in csv.DictReader(f, delimiter="\t"): + path = Path(source_dir) / row["path"] + sentence = row.get("sentence", "").strip() + if path.is_file() and sentence: + rows.append((str(path), sentence)) +if not rows: + raise SystemExit("No usable Common Voice rows found") +rows.sort(key=lambda item: item[0]) +with open(output, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter="\t") + writer.writerow(["path", "sentence"]) + writer.writerows(rows) +PY + +row_count=$(( $(wc -l < "$OUT_DIR/selected.tsv") - 1 )) +(( row_count > 0 )) || { echo "No usable source clips" >&2; exit 2; } + +# Normalize a long deterministic list first. Repeating rows makes it possible +# to construct exact-duration windows even when Common Voice clips are short. +concat_list="$OUT_DIR/concat.txt" +: > "$concat_list" +: > "$OUT_DIR/cv_6m43s.txt" +for i in $(seq 1 160); do + row_number=$((2 + (i - 1) % row_count)) + row="$(sed -n "${row_number}p" "$OUT_DIR/selected.tsv")" + clip="${row%%$'\t'*}" + sentence="${row#*$'\t'}" + normalized="$OUT_DIR/clip-${i}.wav" + ffmpeg -hide_banner -loglevel error -y -i "$clip" -ac 1 -ar 16000 "$normalized" + printf "file '%s'\n" "$normalized" >> "$concat_list" + printf '%s ' "$sentence" >> "$OUT_DIR/cv_6m43s.txt" +done +printf '\n' >> "$OUT_DIR/cv_6m43s.txt" + +# The concat stream is longer than 403 seconds; -t makes all three outputs +# exactly 6:43 (or fails rather than silently producing a short fixture). +ffmpeg -hide_banner -loglevel error -y -f concat -safe 0 -i "$concat_list" \ + -t 10 -ac 1 -ar 16000 "$OUT_DIR/cv_10s.wav" +ffmpeg -hide_banner -loglevel error -y -f concat -safe 0 -i "$concat_list" \ + -t 30 -ac 1 -ar 16000 "$OUT_DIR/cv_30s.wav" +ffmpeg -hide_banner -loglevel error -y -f concat -safe 0 -i "$concat_list" \ + -t 403 -ac 1 -ar 16000 "$OUT_DIR/cv_6m43s.wav" +ffmpeg -hide_banner -loglevel error -y -i "$OUT_DIR/cv_6m43s.wav" "$OUT_DIR/cv_6m43s.flac" +ffmpeg -hide_banner -loglevel error -y -i "$OUT_DIR/cv_6m43s.wav" "$OUT_DIR/cv_6m43s.mp3" + +sha256sum "$OUT_DIR"/cv_*.{wav,flac,mp3} > "$OUT_DIR/SHA256SUMS" +printf '%s\n' "Generated from Common Voice source: $SOURCE_DIR" > "$OUT_DIR/README.generated.txt" +cp "$OUT_DIR/selected.tsv" "$OUT_DIR/references.tsv" diff --git a/backend/tests/test_fixture_contract.py b/backend/tests/test_fixture_contract.py new file mode 100644 index 000000000..47d67da7c --- /dev/null +++ b/backend/tests/test_fixture_contract.py @@ -0,0 +1,20 @@ +"""CI-safe checks for the opt-in audio fixture generator.""" + +from pathlib import Path + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_fixture_generator_is_executable_and_documented(): + generator = FIXTURES / "generate_fixtures.sh" + readme = FIXTURES / "README.md" + + assert generator.is_file() + assert generator.stat().st_mode & 0o111 + assert "Common Voice" in readme.read_text(encoding="utf-8") + assert "generated/" in readme.read_text(encoding="utf-8") + + +def test_no_generated_media_is_checked_in(): + generated = FIXTURES / "generated" + assert not generated.exists() diff --git a/backend/tests/test_generation_unload.py b/backend/tests/test_generation_unload.py new file mode 100644 index 000000000..b12428ca3 --- /dev/null +++ b/backend/tests/test_generation_unload.py @@ -0,0 +1,180 @@ +"""Regression tests for the post-generation VRAM unload. + +Bug: ``run_generation``'s ``finally`` block imported ``unload_all_models`` +from ``backend.backends`` — a function that did not exist. The ImportError +was swallowed by a bare ``except Exception``, so models were never unloaded +and chained generations accumulated VRAM until CUDA OOM. + +These tests pin both halves of the fix: ``unload_all_models`` exists and +actually unloads every registered backend, and ``run_generation`` invokes +it on the success path, the exception path, and even when it raises. +""" + +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest + +import backend.backends as backends +import backend.services.generation as generation +from backend.services import history, profiles +from backend.utils import chunked_tts + + +class _FakeBackend: + """Minimal loaded-backend stand-in for the registry tests.""" + + def __init__(self): + self.unload_calls = 0 + + def is_loaded(self) -> bool: + return True + + def unload_model(self) -> None: + self.unload_calls += 1 + + +@pytest.fixture(autouse=True) +def clean_registry(): + backends.reset_backends() + yield + backends.reset_backends() + + +def test_unload_all_models_exists(): + assert callable(backends.unload_all_models) + + +def test_unload_all_models_unloads_every_registered_backend(monkeypatch): + tts_a, tts_b = _FakeBackend(), _FakeBackend() + stt = _FakeBackend() + llm = _FakeBackend() + monkeypatch.setattr(backends, "_tts_backends", {"a": tts_a, "b": tts_b}) + monkeypatch.setattr(backends, "_stt_backend", stt) + monkeypatch.setattr(backends, "_llm_backends", {"qwen_llm": llm}) + + backends.unload_all_models() + + assert tts_a.unload_calls == 1 + assert tts_b.unload_calls == 1 + assert stt.unload_calls == 1 + assert llm.unload_calls == 1 + # Registry is empty afterwards — the factories recreate instances lazily. + assert backends._tts_backends == {} + assert backends._llm_backends == {} + assert backends._stt_backend is None + + +def test_unload_all_models_keeps_unloading_after_one_fails(monkeypatch): + broken = _FakeBackend() + broken.unload_model = MagicMock(side_effect=RuntimeError("boom")) + healthy = _FakeBackend() + monkeypatch.setattr(backends, "_tts_backends", {"broken": broken, "ok": healthy}) + + backends.unload_all_models() + + broken.unload_model.assert_called_once_with() + assert healthy.unload_calls == 1 + assert backends._tts_backends == {} + + +def test_unload_all_models_without_torch(monkeypatch): + # CPU-only / MLX installs have no torch; the guarded import must not raise. + monkeypatch.setitem(sys.modules, "torch", None) + backends.unload_all_models() + + +def test_unload_all_models_clears_cuda_and_mps_caches(monkeypatch): + cuda = SimpleNamespace(is_available=lambda: True, empty_cache=MagicMock()) + mps = SimpleNamespace(is_available=lambda: True, empty_cache=MagicMock()) + fake_torch = SimpleNamespace(cuda=cuda, mps=mps) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + + backends.unload_all_models() + + cuda.empty_cache.assert_called_once_with() + mps.empty_cache.assert_called_once_with() + + +class _RunGenerationMocks: + """Bundle of spies/stubs for driving ``run_generation`` without backends.""" + + def __init__(self, monkeypatch, *, chunk_error: Exception | None = None): + self.statuses: list[str] = [] + + fake_tts = _FakeBackend() + monkeypatch.setattr(backends, "get_tts_backend_for_engine", lambda engine: fake_tts) + + async def fake_load_engine_model(engine, model_size="default"): + return None + + monkeypatch.setattr(backends, "load_engine_model", fake_load_engine_model) + + async def fake_voice_prompt(profile_id, db, use_cache=True, engine=None): + return {} + + monkeypatch.setattr(profiles, "create_voice_prompt_for_profile", fake_voice_prompt) + + async def fake_update_status(generation_id, status, db, **kwargs): + self.statuses.append(status) + + monkeypatch.setattr(history, "update_generation_status", fake_update_status) + + async def fake_generate_chunked(model, text, voice_prompt, **kwargs): + if chunk_error is not None: + raise chunk_error + return np.zeros(2400, dtype=np.float32), 24000 + + monkeypatch.setattr(chunked_tts, "generate_chunked", fake_generate_chunked) + + monkeypatch.setattr(generation, "_save_generate", lambda **kwargs: "clean.wav") + monkeypatch.setattr( + generation, + "get_task_manager", + lambda: SimpleNamespace(complete_generation=lambda gid: None), + ) + fake_db = SimpleNamespace(close=lambda: None) + monkeypatch.setattr(generation, "get_db", lambda: iter([fake_db])) + + self.unload_spy = MagicMock() + monkeypatch.setattr(backends, "unload_all_models", self.unload_spy) + + +async def _run(**overrides): + kwargs = dict( + generation_id="gen-test", + profile_id="p1", + text="hello", + language="en", + engine="qwen", + model_size="1.7B", + seed=None, + mode="generate", + ) + kwargs.update(overrides) + await generation.run_generation(**kwargs) + + +async def test_run_generation_unloads_models_on_success(monkeypatch): + mocks = _RunGenerationMocks(monkeypatch) + await _run() + assert "completed" in mocks.statuses + mocks.unload_spy.assert_called_once_with() + + +async def test_run_generation_unloads_models_on_exception(monkeypatch): + mocks = _RunGenerationMocks(monkeypatch, chunk_error=RuntimeError("inference boom")) + await _run() + assert "failed" in mocks.statuses + mocks.unload_spy.assert_called_once_with() + + +async def test_run_generation_survives_unload_failure(monkeypatch): + mocks = _RunGenerationMocks(monkeypatch) + mocks.unload_spy.side_effect = RuntimeError("unload boom") + # The except in run_generation's finally must log and swallow. + await _run() + assert "completed" in mocks.statuses + mocks.unload_spy.assert_called_once_with() diff --git a/backend/tests/test_mcp_mount_slashes.py b/backend/tests/test_mcp_mount_slashes.py new file mode 100644 index 000000000..1e25e366f --- /dev/null +++ b/backend/tests/test_mcp_mount_slashes.py @@ -0,0 +1,67 @@ +"""Regression tests for the MCP mount's optional trailing slash. + +Starlette strips a bare ``/mcp`` mount to an empty child path, while +FastMCP's HTTP app routes the endpoint at ``/``. The wrapper must normalize +that empty path without redirecting a POST request. +""" + +from __future__ import annotations + +import json + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +from backend.mcp_server.server import ( + MountRootSlashRewrite, + build_mcp_server, + compose_lifespan, +) + +INITIALIZE = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "voicebox-regression", "version": "1.0"}, + }, +} +HEADERS = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", +} + + +def _build_mcp_only_app() -> FastAPI: + """Build only the MCP surface so this test avoids Voicebox model startup.""" + mcp_app = build_mcp_server().http_app(path="/", transport="http") + app = FastAPI(lifespan=compose_lifespan(mcp_app.router.lifespan_context)) + app.mount("/mcp", MountRootSlashRewrite(mcp_app)) + return app + + +def _response_payload(response) -> dict: + """Decode either FastMCP's JSON or SSE response representation.""" + content_type = response.headers.get("content-type", "") + if "application/json" in content_type: + return response.json() + + for line in response.text.splitlines(): + if line.startswith("data: "): + return json.loads(line[6:]) + raise AssertionError(f"initialize response was not JSON/SSE: {response.text!r}") + + +@pytest.mark.parametrize("path", ["/mcp", "/mcp/"]) +def test_mcp_initialize_accepts_both_slashes(path: str): + with TestClient(_build_mcp_only_app()) as client: + response = client.post(path, json=INITIALIZE, headers=HEADERS) + + assert response.status_code == 200, response.text + payload = _response_payload(response) + assert payload["jsonrpc"] == "2.0" + assert payload["id"] == 1 + assert payload["result"]["serverInfo"]["name"] == "voicebox" diff --git a/backend/tests/test_tada_chunk_cap.py b/backend/tests/test_tada_chunk_cap.py new file mode 100644 index 000000000..0c8d58a70 --- /dev/null +++ b/backend/tests/test_tada_chunk_cap.py @@ -0,0 +1,33 @@ +"""Unit tests for per-engine chunk-size resolution in generation. + +Regression: tada degenerates on long inputs (a 650-char paragraph yielded +a 1.0s clip containing only "y"). GenerationRequest defaults +max_chunk_chars=800, so the cap must apply even when the caller passes +the default value explicitly. +""" + +import pytest + +from backend.services.generation import ( + TADA_MAX_CHUNK_CHARS, + effective_max_chunk_chars, +) + + +def test_tada_none_gets_cap(): + assert effective_max_chunk_chars("tada", None) == TADA_MAX_CHUNK_CHARS + + +def test_tada_default_800_is_capped(): + # The request default (800) must not bypass the tada cap. + assert effective_max_chunk_chars("tada", 800) == TADA_MAX_CHUNK_CHARS + + +def test_tada_smaller_explicit_value_respected(): + assert effective_max_chunk_chars("tada", 100) == 100 + + +def test_other_engines_untouched(): + assert effective_max_chunk_chars("qwen", 800) == 800 + assert effective_max_chunk_chars("kokoro", 800) == 800 + assert effective_max_chunk_chars("qwen", None) is None diff --git a/backend/tests/test_tts_vram_churn.py b/backend/tests/test_tts_vram_churn.py new file mode 100644 index 000000000..4e228501a --- /dev/null +++ b/backend/tests/test_tts_vram_churn.py @@ -0,0 +1,85 @@ +"""Opt-in GPU checks for TADA long text and post-generation cleanup. + +The script-style all-model harness owns profile setup. These tests provide a +small, repeatable endpoint contract for a locally prepared profile: + + VOICEBOX_EVAL_PROFILE_ID= \ + python -m pytest backend/tests/test_tts_vram_churn.py -m 'gpu and e2e' +""" + +from __future__ import annotations + +import asyncio +import json +import os +import time + +import httpx +import pytest + +pytestmark = [pytest.mark.gpu, pytest.mark.e2e] + +SHORT_TEXT = "This is a short deterministic Voicebox evaluation sentence." +LONG_TEXT = ("This is a public deterministic evaluation sentence. " * 40).strip() + + +def _profile_id() -> str: + value = os.environ.get("VOICEBOX_EVAL_PROFILE_ID") + if not value: + pytest.skip("set VOICEBOX_EVAL_PROFILE_ID to a locally prepared cloned profile") + return value + + +async def _wait_for_generation(client: httpx.AsyncClient, base_url: str, generation_id: str) -> dict: + deadline = time.monotonic() + 900 + while time.monotonic() < deadline: + response = await client.get(f"{base_url}/generate/{generation_id}/status") + response.raise_for_status() + payload = None + for line in response.text.splitlines(): + if line.startswith("data: "): + payload = json.loads(line[6:]) + break + if payload and payload.get("status") in {"completed", "failed"}: + return payload + await asyncio.sleep(1) + pytest.fail(f"generation {generation_id} did not finish within 900 seconds") + + +async def _generate( + client: httpx.AsyncClient, + base_url: str, + profile_id: str, + engine: str, + text: str, +) -> dict: + response = await client.post( + f"{base_url}/generate", + json={ + "profile_id": profile_id, + "text": text, + "language": "en", + "engine": engine, + "model_size": "1B" if engine == "tada" else "1.7B", + }, + ) + response.raise_for_status() + return await _wait_for_generation(client, base_url, response.json()["id"]) + + +async def test_alternating_generations_finish_and_release_vram(live_backend: str, gpu_or_skip, vram_mb): + profile_id = _profile_id() + baseline = vram_mb() + async with httpx.AsyncClient(timeout=900.0) as client: + for engine in ("tada", "qwen", "tada", "qwen", "tada"): + result = await _generate(client, live_backend, profile_id, engine, SHORT_TEXT) + assert result["status"] == "completed", result + assert vram_mb() - baseline < 500, f"VRAM retained after {engine}: {vram_mb()} MiB" + + +async def test_tada_long_text_not_truncated(live_backend: str, gpu_or_skip): + profile_id = _profile_id() + async with httpx.AsyncClient(timeout=900.0) as client: + result = await _generate(client, live_backend, profile_id, "tada", LONG_TEXT) + assert result["status"] == "completed", result + assert result.get("duration", 0) > 30, result diff --git a/backend/tests/test_whisper_chunking.py b/backend/tests/test_whisper_chunking.py new file mode 100644 index 000000000..6d754b8de --- /dev/null +++ b/backend/tests/test_whisper_chunking.py @@ -0,0 +1,96 @@ +"""Unit tests for long-audio chunking in the PyTorch STT backend. + +Regression: Whisper's encoder only accepts 30s of audio per pass. Before +chunking, a 6:43 upload silently transcribed just the first 30 seconds. +Also guards the ``return_timestamps=True`` flag, which prevents Whisper +from emitting an early <|endoftext|> after the first sentence of a +window (observed: multi-sentence chunks truncated to one sentence). + +These tests mock the HF processor/model, so they run without GPU or +model downloads. +""" + +import asyncio +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest + +from backend.backends.pytorch_backend import PyTorchSTTBackend + + +class _FakeInputs(dict): + """Mimics the transformers BatchFeature mapping (has .to()).""" + + def to(self, device=None, dtype=None): + return self + + +def _make_backend(chunk_outputs): + """Build a PyTorchSTTBackend with mocked processor/model.""" + backend = PyTorchSTTBackend.__new__(PyTorchSTTBackend) + backend.device = "cpu" + backend.model_size = "small" + + outputs = list(chunk_outputs) + + processor = MagicMock() + processor.side_effect = lambda audio, sampling_rate, return_tensors: _FakeInputs( + input_features=np.zeros((1, 80, 3000), dtype=np.float32) + ) + processor.batch_decode.side_effect = lambda ids, skip_special_tokens: [ + outputs.pop(0) + ] + processor.get_decoder_prompt_ids.return_value = [(1, 50257)] + backend.processor = processor + + model = MagicMock() + model.dtype = "float32" + model.generate.side_effect = lambda feats, **kw: np.array([[0]]) + backend.model = model + return backend + + +def _transcribe(backend, seconds: float, language="es"): + pcm = np.zeros(int(seconds * 16000), dtype=np.float32) + with ( + patch( + "backend.backends.pytorch_backend.load_audio", + return_value=(pcm, 16000), + ), + patch.object( + PyTorchSTTBackend, "load_model_async", return_value=None + ), + patch.object(PyTorchSTTBackend, "is_loaded", return_value=True), + ): + return asyncio.run(backend.transcribe("dummy.wav", language=language)) + + +def test_short_audio_single_chunk(): + backend = _make_backend(["hola mundo"]) + text = _transcribe(backend, seconds=10) + assert text == "hola mundo" + assert backend.model.generate.call_count == 1 + + +def test_long_audio_is_chunked_and_joined(): + backend = _make_backend(["primera parte", "segunda parte", "tercera parte"]) + text = _transcribe(backend, seconds=70) # 3 chunks of 30s + assert backend.model.generate.call_count == 3 + assert text == "primera parte segunda parte tercera parte" + + +def test_timestamps_forced_to_prevent_early_eot(): + backend = _make_backend(["a", "b"]) + _transcribe(backend, seconds=45) + for call in backend.model.generate.call_args_list: + assert call.kwargs.get("return_timestamps") is True + + +def test_language_forcing_still_passed(): + backend = _make_backend(["texto"]) + _transcribe(backend, seconds=5, language="es") + backend.processor.get_decoder_prompt_ids.assert_called_once_with( + language="es", task="transcribe" + ) + assert "forced_decoder_ids" in backend.model.generate.call_args.kwargs diff --git a/backend/tests/test_whisper_long_audio_e2e.py b/backend/tests/test_whisper_long_audio_e2e.py new file mode 100644 index 000000000..2422ae75b --- /dev/null +++ b/backend/tests/test_whisper_long_audio_e2e.py @@ -0,0 +1,64 @@ +"""Opt-in Whisper quality checks for generated Common Voice fixtures. + +Run locally after preparing fixtures and downloading the selected Whisper model: + + python -m pytest backend/tests/test_whisper_long_audio_e2e.py -m 'gpu and e2e' + +These tests deliberately do not download data or models themselves. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import httpx +import pytest + +pytestmark = [pytest.mark.gpu, pytest.mark.e2e] + +FIXTURES = Path(__file__).parent / "fixtures" / "generated" +AUDIO_FILES = [ + "cv_10s.wav", + "cv_30s.wav", + "cv_6m43s.wav", + "cv_6m43s.flac", + "cv_6m43s.mp3", +] + + +def _normalize(text: str) -> str: + return re.sub(r"\s+", " ", text.casefold()).strip() + + +def _require_fixture(name: str) -> Path: + path = FIXTURES / name + if not path.is_file(): + pytest.skip(f"missing local evaluation fixture: {path}") + return path + + +async def _transcribe(base_url: str, path: Path) -> dict: + async with httpx.AsyncClient(timeout=900.0) as client: + with path.open("rb") as audio: + response = await client.post( + f"{base_url}/transcribe", + files={"file": (path.name, audio, "audio/wav")}, + data={"language": "en", "model": "small"}, + ) + if response.status_code == 202: + pytest.fail("Whisper model is not cached; download it in Voicebox before running GPU evaluation") + response.raise_for_status() + return response.json() + + +@pytest.mark.parametrize("filename", AUDIO_FILES) +async def test_whisper_transcribes_each_duration_and_format(filename: str, live_backend: str, gpu_or_skip): + path = _require_fixture(filename) + result = await _transcribe(live_backend, path) + + assert _normalize(result["text"]), f"empty transcription for {filename}" + if filename == "cv_30s.wav": + assert 29.0 <= result["duration"] <= 31.0 + if filename.startswith("cv_6m43s"): + assert result["duration"] >= 400.0 diff --git a/bun.lock b/bun.lock index 74d751ec1..4d85083fe 100644 --- a/bun.lock +++ b/bun.lock @@ -11,6 +11,9 @@ "devDependencies": { "@biomejs/biome": "2.3.12", "@types/node": "^20.0.0", + "husky": "^9.1.7", + "lint-staged": "^17.3.0", + "prettier": "^3.9.6", "tailwindcss": "^4.1.18", "typescript": "^5.6.0", }, @@ -829,6 +832,8 @@ "html-parse-stringify": ["html-parse-stringify@3.0.1", "", { "dependencies": { "void-elements": "3.1.0" } }, "sha512-KknJ50kTInJ7qIScF3jeaFRpMpE8/lfiTdzf/twXyPBLAGrLRTmkz3AdTnKeh40X8k9L2fdYwEp/42WGXIRGcg=="], + "husky": ["husky@9.1.7", "", { "bin": { "husky": "bin.js" } }, "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA=="], + "i18next": ["i18next@26.0.6", "", { "dependencies": { "@babel/runtime": "^7.29.2" }, "peerDependencies": { "typescript": "^5 || ^6" }, "optionalPeers": ["typescript"] }, "sha512-A4U6eCXodIbrhf8EarRurB9/4ebyaurH4+fu4gig9bqxmpSt+fCAFm/GpRQDcN1Xzu/LdFCx4nYHsnM1edIIbg=="], "i18next-browser-languagedetector": ["i18next-browser-languagedetector@8.2.1", "", { "dependencies": { "@babel/runtime": "^7.23.2" } }, "sha512-bZg8+4bdmaOiApD7N7BPT9W8MLZG+nPTOFlLiJiT8uzKXFjhxw4v2ierCXOwB5sFDMtuA5G4kgYZ0AznZxQ/cw=="], @@ -911,6 +916,8 @@ "lines-and-columns": ["lines-and-columns@1.2.4", "", {}, "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="], + "lint-staged": ["lint-staged@17.3.0", "", { "dependencies": { "picomatch": "^4.0.5", "string-argv": "^0.3.2", "tinyexec": "^1.2.4" }, "optionalDependencies": { "yaml": "^2.9.0" }, "bin": { "lint-staged": "bin/lint-staged.js" } }, "sha512-woZS3vNe3UKqBaLPvbLOtKRY4tLANpWQhom12MGWqC8Mh1lCOO+WgSwmX2amjJAqTY9BkXYW87fCUH5H9Ph6xw=="], + "loaders.css": ["loaders.css@0.1.2", "", {}, "sha512-Rhowlq24ey1VOeor+3wYOt9+MjaxBOJm1u4KlQgNC3+0xJ0LS4wq4iG57D/BPzvuD/7HHDGQOWJ+81oR2EI9bQ=="], "locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="], @@ -979,7 +986,7 @@ "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - "picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], "pify": ["pify@2.3.0", "", {}, "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog=="], @@ -1001,6 +1008,8 @@ "prelude-ls": ["prelude-ls@1.2.1", "", {}, "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g=="], + "prettier": ["prettier@3.9.6", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g=="], + "prop-types": ["prop-types@15.8.1", "", { "dependencies": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", "react-is": "^16.13.1" } }, "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg=="], "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], @@ -1069,6 +1078,8 @@ "sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="], + "string-argv": ["string-argv@0.3.2", "", {}, "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q=="], + "strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], "strip-bom-string": ["strip-bom-string@1.0.0", "", {}, "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g=="], @@ -1101,6 +1112,8 @@ "tiny-warning": ["tiny-warning@1.0.3", "", {}, "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA=="], + "tinyexec": ["tinyexec@1.3.0", "", {}, "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ=="], + "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], @@ -1145,6 +1158,8 @@ "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], + "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], + "yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], @@ -1205,16 +1220,22 @@ "@voicebox/web/wavesurfer.js": ["wavesurfer.js@7.12.1", "", {}, "sha512-NswPjVHxk0Q1F/VMRemCPUzSojjuHHisQrBqQiRXg7MVbe3f5vQ6r0rTTXA/a/neC/4hnOEC4YpXca4LpH0SUg=="], + "anymatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "chokidar/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "eslint/js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], "fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], + "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "motion/framer-motion": ["framer-motion@12.29.0", "", { "dependencies": { "motion-dom": "^12.29.0", "motion-utils": "^12.27.2", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-1gEFGXHYV2BD42ZPTFmSU9buehppU+bCuOnHU0AD18DKh9j4DuTx47MvqY5ax+NNWRtK32qIcJf1UxKo1WwjWg=="], "next/postcss": ["postcss@8.4.31", "", { "dependencies": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } }, "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ=="], + "readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "sharp/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], "tinyglobby/picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], diff --git a/docs/evaluation/TEMPLATE.md b/docs/evaluation/TEMPLATE.md new file mode 100644 index 000000000..6a19ccc45 --- /dev/null +++ b/docs/evaluation/TEMPLATE.md @@ -0,0 +1,30 @@ +# + +## 1. Origen del problema + +- Cómo se descubrió. +- Síntomas concretos: mensajes, logs y comportamiento observable. +- Reproducción mínima en la versión sin el fix. + +## 2. Causa raíz técnica + +- Archivo y línea aproximada. +- Mecanismo que produce el fallo. +- Por qué los tests existentes no lo detectaban. + +## 3. Fix aplicado + +- Diff conceptual. +- Decisiones tomadas y alternativas descartadas. + +## 4. Verificación + +- Tests ejecutados, con comandos copiables. +- Salida esperada y salida observada. +- Criterio de regresión: qué volvería a fallar si se revierte el fix. + +## 5. Estado upstream + +- PR o commit relacionado. +- Notas para el maintainer. +- Riesgos y rollback. diff --git a/docs/evaluation/appimage-backend-detection.md b/docs/evaluation/appimage-backend-detection.md new file mode 100644 index 000000000..27476ec38 --- /dev/null +++ b/docs/evaluation/appimage-backend-detection.md @@ -0,0 +1,34 @@ +# Detección de backend de la AppImage + +## 1. Origen del problema + +La aplicación debe reutilizar un backend Voicebox ya activo, rechazar un puerto +ocupado por otra aplicación y arrancar el sidecar cuando el puerto está libre. + +## 2. Causa raíz técnica + +La decisión de Tauri combina el proceso que escucha en el puerto 17493 con una +comprobación del contrato JSON de `/health`. Un test que solo comprobara si el +puerto está libre no distinguiría un backend Voicebox de un servicio ajeno. + +## 3. Fix aplicado + +`test_backend_detection.sh` reproduce el contrato de `main.rs` en tres casos: +backend activo, listener no-Voicebox y puerto libre. Usa un puerto alternativo +mediante `VOICEBOX_PORT` para no interrumpir una instalación activa. + +## 4. Verificación + +```bash +VOICEBOX_PORT=18493 bash scripts/test_backend_detection.sh --case B +VOICEBOX_PORT=18493 bash scripts/test_backend_detection.sh --case C +``` + +El caso A se ejecuta cuando existe un backend real en el puerto seleccionado. +El script no lanza ni mata la AppImage: valida el contrato de decisión y la +presencia del sidecar/build path. + +## 5. Estado upstream + +Pendiente de ejecución con Tauri/AppImage real. El harness es una prueba de +contrato, no reemplaza una prueba visual o de proceso de la aplicación. diff --git a/docs/evaluation/local-setup.md b/docs/evaluation/local-setup.md new file mode 100644 index 000000000..9fa7549b9 --- /dev/null +++ b/docs/evaluation/local-setup.md @@ -0,0 +1,26 @@ +# Runbook local de dictado + +Este flujo es local y no forma parte de la instalación upstream: usa el +script de usuario `~/.local/bin/voicebox-dictate.sh` y herramientas del +entorno gráfico Linux. + +## Preparación + +- Backend Voicebox saludable en `127.0.0.1:17493`. +- Script de dictado instalado en `~/.local/bin/voicebox-dictate.sh`. +- Herramientas disponibles según la sesión: `arecord` o `pw-record`, + `curl`, `wl-copy`, `ydotool` y `notify-send`. + +## Harness + +```bash +VOICEBOX_DICTATE_SCRIPT="$HOME/.local/bin/voicebox-dictate.sh" \ + bash scripts/test_dictate_e2e.sh +``` + +El harness usa stubs y un `HOME` temporal. No modifica la caché de Voicebox, +el portapapeles real ni el script instalado. Valida la cadena de grabación, +transcripción, copia y pegado, además de fallos de cada etapa. + +La prueba no puede validar permisos reales de Wayland, disponibilidad de +`ydotoold`, audio físico ni interacción visual de una sesión de escritorio. diff --git a/docs/evaluation/mcp-trailing-slash.md b/docs/evaluation/mcp-trailing-slash.md new file mode 100644 index 000000000..137ba246a --- /dev/null +++ b/docs/evaluation/mcp-trailing-slash.md @@ -0,0 +1,36 @@ +# Compatibilidad MCP con `/mcp` y `/mcp/` + +## 1. Origen del problema + +Clientes MCP que configuraban la URL sin barra final recibían `405 Method Not +Allowed` al enviar `POST /mcp`; el mismo handshake funcionaba en +`POST /mcp/`. + +## 2. Causa raíz técnica + +FastMCP registra su endpoint interno en `/`. Al montarlo bajo `/mcp`, +Starlette entrega la ruta vacía al sub-application cuando se usa el mount root +sin barra. Esa ruta no coincide con `/` y el POST no llega al handler MCP. + +## 3. Fix aplicado + +`MountRootSlashRewrite` normaliza una ruta interna vacía a `/` antes de delegar +en FastMCP. Se usa tanto en `backend/app.py` como en `mount_into()`. No se usa +un redirect 307 porque algunos clientes MCP no repiten POST tras redirects. + +La documentación y `.mcp.json` usan `/mcp/`, mientras el servidor acepta ambas +formas. + +## 4. Verificación + +```bash +python -m pytest backend/tests/test_mcp_mount_slashes.py -q +``` + +El test monta solo FastMCP, activa su lifespan mediante `TestClient` como +context manager, y envía el handshake `initialize` a `/mcp` y `/mcp/`. + +## 5. Estado upstream + +Pendiente de ejecutar con las dependencias FastAPI/FastMCP del proyecto y de +probar contra el cliente MCP real. El test no necesita GPU ni modelos. diff --git a/docs/evaluation/tada-chunk-cap.md b/docs/evaluation/tada-chunk-cap.md new file mode 100644 index 000000000..491a67242 --- /dev/null +++ b/docs/evaluation/tada-chunk-cap.md @@ -0,0 +1,31 @@ +# Límite de chunks para TADA + +## 1. Origen del problema + +Textos largos para TADA podían degenerar en audio de aproximadamente un +segundo o salida incompleta porque el valor general de 800 caracteres era +demasiado alto para el modelo. + +## 2. Causa raíz técnica + +La petición de generación aplica el valor por defecto global antes de llamar a +la utilidad de TTS chunked. TADA necesita ventanas más pequeñas. + +## 3. Fix aplicado + +`effective_max_chunk_chars()` fuerza un máximo de 250 caracteres para TADA, +respeta valores explícitos inferiores y deja intactos los demás motores. + +## 4. Verificación + +```bash +python -m pytest backend/tests/test_tada_chunk_cap.py -q +``` + +La prueba cubre `None`, el default 800, un valor menor y motores no-TADA. +La duración real de 2.000 caracteres y la ausencia de truncado requieren una +prueba GPU con el modelo TADA descargado. + +## 5. Estado upstream + +Unit test apto para CI; validación acústica pendiente de hardware/modelo. diff --git a/docs/evaluation/vram-unload.md b/docs/evaluation/vram-unload.md new file mode 100644 index 000000000..681a68e61 --- /dev/null +++ b/docs/evaluation/vram-unload.md @@ -0,0 +1,48 @@ +# Liberación de modelos después de una generación + +## 1. Origen del problema + +La rama de VRAM llamaba a `unload_all_models()` desde +`backend/services/generation.py`, pero la función no existía en el registro de +backends. El `ImportError` quedaba oculto por el `except Exception` del bloque +`finally`; por tanto, una generación completada no liberaba los modelos. + +## 2. Causa raíz técnica + +`run_generation()` importaba un símbolo inexistente en +`backend/backends/__init__.py`. La falta de un test que importara y ejercitara +ese símbolo permitió que el fix pareciera correcto mientras era un no-op. + +## 3. Fix aplicado + +`unload_all_models()` ahora: + +1. Recorre los backends TTS, STT y LLM registrados. +2. Llama a `unload_model()` de cada instancia, sin detenerse si una falla. +3. Elimina los registros para que las fábricas creen instancias nuevas bajo + demanda. +4. Vacía las cachés CUDA y MPS cuando están disponibles. +5. Deduplica referencias singleton para no descargar dos veces la misma + instancia. + +El `finally` de `run_generation()` conserva el aislamiento del cleanup: un +fallo al descargar no cambia el resultado ya persistido de la generación. + +## 4. Verificación + +```bash +python -m pytest backend/tests/test_generation_unload.py -q +``` + +La prueba cubre existencia, descarga de TTS/STT/LLM, continuidad después de un +backend defectuoso, cachés CUDA/MPS y llamadas desde `run_generation()` tanto +con éxito como con excepción. + +La ejecución en este checkout requiere instalar las dependencias de +`backend/requirements.txt`; el Python global no tiene SQLAlchemy. + +## 5. Estado upstream + +Fix local pendiente de revisión y de validación en una máquina con GPU. No se +considera evidencia suficiente para afirmar el umbral de VRAM de 500 MiB sin +la suite GPU real. diff --git a/docs/evaluation/whisper-chunking.md b/docs/evaluation/whisper-chunking.md new file mode 100644 index 000000000..149c49a8d --- /dev/null +++ b/docs/evaluation/whisper-chunking.md @@ -0,0 +1,37 @@ +# Transcripción Whisper de audio largo + +## 1. Origen del problema + +Los audios de varios minutos se procesaban como una sola inferencia, aunque el +encoder de Whisper está limitado a ventanas de aproximadamente 30 segundos. +El resultado podía contener solo el inicio del audio. + +## 2. Causa raíz técnica + +La ruta PyTorch de STT no dividía el PCM largo en ventanas antes de llamar a +`model.generate()`. Además, una ventana podía terminar prematuramente si no se +solicitaban timestamps. + +## 3. Fix aplicado + +El backend divide el audio en ventanas de 30 segundos, une las transcripciones +con espacios, fuerza `return_timestamps=True` y conserva el idioma mediante +`forced_decoder_ids`. + +## 4. Verificación + +```bash +python -m pytest backend/tests/test_whisper_chunking.py -q +``` + +La prueba unitaria usa processor/model falsos y cubre audio corto, tres +ventanas para 70 segundos, timestamps y selección de idioma. + +La matriz de calidad WER para audio real de 6:43, formatos WAV/FLAC/MP3 y +carga en frío/caliente requiere `pytest -m gpu` con fixtures locales generadas +por `backend/tests/fixtures/generate_fixtures.sh`. + +## 5. Estado upstream + +La regresión unitaria es apta para CI; la evidencia de WER y latencia queda +pendiente de hardware y modelos descargados. diff --git a/justfile b/justfile index 877f15196..2b0fdf609 100644 --- a/justfile +++ b/justfile @@ -325,6 +325,38 @@ fix-python: _ensure-venv test: _ensure-venv {{ venv_bin }}/python -m pytest {{ backend_dir }}/tests -v +# Run deterministic CI-safe tests (excludes GPU, real-server E2E and slow builds) +[unix] +test-ci: _ensure-venv + {{ venv_bin }}/python -m pytest {{ backend_dir }}/tests -m "not gpu and not e2e and not slow" -v + +[windows] +test-ci: _ensure-venv + & "{{ python }}" -m pytest {{ backend_dir }}/tests -m "not gpu and not e2e and not slow" -v + +# Run local evaluation harnesses; GPU/model tests remain opt-in via --gpu. +[unix] +eval-local: _ensure-venv + #!/usr/bin/env bash + set -euo pipefail + {{ venv_bin }}/python -m pytest {{ backend_dir }}/tests -m "not e2e and not slow" -v + bash scripts/test_backend_detection.sh + if [[ -f "${VOICEBOX_DICTATE_SCRIPT:-$HOME/.local/bin/voicebox-dictate.sh}" ]]; then + VOICEBOX_DICTATE_SCRIPT="${VOICEBOX_DICTATE_SCRIPT:-$HOME/.local/bin/voicebox-dictate.sh}" bash scripts/test_dictate_e2e.sh + else + echo "Skipping dictation harness: no installed script found." + fi + if [[ "${GPU:-0}" == "1" ]]; then + {{ venv_bin }}/python -m pytest {{ backend_dir }}/tests -m gpu -v + else + echo "Skipping GPU suite. Re-run with GPU=1 after model/fixture setup." + fi + +[windows] +eval-local: _ensure-venv + & "{{ python }}" -m pytest {{ backend_dir }}/tests -m "not e2e and not slow" -v + Write-Host "Run scripts/test_backend_detection.sh separately under a Unix shell." + # E2E: generate with every TTS model against the frozen binary (pass extra flags like --only kokoro) [unix] test-models *ARGS: _ensure-venv diff --git a/package.json b/package.json index ba9274f28..37ea7b1f4 100644 --- a/package.json +++ b/package.json @@ -30,11 +30,15 @@ "format:check": "biome format .", "check": "biome check .", "check:fix": "biome check --write .", - "ci": "bun run typecheck && bun run build:web" + "ci": "bun run typecheck && bun run build:web", + "prepare": "husky" }, "devDependencies": { "@biomejs/biome": "2.3.12", "@types/node": "^20.0.0", + "husky": "^9.1.7", + "lint-staged": "^17.3.0", + "prettier": "^3.9.6", "tailwindcss": "^4.1.18", "typescript": "^5.6.0" }, diff --git a/scripts/test_backend_detection.sh b/scripts/test_backend_detection.sh new file mode 100755 index 000000000..f76a1b3f2 --- /dev/null +++ b/scripts/test_backend_detection.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +# test_backend_detection.sh — validates the backend-detection contract that +# tauri/src-tauri/src/main.rs implements in start_server() (Unix path): +# +# Case A: a live Voicebox backend already listens on 17493 +# -> lsof shows a listener; if its command name contains "voicebox" +# the app reuses it directly (and stores the PID); otherwise the +# app GETs /health and reuses the server only if the JSON matches +# {status:"healthy", model_loaded:, gpu_available:}. +# Case B: the port is held by a NON-voicebox process whose /health does not +# match the schema -> the app refuses with "Port 17493 is already +# in use by another application (...)". This script starts its own +# dummy listener (python3 -m http.server) to exercise this. +# Case C: nothing listens on the port -> the app spawns the bundled sidecar +# (binaries/voicebox-server, produced by `bun run build:server`, +# or the GPU onedir backends under /backends/{rocm,cuda}). +# +# Usage: +# scripts/test_backend_detection.sh # auto-detect applicable case +# scripts/test_backend_detection.sh --case A|B|C +# +# Dependencies: bash, curl, lsof, python3 (jq optional — python3 is the +# fallback JSON validator). Exit code 0 = all executed checks passed. +set -euo pipefail + +# VOICEBOX_PORT override exists so the destructive-ish cases (B) can be +# rehearsed on a spare port without stopping a live production backend. +PORT="${VOICEBOX_PORT:-17493}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FORCED_CASE="" +DUMMY_PID="" + +PASS=0 +FAIL=0 + +cleanup() { + if [[ -n "$DUMMY_PID" ]] && kill -0 "$DUMMY_PID" 2>/dev/null; then + kill "$DUMMY_PID" 2>/dev/null || true + wait "$DUMMY_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +ok() { echo " PASS: $*"; PASS=$((PASS + 1)); } +bad() { echo " FAIL: $*"; FAIL=$((FAIL + 1)); } +info() { echo " -> $*"; } + +usage() { + sed -n '2,24p' "$0" + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --case) + [[ $# -ge 2 ]] || usage 1 + FORCED_CASE="${2^^}" + [[ "$FORCED_CASE" =~ ^[ABC]$ ]] || { echo "Invalid case: $2" >&2; usage 1; } + shift 2 + ;; + -h|--help) usage 0 ;; + *) echo "Unknown argument: $1" >&2; usage 1 ;; + esac +done + +for cmd in curl lsof python3; do + command -v "$cmd" >/dev/null 2>&1 || { echo "Missing dependency: $cmd" >&2; exit 2; } +done + +# --- helpers mirroring the Rust logic ------------------------------------- + +# First listener row on the port: prints " " or nothing. +port_listener() { + # `|| true`: lsof exits 1 when nothing matches; with `set -o pipefail` + # that would otherwise abort callers running outside a condition. + { lsof -i ":$PORT" -sTCP:LISTEN 2>/dev/null || true; } | awk 'NR>1 && NF>=2 {print $1, $2; exit}' +} + +# Schema check identical to check_health() in main.rs: +# status == "healthy" AND model_loaded is bool AND gpu_available is bool +health_matches_voicebox_schema() { + local body="$1" + [[ -n "$body" ]] || return 1 + if command -v jq >/dev/null 2>&1; then + jq -e ' + .status == "healthy" + and (.model_loaded | type == "boolean") + and (.gpu_available | type == "boolean") + ' >/dev/null 2>&1 <<<"$body" + else + python3 -c ' +import sys, json +try: + d = json.loads(sys.argv[1]) +except Exception: + sys.exit(1) +ok = (isinstance(d, dict) and d.get("status") == "healthy" + and isinstance(d.get("model_loaded"), bool) + and isinstance(d.get("gpu_available"), bool)) +sys.exit(0 if ok else 1) +' "$body" 2>/dev/null + fi +} + +# --- cases ------------------------------------------------------------------ + +case_a() { + echo "CASE A: live backend already listening on port $PORT" + local row command pid + row="$(port_listener)" + if [[ -z "$row" ]]; then + bad "no process listening on port $PORT (case A not applicable right now)" + return + fi + command="${row%% *}" + pid="${row##* }" + ok "lsof shows listener on :$PORT (command='$command' pid=$pid)" + + local body + body="$(curl -s --max-time 3 "http://127.0.0.1:$PORT/health" || true)" + if health_matches_voicebox_schema "$body"; then + ok "/health matches voicebox schema (status=healthy, model_loaded/gpu_available booleans)" + else + bad "/health does NOT match voicebox schema; body: ${body:0:200}" + return + fi + + if [[ "$command" == *voicebox* ]]; then + info "command name contains 'voicebox' -> app reuses it directly and stores PID $pid (main.rs:301)" + else + info "command '$command' is not a voicebox binary, but the health check passed -> app reuses this external server (main.rs:312)" + fi + info "app decision: REUSE http://127.0.0.1:$PORT (no sidecar spawned)" +} + +case_b() { + echo "CASE B: port $PORT held by a NON-voicebox process" + if [[ -n "$(port_listener)" ]]; then + bad "port $PORT is already in use; cannot start dummy listener (stop the backend or run without --case B when it is up)" + return + fi + + python3 -m http.server "$PORT" --bind 127.0.0.1 >/dev/null 2>&1 & + DUMMY_PID=$! + local i + for i in {1..50}; do + [[ -n "$(port_listener)" ]] && break + sleep 0.1 + done + + local row command + row="$(port_listener)" + if [[ -z "$row" ]]; then + bad "dummy listener (python3 -m http.server) failed to bind port $PORT" + return + fi + command="${row%% *}" + ok "dummy listener up (command='$command' pid=$DUMMY_PID)" + if [[ "$command" == *voicebox* ]]; then + bad "dummy command name unexpectedly contains 'voicebox'" + return + fi + ok "command name '$command' does not contain 'voicebox' -> app falls through to the health check" + + local body + body="$(curl -s --max-time 3 "http://127.0.0.1:$PORT/health" || true)" + if health_matches_voicebox_schema "$body"; then + bad "dummy server's /health unexpectedly MATCHES the voicebox schema" + else + ok "dummy server's /health fails schema validation (as expected)" + info "app decision: REFUSE with 'Port $PORT is already in use by another application ($command)' (main.rs:317)" + fi + + cleanup + DUMMY_PID="" +} + +case_c() { + echo "CASE C: nothing listening on port $PORT -> app would spawn the bundled sidecar" + if [[ -n "$(port_listener)" ]]; then + info "note: port $PORT is currently busy, so the app would not reach the spawn path right now; checking sidecar availability anyway" + else + ok "port $PORT is free" + fi + + local found="" + local candidate + for candidate in "$REPO_ROOT"/tauri/src-tauri/binaries/voicebox-server*; do + [[ -e "$candidate" ]] && { found="$candidate"; break; } + done + + if [[ -n "$found" ]]; then + ok "bundled sidecar exists: $found" + elif [[ -x "$REPO_ROOT/scripts/build-server.sh" ]]; then + ok "sidecar not present in tauri/src-tauri/binaries/ (dev tree) — it is produced by 'bun run build:server' (scripts/build-server.sh) and bundled via tauri.conf.json externalBin" + else + bad "no sidecar binary and scripts/build-server.sh missing" + return + fi + info "app decision: SPAWN sidecar 'voicebox-server' (or GPU onedir backend under /backends/{rocm,cuda})" +} + +# --- dispatch --------------------------------------------------------------- + +echo "Backend detection contract test (port $PORT)" +echo "============================================" + +if [[ -n "$FORCED_CASE" ]]; then + "case_${FORCED_CASE,,}" +else + if [[ -n "$(port_listener)" ]]; then + echo "Auto-detect: port $PORT is busy -> running case A" + echo + case_a + else + echo "Auto-detect: port $PORT is free -> running cases B and C" + echo + case_b + echo + case_c + fi +fi + +echo +echo "============================================" +echo "SUMMARY: $PASS passed, $FAIL failed" +[[ "$FAIL" -eq 0 ]] diff --git a/scripts/test_dictate_e2e.sh b/scripts/test_dictate_e2e.sh new file mode 100755 index 000000000..f678754d7 --- /dev/null +++ b/scripts/test_dictate_e2e.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +# test_dictate_e2e.sh — end-to-end harness for the dictation toggle script. +# +# NOTE: the script under test is user-local tooling that lives OUTSIDE this +# repo at ~/.local/bin/voicebox-dictate.sh (Super+Alt+V -> record/stop -> +# transcribe via the Voicebox backend -> paste via wl-copy + ydotool). +# This harness tests the INSTALLED copy; it does not modify it. +# +# How it works: +# * Creates a temp dir with stub executables for every external command the +# script uses (arecord, curl, wl-copy, wl-paste, ydotool, xclip, xdotool, +# wtype, notify-send, systemctl) and prepends it to PATH. +# * Each stub appends its argv to calls.log. The arecord stub stays alive +# like a real recorder and, on SIGTERM, drops a generated 3 s tone WAV +# (python3 wave module; the repo ships no WAV fixtures) at the output +# path. The curl stub answers /health and returns a canned successful +# /transcribe JSON. +# * HOME is pointed at a temp dir so the script's real state +# (~/.cache/voicebox-dictate) is never touched. +# +# Scenarios: +# happy run1 starts recording, run2 stops -> transcribe -> clipboard +# -> ydotool paste; asserts the full chain ran in order, exit 0. +# fail-record arecord exits 1 at start -> "No se pudo iniciar la grabación" +# fail-save recorder dies without WAV -> "No se guardó el audio" +# fail-http curl fails on /transcribe -> "Falló la petición al backend" +# fail-wlcopy wl-copy exits 1 -> later stages (ydotool) never run +# fail-ydotool ydotool exits 1 -> text stays in clipboard fallback +# +# Dependencies: bash, python3. Exit code 0 = all scenarios passed. +set -euo pipefail + +DICTATE_SCRIPT="${VOICEBOX_DICTATE_SCRIPT:-$HOME/.local/bin/voicebox-dictate.sh}" +[[ -f "$DICTATE_SCRIPT" ]] || { echo "Dictate script not found: $DICTATE_SCRIPT" >&2; exit 2; } + +WORK="$(mktemp -d /tmp/voicebox-dictate-e2e.XXXXXX)" +STUB_DIR="$WORK/stubs" +FAKE_HOME="$WORK/home" +CALLS_LOG="$WORK/calls.log" +CLIP_FILE="$WORK/clipboard.txt" +FIXTURE="$WORK/fixture.wav" +LOG_FILE="$FAKE_HOME/.cache/voicebox-dictate/dictate.log" + +PASS=0 +FAIL=0 + +cleanup() { rm -rf "$WORK"; } +trap cleanup EXIT + +ok() { echo " PASS: $*"; PASS=$((PASS + 1)); } +bad() { echo " FAIL: $*"; FAIL=$((FAIL + 1)); } +section() { echo; echo "== $*"; } + +# --- fixture WAV: 3 s of 440 Hz tone, 16 kHz mono s16 (~96 KB > 10000-byte +# minimum the script validates) ----------------------------------------- +python3 - "$FIXTURE" <<'PYEOF' +import math, struct, sys, wave +path = sys.argv[1] +rate, secs = 16000, 3 +with wave.open(path, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(rate) + frames = b"".join( + struct.pack(" "$STUB_DIR/arecord" <<'EOF' +#!/usr/bin/env bash +echo "arecord $*" >> "$CALLS_LOG" +out="${@: -1}" +if [[ "${VB_STUB_ARECORD_FAIL:-0}" == "1" ]]; then + exit 1 +fi +if [[ "${VB_STUB_ARECORD_NOWAV:-0}" == "1" ]]; then + trap 'exit 0' TERM +else + trap 'cp "$VB_FIXTURE_WAV" "$out" 2>/dev/null; exit 0' TERM +fi +while :; do sleep 1; done +EOF + +cat > "$STUB_DIR/curl" <<'EOF' +#!/usr/bin/env bash +echo "curl $*" >> "$CALLS_LOG" +url="" +ofile="" +prev="" +for a in "$@"; do + [[ "$prev" == "-o" ]] && ofile="$a" + [[ "$a" == http* ]] && url="$a" + prev="$a" +done +if [[ "$url" == */health ]]; then + echo '{"status":"healthy","model_loaded":true,"gpu_available":true}' + exit 0 +fi +if [[ "$url" == */transcribe ]]; then + if [[ "${VB_STUB_CURL_FAIL:-0}" == "1" ]]; then + exit 7 + fi + [[ -n "$ofile" ]] && printf '%s' '{"text":"hola mundo esto es una prueba"}' > "$ofile" + printf '200' + exit 0 +fi +exit 0 +EOF + +cat > "$STUB_DIR/wl-copy" <<'EOF' +#!/usr/bin/env bash +echo "wl-copy $*" >> "$CALLS_LOG" +[[ "${VB_STUB_WLCOPY_FAIL:-0}" == "1" ]] && exit 1 +cat > "$CLIP_FILE" +EOF + +cat > "$STUB_DIR/wl-paste" <<'EOF' +#!/usr/bin/env bash +echo "wl-paste $*" >> "$CALLS_LOG" +cat "$CLIP_FILE" 2>/dev/null || true +EOF + +cat > "$STUB_DIR/ydotool" <<'EOF' +#!/usr/bin/env bash +echo "ydotool $*" >> "$CALLS_LOG" +[[ "${VB_STUB_YDOTOOL_FAIL:-0}" == "1" ]] && exit 1 +exit 0 +EOF + +# Deterministic failing fallbacks so wl-copy/ydotool failures cannot leak into +# real xclip/xdotool/wtype binaries present on the host. +for tool in xclip xdotool wtype; do + cat > "$STUB_DIR/$tool" <> "\$CALLS_LOG" +exit 1 +EOF +done + +for tool in notify-send systemctl; do + cat > "$STUB_DIR/$tool" <> "\$CALLS_LOG" +exit 0 +EOF +done + +chmod +x "$STUB_DIR"/* + +# --- helpers ---------------------------------------------------------------- + +# Runs the installed dictate script in a fully controlled environment. +run_dictate() { + local rc=0 + env -i \ + HOME="$FAKE_HOME" \ + PATH="$STUB_DIR:/usr/bin:/bin" \ + WAYLAND_DISPLAY="wayland-vbtest" \ + CALLS_LOG="$CALLS_LOG" \ + CLIP_FILE="$CLIP_FILE" \ + VB_FIXTURE_WAV="$FIXTURE" \ + VB_STUB_ARECORD_FAIL="${VB_STUB_ARECORD_FAIL:-0}" \ + VB_STUB_ARECORD_NOWAV="${VB_STUB_ARECORD_NOWAV:-0}" \ + VB_STUB_CURL_FAIL="${VB_STUB_CURL_FAIL:-0}" \ + VB_STUB_WLCOPY_FAIL="${VB_STUB_WLCOPY_FAIL:-0}" \ + VB_STUB_YDOTOOL_FAIL="${VB_STUB_YDOTOOL_FAIL:-0}" \ + "$DICTATE_SCRIPT" >/dev/null 2>&1 || rc=$? + return "$rc" +} + +reset_scenario() { + rm -rf "$FAKE_HOME" + mkdir -p "$FAKE_HOME" + : > "$CALLS_LOG" + : > "$CLIP_FILE" + VB_STUB_ARECORD_FAIL=0 + VB_STUB_ARECORD_NOWAV=0 + VB_STUB_CURL_FAIL=0 + VB_STUB_WLCOPY_FAIL=0 + VB_STUB_YDOTOOL_FAIL=0 +} + +called() { grep -q "^$1 " "$CALLS_LOG"; } +not_called() { ! grep -q "^$1 " "$CALLS_LOG"; } +notify_has() { grep -q "^notify-send .*$1" "$CALLS_LOG"; } +script_log_has() { [[ -f "$LOG_FILE" ]] && grep -qF "$1" "$LOG_FILE"; } +first_line() { grep -n -m1 "$1" "$CALLS_LOG" | cut -d: -f1; } +last_line() { grep -n "$1" "$CALLS_LOG" | tail -1 | cut -d: -f1; } + +# --- scenarios ---------------------------------------------------------------- + +echo "Dictate E2E harness" +echo "script under test: $DICTATE_SCRIPT" +echo "workdir: $WORK" + +section "happy path: record -> transcribe -> wl-copy -> ydotool" +reset_scenario + +rc=0; run_dictate || rc=$? +if [[ "$rc" -eq 0 ]] && called arecord; then + ok "first toggle started recording (exit 0, arecord spawned)" +else + bad "first toggle: exit=$rc, arecord called=$(called arecord && echo yes || echo no)" +fi + +rc=0; run_dictate || rc=$? +[[ "$rc" -eq 0 ]] && ok "second toggle finished OK (exit 0)" || bad "second toggle exit=$rc" + +a="$(last_line '^arecord')" +t="$(first_line '/transcribe')" +c="$(first_line '^wl-copy')" +y="$(first_line '^ydotool')" +if [[ -n "$a" && -n "$t" && -n "$c" && -n "$y" && "$a" -lt "$t" && "$t" -lt "$c" && "$c" -lt "$y" ]]; then + ok "chain ran in order: arecord(L$a) -> /transcribe(L$t) -> wl-copy(L$c) -> ydotool(L$y)" +else + bad "chain order wrong: arecord=${a:-none} transcribe=${t:-none} wl-copy=${c:-none} ydotool=${y:-none}" +fi + +grep -Fq 'ydotool key ctrl+v' "$CALLS_LOG" \ + && ok "ydotool invoked as 'key ctrl+v'" \ + || bad "ydotool invocation missing/wrong: $(grep '^ydotool' "$CALLS_LOG" || echo none)" +script_log_has "Pasted text with ydotool (Ctrl+V)" \ + && ok "script log confirms paste" \ + || bad "script log missing 'Pasted text with ydotool'" +notify_has "Dictado listo" \ + && ok "user notified: 'Dictado listo'" \ + || bad "missing 'Dictado listo' notification" + +section "failure injection: recorder fails to start" +reset_scenario +VB_STUB_ARECORD_FAIL=1 +rc=0; run_dictate || rc=$? +[[ "$rc" -eq 1 ]] && ok "aborts with exit 1" || bad "expected exit 1, got $rc" +notify_has "No se pudo iniciar la grabación" \ + && ok "documented error: 'No se pudo iniciar la grabación'" \ + || bad "missing recorder-error notification" +not_called wl-copy && ok "later stages never called" || bad "wl-copy ran despite recorder failure" + +section "failure injection: recording leaves no WAV" +reset_scenario +VB_STUB_ARECORD_NOWAV=1 +run_dictate >/dev/null 2>&1 || true # start (recorder stays up, writes nothing on TERM) +rc=0; run_dictate || rc=$? +[[ "$rc" -eq 1 ]] && ok "aborts with exit 1" || bad "expected exit 1, got $rc" +notify_has "No se guardó el audio" \ + && ok "documented error: 'No se guardó el audio'" \ + || bad "missing save-error notification" +if not_called ydotool && ! grep -q '/transcribe' "$CALLS_LOG"; then + ok "transcribe/paste stages never called" +else + bad "later stages ran despite missing WAV" +fi + +section "failure injection: /transcribe request fails" +reset_scenario +VB_STUB_CURL_FAIL=1 +run_dictate >/dev/null 2>&1 || true +rc=0; run_dictate || rc=$? +[[ "$rc" -eq 1 ]] && ok "aborts with exit 1" || bad "expected exit 1, got $rc" +notify_has "Falló la petición al backend" \ + && ok "documented error: 'Falló la petición al backend'" \ + || bad "missing http-error notification" +not_called wl-copy && ok "wl-copy never called" || bad "wl-copy ran despite transcribe failure" +not_called ydotool && ok "ydotool never called" || bad "ydotool ran despite transcribe failure" + +section "failure injection: wl-copy fails" +reset_scenario +VB_STUB_WLCOPY_FAIL=1 +run_dictate >/dev/null 2>&1 || true +rc=0; run_dictate || rc=$? +# Documented behavior: type_text() fails -> error notification, but the +# script still exits 0 after cleaning up the WAV. +[[ "$rc" -eq 0 ]] && ok "script exits 0 (documented: error is notified, not fatal)" || bad "expected exit 0, got $rc" +script_log_has "wl-copy falló" \ + && ok "script log: 'wl-copy falló'" \ + || bad "script log missing 'wl-copy falló'" +notify_has "No se pudo escribir ni copiar el texto" \ + && ok "documented error: 'No se pudo escribir ni copiar el texto'" \ + || bad "missing clipboard-error notification" +not_called ydotool && ok "ydotool (later stage) never called" || bad "ydotool ran despite wl-copy failure" + +section "failure injection: ydotool fails" +reset_scenario +VB_STUB_YDOTOOL_FAIL=1 +run_dictate >/dev/null 2>&1 || true +rc=0; run_dictate || rc=$? +# Documented behavior: text stays in the clipboard, user told to paste +# manually, type_text() still returns success -> exit 0. +[[ "$rc" -eq 0 ]] && ok "script exits 0 (documented clipboard fallback)" || bad "expected exit 0, got $rc" +called wl-copy && ok "wl-copy ran (text left in clipboard)" || bad "wl-copy did not run" +script_log_has "ydotool paste failed, leaving text in clipboard" \ + && ok "script log: 'ydotool paste failed, leaving text in clipboard'" \ + || bad "script log missing ydotool-failure entry" +notify_has "Texto en portapapeles" \ + && ok "documented fallback: 'Texto en portapapeles' (paste manually)" \ + || bad "missing clipboard-fallback notification" + +echo +echo "============================================" +echo "SUMMARY: $PASS passed, $FAIL failed" +[[ "$FAIL" -eq 0 ]] diff --git a/tauri/src-tauri/src/main.rs b/tauri/src-tauri/src/main.rs index 0f44ac90f..1bc0496ba 100644 --- a/tauri/src-tauri/src/main.rs +++ b/tauri/src-tauri/src/main.rs @@ -1655,5 +1655,17 @@ pub fn run() { } fn main() { + // On Wayland + NVIDIA, WebKitGTK's DMA-BUF renderer produces blank or + // corrupted webviews unless it is disabled via this env var (upstream + // webkitgtk bug: https://bugs.webkit.org/show_bug.cgi?id=261037). + // Without it the AppImage renders a broken window unless the user sets + // the variable manually. Set it here, before any GTK/WebKit init, so + // double-clicking the AppImage just works. An explicit user override + // (including `=0` to re-enable the renderer) is respected. + #[cfg(target_os = "linux")] + if std::env::var_os("WEBKIT_DISABLE_DMABUF_RENDERER").is_none() { + std::env::set_var("WEBKIT_DISABLE_DMABUF_RENDERER", "1"); + } + run(); }