From d9d9d64eefa9c2fb3189ac133ab71a8b6669ce78 Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 08:47:26 +0200 Subject: [PATCH 01/15] Add MMS-TTS backend engine with Romanian support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New 'mms' engine wrapping facebook/mms-tts-ron (Meta MMS, VITS architecture) — pure PyTorch via the already-pinned transformers dependency, zero new packages. Single preset voice, no cloning, modeled on the Kokoro preset-voice pattern. The mms-tts-ron vocab mixes Romanian diacritic conventions (comma-below s U+0219 but cedilla t U+0163), and the character-level VitsTokenizer silently drops out-of-vocab characters, so input text is NFC-normalized and both real-world variants are mapped onto the trained forms before tokenizing. Registered in the model config registry, engine factory, request-model regexes (language 'ro', engine 'mms'), preset-voice endpoints, and the PyInstaller hidden imports. Co-Authored-By: Claude Opus 4.8 --- backend/backends/__init__.py | 13 ++ backend/backends/mms_backend.py | 210 ++++++++++++++++++++++++++++++++ backend/build_binary.py | 4 + backend/models.py | 14 +-- backend/routes/profiles.py | 15 +++ backend/services/profiles.py | 5 + 6 files changed, 254 insertions(+), 7 deletions(-) create mode 100644 backend/backends/mms_backend.py diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index 2437a87b3..5a0bfdc2c 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -215,6 +215,7 @@ def is_loaded(self) -> bool: "chatterbox_turbo": "Chatterbox Turbo", "tada": "TADA", "kokoro": "Kokoro", + "mms": "MMS TTS", } LLM_ENGINES = { @@ -364,6 +365,14 @@ def _get_non_qwen_tts_configs() -> list[ModelConfig]: size_mb=350, languages=["en", "es", "fr", "hi", "it", "pt", "ja", "zh"], ), + ModelConfig( + model_name="mms-tts-ron", + display_name="MMS Romanian (Meta)", + engine="mms", + hf_repo_id="facebook/mms-tts-ron", + size_mb=150, + languages=["ro"], + ), ] @@ -704,6 +713,10 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend: from .kokoro_backend import KokoroTTSBackend backend = KokoroTTSBackend() + elif engine == "mms": + from .mms_backend import MMSTTSBackend + + backend = MMSTTSBackend() elif engine == "qwen_custom_voice": from .qwen_custom_voice_backend import QwenCustomVoiceBackend diff --git a/backend/backends/mms_backend.py b/backend/backends/mms_backend.py new file mode 100644 index 000000000..4e72d92ad --- /dev/null +++ b/backend/backends/mms_backend.py @@ -0,0 +1,210 @@ +""" +MMS-TTS backend implementation. + +Wraps Meta's Massively Multilingual Speech (MMS) TTS checkpoints — one VITS +model per language (``facebook/mms-tts-{iso3}``). Pure PyTorch via +``transformers``, CPU realtime, 16kHz output, CC-BY-NC 4.0 license. + +MMS has no concept of voice cloning: each checkpoint is a single preset +speaker, so profiles follow the preset-voice pattern (like Kokoro). + +Languages supported: + - Romanian (ro) — ``facebook/mms-tts-ron`` + +Adding a language is a one-line addition to ``MMS_HF_REPOS`` plus a voice +entry in ``MMS_VOICES`` and a ``ModelConfig`` registration. +""" + +import asyncio +import logging +import unicodedata + +import numpy as np + +from .base import ( + combine_voice_prompts as _combine_voice_prompts, + empty_device_cache, + get_torch_device, + is_model_cached, + manual_seed, + model_load_progress, +) + +logger = logging.getLogger(__name__) + +# Our ISO 639-1 language codes -> MMS per-language HF checkpoints (ISO 639-3) +MMS_HF_REPOS = {"ro": "facebook/mms-tts-ron"} +MMS_DEFAULT_LANGUAGE = "ro" + +# Confirmed against model.config.sampling_rate for mms-tts-ron +MMS_SAMPLE_RATE = 16000 + +# Default voice if none specified +MMS_DEFAULT_VOICE = "mms_ro_default" + +# All available MMS voices: (voice_id, display_name, gender, lang_code). +# One entry per language checkpoint — MMS ships a single speaker each. +MMS_VOICES = [ + ("mms_ro_default", "Romanian (MMS)", "male", "ro"), +] + +# The mms-tts-ron vocab mixes Romanian diacritic conventions: it contains +# comma-below ș (U+0219) but cedilla ţ (U+0163), and lacks their +# counterparts ş (U+015F) and ț (U+021B). The character-level VitsTokenizer +# silently drops characters outside its vocab, so both real-world variants +# must be mapped onto the trained form. Uppercase variants are included — +# the tokenizer lowercases after this mapping (Ș -> ș, Ţ -> ţ). +_RO_DIACRITICS_TRANSLATION = str.maketrans( + { + "ş": "ș", # ş (s-cedilla) -> ș (s-comma-below) + "Ş": "Ș", # Ş (S-cedilla) -> Ș (S-comma-below) + "ț": "ţ", # ț (t-comma-below) -> ţ (t-cedilla) + "Ț": "Ţ", # Ț (T-comma-below) -> Ţ (T-cedilla) + } +) + + +def normalize_romanian_text(text: str) -> str: + """Normalize Romanian text to the diacritic forms in the MMS vocab. + + Applies NFC normalization first (composing any decomposed + letter + combining-mark sequences), then maps cedilla/comma-below + s and t variants onto the forms the ``facebook/mms-tts-ron`` + checkpoint was trained on, so no diacritic is silently dropped + by the tokenizer. + """ + return unicodedata.normalize("NFC", text).translate(_RO_DIACRITICS_TRANSLATION) + + +class MMSTTSBackend: + """Meta MMS-TTS backend — per-language VITS checkpoints, single preset voice.""" + + def __init__(self): + self._model = None + self._tokenizer = None + self._device: str | None = None + self.model_size = "default" + + @property + def device(self) -> str: + if self._device is None: + # CPU is realtime for a ~100M-param VITS; skip MPS like Kokoro. + self._device = get_torch_device(allow_mps=False) + return self._device + + def is_loaded(self) -> bool: + return self._model is not None + + def _get_model_path(self, model_size: str) -> str: + return MMS_HF_REPOS[MMS_DEFAULT_LANGUAGE] + + def _is_model_cached(self, model_size: str = "default") -> bool: + """Check if MMS model files are cached locally.""" + return is_model_cached(MMS_HF_REPOS[MMS_DEFAULT_LANGUAGE]) + + async def load_model(self, model_size: str = "default") -> None: + """Load the MMS model and tokenizer.""" + if self._model is not None: + return + await asyncio.to_thread(self._load_model_sync) + + def _load_model_sync(self): + """Synchronous model loading.""" + model_name = "mms-tts-ron" + is_cached = self._is_model_cached() + + with model_load_progress(model_name, is_cached): + from transformers import AutoTokenizer, VitsModel # lazy: heavy import + + repo = MMS_HF_REPOS[MMS_DEFAULT_LANGUAGE] + device = self.device + logger.info("Loading MMS-TTS (%s) on %s...", repo, device) + + self._tokenizer = AutoTokenizer.from_pretrained(repo) + self._model = VitsModel.from_pretrained(repo).to(device).eval() + + logger.info("MMS-TTS loaded successfully") + + def unload_model(self) -> None: + """Unload model to free memory.""" + if self._model is not None: + del self._model + self._model = None + self._tokenizer = None + empty_device_cache(self.device) + logger.info("MMS-TTS unloaded") + + async def create_voice_prompt( + self, + audio_path: str, + reference_text: str, + use_cache: bool = True, + ) -> tuple[dict, bool]: + """ + Create voice prompt for MMS. + + MMS doesn't do voice cloning — each checkpoint is one fixed speaker. + When called for a cloned profile (fallback), uses the default voice. + For preset profiles, the voice_prompt dict is built by the profile + service and bypasses this method entirely. + """ + return { + "voice_type": "preset", + "preset_engine": "mms", + "preset_voice_id": MMS_DEFAULT_VOICE, + }, False + + async def combine_voice_prompts( + self, + audio_paths: list[str], + reference_texts: list[str], + ) -> tuple[np.ndarray, str]: + """Combine voice prompts — uses base implementation for audio concatenation.""" + return await _combine_voice_prompts(audio_paths, reference_texts, sample_rate=MMS_SAMPLE_RATE) + + async def generate( + self, + text: str, + voice_prompt: dict, + language: str = "ro", + seed: int | None = None, + instruct: str | None = None, + ) -> tuple[np.ndarray, int]: + """ + Generate audio from text using MMS-TTS. + + Args: + text: Text to synthesize + voice_prompt: Preset voice dict (single speaker — informational only) + language: Language code + seed: Random seed for reproducibility (VITS sampling is stochastic) + instruct: Not supported by MMS (ignored) + + Returns: + Tuple of (audio_array, sample_rate) + """ + await self.load_model() + + def _generate_sync(): + import torch # lazy: heavy import + + if seed is not None: + manual_seed(seed, self.device) + + normalized = normalize_romanian_text(text) if language == "ro" else unicodedata.normalize("NFC", text) + + inputs = self._tokenizer(normalized, return_tensors="pt").to(self.device) + sample_rate = getattr(self._model.config, "sampling_rate", MMS_SAMPLE_RATE) + + # Text made entirely of out-of-vocab characters tokenizes to an + # empty sequence — return 1 second of silence as fallback. + if inputs["input_ids"].shape[-1] == 0: + return np.zeros(sample_rate, dtype=np.float32), sample_rate + + with torch.no_grad(): + waveform = self._model(**inputs).waveform + + audio = waveform.squeeze().detach().cpu().numpy().astype(np.float32) + return audio, sample_rate + + return await asyncio.to_thread(_generate_sync) diff --git a/backend/build_binary.py b/backend/build_binary.py index 90829c73d..a110dde3d 100644 --- a/backend/build_binary.py +++ b/backend/build_binary.py @@ -276,6 +276,10 @@ def build_server(cuda=False, rocm=False): "backend.backends.kokoro_backend", "--collect-all", "kokoro", + # MMS-TTS — pure transformers VITS; the transformers hook already + # bundles the model classes, only the backend module is needed. + "--hidden-import", + "backend.backends.mms_backend", # misaki ships G2P data files (dictionaries, phoneme tables) # that must be bundled for espeak/en/ja/zh G2P to work "--collect-all", diff --git a/backend/models.py b/backend/models.py index 7970ce41e..b1a6a7d47 100644 --- a/backend/models.py +++ b/backend/models.py @@ -18,7 +18,7 @@ class VoiceProfileCreate(BaseModel): name: str = Field(..., min_length=1, max_length=100) description: Optional[str] = Field(None, max_length=500) language: str = Field( - default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$" + default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr|ro)$" ) voice_type: Optional[str] = Field(default="cloned", pattern="^(cloned|preset|designed)$") preset_engine: Optional[str] = Field(None, max_length=50) @@ -81,11 +81,11 @@ class GenerationRequest(BaseModel): profile_id: str text: str = Field(..., min_length=1, max_length=50000) - language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$") + language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr|ro)$") seed: Optional[int] = Field(None, ge=0) model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$") instruct: Optional[str] = Field(None, max_length=500) - engine: Optional[str] = Field(default="qwen", pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$") + engine: Optional[str] = Field(default="qwen", pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$") personality: bool = Field( default=False, description="When true and the profile has a personality prompt, the input text is rewritten in-character before TTS.", @@ -317,7 +317,7 @@ class MCPClientBindingResponse(BaseModel): profile_id: Optional[str] = None default_engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$", + pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$", ) default_personality: bool = False last_seen_at: Optional[datetime] = None @@ -336,7 +336,7 @@ class MCPClientBindingUpsert(BaseModel): profile_id: Optional[str] = None default_engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$", + pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$", ) default_personality: bool = False @@ -355,7 +355,7 @@ class SpeakRequest(BaseModel): ) engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$", + pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$", ) personality: Optional[bool] = Field( None, @@ -363,7 +363,7 @@ class SpeakRequest(BaseModel): ) language: Optional[str] = Field( None, - pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$", + pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr|ro)$", ) diff --git a/backend/routes/profiles.py b/backend/routes/profiles.py index e0f7f7fd7..241bd0b1e 100644 --- a/backend/routes/profiles.py +++ b/backend/routes/profiles.py @@ -104,6 +104,21 @@ async def list_preset_voices(engine: str): for speaker_id, display_name, gender, lang, _desc in QWEN_CUSTOM_VOICES ], } + if engine == "mms": + from ..backends.mms_backend import MMS_VOICES + + return { + "engine": engine, + "voices": [ + { + "voice_id": vid, + "name": name, + "gender": gender, + "language": lang, + } + for vid, name, gender, lang in MMS_VOICES + ], + } return {"engine": engine, "voices": []} @router.get("/profiles/{profile_id}", response_model=models.VoiceProfileResponse) diff --git a/backend/services/profiles.py b/backend/services/profiles.py index d7d32fa0f..ac1340b20 100644 --- a/backend/services/profiles.py +++ b/backend/services/profiles.py @@ -73,6 +73,11 @@ def _get_preset_voice_ids(engine: str) -> set[str]: return {voice_id for voice_id, _name, _gender, _lang, _desc in QWEN_CUSTOM_VOICES} + if engine == "mms": + from ..backends.mms_backend import MMS_VOICES + + return {voice_id for voice_id, _name, _gender, _lang in MMS_VOICES} + return set() From eddb7a286445e77d186019f0bc59620117926e3a Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 08:47:36 +0200 Subject: [PATCH 02/15] Add MMS Romanian engine to frontend Romanian added to ALL_LANGUAGES, 'mms' wired through every engine enumeration: engine union types, zod schema, engine picker, display names, preset-engine sets (profile form/list/card, floating generate box), model management filter, and captures play-as union. MMS is treated as a preset-voice engine like Kokoro and Qwen CustomVoice. Co-Authored-By: Claude Opus 4.8 --- .../components/CapturesTab/CapturesTab.tsx | 2 +- .../Generation/EngineModelSelector.tsx | 2 ++ .../Generation/FloatingGenerateBox.tsx | 5 ++-- .../ServerSettings/ModelManagement.tsx | 3 ++- .../components/VoiceProfiles/ProfileCard.tsx | 1 + .../components/VoiceProfiles/ProfileForm.tsx | 4 ++- .../components/VoiceProfiles/ProfileList.tsx | 2 +- app/src/lib/api/types.ts | 3 ++- app/src/lib/constants/languages.ts | 3 +++ app/src/lib/hooks/useGenerationForm.ts | 25 +++++++++++-------- app/src/lib/utils/format.ts | 1 + 11 files changed, 34 insertions(+), 17 deletions(-) diff --git a/app/src/components/CapturesTab/CapturesTab.tsx b/app/src/components/CapturesTab/CapturesTab.tsx index 7492d320d..6dcf99b3e 100644 --- a/app/src/components/CapturesTab/CapturesTab.tsx +++ b/app/src/components/CapturesTab/CapturesTab.tsx @@ -269,7 +269,7 @@ export function CapturesTab() { // override fall through to whatever the backend picks. const engine = voice.default_engine as | 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' - | 'chatterbox_turbo' | 'tada' | 'kokoro' + | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'mms' | undefined; return apiClient.generateSpeech({ profile_id: voice.id, diff --git a/app/src/components/Generation/EngineModelSelector.tsx b/app/src/components/Generation/EngineModelSelector.tsx index 7f4f600b5..ef3e2a918 100644 --- a/app/src/components/Generation/EngineModelSelector.tsx +++ b/app/src/components/Generation/EngineModelSelector.tsx @@ -27,6 +27,7 @@ const ENGINE_OPTIONS = [ { value: 'tada:1B', label: 'TADA 1B', engine: 'tada' }, { value: 'tada:3B', label: 'TADA 3B Multilingual', engine: 'tada' }, { value: 'kokoro', label: 'Kokoro 82M', engine: 'kokoro' }, + { value: 'mms', label: 'MMS Romanian', engine: 'mms' }, ] as const; const ENGINE_DESCRIPTIONS: Record = { @@ -37,6 +38,7 @@ const ENGINE_DESCRIPTIONS: Record = { chatterbox_turbo: 'English, [laugh] [cough] tags', tada: 'HumeAI, 700s+ coherent audio', kokoro: '82M params, CPU realtime, 8 langs', + mms: 'Meta MMS, Romanian, CPU realtime', }; /** Engines that only support English and should force language to 'en' on select. */ diff --git a/app/src/components/Generation/FloatingGenerateBox.tsx b/app/src/components/Generation/FloatingGenerateBox.tsx index 7618490b4..31157d67f 100644 --- a/app/src/components/Generation/FloatingGenerateBox.tsx +++ b/app/src/components/Generation/FloatingGenerateBox.tsx @@ -151,7 +151,8 @@ export function FloatingGenerateBox({ | 'chatterbox_turbo' | 'tada' | 'kokoro' - | 'qwen_custom_voice'; + | 'qwen_custom_voice' + | 'mms'; useEffect(() => { if (selectedProfile?.language) { form.setValue('language', selectedProfile.language as LanguageCode); @@ -163,7 +164,7 @@ export function FloatingGenerateBox({ } else if (selectedProfile && selectedProfile.voice_type !== 'preset') { // Cloned/designed profile with no default — ensure a compatible (non-preset) engine const currentEngine = form.getValues('engine'); - const presetEngines = new Set(['kokoro', 'qwen_custom_voice']); + const presetEngines = new Set(['kokoro', 'qwen_custom_voice', 'mms']); if (currentEngine && presetEngines.has(currentEngine)) { form.setValue('engine', 'qwen'); } diff --git a/app/src/components/ServerSettings/ModelManagement.tsx b/app/src/components/ServerSettings/ModelManagement.tsx index b06783f96..8ac5886c4 100644 --- a/app/src/components/ServerSettings/ModelManagement.tsx +++ b/app/src/components/ServerSettings/ModelManagement.tsx @@ -414,7 +414,8 @@ export function ModelManagement() { m.model_name.startsWith('luxtts') || m.model_name.startsWith('chatterbox') || m.model_name.startsWith('tada') || - m.model_name.startsWith('kokoro'), + m.model_name.startsWith('kokoro') || + m.model_name.startsWith('mms'), ) ?? []; const whisperModels = modelStatus?.models.filter((m) => m.model_name.startsWith('whisper')) ?? []; const llmModels = modelStatus?.models.filter((m) => m.model_name.startsWith('qwen3-')) ?? []; diff --git a/app/src/components/VoiceProfiles/ProfileCard.tsx b/app/src/components/VoiceProfiles/ProfileCard.tsx index e9042a571..d850302f2 100644 --- a/app/src/components/VoiceProfiles/ProfileCard.tsx +++ b/app/src/components/VoiceProfiles/ProfileCard.tsx @@ -22,6 +22,7 @@ import { useUIStore } from '@/stores/uiStore'; const ENGINE_DISPLAY_NAMES: Record = { kokoro: 'Kokoro', qwen_custom_voice: 'CustomVoice', + mms: 'MMS', }; interface ProfileCardProps { diff --git a/app/src/components/VoiceProfiles/ProfileForm.tsx b/app/src/components/VoiceProfiles/ProfileForm.tsx index 7ef4651c1..4ed511341 100644 --- a/app/src/components/VoiceProfiles/ProfileForm.tsx +++ b/app/src/components/VoiceProfiles/ProfileForm.tsx @@ -61,7 +61,7 @@ import { AudioSampleUpload } from './AudioSampleUpload'; import { SampleList } from './SampleList'; const MAX_AUDIO_DURATION_SECONDS = 30; -const PRESET_ONLY_ENGINES = new Set(['kokoro', 'qwen_custom_voice']); +const PRESET_ONLY_ENGINES = new Set(['kokoro', 'qwen_custom_voice', 'mms']); const DEFAULT_ENGINE_OPTIONS = [ { value: 'qwen', label: 'Qwen3-TTS' }, { value: 'qwen_custom_voice', label: 'Qwen CustomVoice' }, @@ -70,6 +70,7 @@ const DEFAULT_ENGINE_OPTIONS = [ { value: 'chatterbox_turbo', label: 'Chatterbox Turbo' }, { value: 'tada', label: 'TADA' }, { value: 'kokoro', label: 'Kokoro 82M' }, + { value: 'mms', label: 'MMS Romanian' }, ] as const; function makeProfileSchema(t: (key: string) => string) { @@ -898,6 +899,7 @@ export function ProfileForm() { Kokoro 82M Qwen CustomVoice + MMS Romanian diff --git a/app/src/components/VoiceProfiles/ProfileList.tsx b/app/src/components/VoiceProfiles/ProfileList.tsx index 3bfad014f..86e9be944 100644 --- a/app/src/components/VoiceProfiles/ProfileList.tsx +++ b/app/src/components/VoiceProfiles/ProfileList.tsx @@ -9,7 +9,7 @@ import { ProfileCard } from './ProfileCard'; import { ProfileForm } from './ProfileForm'; /** Engines that use preset (built-in) voices instead of cloned profiles. */ -const PRESET_ENGINES = new Set(['kokoro', 'qwen_custom_voice']); +const PRESET_ENGINES = new Set(['kokoro', 'qwen_custom_voice', 'mms']); export function ProfileList() { const { t } = useTranslation(); diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index 4a9707498..55d8792a5 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -78,7 +78,8 @@ export interface GenerationRequest { | 'chatterbox' | 'chatterbox_turbo' | 'tada' - | 'kokoro'; + | 'kokoro' + | 'mms'; instruct?: string; /** When true and the profile has a personality prompt, input text is rewritten in-character before TTS. */ personality?: boolean; diff --git a/app/src/lib/constants/languages.ts b/app/src/lib/constants/languages.ts index e28c519bc..ca8534030 100644 --- a/app/src/lib/constants/languages.ts +++ b/app/src/lib/constants/languages.ts @@ -6,6 +6,7 @@ * Chatterbox Multilingual supports 23 languages. * Chatterbox Turbo is English-only. * Kokoro supports 8 languages. + * MMS is Romanian-only (per-language Meta checkpoints). */ /** All languages that any engine supports. */ @@ -28,6 +29,7 @@ export const ALL_LANGUAGES = { no: 'Norwegian', pl: 'Polish', pt: 'Portuguese', + ro: 'Romanian', ru: 'Russian', sv: 'Swedish', sw: 'Swahili', @@ -70,6 +72,7 @@ export const ENGINE_LANGUAGES: Record = { tada: ['en', 'ar', 'zh', 'de', 'es', 'fr', 'it', 'ja', 'pl', 'pt'], kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'], qwen_custom_voice: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'], + mms: ['ro'], } as const; /** Helper: get language options for a given engine. */ diff --git a/app/src/lib/hooks/useGenerationForm.ts b/app/src/lib/hooks/useGenerationForm.ts index e90320e93..3e8a9f714 100644 --- a/app/src/lib/hooks/useGenerationForm.ts +++ b/app/src/lib/hooks/useGenerationForm.ts @@ -27,6 +27,7 @@ const generationSchema = z.object({ 'chatterbox_turbo', 'tada', 'kokoro', + 'mms', ]) .optional(), personality: z.boolean().optional(), @@ -100,9 +101,11 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { : 'tada-1b' : engine === 'kokoro' ? 'kokoro' - : engine === 'qwen_custom_voice' - ? `qwen-custom-voice-${data.modelSize}` - : `qwen-tts-${data.modelSize}`; + : engine === 'mms' + ? 'mms-tts-ron' + : engine === 'qwen_custom_voice' + ? `qwen-custom-voice-${data.modelSize}` + : `qwen-tts-${data.modelSize}`; const displayName = engine === 'luxtts' ? 'LuxTTS' @@ -116,13 +119,15 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { : 'TADA 1B' : engine === 'kokoro' ? 'Kokoro 82M' - : engine === 'qwen_custom_voice' - ? data.modelSize === '1.7B' - ? 'Qwen CustomVoice 1.7B' - : 'Qwen CustomVoice 0.6B' - : data.modelSize === '1.7B' - ? 'Qwen TTS 1.7B' - : 'Qwen TTS 0.6B'; + : engine === 'mms' + ? 'MMS Romanian (Meta)' + : engine === 'qwen_custom_voice' + ? data.modelSize === '1.7B' + ? 'Qwen CustomVoice 1.7B' + : 'Qwen CustomVoice 0.6B' + : data.modelSize === '1.7B' + ? 'Qwen TTS 1.7B' + : 'Qwen TTS 0.6B'; // Check if model needs downloading try { diff --git a/app/src/lib/utils/format.ts b/app/src/lib/utils/format.ts index 0d01a7fdd..9d3dcafcf 100644 --- a/app/src/lib/utils/format.ts +++ b/app/src/lib/utils/format.ts @@ -57,6 +57,7 @@ const ENGINE_DISPLAY_NAMES: Record = { luxtts: 'LuxTTS', chatterbox: 'Chatterbox', chatterbox_turbo: 'Chatterbox Turbo', + mms: 'MMS TTS', }; export function formatEngineName(engine?: string, modelSize?: string): string { From dd0c200fabf9b159d1b31402f6e7fcf8c1576ca0 Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 08:47:36 +0200 Subject: [PATCH 03/15] Add MMS backend tests Unit coverage for the Romanian diacritics normalization (cedilla vs comma-below variants, NFC composition, lossless handling of unknown chars), engine registration surfaces, and request-model regexes. End-to-end generation tests (model download ~150MB) are opt-in via VOICEBOX_MMS_E2E=1: Romanian text with both diacritic conventions produces valid 16kHz float32 audio, both spellings tokenize identically, seeded generation is deterministic, and fully out-of-vocab text falls back to silence. Co-Authored-By: Claude Opus 4.8 --- backend/tests/test_mms_backend.py | 287 ++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 backend/tests/test_mms_backend.py diff --git a/backend/tests/test_mms_backend.py b/backend/tests/test_mms_backend.py new file mode 100644 index 000000000..c76d2ba8f --- /dev/null +++ b/backend/tests/test_mms_backend.py @@ -0,0 +1,287 @@ +""" +Tests for the MMS-TTS backend (Romanian). + +Unit tests cover the Romanian diacritics normalization (the mms-tts-ron +vocab mixes comma-below ș with cedilla ţ, and the character-level +VitsTokenizer silently drops out-of-vocab characters) and the engine +registration surfaces (model config registry, backend factory, request +model regexes, preset voice endpoints). + +The end-to-end generation test downloads the ~150MB model on first run +and is opt-in: + + VOICEBOX_MMS_E2E=1 python -m pytest backend/tests/test_mms_backend.py -v +""" + +import os +import sys +import unicodedata +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from backend.backends import ( + TTS_ENGINES, + get_model_config, + get_tts_backend_for_engine, + reset_backends, +) +from backend.backends.mms_backend import ( + MMS_DEFAULT_VOICE, + MMS_HF_REPOS, + MMS_SAMPLE_RATE, + MMS_VOICES, + MMSTTSBackend, + normalize_romanian_text, +) + +S_CEDILLA = "ş" # ş — not in the mms-tts-ron vocab +S_CEDILLA_UPPER = "Ş" # Ş +S_COMMA = "ș" # ș — in vocab +S_COMMA_UPPER = "Ș" # Ș +T_CEDILLA = "ţ" # ţ — in vocab +T_CEDILLA_UPPER = "Ţ" # Ţ +T_COMMA = "ț" # ț — not in vocab +T_COMMA_UPPER = "Ț" # Ț +COMBINING_COMMA_BELOW = "̦" +COMBINING_BREVE = "̆" + +MMS_PRESET_PROMPT = { + "voice_type": "preset", + "preset_engine": "mms", + "preset_voice_id": MMS_DEFAULT_VOICE, +} + + +class TestRomanianDiacriticsNormalization: + """The tokenizer drops unknown chars silently — every real-world + diacritic variant must be mapped onto the form in the vocab.""" + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + (S_CEDILLA, S_COMMA), + (S_CEDILLA_UPPER, S_COMMA_UPPER), + (T_COMMA, T_CEDILLA), + (T_COMMA_UPPER, T_CEDILLA_UPPER), + ], + ) + def test_wrong_variant_mapped_to_vocab_form(self, raw, expected): + assert normalize_romanian_text(raw) == expected + + @pytest.mark.parametrize("char", [S_COMMA, T_CEDILLA, "ă", "â", "î", "a", "b", " "]) + def test_vocab_forms_pass_through_unchanged(self, char): + assert normalize_romanian_text(char) == char + + def test_nfd_sequences_are_composed(self): + # s/t + combining comma below compose (via NFC) to the comma-below + # letters, which then go through the same variant mapping. + assert normalize_romanian_text("s" + COMBINING_COMMA_BELOW) == S_COMMA + assert normalize_romanian_text("t" + COMBINING_COMMA_BELOW) == T_CEDILLA + assert normalize_romanian_text("a" + COMBINING_BREVE) == "ă" + + def test_mixed_sentence_contains_only_vocab_diacritics(self): + # Both variant families in one string, as found in the wild. + text = f"{S_CEDILLA_UPPER}tii c{S_CEDILLA} {T_COMMA}ara {T_CEDILLA}ine pa{S_COMMA}ii" + normalized = normalize_romanian_text(text) + lowered = normalized.lower() + assert S_CEDILLA not in lowered + assert T_COMMA not in lowered + assert lowered.count(S_COMMA) == 3 + assert lowered.count(T_CEDILLA) == 2 + + def test_plain_text_untouched(self): + text = "Salut, ce mai faci? 1-2!" + assert normalize_romanian_text(text) == text + + def test_unknown_chars_are_preserved(self): + # Normalization is not lossy — dropping out-of-vocab characters is + # the tokenizer's job, not ours. + text = "kiwi & yoga 42" + assert normalize_romanian_text(text) == text + + def test_output_is_nfc(self): + decomposed = unicodedata.normalize("NFD", "Bună ziua, țară") + assert unicodedata.is_normalized("NFC", normalize_romanian_text(decomposed)) + + +class TestMMSRegistration: + def test_engine_listed(self): + assert "mms" in TTS_ENGINES + + def test_model_config_resolves(self): + cfg = get_model_config("mms-tts-ron") + assert cfg is not None + assert cfg.engine == "mms" + assert cfg.hf_repo_id == "facebook/mms-tts-ron" + assert cfg.languages == ["ro"] + assert cfg.size_mb == 150 + assert cfg.supports_instruct is False + assert cfg.needs_trim is False + + def test_backend_factory_returns_mms_backend(self): + try: + backend = get_tts_backend_for_engine("mms") + assert isinstance(backend, MMSTTSBackend) + # Factory caches instances per engine + assert get_tts_backend_for_engine("mms") is backend + finally: + reset_backends() + + def test_voice_catalog_shape(self): + # Same tuple shape as KOKORO_VOICES: (voice_id, name, gender, lang) + for voice_id, name, gender, lang in MMS_VOICES: + assert voice_id + assert name + assert gender in ("male", "female") + assert lang in MMS_HF_REPOS + assert MMS_DEFAULT_VOICE in {v[0] for v in MMS_VOICES} + + def test_preset_voice_ids_service(self): + from backend.services.profiles import _get_preset_voice_ids + + assert _get_preset_voice_ids("mms") == {MMS_DEFAULT_VOICE} + + async def test_preset_voices_route(self): + # routes/profiles imports ..app at module scope; importing the app + # first (the normal boot order) avoids a circular import. + import backend.app # noqa: F401 -- side-effect import initializes routers + from backend.routes.profiles import list_preset_voices + + result = await list_preset_voices("mms") + assert result["engine"] == "mms" + assert result["voices"] == [ + { + "voice_id": MMS_DEFAULT_VOICE, + "name": "Romanian (MMS)", + "gender": "male", + "language": "ro", + } + ] + + +class TestRequestModelRegexes: + def test_generation_request_accepts_ro_and_mms(self): + from backend.models import GenerationRequest + + req = GenerationRequest(profile_id="p1", text="Bună ziua", language="ro", engine="mms") + assert req.language == "ro" + assert req.engine == "mms" + + def test_generation_request_rejects_unknown_engine(self): + from pydantic import ValidationError + + from backend.models import GenerationRequest + + with pytest.raises(ValidationError): + GenerationRequest(profile_id="p1", text="hi", engine="mms2") + + def test_profile_create_accepts_ro(self): + from backend.models import VoiceProfileCreate + + profile = VoiceProfileCreate( + name="Vocea", + language="ro", + voice_type="preset", + preset_engine="mms", + preset_voice_id=MMS_DEFAULT_VOICE, + default_engine="mms", + ) + assert profile.language == "ro" + assert profile.default_engine == "mms" + + def test_speak_request_accepts_ro_and_mms(self): + from backend.models import SpeakRequest + + req = SpeakRequest(text="Bună", engine="mms", language="ro") + assert req.engine == "mms" + assert req.language == "ro" + + def test_mcp_binding_accepts_mms(self): + from backend.models import MCPClientBindingUpsert + + binding = MCPClientBindingUpsert(client_id="client-1", default_engine="mms") + assert binding.default_engine == "mms" + + +class TestMMSBackendUnit: + def test_initial_state(self): + backend = MMSTTSBackend() + assert not backend.is_loaded() + assert backend._get_model_path("default") == "facebook/mms-tts-ron" + + async def test_create_voice_prompt_returns_preset_fallback(self): + backend = MMSTTSBackend() + prompt, was_cached = await backend.create_voice_prompt("/tmp/none.wav", "text") + assert prompt == MMS_PRESET_PROMPT + assert was_cached is False + + def test_unload_without_load_is_noop(self): + backend = MMSTTSBackend() + backend.unload_model() + assert not backend.is_loaded() + + +RUN_MMS_E2E = os.environ.get("VOICEBOX_MMS_E2E") == "1" + + +@pytest.mark.skipif(not RUN_MMS_E2E, reason="set VOICEBOX_MMS_E2E=1 to run (downloads ~150MB model)") +class TestMMSGenerationE2E: + """Full generation through the real model — both diacritic conventions + must produce identical tokens and valid audio at 16kHz.""" + + async def test_generate_romanian_with_both_diacritic_variants(self): + backend = MMSTTSBackend() + try: + text = ( + "Bună ziua! Ce mai faceți? Știți că țara noastră e frumoasă? " + f"{S_CEDILLA_UPPER}i {T_CEDILLA}ine{T_COMMA}i minte pa{S_CEDILLA}ii." + ) + audio, sample_rate = await backend.generate(text, MMS_PRESET_PROMPT, "ro") + + assert sample_rate == MMS_SAMPLE_RATE == 16000 + assert isinstance(audio, np.ndarray) + assert audio.dtype == np.float32 + assert audio.ndim == 1 + assert len(audio) > sample_rate, "expected more than 1s of audio" + assert not np.isnan(audio).any() + assert float(np.abs(audio).max()) > 0.01, "audio should not be silence" + finally: + backend.unload_model() + + async def test_diacritic_variants_tokenize_identically(self): + backend = MMSTTSBackend() + try: + await backend.load_model() + tokenizer = backend._tokenizer + + cedilla_ids = tokenizer(normalize_romanian_text("ştiţi paşii ţară"))["input_ids"] + comma_ids = tokenizer(normalize_romanian_text("știți pașii țară"))["input_ids"] + assert cedilla_ids == comma_ids + # Nothing was dropped: with add_blank the tokenizer interleaves a + # blank between characters -> 2 * len(text) + 1 tokens. + assert len(comma_ids) == 2 * len("știți pașii țară") + 1 + finally: + backend.unload_model() + + async def test_seeded_generation_is_deterministic(self): + backend = MMSTTSBackend() + try: + first, _ = await backend.generate("Bună ziua!", MMS_PRESET_PROMPT, "ro", seed=42) + second, _ = await backend.generate("Bună ziua!", MMS_PRESET_PROMPT, "ro", seed=42) + np.testing.assert_allclose(first, second) + finally: + backend.unload_model() + + async def test_fully_out_of_vocab_text_returns_silence(self): + backend = MMSTTSBackend() + try: + audio, sample_rate = await backend.generate("!!!", MMS_PRESET_PROMPT, "ro") + assert sample_rate == MMS_SAMPLE_RATE + assert len(audio) == sample_rate + assert not audio.any() + finally: + backend.unload_model() From b587d84e7eabb519a10ae7717e0e9fe2d5206a5d Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 08:49:02 +0200 Subject: [PATCH 04/15] Document the MMS Romanian engine README engine/language counts (7 to 8 engines, 23 to 24 languages) and tables, preset-voices docs page with the MMS section, PROJECT_STATUS engine tracking, and a CHANGELOG entry under Unreleased. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 10 +++++++ README.md | 11 ++++---- docs/PROJECT_STATUS.md | 9 +++++-- docs/content/docs/overview/preset-voices.mdx | 28 ++++++++++++++++++-- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b03f5e46..9519de2a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ ## [Unreleased] +### Voice Generation + +- **Romanian TTS via a new MMS engine.** A new `mms` engine wraps Meta's + MMS-TTS Romanian checkpoint (`facebook/mms-tts-ron`) — a ~150MB VITS model + that runs realtime on CPU, using the already-bundled transformers runtime + (zero new dependencies). It ships as a preset voice (like Kokoro): create a + profile with the Romanian MMS voice and generate. Romanian text using either + diacritic convention (cedilla ş/ţ or comma-below ș/ț) is normalized + automatically so no character is silently dropped by the tokenizer. + ### Linux - **ROCm setup works on Linux AMD systems.** Docker ROCm builds now keep PyTorch diff --git a/README.md b/README.md index 34ef1c0c8..f7848dad9 100644 --- a/README.md +++ b/README.md @@ -67,14 +67,14 @@ ## What is Voicebox? -Voicebox is a **local-first AI voice studio** — a free and open-source alternative to **ElevenLabs** and **WisprFlow** in one app. Clone voices from a few seconds of audio, generate speech in 23 languages across 7 TTS engines, dictate into any text field with a global hotkey, and give any MCP-aware AI agent a voice of your choosing. +Voicebox is a **local-first AI voice studio** — a free and open-source alternative to **ElevenLabs** and **WisprFlow** in one app. Clone voices from a few seconds of audio, generate speech in 24 languages across 8 TTS engines, dictate into any text field with a global hotkey, and give any MCP-aware AI agent a voice of your choosing. The two cloud incumbents sit on opposite halves of the voice I/O loop — ElevenLabs on output, WisprFlow on input. Voicebox does both, bridges them with a bundled local LLM for refinement and per-profile personas, and runs the whole thing on your machine. - **Complete privacy** — models, voice data, and captures never leave your machine -- **7 TTS engines** — Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, and Kokoro +- **8 TTS engines** — Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro, and MMS - **Voice cloning and preset voices** — zero-shot cloning from a reference sample, or 50+ curated preset voices via Kokoro and Qwen CustomVoice -- **23 languages** — from English to Arabic, Japanese, Hindi, Swahili, and more +- **24 languages** — from English to Arabic, Japanese, Hindi, Romanian, Swahili, and more - **Post-processing effects** — pitch shift, reverb, delay, chorus, compression, and filters - **Expressive speech** — paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice - **Unlimited length** — auto-chunking with crossfade for scripts, articles, and chapters @@ -109,7 +109,7 @@ The two cloud incumbents sit on opposite halves of the voice I/O loop — Eleven ### Multi-Engine Voice Cloning -Seven TTS engines with different strengths, switchable per-generation: +Eight TTS engines with different strengths, switchable per-generation: | Engine | Languages | Strengths | | --------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------- | @@ -120,6 +120,7 @@ Seven TTS engines with different strengths, switchable per-generation: | **Chatterbox Turbo** | English | Fast 350M model with paralinguistic emotion/sound tags | | **TADA** (1B / 3B) | 10 | HumeAI speech-language model — 700s+ coherent audio, text-acoustic dual alignment | | **Kokoro** | 8 | 50 curated preset voices, tiny 82M model, fast CPU inference | +| **MMS** (Meta) | Romanian | Per-language VITS checkpoints (~150MB), CPU realtime, preset voice — the only engine with Romanian | ### Emotions & Paralinguistic Tags @@ -369,7 +370,7 @@ Full API documentation available at `http://127.0.0.1:17493/docs`. | Frontend | React, TypeScript, Tailwind CSS | | State | Zustand, React Query | | Backend | FastAPI (Python) | -| TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro | +| TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro, MMS | | STT | Whisper / Whisper Turbo (PyTorch or MLX) | | Local LLM | Qwen3 (0.6B / 1.7B / 4B), shared runtime with TTS / STT | | MCP Server | FastMCP mounted at `/mcp` (Streamable HTTP) + bundled stdio shim binary | diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index eef11d8a6..c4a48ea5d 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -23,7 +23,7 @@ The backend exposes: -- **`TTSBackend` Protocol** with seven concrete engine implementations: +- **`TTSBackend` Protocol** with eight concrete engine implementations: - Qwen3-TTS (PyTorch or MLX depending on platform) - Qwen CustomVoice (predefined speakers with instruct) - LuxTTS (fast, CPU-friendly) @@ -31,6 +31,7 @@ The backend exposes: - Chatterbox Turbo (English, paralinguistic tags) - TADA (1B English, 3B multilingual via HumeAI) - Kokoro 82M (pre-built voices, CPU realtime) + - MMS (Meta per-language VITS checkpoints — Romanian, preset voice, CPU realtime) - **`STTBackend` Protocol** for Whisper (PyTorch or MLX-Whisper) - **Profiles / History / Stories** services for persistence and timeline editing @@ -49,6 +50,7 @@ The backend exposes: | Chatterbox Turbo | `backend/backends/chatterbox_turbo_backend.py` | Chatterbox Turbo — English, paralinguistic tags | | TADA | `backend/backends/hume_backend.py` | HumeAI TADA — 1B English + 3B Multilingual | | Kokoro | `backend/backends/kokoro_backend.py` | Kokoro 82M — CPU realtime, pre-built voices | +| MMS | `backend/backends/mms_backend.py` | Meta MMS — Romanian VITS, preset voice, diacritics normalization | | Qwen CustomVoice | `backend/backends/qwen_custom_voice_backend.py` | Qwen CustomVoice — predefined speakers with instruct | | Platform detect | `backend/platform_detect.py` | Apple Silicon → MLX, else → PyTorch | | API types | `backend/models.py` | Pydantic request/response models | @@ -156,6 +158,7 @@ Shipped 2026-04-25 (PR #544). Voicebox went from a voice-cloning studio to a ful - Chatterbox Turbo — paralinguistic tags, low latency English (PR #258) - HumeAI TADA — 1B English + 3B Multilingual (PR #296) - Kokoro 82M — CPU-realtime, 8 languages, Apache 2.0 (PR #325) +- MMS (Meta) — Romanian preset voice, ~150MB VITS, CPU realtime, diacritics normalization - Multi-engine architecture with thread-safe backend registry (PR #254) - Chunked TTS generation — engine-agnostic, removes ~500 char limit (PR #266) - Async generation queue (PR #269) @@ -255,12 +258,13 @@ Shipped 2026-04-25 (PR #544). Voicebox went from a voice-cloning studio to a ful | TADA 1B | `tada-1b` | Cloned | English | ~4 GB | HumeAI speech-language model, 700s+ coherent audio | None | | TADA 3B Multilingual | `tada-3b-ml` | Cloned | 10 (en, ar, zh, de, es, fr, it, ja, pl, pt) | ~8 GB | Multilingual, text-acoustic dual alignment | None | | Kokoro 82M | `kokoro` | Preset | 8 (en, es, fr, hi, it, pt, ja, zh) | ~350 MB | 82M params, CPU realtime, Apache 2.0, pre-built voices | None | +| MMS Romanian | `mms-tts-ron` | Preset | Romanian (ro) | ~150 MB | Meta MMS VITS, 16 kHz, CPU realtime, cedilla/comma-below diacritics normalization | None | ### Multi-Engine Architecture (Shipped) - **Thread-safe backend registry** (`_tts_backends` dict + `_tts_backends_lock`) with double-checked locking - **Per-engine backend instances** — each engine gets its own singleton, loaded lazily -- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro'` +- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'mms'` - **Per-engine language filtering** — `ENGINE_LANGUAGES` map in frontend, backend regex accepts all languages - **Per-engine voice prompts** — `create_voice_prompt_for_profile()` dispatches to the correct backend - **Profile type system** — preset vs cloned profiles, UI grays out incompatible engines and auto-switches on selection @@ -577,6 +581,7 @@ Notable: | **Chatterbox Turbo** | 5s zero-shot | Fast | 24 kHz | English | Low | Partial — inline tags | CPU/CUDA | **Shipped** (PR #258) | | **HumeAI TADA 1B/3B** | Zero-shot | 5x faster than LLM-TTS | 24 kHz | EN (1B), 10 (3B) | Medium | Partial — prosody | PyTorch | **Shipped** (PR #296) | | **Kokoro-82M** | Preset voices | CPU realtime | 24 kHz | 8 | Tiny (82M) | None | All | **Shipped** (PR #325) | +| **MMS (Meta)** | Preset voice (1/lang) | CPU realtime | 16 kHz | Romanian (extensible per-checkpoint) | Tiny (~100M) | None | All | **Shipped** — zero new deps (transformers VITS) | | ~~**CosyVoice2-0.5B**~~ | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | **Yes** | — | **Abandoned** (PR #311) — poor output quality | | ~~**VoxCPM2**~~ | Zero-shot | ~0.15 RTF streaming | 48 kHz | 30 | Medium | Partial — parenthetical style | **CUDA-only in practice** | **Backlogged** (2026-04-18) — see notes above | | **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | **Yes** — word-level inline | All | Candidate — license TBD | diff --git a/docs/content/docs/overview/preset-voices.mdx b/docs/content/docs/overview/preset-voices.mdx index f7b3c92bd..8cf9458df 100644 --- a/docs/content/docs/overview/preset-voices.mdx +++ b/docs/content/docs/overview/preset-voices.mdx @@ -7,12 +7,13 @@ description: "Use built-in, ready-made voices without recording audio samples" Some Voicebox engines ship with a curated set of pre-built voices. Instead of cloning from your own audio sample, you pick a voice from a fixed catalog and the model speaks in that voice. No recording, no upload, no per-voice training required. -Two engines in 0.4 ship preset voices: +Three engines ship preset voices: | Engine | Voices | Languages | Strengths | | --------------------- | ----------------------- | --------- | ------------------------------------------------------- | | **Kokoro 82M** | 50 | 9 | Tiny model, CPU-friendly, lowest VRAM of any engine | | **Qwen CustomVoice** | 9 (premium curated) | 4 | Natural-language style control over tone, emotion, pace | +| **MMS (Meta)** | 1 per language | Romanian | The only engine with Romanian — ~150MB, CPU realtime | Looking for cloning a specific person's voice instead? See [Voice Cloning](/overview/voice-cloning). @@ -42,7 +43,7 @@ Two engines in 0.4 ship preset voices: Same entry point as cloning profiles - Select **Kokoro** or **Qwen CustomVoice** from the engine dropdown + Select **Kokoro**, **Qwen CustomVoice**, or **MMS Romanian** from the engine dropdown The voice catalog for the chosen engine appears — preview each by clicking it @@ -167,6 +168,29 @@ The full Generate page also surfaces the instruct field as a separate input. | Instruct | Yes — natural-language style control | | Cloning | No — paired Base Qwen3-TTS engine handles cloning | +## MMS Romanian — Meta's Massively Multilingual Speech + +MMS wraps Meta's per-language VITS checkpoints (`facebook/mms-tts-{lang}`). Each checkpoint is a single fixed speaker, so there is exactly one preset voice per language. Voicebox ships the Romanian checkpoint — the only engine in the app that speaks Romanian. + +**Repository:** [`facebook/mms-tts-ron`](https://huggingface.co/facebook/mms-tts-ron) · CC-BY-NC 4.0 licensed + +| Voice | Gender | Language | +| --------------- | ------ | --------------- | +| Romanian (MMS) | male | Romanian (`ro`) | + +Romanian text in the wild mixes cedilla diacritics (ş/ţ) with the correct comma-below forms (ș/ț). Voicebox normalizes both conventions automatically before synthesis, so either spelling is pronounced correctly. + +### MMS at a Glance + +| Property | Value | +| --------------- | ---------------------------------------------- | +| Architecture | VITS (~100M params per language) | +| Sample rate | 16 kHz | +| Size on disk | ~150 MB | +| Speed | Realtime on CPU | +| Instruct | Not supported (single fixed speaker) | +| License | CC-BY-NC 4.0 | + ## Cloning vs Preset — Quick Decision | You want… | Use | From 6a8269831e8bac0982c2aa96f00267c51d154090 Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 11:13:57 +0200 Subject: [PATCH 05/15] Address review feedback on MMS engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CapturesTab: only honor a capture's language when the target engine supports it (MMS is Romanian-only; also hardens Kokoro playback), falling back to the profile's language otherwise - models.py: extract shared TTS_LANGUAGE_PATTERN / TTS_ENGINE_PATTERN constants to replace seven duplicated inline regexes - mms_backend: correct module docstring — adding a language also needs the checkpoint resolution threaded through model loading - README: mention MMS (Romanian) in the preset-voices bullet Co-Authored-By: Claude Opus 4.8 --- README.md | 2 +- .../components/CapturesTab/CapturesTab.tsx | 13 ++++++++++-- backend/backends/mms_backend.py | 6 ++++-- backend/models.py | 21 +++++++++++-------- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f7848dad9..eb13001eb 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ The two cloud incumbents sit on opposite halves of the voice I/O loop — Eleven - **Complete privacy** — models, voice data, and captures never leave your machine - **8 TTS engines** — Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro, and MMS -- **Voice cloning and preset voices** — zero-shot cloning from a reference sample, or 50+ curated preset voices via Kokoro and Qwen CustomVoice +- **Voice cloning and preset voices** — zero-shot cloning from a reference sample, or 50+ curated preset voices via Kokoro, Qwen CustomVoice, and MMS (Romanian) - **24 languages** — from English to Arabic, Japanese, Hindi, Romanian, Swahili, and more - **Post-processing effects** — pitch shift, reverb, delay, chorus, compression, and filters - **Expressive speech** — paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice diff --git a/app/src/components/CapturesTab/CapturesTab.tsx b/app/src/components/CapturesTab/CapturesTab.tsx index 6dcf99b3e..0f69a369c 100644 --- a/app/src/components/CapturesTab/CapturesTab.tsx +++ b/app/src/components/CapturesTab/CapturesTab.tsx @@ -64,7 +64,7 @@ import type { CaptureSource, VoiceProfileResponse, } from '@/lib/api/types'; -import type { LanguageCode } from '@/lib/constants/languages'; +import { ENGINE_LANGUAGES, type LanguageCode } from '@/lib/constants/languages'; import { BOTTOM_SAFE_AREA_PADDING } from '@/lib/constants/ui'; import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession'; import { useDictationReadiness } from '@/lib/hooks/useDictationReadiness'; @@ -263,7 +263,6 @@ export function CapturesTab() { mutationFn: async ({ capture, voice }: { capture: CaptureResponse; voice: VoiceProfileResponse }) => { const text = capture.transcript_refined || capture.transcript_raw; if (!text.trim()) throw new Error(t('captures.noTranscriptError')); - const language = (capture.language || voice.language) as LanguageCode; // Preset profiles (Kokoro etc.) reject the qwen default — honor the // profile's stored engine preference. Cloned profiles without an // override fall through to whatever the backend picks. @@ -271,6 +270,16 @@ export function CapturesTab() { | 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'mms' | undefined; + // Prefer the capture's language, but only if the target engine can + // speak it (e.g. MMS is Romanian-only, Kokoro covers 8 languages) — + // otherwise fall back to the profile's own language. + const captureLanguage = capture.language as LanguageCode | undefined; + const supported = engine ? ENGINE_LANGUAGES[engine] : undefined; + const language = ( + captureLanguage && (!supported || supported.includes(captureLanguage)) + ? captureLanguage + : voice.language + ) as LanguageCode; return apiClient.generateSpeech({ profile_id: voice.id, text, diff --git a/backend/backends/mms_backend.py b/backend/backends/mms_backend.py index 4e72d92ad..cec49d6c7 100644 --- a/backend/backends/mms_backend.py +++ b/backend/backends/mms_backend.py @@ -11,8 +11,10 @@ Languages supported: - Romanian (ro) — ``facebook/mms-tts-ron`` -Adding a language is a one-line addition to ``MMS_HF_REPOS`` plus a voice -entry in ``MMS_VOICES`` and a ``ModelConfig`` registration. +Adding a language requires an entry in ``MMS_HF_REPOS`` and ``MMS_VOICES``, +a ``ModelConfig`` registration, and threading the requested language through +``_get_model_path``/``_load_model_sync`` — today those resolve to +``MMS_DEFAULT_LANGUAGE`` because only one checkpoint ships. """ import asyncio diff --git a/backend/models.py b/backend/models.py index b1a6a7d47..7dab324c1 100644 --- a/backend/models.py +++ b/backend/models.py @@ -11,15 +11,18 @@ default_toggle_to_talk_chord, ) +# Shared validation patterns for TTS requests. The Qwen-specific 10-language +# patterns further down are intentionally separate — do not merge them. +TTS_LANGUAGE_PATTERN = "^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr|ro)$" +TTS_ENGINE_PATTERN = "^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$" + class VoiceProfileCreate(BaseModel): """Request model for creating a voice profile.""" name: str = Field(..., min_length=1, max_length=100) description: Optional[str] = Field(None, max_length=500) - language: str = Field( - default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr|ro)$" - ) + language: str = Field(default="en", pattern=TTS_LANGUAGE_PATTERN) voice_type: Optional[str] = Field(default="cloned", pattern="^(cloned|preset|designed)$") preset_engine: Optional[str] = Field(None, max_length=50) preset_voice_id: Optional[str] = Field(None, max_length=100) @@ -81,11 +84,11 @@ class GenerationRequest(BaseModel): profile_id: str text: str = Field(..., min_length=1, max_length=50000) - language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr|ro)$") + language: str = Field(default="en", pattern=TTS_LANGUAGE_PATTERN) seed: Optional[int] = Field(None, ge=0) model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$") instruct: Optional[str] = Field(None, max_length=500) - engine: Optional[str] = Field(default="qwen", pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$") + engine: Optional[str] = Field(default="qwen", pattern=TTS_ENGINE_PATTERN) personality: bool = Field( default=False, description="When true and the profile has a personality prompt, the input text is rewritten in-character before TTS.", @@ -317,7 +320,7 @@ class MCPClientBindingResponse(BaseModel): profile_id: Optional[str] = None default_engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$", + pattern=TTS_ENGINE_PATTERN, ) default_personality: bool = False last_seen_at: Optional[datetime] = None @@ -336,7 +339,7 @@ class MCPClientBindingUpsert(BaseModel): profile_id: Optional[str] = None default_engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$", + pattern=TTS_ENGINE_PATTERN, ) default_personality: bool = False @@ -355,7 +358,7 @@ class SpeakRequest(BaseModel): ) engine: Optional[str] = Field( None, - pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$", + pattern=TTS_ENGINE_PATTERN, ) personality: Optional[bool] = Field( None, @@ -363,7 +366,7 @@ class SpeakRequest(BaseModel): ) language: Optional[str] = Field( None, - pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr|ro)$", + pattern=TTS_LANGUAGE_PATTERN, ) From ad7464fc83f4b03566266de59b0f88ece465cc2a Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 22:27:00 +0200 Subject: [PATCH 06/15] Add F5-TTS backend engine with Romanian voice cloning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps the community Romanian F5-TTS fine-tune (MihaiPopa-1/F5-TTS-Romanian, Apache-2.0) for zero-shot voice cloning — the first engine with true Romanian cloning. Follows the Chatterbox cloning pattern: the voice prompt stores the reference audio path + transcript, processed at generation time. - References over 12s are trimmed at the quietest 300ms window between 8-12s (F5 conditioning degrades with long refs; upstream hard-clips at 12s), with the transcript proportionally truncated to keep correspondence and avoid f5-tts's Whisper auto-transcription download. - Romanian diacritics are normalized to the comma-below forms: the fine-tune's vocab lacks cedilla ţ (U+0163) and f5-tts silently maps out-of-vocab chars to space. - MPS enabled: verified stable on this checkpoint with memory free and ~2x faster than CPU (65.6s vs 125.8s for 5.5s of audio). - Cached-state check covers both the fine-tune repo and the Vocos vocoder that f5-tts downloads on init. - New dependency: f5-tts==1.1.21 (pinned), imported lazily. Co-Authored-By: Claude Opus 4.8 --- backend/backends/f5_backend.py | 335 +++++++++++++++++++++++++++++++++ backend/requirements.txt | 5 + 2 files changed, 340 insertions(+) create mode 100644 backend/backends/f5_backend.py diff --git a/backend/backends/f5_backend.py b/backend/backends/f5_backend.py new file mode 100644 index 000000000..16e26b844 --- /dev/null +++ b/backend/backends/f5_backend.py @@ -0,0 +1,335 @@ +""" +F5-TTS Romanian backend implementation. + +Wraps the community Romanian fine-tune of F5-TTS +(``MihaiPopa-1/F5-TTS-Romanian``, Apache-2.0, base ``SWivid/F5-TTS``) for +zero-shot voice cloning — the first engine offering true Romanian cloning +from a reference sample. The fine-tune retains English. + +Like Chatterbox, this is a cloning engine: the voice prompt stores the +reference audio path and transcript, and the audio is processed at +generation time. References longer than 12 seconds are trimmed at the +quietest moment (F5 conditioning degrades with long references) and the +transcript is proportionally truncated to keep text/audio correspondence. + +Output is 24 kHz mono float32 via the Vocos vocoder, which f5-tts +downloads from ``charactr/vocos-mel-24khz`` on first load. +""" + +import asyncio +import hashlib +import logging +import unicodedata +from pathlib import Path + +import numpy as np + +from .. import config +from .base import ( + combine_voice_prompts as _combine_voice_prompts, + empty_device_cache, + get_torch_device, + is_model_cached, + model_load_progress, +) + +logger = logging.getLogger(__name__) + +F5_HF_REPO = "MihaiPopa-1/F5-TTS-Romanian" +F5_CKPT_FILE = "model_750_pruned.safetensors" +F5_VOCAB_FILE = "vocab.txt" +# f5_tts.api.F5TTS downloads the Vocos vocoder from this repo on init; +# _is_model_cached must account for it so the UI "downloaded" state is truthful. +F5_VOCODER_HF_REPO = "charactr/vocos-mel-24khz" +F5_SAMPLE_RATE = 24000 +F5_NFE_STEPS = 32 + +# F5 conditioning degrades with references over ~12s (upstream also hard-clips +# at 12s). Trim at the quietest 300ms window found between 8s and 12s so the +# cut lands in a natural pause and upstream's cruder clipper never fires. +F5_MAX_REF_SECONDS = 12.0 +F5_MIN_REF_SECONDS = 8.0 +F5_TRIM_WINDOW_SECONDS = 0.3 +_TRIM_HOP_SECONDS = 0.01 + +# The fine-tune's vocab.txt contains comma-below ș (U+0219) and ț (U+021B) +# plus cedilla ş (U+015F), but NOT cedilla ţ (U+0163). f5-tts maps +# out-of-vocab characters to index 0 (space), silently dropping them, so +# both real-world diacritic families must be mapped onto the comma-below +# forms the model was trained on. (ş is technically in the vocab but the +# comma-below family is the canonical Romanian form; unifying onto it +# keeps ref/gen text consistent.) +_RO_DIACRITICS_TRANSLATION = str.maketrans( + { + "ş": "ș", # ş (s-cedilla) -> ș (s-comma-below) + "Ş": "Ș", # Ş (S-cedilla) -> Ș (S-comma-below) + "ţ": "ț", # ţ (t-cedilla, NOT in vocab) -> ț (t-comma-below) + "Ţ": "Ț", # Ţ (T-cedilla, NOT in vocab) -> Ț (T-comma-below) + } +) + + +def normalize_romanian_text(text: str) -> str: + """Normalize Romanian text to the diacritic forms in the F5 vocab. + + Applies NFC normalization first (composing any decomposed + letter + combining-mark sequences), then maps cedilla s and t + variants onto the comma-below forms the fine-tune was trained on, + so no diacritic is silently dropped by the character tokenizer. + """ + return unicodedata.normalize("NFC", text).translate(_RO_DIACRITICS_TRANSLATION) + + +def trim_reference_audio( + audio: np.ndarray, + sample_rate: int, + *, + max_seconds: float = F5_MAX_REF_SECONDS, + min_seconds: float = F5_MIN_REF_SECONDS, + window_seconds: float = F5_TRIM_WINDOW_SECONDS, +) -> tuple[np.ndarray, float]: + """Trim reference audio to at most ``max_seconds`` at the quietest moment. + + Slides a ``window_seconds`` RMS window (10ms hop) over the span between + ``min_seconds`` and ``max_seconds`` and cuts at the centre of the + quietest window, so the cut lands in a pause rather than mid-word. + + Args: + audio: Mono audio array. + sample_rate: Sample rate of ``audio``. + max_seconds: Hard upper bound for the trimmed length. + min_seconds: Earliest allowed cut point. + window_seconds: RMS window length used to find the quietest moment. + + Returns: + Tuple of (trimmed_audio, kept_fraction). ``kept_fraction`` is 1.0 + when the audio was already short enough and untouched. + """ + if len(audio) <= max_seconds * sample_rate: + return audio, 1.0 + + start = int(min_seconds * sample_rate) + end = min(int(max_seconds * sample_rate), len(audio)) + window = max(1, int(window_seconds * sample_rate)) + hop = max(1, int(_TRIM_HOP_SECONDS * sample_rate)) + + segment = audio[start:end].astype(np.float64) + window_starts = np.arange(0, len(segment) - window, hop) + if len(window_starts) == 0: + cut = end + else: + # Windowed energy via cumulative sum — O(n) instead of a python loop. + cumulative = np.concatenate(([0.0], np.cumsum(segment * segment))) + energies = cumulative[window_starts + window] - cumulative[window_starts] + quietest = int(window_starts[np.argmin(energies)]) + cut = start + quietest + window // 2 + + return audio[:cut], cut / len(audio) + + +def trim_reference_text(text: str, kept_fraction: float) -> str: + """Truncate a transcript to match trimmed reference audio. + + Keeps the leading ``kept_fraction`` of words (assuming roughly constant + speech rate) and cuts at a word boundary. This is an approximation — + the exact spoken-word boundary isn't known without ASR, and f5-tts's + auto-transcription path would download a ~1.6GB Whisper model. + """ + if kept_fraction >= 1.0: + return text + words = text.split() + keep = max(1, round(len(words) * kept_fraction)) + return " ".join(words[:keep]).rstrip(",;:- ") + + +class F5TTSBackend: + """F5-TTS Romanian backend for zero-shot voice cloning.""" + + def __init__(self): + self.model = None + self.model_size = "default" + self._device: str | None = None + self._model_load_lock = asyncio.Lock() + + def _get_device(self) -> str: + # MPS verified stable on this checkpoint with memory free and ~2x + # faster than CPU (CPU is ~20x slower than realtime for F5's 32-step + # flow matching, so every bit helps). + return get_torch_device(allow_mps=True) + + def is_loaded(self) -> bool: + return self.model is not None + + def _get_model_path(self, model_size: str = "default") -> str: + return F5_HF_REPO + + def _is_model_cached(self, model_size: str = "default") -> bool: + """Check both the fine-tune checkpoint and the Vocos vocoder cache.""" + return is_model_cached(F5_HF_REPO, required_files=[F5_CKPT_FILE, F5_VOCAB_FILE]) and is_model_cached( + F5_VOCODER_HF_REPO + ) + + async def load_model(self, model_size: str = "default") -> None: + """Load the F5-TTS Romanian model.""" + if self.model is not None: + return + async with self._model_load_lock: + if self.model is not None: + return + await asyncio.to_thread(self._load_model_sync) + + def _load_model_sync(self): + """Synchronous model loading.""" + model_name = "f5-tts-romanian" + is_cached = self._is_model_cached() + + with model_load_progress(model_name, is_cached): + from huggingface_hub import hf_hub_download # lazy: heavy import + + ckpt_file = hf_hub_download(F5_HF_REPO, F5_CKPT_FILE) + vocab_file = hf_hub_download(F5_HF_REPO, F5_VOCAB_FILE) + + device = self._get_device() + self._device = device + logger.info("Loading F5-TTS Romanian on %s...", device) + + from f5_tts.api import F5TTS # lazy: heavy import + + # The pruned checkpoint stores plain (non-EMA-prefixed) weights, + # so use_ema=False is the semantically correct load path. + self.model = F5TTS( + model="F5TTS_v1_Base", + ckpt_file=ckpt_file, + vocab_file=vocab_file, + use_ema=False, + device=device, + ) + + logger.info("F5-TTS Romanian loaded successfully") + + def unload_model(self) -> None: + """Unload model to free memory.""" + if self.model is not None: + device = self._device + del self.model + self.model = None + self._device = None + empty_device_cache(device) + logger.info("F5-TTS Romanian unloaded") + + async def create_voice_prompt( + self, + audio_path: str, + reference_text: str, + use_cache: bool = True, + ) -> tuple[dict, bool]: + """ + Create voice prompt from reference audio. + + Like Chatterbox, F5 processes reference audio at generation time, + so the prompt just stores the file path and transcript. Trimming + to the 12s reference limit also happens at generation time. + """ + voice_prompt = { + "ref_audio": str(audio_path), + "ref_text": reference_text, + } + return voice_prompt, False + + async def combine_voice_prompts( + self, + audio_paths: list[str], + reference_texts: list[str], + ) -> tuple[np.ndarray, str]: + return await _combine_voice_prompts(audio_paths, reference_texts, sample_rate=F5_SAMPLE_RATE) + + def _prepare_reference(self, ref_audio: str, ref_text: str) -> tuple[str, str]: + """Trim the reference to the F5 limit, returning (audio_path, text). + + References at or under the limit pass through untouched. Longer ones + are cut at the quietest moment between 8s and 12s, written to the + cache directory (content-keyed, reused across generations), and the + transcript is proportionally truncated to keep correspondence. + """ + import librosa # lazy: heavy import + import soundfile as sf # lazy: heavy import + + audio, sample_rate = librosa.load(ref_audio, sr=None, mono=True) + trimmed, kept_fraction = trim_reference_audio(audio, sample_rate) + if kept_fraction >= 1.0: + return ref_audio, ref_text + + stat = Path(ref_audio).stat() + cache_key = hashlib.md5(f"{ref_audio}:{stat.st_mtime_ns}:{stat.st_size}".encode()).hexdigest()[:16] + trimmed_path = config.get_cache_dir() / f"f5_ref_{cache_key}.wav" + if not trimmed_path.exists(): + sf.write(str(trimmed_path), trimmed, sample_rate) + + trimmed_text = trim_reference_text(ref_text, kept_fraction) + logger.info( + "[F5] Trimmed reference %.1fs -> %.1fs (kept %.0f%%)", + len(audio) / sample_rate, + len(trimmed) / sample_rate, + kept_fraction * 100, + ) + return str(trimmed_path), trimmed_text + + async def generate( + self, + text: str, + voice_prompt: dict, + language: str = "ro", + seed: int | None = None, + instruct: str | None = None, + ) -> tuple[np.ndarray, int]: + """ + Generate audio using F5-TTS Romanian. + + Args: + text: Text to synthesize + voice_prompt: Dict with ref_audio path and ref_text transcript + language: Language code ("ro" or "en") + seed: Random seed for reproducibility + instruct: Not supported by F5 (ignored) + + Returns: + Tuple of (audio_array, sample_rate) + """ + await self.load_model() + + ref_audio = voice_prompt.get("ref_audio") + ref_text = voice_prompt.get("ref_text") or "" + if not ref_audio or not Path(ref_audio).exists(): + raise ValueError(f"F5-TTS requires reference audio for voice cloning (missing: {ref_audio})") + # An empty ref_text makes f5-tts auto-transcribe with Whisper + # large-v3-turbo — a surprise ~1.6GB download. Refuse instead. + if not ref_text.strip(): + raise ValueError("F5-TTS requires the reference transcript (profile sample reference_text)") + + def _generate_sync(): + ref_file, trimmed_text = self._prepare_reference(ref_audio, ref_text) + + if language == "ro": + gen_text = normalize_romanian_text(text) + prompt_text = normalize_romanian_text(trimmed_text) + else: + gen_text = unicodedata.normalize("NFC", text) + prompt_text = unicodedata.normalize("NFC", trimmed_text) + + logger.info("[F5] Generating: lang=%s", language) + + # F5TTS.infer seeds torch/numpy/random itself (seed_everything); + # passing seed=None picks a fresh random seed. + wav, sample_rate, _spec = self.model.infer( + ref_file, + prompt_text, + gen_text, + nfe_step=F5_NFE_STEPS, + seed=seed, + show_info=logger.debug, + ) + + audio = np.asarray(wav, dtype=np.float32) + return audio, int(sample_rate) + + return await asyncio.to_thread(_generate_sync) diff --git a/backend/requirements.txt b/backend/requirements.txt index caafc0e7e..29a1440e3 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -52,6 +52,11 @@ en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_ # for the same reason en_core_web_sm does. unidic-lite>=1.0.8 +# F5-TTS (Romanian voice-cloning fine-tune, MihaiPopa-1/F5-TTS-Romanian). +# Pinned: pulls a large dep tree (datasets, wandb, cached_path) — bump +# deliberately. Its torchaudio I/O path needs torchcodec + system FFmpeg. +f5-tts==1.1.21 + # Audio processing librosa>=0.10.0 soundfile>=0.12.0 From 09d226ba2de41691606f92f31ccb530b52f36823 Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 22:27:10 +0200 Subject: [PATCH 07/15] Register the F5 engine across backend touch-points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TTS_ENGINES entry, ModelConfig (f5-tts-romanian, ro+en), and factory branch in backends/__init__.py - f5 added to TTS_ENGINE_PATTERN (covers all validation sites) - f5 added to CLONING_ENGINES so cloned profiles accept it; no preset branches — F5 is a cloning engine - PyInstaller: hidden-import + collect-all f5_tts (yaml model configs are data files read via importlib.resources); excluded from the MCP shim Co-Authored-By: Claude Opus 4.8 --- backend/backends/__init__.py | 13 +++++++++++++ backend/build_binary.py | 9 +++++++++ backend/models.py | 2 +- backend/services/profiles.py | 2 +- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index 5a0bfdc2c..8b7eb8240 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -216,6 +216,7 @@ def is_loaded(self) -> bool: "tada": "TADA", "kokoro": "Kokoro", "mms": "MMS TTS", + "f5": "F5-TTS", } LLM_ENGINES = { @@ -373,6 +374,14 @@ def _get_non_qwen_tts_configs() -> list[ModelConfig]: size_mb=150, languages=["ro"], ), + ModelConfig( + model_name="f5-tts-romanian", + display_name="F5-TTS Romanian (community)", + engine="f5", + hf_repo_id="MihaiPopa-1/F5-TTS-Romanian", + size_mb=1200, + languages=["ro", "en"], + ), ] @@ -717,6 +726,10 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend: from .mms_backend import MMSTTSBackend backend = MMSTTSBackend() + elif engine == "f5": + from .f5_backend import F5TTSBackend + + backend = F5TTSBackend() elif engine == "qwen_custom_voice": from .qwen_custom_voice_backend import QwenCustomVoiceBackend diff --git a/backend/build_binary.py b/backend/build_binary.py index a110dde3d..da69cf52e 100644 --- a/backend/build_binary.py +++ b/backend/build_binary.py @@ -280,6 +280,13 @@ def build_server(cuda=False, rocm=False): # bundles the model classes, only the backend module is needed. "--hidden-import", "backend.backends.mms_backend", + # F5-TTS — model configs ship as yaml data files read via + # importlib.resources at runtime (configs/F5TTS_v1_Base.yaml), + # which hidden-import alone won't bundle. + "--hidden-import", + "backend.backends.f5_backend", + "--collect-all", + "f5_tts", # misaki ships G2P data files (dictionaries, phoneme tables) # that must be bundled for espeak/en/ja/zh G2P to work "--collect-all", @@ -722,6 +729,8 @@ def build_shim(): "--exclude-module", "kokoro", "--exclude-module", + "f5_tts", + "--exclude-module", "misaki", "--exclude-module", "spacy", diff --git a/backend/models.py b/backend/models.py index 7dab324c1..97649a565 100644 --- a/backend/models.py +++ b/backend/models.py @@ -14,7 +14,7 @@ # Shared validation patterns for TTS requests. The Qwen-specific 10-language # patterns further down are intentionally separate — do not merge them. TTS_LANGUAGE_PATTERN = "^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr|ro)$" -TTS_ENGINE_PATTERN = "^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms)$" +TTS_ENGINE_PATTERN = "^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro|mms|f5)$" class VoiceProfileCreate(BaseModel): diff --git a/backend/services/profiles.py b/backend/services/profiles.py index ac1340b20..880c97db8 100644 --- a/backend/services/profiles.py +++ b/backend/services/profiles.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) -CLONING_ENGINES = {"qwen", "luxtts", "chatterbox", "chatterbox_turbo", "tada"} +CLONING_ENGINES = {"qwen", "luxtts", "chatterbox", "chatterbox_turbo", "tada", "f5"} def _profile_to_response( From 1265e6e524b89438d73429f10c19373a9ecbba46 Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 22:27:10 +0200 Subject: [PATCH 08/15] Add F5-TTS Romanian engine to frontend Cloning-engine treatment (like Chatterbox), not preset: added to the engine unions, zod enum, model-name/display chains, engine selector (+ CLONING_ENGINES set), cloned-profile default-engine options, model management filter, and ENGINE_LANGUAGES (ro, en). Deliberately NOT in PRESET_ONLY_ENGINES / PRESET_ENGINES / preset badge maps. Co-Authored-By: Claude Opus 4.8 --- .../components/CapturesTab/CapturesTab.tsx | 2 +- .../Generation/EngineModelSelector.tsx | 4 ++- .../Generation/FloatingGenerateBox.tsx | 14 ++++++++--- .../ServerSettings/ModelManagement.tsx | 3 ++- .../components/VoiceProfiles/ProfileForm.tsx | 5 ++-- app/src/lib/api/types.ts | 3 ++- app/src/lib/constants/languages.ts | 2 ++ app/src/lib/hooks/useGenerationForm.ts | 25 +++++++++++-------- app/src/lib/utils/format.ts | 1 + 9 files changed, 38 insertions(+), 21 deletions(-) diff --git a/app/src/components/CapturesTab/CapturesTab.tsx b/app/src/components/CapturesTab/CapturesTab.tsx index 0f69a369c..481eb1f1c 100644 --- a/app/src/components/CapturesTab/CapturesTab.tsx +++ b/app/src/components/CapturesTab/CapturesTab.tsx @@ -268,7 +268,7 @@ export function CapturesTab() { // override fall through to whatever the backend picks. const engine = voice.default_engine as | 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' - | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'mms' + | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'mms' | 'f5' | undefined; // Prefer the capture's language, but only if the target engine can // speak it (e.g. MMS is Romanian-only, Kokoro covers 8 languages) — diff --git a/app/src/components/Generation/EngineModelSelector.tsx b/app/src/components/Generation/EngineModelSelector.tsx index ef3e2a918..b12ef6ac0 100644 --- a/app/src/components/Generation/EngineModelSelector.tsx +++ b/app/src/components/Generation/EngineModelSelector.tsx @@ -28,6 +28,7 @@ const ENGINE_OPTIONS = [ { value: 'tada:3B', label: 'TADA 3B Multilingual', engine: 'tada' }, { value: 'kokoro', label: 'Kokoro 82M', engine: 'kokoro' }, { value: 'mms', label: 'MMS Romanian', engine: 'mms' }, + { value: 'f5', label: 'F5-TTS Romanian', engine: 'f5' }, ] as const; const ENGINE_DESCRIPTIONS: Record = { @@ -39,13 +40,14 @@ const ENGINE_DESCRIPTIONS: Record = { tada: 'HumeAI, 700s+ coherent audio', kokoro: '82M params, CPU realtime, 8 langs', mms: 'Meta MMS, Romanian, CPU realtime', + f5: 'Romanian voice cloning, slow but faithful', }; /** Engines that only support English and should force language to 'en' on select. */ const ENGLISH_ONLY_ENGINES = new Set(['luxtts', 'chatterbox_turbo']); /** Engines that support cloned (reference audio) profiles. */ -const CLONING_ENGINES = new Set(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo', 'tada']); +const CLONING_ENGINES = new Set(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo', 'tada', 'f5']); function getAvailableOptions(selectedProfile?: VoiceProfileResponse | null) { if (!selectedProfile) return ENGINE_OPTIONS; diff --git a/app/src/components/Generation/FloatingGenerateBox.tsx b/app/src/components/Generation/FloatingGenerateBox.tsx index 31157d67f..91ac722ad 100644 --- a/app/src/components/Generation/FloatingGenerateBox.tsx +++ b/app/src/components/Generation/FloatingGenerateBox.tsx @@ -152,7 +152,8 @@ export function FloatingGenerateBox({ | 'tada' | 'kokoro' | 'qwen_custom_voice' - | 'mms'; + | 'mms' + | 'f5'; useEffect(() => { if (selectedProfile?.language) { form.setValue('language', selectedProfile.language as LanguageCode); @@ -419,13 +420,19 @@ export function FloatingGenerateBox({ ? 'bg-accent text-accent-foreground border border-accent hover:bg-accent/90' : 'bg-card border border-border hover:bg-background/50', )} - aria-label={active ? t('generation.persona.ariaLabelActive') : t('generation.persona.ariaLabelInactive')} + aria-label={ + active + ? t('generation.persona.ariaLabelActive') + : t('generation.persona.ariaLabelInactive') + } aria-pressed={active} > - {active ? t('generation.persona.tooltipActive') : t('generation.persona.tooltipInactive')} + {active + ? t('generation.persona.tooltipActive') + : t('generation.persona.tooltipInactive')} @@ -567,7 +574,6 @@ export function FloatingGenerateBox({ )} - m.model_name.startsWith('whisper')) ?? []; const llmModels = modelStatus?.models.filter((m) => m.model_name.startsWith('qwen3-')) ?? []; diff --git a/app/src/components/VoiceProfiles/ProfileForm.tsx b/app/src/components/VoiceProfiles/ProfileForm.tsx index 4ed511341..bb672f4bd 100644 --- a/app/src/components/VoiceProfiles/ProfileForm.tsx +++ b/app/src/components/VoiceProfiles/ProfileForm.tsx @@ -71,6 +71,7 @@ const DEFAULT_ENGINE_OPTIONS = [ { value: 'tada', label: 'TADA' }, { value: 'kokoro', label: 'Kokoro 82M' }, { value: 'mms', label: 'MMS Romanian' }, + { value: 'f5', label: 'F5-TTS Romanian' }, ] as const; function makeProfileSchema(t: (key: string) => string) { @@ -1209,9 +1210,7 @@ export function ProfileForm() { {...field} /> - - {t('profileForm.fields.personalityHint')} - + {t('profileForm.fields.personalityHint')} )} diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index 55d8792a5..a7be7c6b0 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -79,7 +79,8 @@ export interface GenerationRequest { | 'chatterbox_turbo' | 'tada' | 'kokoro' - | 'mms'; + | 'mms' + | 'f5'; instruct?: string; /** When true and the profile has a personality prompt, input text is rewritten in-character before TTS. */ personality?: boolean; diff --git a/app/src/lib/constants/languages.ts b/app/src/lib/constants/languages.ts index ca8534030..25e8e145f 100644 --- a/app/src/lib/constants/languages.ts +++ b/app/src/lib/constants/languages.ts @@ -73,6 +73,8 @@ export const ENGINE_LANGUAGES: Record = { kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'], qwen_custom_voice: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'], mms: ['ro'], + // F5-TTS Romanian community fine-tune — Romanian cloning; retains English. + f5: ['ro', 'en'], } as const; /** Helper: get language options for a given engine. */ diff --git a/app/src/lib/hooks/useGenerationForm.ts b/app/src/lib/hooks/useGenerationForm.ts index 3e8a9f714..229c9114c 100644 --- a/app/src/lib/hooks/useGenerationForm.ts +++ b/app/src/lib/hooks/useGenerationForm.ts @@ -28,6 +28,7 @@ const generationSchema = z.object({ 'tada', 'kokoro', 'mms', + 'f5', ]) .optional(), personality: z.boolean().optional(), @@ -103,9 +104,11 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { ? 'kokoro' : engine === 'mms' ? 'mms-tts-ron' - : engine === 'qwen_custom_voice' - ? `qwen-custom-voice-${data.modelSize}` - : `qwen-tts-${data.modelSize}`; + : engine === 'f5' + ? 'f5-tts-romanian' + : engine === 'qwen_custom_voice' + ? `qwen-custom-voice-${data.modelSize}` + : `qwen-tts-${data.modelSize}`; const displayName = engine === 'luxtts' ? 'LuxTTS' @@ -121,13 +124,15 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { ? 'Kokoro 82M' : engine === 'mms' ? 'MMS Romanian (Meta)' - : engine === 'qwen_custom_voice' - ? data.modelSize === '1.7B' - ? 'Qwen CustomVoice 1.7B' - : 'Qwen CustomVoice 0.6B' - : data.modelSize === '1.7B' - ? 'Qwen TTS 1.7B' - : 'Qwen TTS 0.6B'; + : engine === 'f5' + ? 'F5-TTS Romanian (community)' + : engine === 'qwen_custom_voice' + ? data.modelSize === '1.7B' + ? 'Qwen CustomVoice 1.7B' + : 'Qwen CustomVoice 0.6B' + : data.modelSize === '1.7B' + ? 'Qwen TTS 1.7B' + : 'Qwen TTS 0.6B'; // Check if model needs downloading try { diff --git a/app/src/lib/utils/format.ts b/app/src/lib/utils/format.ts index 9d3dcafcf..6c32c7f32 100644 --- a/app/src/lib/utils/format.ts +++ b/app/src/lib/utils/format.ts @@ -58,6 +58,7 @@ const ENGINE_DISPLAY_NAMES: Record = { chatterbox: 'Chatterbox', chatterbox_turbo: 'Chatterbox Turbo', mms: 'MMS TTS', + f5: 'F5-TTS', }; export function formatEngineName(engine?: string, modelSize?: string): string { From 1594d0c1f961e139113a0d07e34bfe26e7294ae9 Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 22:31:34 +0200 Subject: [PATCH 09/15] Add F5 backend tests Unit coverage: diacritics normalization (cedilla -> comma-below, the reverse direction from MMS), vocab coverage pinning (skips when the checkpoint isn't cached), reference trimming (cut lands in a synthetic silence gap, never exceeds 12s, proportional text truncation), engine registration surfaces, and prompt validation (missing ref audio and empty ref text are refused before the model loads). E2E gated behind VOICEBOX_F5_E2E=1: generates the Romanian reference in-test with the MMS engine, clones through the real checkpoint on MPS, asserts 24kHz float32 non-NaN output and seeded determinism (passed in 96s on this machine). Co-Authored-By: Claude Opus 4.8 --- backend/backends/f5_backend.py | 5 +- backend/tests/test_f5_backend.py | 375 +++++++++++++++++++++++++++++++ 2 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_f5_backend.py diff --git a/backend/backends/f5_backend.py b/backend/backends/f5_backend.py index 16e26b844..6d3d10e83 100644 --- a/backend/backends/f5_backend.py +++ b/backend/backends/f5_backend.py @@ -295,8 +295,7 @@ async def generate( Returns: Tuple of (audio_array, sample_rate) """ - await self.load_model() - + # Validate the prompt before the expensive model load. ref_audio = voice_prompt.get("ref_audio") ref_text = voice_prompt.get("ref_text") or "" if not ref_audio or not Path(ref_audio).exists(): @@ -306,6 +305,8 @@ async def generate( if not ref_text.strip(): raise ValueError("F5-TTS requires the reference transcript (profile sample reference_text)") + await self.load_model() + def _generate_sync(): ref_file, trimmed_text = self._prepare_reference(ref_audio, ref_text) diff --git a/backend/tests/test_f5_backend.py b/backend/tests/test_f5_backend.py new file mode 100644 index 000000000..9c7bff6b3 --- /dev/null +++ b/backend/tests/test_f5_backend.py @@ -0,0 +1,375 @@ +""" +Tests for the F5-TTS Romanian backend. + +Unit tests cover the Romanian diacritics normalization (the fine-tune's +vocab lacks cedilla ţ and f5-tts maps out-of-vocab characters to space), +the reference-audio trimming (F5 degrades with references over 12s), and +the engine registration surfaces (model config registry, backend factory, +request model regexes, cloning-profile validation). + +The end-to-end cloning test downloads the ~1.2GB checkpoint on first run +and is opt-in: + + VOICEBOX_F5_E2E=1 python -m pytest backend/tests/test_f5_backend.py -v +""" + +import os +import sys +import unicodedata +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from backend.backends import ( + TTS_ENGINES, + get_model_config, + get_tts_backend_for_engine, + reset_backends, +) +from backend.backends.f5_backend import ( + F5_HF_REPO, + F5_MAX_REF_SECONDS, + F5_SAMPLE_RATE, + F5_VOCAB_FILE, + F5TTSBackend, + normalize_romanian_text, + trim_reference_audio, + trim_reference_text, +) + +S_CEDILLA = "ş" # ş — in the F5 vocab, but not the canonical form +S_CEDILLA_UPPER = "Ş" # Ş +S_COMMA = "ș" # ș — in vocab (canonical) +S_COMMA_UPPER = "Ș" # Ș +T_CEDILLA = "ţ" # ţ — NOT in the F5 vocab +T_CEDILLA_UPPER = "Ţ" # Ţ +T_COMMA = "ț" # ț — in vocab (canonical) +T_COMMA_UPPER = "Ț" # Ț +COMBINING_COMMA_BELOW = "̦" +COMBINING_BREVE = "̆" + + +class TestRomanianDiacriticsNormalization: + """f5-tts maps out-of-vocab chars to index 0 (space) silently — both + real-world diacritic families must land on the comma-below vocab forms. + Note the mapping direction is the opposite of the MMS backend's: this + vocab keeps comma-below ț and lacks cedilla ţ.""" + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + (S_CEDILLA, S_COMMA), + (S_CEDILLA_UPPER, S_COMMA_UPPER), + (T_CEDILLA, T_COMMA), + (T_CEDILLA_UPPER, T_COMMA_UPPER), + ], + ) + def test_cedilla_variants_mapped_to_comma_below(self, raw, expected): + assert normalize_romanian_text(raw) == expected + + @pytest.mark.parametrize("char", [S_COMMA, T_COMMA, "ă", "â", "î", "Ă", "Â", "Î", "a", " "]) + def test_vocab_forms_pass_through_unchanged(self, char): + assert normalize_romanian_text(char) == char + + def test_nfd_sequences_are_composed(self): + # s/t + combining comma below compose (via NFC) to the comma-below + # letters, which are already the vocab forms. + assert normalize_romanian_text("s" + COMBINING_COMMA_BELOW) == S_COMMA + assert normalize_romanian_text("t" + COMBINING_COMMA_BELOW) == T_COMMA + assert normalize_romanian_text("a" + COMBINING_BREVE) == "ă" + + def test_mixed_sentence_contains_only_vocab_diacritics(self): + text = f"{S_CEDILLA_UPPER}tii c{S_CEDILLA} {T_CEDILLA}ara {T_COMMA}ine pa{S_COMMA}ii" + normalized = normalize_romanian_text(text) + lowered = normalized.lower() + assert S_CEDILLA not in lowered + assert T_CEDILLA not in lowered + assert lowered.count(S_COMMA) == 3 + assert lowered.count(T_COMMA) == 2 + + def test_plain_text_untouched(self): + text = "Salut, ce mai faci? 1-2!" + assert normalize_romanian_text(text) == text + + def test_output_is_nfc(self): + decomposed = unicodedata.normalize("NFD", "Bună ziua, țară") + assert unicodedata.is_normalized("NFC", normalize_romanian_text(decomposed)) + + +class TestVocabDiacriticCoverage: + """Pin the vocab facts the normalization is built on — if the upstream + vocab.txt ever changes, this fails loudly instead of silently dropping + diacritics at generation time.""" + + @pytest.fixture(scope="class") + def vocab(self) -> set[str]: + from huggingface_hub import hf_hub_download + from huggingface_hub.errors import LocalEntryNotFoundError + + try: + path = hf_hub_download(F5_HF_REPO, F5_VOCAB_FILE, local_files_only=True) + except LocalEntryNotFoundError: + pytest.skip(f"{F5_HF_REPO} vocab not cached locally") + return set(Path(path).read_text(encoding="utf-8").split("\n")) + + def test_comma_below_family_in_vocab(self, vocab): + for char in (S_COMMA, S_COMMA_UPPER, T_COMMA, T_COMMA_UPPER): + assert char in vocab, f"{char!r} missing from vocab — normalization target invalid" + + def test_cedilla_t_not_in_vocab(self, vocab): + # The reason the normalization exists: cedilla ţ is absent. + assert T_CEDILLA not in vocab + assert T_CEDILLA_UPPER not in vocab + + def test_other_romanian_diacritics_in_vocab(self, vocab): + for char in ("ă", "Ă", "â", "Â", "î", "Î"): + assert char in vocab + + def test_normalized_text_fully_covered(self, vocab): + text = normalize_romanian_text("Ştiţi că ţara Ţării are paşi şi înţelegere, Bună ziua!") + missing = {c for c in text if c not in vocab and not c.isspace()} + assert not missing, f"normalized text still has out-of-vocab chars: {missing}" + + +class TestReferenceTrimming: + SR = 24000 + + def _speech_like(self, seconds: float, level: float = 0.3) -> np.ndarray: + rng = np.random.default_rng(0) + return (rng.standard_normal(int(seconds * self.SR)) * level).astype(np.float32) + + def test_short_audio_untouched(self): + audio = self._speech_like(5.0) + trimmed, kept = trim_reference_audio(audio, self.SR) + assert kept == 1.0 + assert trimmed is audio + + def test_audio_at_limit_untouched(self): + audio = self._speech_like(F5_MAX_REF_SECONDS) + trimmed, kept = trim_reference_audio(audio, self.SR) + assert kept == 1.0 + assert len(trimmed) == len(audio) + + def test_cut_lands_in_silence_gap(self): + # 20s of "speech" with a known 500ms silence gap at 9.5s-10.0s — + # the quietest-window search must cut inside the gap. + audio = self._speech_like(20.0) + gap_start, gap_end = int(9.5 * self.SR), int(10.0 * self.SR) + audio[gap_start:gap_end] = 0.0 + + trimmed, kept = trim_reference_audio(audio, self.SR) + + assert gap_start <= len(trimmed) <= gap_end + assert 0.0 < kept < 1.0 + assert kept == pytest.approx(len(trimmed) / len(audio)) + + def test_result_never_exceeds_max(self): + # No gaps at all — uniform noise. The cut must still be <= 12s. + audio = self._speech_like(30.0) + trimmed, _ = trim_reference_audio(audio, self.SR) + assert len(trimmed) <= F5_MAX_REF_SECONDS * self.SR + + def test_cut_not_before_min_seconds(self): + # Silence at the very start must not produce a uselessly short ref. + audio = self._speech_like(20.0) + audio[: int(2 * self.SR)] = 0.0 + trimmed, _ = trim_reference_audio(audio, self.SR) + assert len(trimmed) >= 8.0 * self.SR + + def test_text_untrimmed_when_audio_kept(self): + assert trim_reference_text("O propoziție întreagă.", 1.0) == "O propoziție întreagă." + + def test_text_proportionally_truncated_at_word_boundary(self): + text = "unu doi trei patru cinci șase șapte opt nouă zece" + result = trim_reference_text(text, 0.5) + assert result == "unu doi trei patru cinci" + + def test_text_trailing_punctuation_stripped(self): + text = "unu doi trei patru, cinci șase opt nouă" + result = trim_reference_text(text, 0.5) + assert result == "unu doi trei patru" + + def test_text_never_empty(self): + assert trim_reference_text("cuvânt lung aici", 0.01) == "cuvânt" + + +class TestF5Registration: + def test_engine_listed(self): + assert "f5" in TTS_ENGINES + + def test_model_config_resolves(self): + cfg = get_model_config("f5-tts-romanian") + assert cfg is not None + assert cfg.engine == "f5" + assert cfg.hf_repo_id == F5_HF_REPO + assert cfg.languages == ["ro", "en"] + assert cfg.size_mb == 1200 + assert cfg.supports_instruct is False + assert cfg.needs_trim is False + + def test_backend_factory_returns_f5_backend(self): + try: + backend = get_tts_backend_for_engine("f5") + assert isinstance(backend, F5TTSBackend) + assert get_tts_backend_for_engine("f5") is backend + finally: + reset_backends() + + def test_f5_is_a_cloning_engine(self): + from backend.services.profiles import CLONING_ENGINES, _get_preset_voice_ids + + assert "f5" in CLONING_ENGINES + # No preset voices — F5 clones from reference audio only. + assert _get_preset_voice_ids("f5") == set() + + def test_cloned_profile_validation_accepts_f5(self): + from types import SimpleNamespace + + from backend.services.profiles import validate_profile_engine + + profile = SimpleNamespace(id="p1", voice_type="cloned") + validate_profile_engine(profile, "f5") # must not raise + + +class TestRequestModelRegexes: + def test_generation_request_accepts_ro_and_f5(self): + from backend.models import GenerationRequest + + req = GenerationRequest(profile_id="p1", text="Bună ziua", language="ro", engine="f5") + assert req.language == "ro" + assert req.engine == "f5" + + def test_generation_request_rejects_unknown_engine(self): + from pydantic import ValidationError + + from backend.models import GenerationRequest + + with pytest.raises(ValidationError): + GenerationRequest(profile_id="p1", text="hi", engine="f5x") + + def test_profile_create_accepts_f5_default_engine(self): + from backend.models import VoiceProfileCreate + + profile = VoiceProfileCreate(name="Vocea", language="ro", default_engine="f5") + assert profile.default_engine == "f5" + + def test_speak_request_accepts_ro_and_f5(self): + from backend.models import SpeakRequest + + req = SpeakRequest(text="Bună", engine="f5", language="ro") + assert req.engine == "f5" + assert req.language == "ro" + + def test_mcp_binding_accepts_f5(self): + from backend.models import MCPClientBindingUpsert + + binding = MCPClientBindingUpsert(client_id="client-1", default_engine="f5") + assert binding.default_engine == "f5" + + +class TestF5BackendUnit: + def test_initial_state(self): + backend = F5TTSBackend() + assert not backend.is_loaded() + assert backend._get_model_path("default") == F5_HF_REPO + + async def test_create_voice_prompt_stores_reference(self): + backend = F5TTSBackend() + prompt, was_cached = await backend.create_voice_prompt("/tmp/sample.wav", "Bună ziua") + assert prompt == {"ref_audio": "/tmp/sample.wav", "ref_text": "Bună ziua"} + assert was_cached is False + + def test_unload_without_load_is_noop(self): + backend = F5TTSBackend() + backend.unload_model() + assert not backend.is_loaded() + + async def test_generate_rejects_missing_reference_audio(self): + backend = F5TTSBackend() + with pytest.raises(ValueError, match="reference audio"): + await backend.generate("text", {"ref_audio": "/nonexistent.wav", "ref_text": "x"}, "ro") + + async def test_generate_rejects_empty_reference_text(self, tmp_path): + # Empty ref_text would trigger f5-tts's Whisper auto-transcription + # (a ~1.6GB surprise download) — the backend must refuse instead. + import soundfile as sf + + wav = tmp_path / "ref.wav" + sf.write(str(wav), np.zeros(24000, dtype=np.float32), 24000) + backend = F5TTSBackend() + with pytest.raises(ValueError, match="transcript"): + await backend.generate("text", {"ref_audio": str(wav), "ref_text": " "}, "ro") + + +RUN_F5_E2E = os.environ.get("VOICEBOX_F5_E2E") == "1" + +ROMANIAN_REF_SENTENCE = "Bună ziua, mă numesc Adrian și locuiesc în București de mulți ani." + + +@pytest.mark.skipif(not RUN_F5_E2E, reason="set VOICEBOX_F5_E2E=1 to run (downloads ~1.2GB model)") +class TestF5CloningE2E: + """Full cloning through the real model. The reference audio is produced + in-test by the MMS engine (cached, CPU-realtime) so the test needs no + fixture files: one Romanian sentence serves as both ref audio and + ref text.""" + + @pytest.fixture(scope="class") + def reference(self, tmp_path_factory) -> tuple[str, str]: + import asyncio + + import soundfile as sf + + from backend.backends.mms_backend import MMSTTSBackend + + mms = MMSTTSBackend() + try: + audio, sample_rate = asyncio.run( + mms.generate( + ROMANIAN_REF_SENTENCE, + {"voice_type": "preset", "preset_engine": "mms", "preset_voice_id": "mms_ro_default"}, + "ro", + seed=7, + ) + ) + finally: + mms.unload_model() + path = tmp_path_factory.mktemp("f5_ref") / "ref.wav" + sf.write(str(path), audio, sample_rate) + return str(path), ROMANIAN_REF_SENTENCE + + @pytest.fixture(scope="class") + def backend(self): + backend = F5TTSBackend() + yield backend + backend.unload_model() + + async def test_clone_generates_romanian_audio(self, backend, reference): + ref_audio, ref_text = reference + prompt, _ = await backend.create_voice_prompt(ref_audio, ref_text) + + audio, sample_rate = await backend.generate( + "Ștefan cel Mare a domnit în Moldova aproape cincizeci de ani.", + prompt, + "ro", + seed=42, + ) + + assert sample_rate == F5_SAMPLE_RATE == 24000 + assert isinstance(audio, np.ndarray) + assert audio.dtype == np.float32 + assert audio.ndim == 1 + assert len(audio) > sample_rate, "expected more than 1s of audio" + assert not np.isnan(audio).any() + assert not np.isinf(audio).any() + assert float(np.abs(audio).max()) > 0.01, "audio should not be silence" + + async def test_seeded_generation_is_deterministic(self, backend, reference): + ref_audio, ref_text = reference + prompt, _ = await backend.create_voice_prompt(ref_audio, ref_text) + + first, _ = await backend.generate("Bună ziua!", prompt, "ro", seed=42) + second, _ = await backend.generate("Bună ziua!", prompt, "ro", seed=42) + np.testing.assert_allclose(first, second) From c33464a928533f805ba823160f9d13ef7cc35535 Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Sun, 19 Jul 2026 22:35:38 +0200 Subject: [PATCH 10/15] Document the F5-TTS Romanian engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README engine counts (8 to 9) and tables — Romanian now has true voice cloning, so the MMS row drops the 'only engine with Romanian' claim. PROJECT_STATUS engine tracking and model matrix, and a CHANGELOG entry under Unreleased that flags the new f5-tts dependency, the FFmpeg system requirement, and the honest speed caveat (~12x realtime on MPS, ~20x on CPU). preset-voices.mdx is untouched — F5 is a cloning engine. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 11 +++++++++++ README.md | 11 ++++++----- docs/PROJECT_STATUS.md | 9 +++++++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9519de2a0..1cd4972bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ ### Voice Generation +- **Romanian voice cloning via a new F5-TTS engine.** A new `f5` engine wraps + the community Romanian fine-tune of F5-TTS (`MihaiPopa-1/F5-TTS-Romanian`, + Apache-2.0) — the first engine with true Romanian voice cloning: record a + cloned profile and generate Romanian (or English) speech in that voice, + zero-shot. References longer than 12s are trimmed at the quietest pause and + both Romanian diacritic conventions are normalized to the model's vocab. + Heads-up: generation uses 32-step flow matching and is well below realtime + (~12x slower on Apple Silicon MPS, ~20x on CPU); the ~1.2GB checkpoint plus + the Vocos vocoder download on first use. Adds one Python dependency + (`f5-tts`, pinned); FFmpeg is required on the system (torchcodec audio I/O). + - **Romanian TTS via a new MMS engine.** A new `mms` engine wraps Meta's MMS-TTS Romanian checkpoint (`facebook/mms-tts-ron`) — a ~150MB VITS model that runs realtime on CPU, using the already-bundled transformers runtime diff --git a/README.md b/README.md index eb13001eb..8b82abca2 100644 --- a/README.md +++ b/README.md @@ -67,12 +67,12 @@ ## What is Voicebox? -Voicebox is a **local-first AI voice studio** — a free and open-source alternative to **ElevenLabs** and **WisprFlow** in one app. Clone voices from a few seconds of audio, generate speech in 24 languages across 8 TTS engines, dictate into any text field with a global hotkey, and give any MCP-aware AI agent a voice of your choosing. +Voicebox is a **local-first AI voice studio** — a free and open-source alternative to **ElevenLabs** and **WisprFlow** in one app. Clone voices from a few seconds of audio, generate speech in 24 languages across 9 TTS engines, dictate into any text field with a global hotkey, and give any MCP-aware AI agent a voice of your choosing. The two cloud incumbents sit on opposite halves of the voice I/O loop — ElevenLabs on output, WisprFlow on input. Voicebox does both, bridges them with a bundled local LLM for refinement and per-profile personas, and runs the whole thing on your machine. - **Complete privacy** — models, voice data, and captures never leave your machine -- **8 TTS engines** — Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro, and MMS +- **9 TTS engines** — Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro, MMS, and F5-TTS - **Voice cloning and preset voices** — zero-shot cloning from a reference sample, or 50+ curated preset voices via Kokoro, Qwen CustomVoice, and MMS (Romanian) - **24 languages** — from English to Arabic, Japanese, Hindi, Romanian, Swahili, and more - **Post-processing effects** — pitch shift, reverb, delay, chorus, compression, and filters @@ -109,7 +109,7 @@ The two cloud incumbents sit on opposite halves of the voice I/O loop — Eleven ### Multi-Engine Voice Cloning -Eight TTS engines with different strengths, switchable per-generation: +Nine TTS engines with different strengths, switchable per-generation: | Engine | Languages | Strengths | | --------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------- | @@ -120,7 +120,8 @@ Eight TTS engines with different strengths, switchable per-generation: | **Chatterbox Turbo** | English | Fast 350M model with paralinguistic emotion/sound tags | | **TADA** (1B / 3B) | 10 | HumeAI speech-language model — 700s+ coherent audio, text-acoustic dual alignment | | **Kokoro** | 8 | 50 curated preset voices, tiny 82M model, fast CPU inference | -| **MMS** (Meta) | Romanian | Per-language VITS checkpoints (~150MB), CPU realtime, preset voice — the only engine with Romanian | +| **MMS** (Meta) | Romanian | Per-language VITS checkpoints (~150MB), CPU realtime, preset voice | +| **F5-TTS Romanian** | Romanian, English | Community fine-tune with true Romanian voice cloning — zero-shot from a reference sample (slow: flow matching, ~12-20x realtime) | ### Emotions & Paralinguistic Tags @@ -370,7 +371,7 @@ Full API documentation available at `http://127.0.0.1:17493/docs`. | Frontend | React, TypeScript, Tailwind CSS | | State | Zustand, React Query | | Backend | FastAPI (Python) | -| TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro, MMS | +| TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro, MMS, F5-TTS | | STT | Whisper / Whisper Turbo (PyTorch or MLX) | | Local LLM | Qwen3 (0.6B / 1.7B / 4B), shared runtime with TTS / STT | | MCP Server | FastMCP mounted at `/mcp` (Streamable HTTP) + bundled stdio shim binary | diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index c4a48ea5d..955a53d8f 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -23,7 +23,7 @@ The backend exposes: -- **`TTSBackend` Protocol** with eight concrete engine implementations: +- **`TTSBackend` Protocol** with nine concrete engine implementations: - Qwen3-TTS (PyTorch or MLX depending on platform) - Qwen CustomVoice (predefined speakers with instruct) - LuxTTS (fast, CPU-friendly) @@ -32,6 +32,7 @@ The backend exposes: - TADA (1B English, 3B multilingual via HumeAI) - Kokoro 82M (pre-built voices, CPU realtime) - MMS (Meta per-language VITS checkpoints — Romanian, preset voice, CPU realtime) + - F5-TTS Romanian (community fine-tune — Romanian voice cloning, MPS/CPU, slow) - **`STTBackend` Protocol** for Whisper (PyTorch or MLX-Whisper) - **Profiles / History / Stories** services for persistence and timeline editing @@ -51,6 +52,7 @@ The backend exposes: | TADA | `backend/backends/hume_backend.py` | HumeAI TADA — 1B English + 3B Multilingual | | Kokoro | `backend/backends/kokoro_backend.py` | Kokoro 82M — CPU realtime, pre-built voices | | MMS | `backend/backends/mms_backend.py` | Meta MMS — Romanian VITS, preset voice, diacritics normalization | +| F5-TTS | `backend/backends/f5_backend.py` | F5-TTS Romanian fine-tune — cloning, 12s reference trimming, diacritics normalization | | Qwen CustomVoice | `backend/backends/qwen_custom_voice_backend.py` | Qwen CustomVoice — predefined speakers with instruct | | Platform detect | `backend/platform_detect.py` | Apple Silicon → MLX, else → PyTorch | | API types | `backend/models.py` | Pydantic request/response models | @@ -159,6 +161,7 @@ Shipped 2026-04-25 (PR #544). Voicebox went from a voice-cloning studio to a ful - HumeAI TADA — 1B English + 3B Multilingual (PR #296) - Kokoro 82M — CPU-realtime, 8 languages, Apache 2.0 (PR #325) - MMS (Meta) — Romanian preset voice, ~150MB VITS, CPU realtime, diacritics normalization +- F5-TTS Romanian — community fine-tune, first true Romanian cloning, 24 kHz, MPS/CPU - Multi-engine architecture with thread-safe backend registry (PR #254) - Chunked TTS generation — engine-agnostic, removes ~500 char limit (PR #266) - Async generation queue (PR #269) @@ -259,12 +262,13 @@ Shipped 2026-04-25 (PR #544). Voicebox went from a voice-cloning studio to a ful | TADA 3B Multilingual | `tada-3b-ml` | Cloned | 10 (en, ar, zh, de, es, fr, it, ja, pl, pt) | ~8 GB | Multilingual, text-acoustic dual alignment | None | | Kokoro 82M | `kokoro` | Preset | 8 (en, es, fr, hi, it, pt, ja, zh) | ~350 MB | 82M params, CPU realtime, Apache 2.0, pre-built voices | None | | MMS Romanian | `mms-tts-ron` | Preset | Romanian (ro) | ~150 MB | Meta MMS VITS, 16 kHz, CPU realtime, cedilla/comma-below diacritics normalization | None | +| F5-TTS Romanian | `f5-tts-romanian` | Cloned | Romanian, English (ro, en) | ~1.2 GB (+Vocos vocoder) | Community F5-TTS fine-tune (`MihaiPopa-1/F5-TTS-Romanian`), 24 kHz, 32-step flow matching (~12x realtime on MPS, ~20x on CPU), 12s reference trimming, diacritics normalization | None | ### Multi-Engine Architecture (Shipped) - **Thread-safe backend registry** (`_tts_backends` dict + `_tts_backends_lock`) with double-checked locking - **Per-engine backend instances** — each engine gets its own singleton, loaded lazily -- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'mms'` +- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'mms' | 'f5'` - **Per-engine language filtering** — `ENGINE_LANGUAGES` map in frontend, backend regex accepts all languages - **Per-engine voice prompts** — `create_voice_prompt_for_profile()` dispatches to the correct backend - **Profile type system** — preset vs cloned profiles, UI grays out incompatible engines and auto-switches on selection @@ -582,6 +586,7 @@ Notable: | **HumeAI TADA 1B/3B** | Zero-shot | 5x faster than LLM-TTS | 24 kHz | EN (1B), 10 (3B) | Medium | Partial — prosody | PyTorch | **Shipped** (PR #296) | | **Kokoro-82M** | Preset voices | CPU realtime | 24 kHz | 8 | Tiny (82M) | None | All | **Shipped** (PR #325) | | **MMS (Meta)** | Preset voice (1/lang) | CPU realtime | 16 kHz | Romanian (extensible per-checkpoint) | Tiny (~100M) | None | All | **Shipped** — zero new deps (transformers VITS) | +| **F5-TTS Romanian** | Zero-shot cloning (<=12s ref) | Slow (~12x RT on MPS) | 24 kHz | Romanian + English | Medium (336M + Vocos) | None | MPS/CPU (CUDA untested) | **Shipped** — one new dep (`f5-tts`), community fine-tune | | ~~**CosyVoice2-0.5B**~~ | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | **Yes** | — | **Abandoned** (PR #311) — poor output quality | | ~~**VoxCPM2**~~ | Zero-shot | ~0.15 RTF streaming | 48 kHz | 30 | Medium | Partial — parenthetical style | **CUDA-only in practice** | **Backlogged** (2026-04-18) — see notes above | | **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | **Yes** — word-level inline | All | Candidate — license TBD | From c6fd2110cf960919941d1945ca38637a2b6523cb Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Wed, 29 Jul 2026 23:11:21 +0200 Subject: [PATCH 11/15] Address review feedback on F5 engine - CapturesTab: resolve effective engine as default_engine ?? preset_engine (matches FloatingGenerateBox) so preset-only profiles keep their engine and language guard during capture playback - README: add FFmpeg to Quick Start prerequisites - preset-voices.mdx: MMS is the only *preset* engine with Romanian (F5 clones it); fix Kokoro language count 9 -> 8 - PROJECT_STATUS: add mms | f5 to the documented engine resolution flow Co-Authored-By: Claude Opus 4.8 --- README.md | 2 +- app/src/components/CapturesTab/CapturesTab.tsx | 2 +- docs/PROJECT_STATUS.md | 2 +- docs/content/docs/overview/preset-voices.mdx | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8b82abca2..d34183956 100644 --- a/README.md +++ b/README.md @@ -418,7 +418,7 @@ just dev # starts backend + desktop app Install [just](https://github.com/casey/just): `brew install just` or `cargo install just`. Run `just --list` to see all commands. -**Prerequisites:** [Bun](https://bun.sh), [Rust](https://rustup.rs), [Python 3.11+](https://python.org), [Tauri Prerequisites](https://v2.tauri.app/start/prerequisites/), and [Xcode](https://developer.apple.com/xcode/) on macOS. +**Prerequisites:** [Bun](https://bun.sh), [Rust](https://rustup.rs), [Python 3.11+](https://python.org), [FFmpeg](https://ffmpeg.org) (`brew install ffmpeg` — required for F5-TTS generation and torchaudio ≥2.11 audio I/O), [Tauri Prerequisites](https://v2.tauri.app/start/prerequisites/), and [Xcode](https://developer.apple.com/xcode/) on macOS. The repo ships a pre-wired `.mcp.json` at the root — running Claude Code inside this checkout picks up the Voicebox MCP tools automatically once the dev app is running. diff --git a/app/src/components/CapturesTab/CapturesTab.tsx b/app/src/components/CapturesTab/CapturesTab.tsx index 481eb1f1c..d872ace63 100644 --- a/app/src/components/CapturesTab/CapturesTab.tsx +++ b/app/src/components/CapturesTab/CapturesTab.tsx @@ -266,7 +266,7 @@ export function CapturesTab() { // Preset profiles (Kokoro etc.) reject the qwen default — honor the // profile's stored engine preference. Cloned profiles without an // override fall through to whatever the backend picks. - const engine = voice.default_engine as + const engine = (voice.default_engine ?? voice.preset_engine) as | 'qwen' | 'qwen_custom_voice' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro' | 'mms' | 'f5' | undefined; diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 955a53d8f..1eb5195fa 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -73,7 +73,7 @@ The backend exposes: ``` POST /generate 1. Look up voice profile from DB - 2. Resolve engine from request (qwen | qwen_custom_voice | luxtts | chatterbox | chatterbox_turbo | tada | kokoro) + 2. Resolve engine from request (qwen | qwen_custom_voice | luxtts | chatterbox | chatterbox_turbo | tada | kokoro | mms | f5) 3. Get backend: get_tts_backend_for_engine(engine) # thread-safe singleton per engine 4. Check model cache → if missing, trigger background download, return HTTP 202 5. Load model (lazy): tts_backend.load_model(model_size) diff --git a/docs/content/docs/overview/preset-voices.mdx b/docs/content/docs/overview/preset-voices.mdx index 8cf9458df..da577d6ca 100644 --- a/docs/content/docs/overview/preset-voices.mdx +++ b/docs/content/docs/overview/preset-voices.mdx @@ -11,9 +11,9 @@ Three engines ship preset voices: | Engine | Voices | Languages | Strengths | | --------------------- | ----------------------- | --------- | ------------------------------------------------------- | -| **Kokoro 82M** | 50 | 9 | Tiny model, CPU-friendly, lowest VRAM of any engine | +| **Kokoro 82M** | 50 | 8 | Tiny model, CPU-friendly, lowest VRAM of any engine | | **Qwen CustomVoice** | 9 (premium curated) | 4 | Natural-language style control over tone, emotion, pace | -| **MMS (Meta)** | 1 per language | Romanian | The only engine with Romanian — ~150MB, CPU realtime | +| **MMS (Meta)** | 1 per language | Romanian | The only preset engine with Romanian — ~150MB, CPU realtime | Looking for cloning a specific person's voice instead? See [Voice Cloning](/overview/voice-cloning). @@ -170,7 +170,7 @@ The full Generate page also surfaces the instruct field as a separate input. ## MMS Romanian — Meta's Massively Multilingual Speech -MMS wraps Meta's per-language VITS checkpoints (`facebook/mms-tts-{lang}`). Each checkpoint is a single fixed speaker, so there is exactly one preset voice per language. Voicebox ships the Romanian checkpoint — the only engine in the app that speaks Romanian. +MMS wraps Meta's per-language VITS checkpoints (`facebook/mms-tts-{lang}`). Each checkpoint is a single fixed speaker, so there is exactly one preset voice per language. Voicebox ships the Romanian checkpoint — the only preset engine that speaks Romanian (for Romanian voice cloning, see the F5-TTS engine). **Repository:** [`facebook/mms-tts-ron`](https://huggingface.co/facebook/mms-tts-ron) · CC-BY-NC 4.0 licensed From 61809ce3fb578271b348db6ce956abed66886484 Mon Sep 17 00:00:00 2001 From: Adrian Popa Date: Thu, 30 Jul 2026 08:10:08 +0200 Subject: [PATCH 12/15] Support local F5 checkpoint override via VOICEBOX_F5_CKPT Lets users point the f5 engine at a personal fine-tuned checkpoint (pruned safetensors) without code changes; vocab optionally overridable via VOICEBOX_F5_VOCAB. Missing paths log a warning and fall back to the HF checkpoint. Unit-tested. Co-Authored-By: Claude Opus 4.8 --- backend/backends/f5_backend.py | 29 +++++++++++++++++++++++++++-- backend/tests/test_f5_backend.py | 16 ++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/backend/backends/f5_backend.py b/backend/backends/f5_backend.py index 6d3d10e83..99e7c72ed 100644 --- a/backend/backends/f5_backend.py +++ b/backend/backends/f5_backend.py @@ -19,6 +19,7 @@ import asyncio import hashlib import logging +import os import unicodedata from pathlib import Path @@ -38,6 +39,12 @@ F5_HF_REPO = "MihaiPopa-1/F5-TTS-Romanian" F5_CKPT_FILE = "model_750_pruned.safetensors" F5_VOCAB_FILE = "vocab.txt" +# Local checkpoint override for personal fine-tunes: point VOICEBOX_F5_CKPT at +# a pruned .safetensors (and optionally VOICEBOX_F5_VOCAB at a matching +# vocab.txt — defaults to the repo vocab, which personal fine-tunes based on +# it share). When set, the checkpoint download is skipped entirely. +F5_CKPT_OVERRIDE_ENV = "VOICEBOX_F5_CKPT" +F5_VOCAB_OVERRIDE_ENV = "VOICEBOX_F5_VOCAB" # f5_tts.api.F5TTS downloads the Vocos vocoder from this repo on init; # _is_model_cached must account for it so the UI "downloaded" state is truthful. F5_VOCODER_HF_REPO = "charactr/vocos-mel-24khz" @@ -163,8 +170,22 @@ def is_loaded(self) -> bool: def _get_model_path(self, model_size: str = "default") -> str: return F5_HF_REPO + @staticmethod + def _ckpt_override() -> str | None: + """Local checkpoint path from VOICEBOX_F5_CKPT, if set and existing.""" + path = os.environ.get(F5_CKPT_OVERRIDE_ENV) + if path and Path(path).is_file(): + return path + if path: + logger.warning("%s=%s does not exist — falling back to the HF checkpoint", F5_CKPT_OVERRIDE_ENV, path) + return None + def _is_model_cached(self, model_size: str = "default") -> bool: """Check both the fine-tune checkpoint and the Vocos vocoder cache.""" + if self._ckpt_override(): + # Local fine-tune supplies the checkpoint; only vocab + vocoder + # still come from the cache. + return is_model_cached(F5_HF_REPO, required_files=[F5_VOCAB_FILE]) and is_model_cached(F5_VOCODER_HF_REPO) return is_model_cached(F5_HF_REPO, required_files=[F5_CKPT_FILE, F5_VOCAB_FILE]) and is_model_cached( F5_VOCODER_HF_REPO ) @@ -186,8 +207,12 @@ def _load_model_sync(self): with model_load_progress(model_name, is_cached): from huggingface_hub import hf_hub_download # lazy: heavy import - ckpt_file = hf_hub_download(F5_HF_REPO, F5_CKPT_FILE) - vocab_file = hf_hub_download(F5_HF_REPO, F5_VOCAB_FILE) + ckpt_file = self._ckpt_override() + if ckpt_file: + logger.info("Using local F5 checkpoint override: %s", ckpt_file) + else: + ckpt_file = hf_hub_download(F5_HF_REPO, F5_CKPT_FILE) + vocab_file = os.environ.get(F5_VOCAB_OVERRIDE_ENV) or hf_hub_download(F5_HF_REPO, F5_VOCAB_FILE) device = self._get_device() self._device = device diff --git a/backend/tests/test_f5_backend.py b/backend/tests/test_f5_backend.py index 9c7bff6b3..750718ef5 100644 --- a/backend/tests/test_f5_backend.py +++ b/backend/tests/test_f5_backend.py @@ -304,6 +304,22 @@ async def test_generate_rejects_empty_reference_text(self, tmp_path): await backend.generate("text", {"ref_audio": str(wav), "ref_text": " "}, "ro") +class TestF5CheckpointOverride: + def test_no_override_by_default(self, monkeypatch): + monkeypatch.delenv("VOICEBOX_F5_CKPT", raising=False) + assert F5TTSBackend._ckpt_override() is None + + def test_missing_path_falls_back(self, monkeypatch): + monkeypatch.setenv("VOICEBOX_F5_CKPT", "/nonexistent/model.safetensors") + assert F5TTSBackend._ckpt_override() is None + + def test_existing_path_wins(self, monkeypatch, tmp_path): + ckpt = tmp_path / "personal.safetensors" + ckpt.write_bytes(b"fake") + monkeypatch.setenv("VOICEBOX_F5_CKPT", str(ckpt)) + assert F5TTSBackend._ckpt_override() == str(ckpt) + + RUN_F5_E2E = os.environ.get("VOICEBOX_F5_E2E") == "1" ROMANIAN_REF_SENTENCE = "Bună ziua, mă numesc Adrian și locuiesc în București de mulți ani." From 26a74ca967bb9d8b3e2ff86b1a4313efd9f1dc8c Mon Sep 17 00:00:00 2001 From: fsq5279 <> Date: Sat, 1 Aug 2026 10:45:35 +0200 Subject: [PATCH 13/15] Spell out numbers and add speech-rate override in the F5 engine Digit characters are in the fine-tune's vocab but effectively untrained (the training data writes numbers in letters), so raw digits came out garbled. Romanian text now gets numbers spelled out before synthesis: cardinals with gender agreement and the "de" linker, decimal commas, clock times and percentages. Also adds VOICEBOX_F5_SPEED (0.3-2.0, default 1.0) to compensate fine-tunes whose training data was read faster than the desired output pace. Co-Authored-By: Claude Fable 5 --- backend/backends/f5_backend.py | 132 ++++++++++++++++++++++++++++++- backend/tests/test_f5_backend.py | 96 +++++++++++++++++++++- 2 files changed, 223 insertions(+), 5 deletions(-) diff --git a/backend/backends/f5_backend.py b/backend/backends/f5_backend.py index 99e7c72ed..be8334992 100644 --- a/backend/backends/f5_backend.py +++ b/backend/backends/f5_backend.py @@ -20,6 +20,7 @@ import hashlib import logging import os +import re import unicodedata from pathlib import Path @@ -50,6 +51,25 @@ F5_VOCODER_HF_REPO = "charactr/vocos-mel-24khz" F5_SAMPLE_RATE = 24000 F5_NFE_STEPS = 32 +# Speech-rate compensation for fine-tunes whose training data was read +# faster than the desired output pace (1.0 = the model's natural rate, +# lower = slower). Personal v3 checkpoint pairs with 0.85. +F5_SPEED_ENV = "VOICEBOX_F5_SPEED" + + +def _f5_speed() -> float: + raw = os.environ.get(F5_SPEED_ENV) + if not raw: + return 1.0 + try: + speed = float(raw) + except ValueError: + logger.warning("Invalid %s=%r, using 1.0", F5_SPEED_ENV, raw) + return 1.0 + if not 0.3 <= speed <= 2.0: + logger.warning("%s=%s outside [0.3, 2.0], using 1.0", F5_SPEED_ENV, speed) + return 1.0 + return speed # F5 conditioning degrades with references over ~12s (upstream also hard-clips # at 12s). Trim at the quietest 300ms window found between 8s and 12s so the @@ -76,15 +96,118 @@ ) +# The fine-tune (and the dataset it was trained on) spells numbers out in +# letters; digit characters are in the vocab but effectively untrained, so +# raw digits come out garbled. Spell them out the way a Romanian reader would. +_RO_UNITS = ["", "unu", "doi", "trei", "patru", "cinci", "șase", "șapte", "opt", "nouă"] +_RO_UNITS_F = ["", "una", "două", "trei", "patru", "cinci", "șase", "șapte", "opt", "nouă"] +_RO_TEENS = [ + "zece", "unsprezece", "doisprezece", "treisprezece", "paisprezece", + "cincisprezece", "șaisprezece", "șaptesprezece", "optsprezece", "nouăsprezece", +] +_RO_TENS = ["", "", "douăzeci", "treizeci", "patruzeci", "cincizeci", + "șaizeci", "șaptezeci", "optzeci", "nouăzeci"] +# (value, singular, plural) — group words for thousands and up +_RO_SCALES = [ + (1_000_000_000, "un miliard", "miliarde"), + (1_000_000, "un milion", "milioane"), + (1_000, "o mie", "mii"), +] + + +def _ro_under_100(n: int, feminine: bool) -> str: + units = _RO_UNITS_F if feminine else _RO_UNITS + if n < 10: + return units[n] + if n < 20: + if n == 12 and feminine: + return "douăsprezece" + return _RO_TEENS[n - 10] + tens, unit = divmod(n, 10) + return _RO_TENS[tens] + (f" și {units[unit]}" if unit else "") + + +def _ro_under_1000(n: int, feminine: bool) -> str: + hundreds, rest = divmod(n, 100) + parts = [] + if hundreds == 1: + parts.append("o sută") + elif hundreds == 2: + parts.append("două sute") + elif hundreds: + parts.append(f"{_RO_UNITS[hundreds]} sute") + if rest: + parts.append(_ro_under_100(rest, feminine)) + return " ".join(parts) + + +def _ro_int_to_words(n: int, feminine: bool = False) -> str: + if n == 0: + return "zero" + parts = [] + for value, singular, plural in _RO_SCALES: + group, n = divmod(n, value) + if not group: + continue + if group == 1: + parts.append(singular) + else: + # groups counting mii/milioane are grammatically feminine, and + # 20+ links with "de": "douăzeci de mii", but "douăsprezece mii" + link = " " if 1 <= group % 100 <= 19 else " de " + parts.append(_ro_under_1000(group, feminine=True) + link + plural) + if n: + parts.append(_ro_under_1000(n, feminine)) + return " ".join(parts) + + +# time (18:30) | thousand-separated (1.650) | decimal comma (4,9) | plain int — +# each optionally followed by % (spoken "la sută") +_RO_NUMBER_RE = re.compile( + r"\b(?:(?P\d{1,2}):(?P\d{2})\b" + r"|(?P\d{1,3}(?:\.\d{3})+|\d+)(?:,(?P\d+))?\b(?P\s?%)?)" +) + + +def _spell_number_match(m: re.Match) -> str: + if m.group("h") is not None: + hour, minute = int(m.group("h")), int(m.group("m")) + if hour > 23 or minute > 59: # not a plausible time (e.g. 45:99) + return m.group(0) + words = _ro_int_to_words(hour) + if minute: + words += f" și {_ro_int_to_words(minute)}" + return words + words = _ro_int_to_words(int(m.group("num").replace(".", ""))) + frac = m.group("frac") + if frac: + # leading zeros are read digit by digit: 0,05 -> "zero virgulă zero cinci" + if frac.startswith("0"): + frac_words = " ".join("zero" if d == "0" else _RO_UNITS[int(d)] for d in frac) + else: + frac_words = _ro_int_to_words(int(frac)) + words += f" virgulă {frac_words}" + if m.group("pct"): + words += " la sută" + return words + + +def spell_romanian_numbers(text: str) -> str: + """Spell digits out in Romanian words (cardinals, decimals, times, %).""" + return _RO_NUMBER_RE.sub(_spell_number_match, text) + + def normalize_romanian_text(text: str) -> str: - """Normalize Romanian text to the diacritic forms in the F5 vocab. + """Normalize Romanian text to the forms the F5 fine-tune was trained on. Applies NFC normalization first (composing any decomposed - letter + combining-mark sequences), then maps cedilla s and t + letter + combining-mark sequences), maps cedilla s and t variants onto the comma-below forms the fine-tune was trained on, - so no diacritic is silently dropped by the character tokenizer. + so no diacritic is silently dropped by the character tokenizer, + and spells out numbers, which the model only saw written in letters. """ - return unicodedata.normalize("NFC", text).translate(_RO_DIACRITICS_TRANSLATION) + normalized = unicodedata.normalize("NFC", text).translate(_RO_DIACRITICS_TRANSLATION) + return spell_romanian_numbers(normalized) def trim_reference_audio( @@ -352,6 +475,7 @@ def _generate_sync(): gen_text, nfe_step=F5_NFE_STEPS, seed=seed, + speed=_f5_speed(), show_info=logger.debug, ) diff --git a/backend/tests/test_f5_backend.py b/backend/tests/test_f5_backend.py index 750718ef5..3c0c5a556 100644 --- a/backend/tests/test_f5_backend.py +++ b/backend/tests/test_f5_backend.py @@ -36,6 +36,7 @@ F5_VOCAB_FILE, F5TTSBackend, normalize_romanian_text, + spell_romanian_numbers, trim_reference_audio, trim_reference_text, ) @@ -91,7 +92,9 @@ def test_mixed_sentence_contains_only_vocab_diacritics(self): assert lowered.count(T_COMMA) == 2 def test_plain_text_untouched(self): - text = "Salut, ce mai faci? 1-2!" + # no digits here: numbers are intentionally rewritten (see + # TestSpellRomanianNumbers); this test guards diacritic pass-through + text = "Salut, ce mai faci? Bine!" assert normalize_romanian_text(text) == text def test_output_is_nfc(self): @@ -99,6 +102,97 @@ def test_output_is_nfc(self): assert unicodedata.is_normalized("NFC", normalize_romanian_text(decomposed)) +class TestF5Speed: + """VOICEBOX_F5_SPEED compensates fine-tunes trained on fast-read data.""" + + def test_default_is_full_speed(self, monkeypatch): + from backend.backends.f5_backend import F5_SPEED_ENV, _f5_speed + + monkeypatch.delenv(F5_SPEED_ENV, raising=False) + assert _f5_speed() == 1.0 + + def test_valid_value_used(self, monkeypatch): + from backend.backends.f5_backend import F5_SPEED_ENV, _f5_speed + + monkeypatch.setenv(F5_SPEED_ENV, "0.85") + assert _f5_speed() == 0.85 + + @pytest.mark.parametrize("raw", ["abc", "", "0.1", "5.0"]) + def test_invalid_or_out_of_range_falls_back(self, monkeypatch, raw): + from backend.backends.f5_backend import F5_SPEED_ENV, _f5_speed + + monkeypatch.setenv(F5_SPEED_ENV, raw) + assert _f5_speed() == 1.0 + + +class TestSpellRomanianNumbers: + """Digits are effectively untrained in the fine-tune; they must be + spelled out the way the training data wrote them (in letters).""" + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ("0", "zero"), + ("1", "unu"), + ("12", "doisprezece"), + ("19", "nouăsprezece"), + ("21", "douăzeci și unu"), + ("100", "o sută"), + ("101", "o sută unu"), + ("200", "două sute"), + ("387", "trei sute optzeci și șapte"), + ("1000", "o mie"), + ("2000", "două mii"), + ("12000", "douăsprezece mii"), + ("20000", "douăzeci de mii"), + ("21000", "douăzeci și una de mii"), + ("17493", "șaptesprezece mii patru sute nouăzeci și trei"), + ("1000000", "un milion"), + ("2000000", "două milioane"), + ], + ) + def test_cardinals(self, raw, expected): + assert spell_romanian_numbers(raw) == expected + + def test_thousand_separator_dots(self): + assert spell_romanian_numbers("1.650") == "o mie șase sute cincizeci" + + def test_decimal_comma(self): + assert spell_romanian_numbers("4,9") == "patru virgulă nouă" + + def test_decimal_with_leading_zero_read_digitwise(self): + assert spell_romanian_numbers("0,05") == "zero virgulă zero cinci" + + def test_percent(self): + assert spell_romanian_numbers("4,9%") == "patru virgulă nouă la sută" + + def test_time(self): + assert spell_romanian_numbers("18:30") == "optsprezece și treizeci" + + def test_time_on_the_hour_drops_minutes(self): + assert spell_romanian_numbers("18:00") == "optsprezece" + + def test_implausible_time_left_alone(self): + assert spell_romanian_numbers("45:99") == "45:99" + + def test_number_inside_sentence(self): + assert ( + spell_romanian_numbers("Factura de 387 de lei e scadentă pe 25 august.") + == "Factura de trei sute optzeci și șapte de lei e scadentă pe douăzeci și cinci august." + ) + + def test_text_without_digits_untouched(self): + text = "Bună dimineața, ce mai faci?" + assert spell_romanian_numbers(text) == text + + def test_normalize_romanian_text_spells_numbers(self): + assert "șaptesprezece mii" in normalize_romanian_text("portul 17493") + + def test_normalized_output_has_no_digits(self): + out = normalize_romanian_text("La 18:30 plătesc 1.650 de lei, adică 4,9% din 34.000.") + assert not any(ch.isdigit() for ch in out) + + class TestVocabDiacriticCoverage: """Pin the vocab facts the normalization is built on — if the upstream vocab.txt ever changes, this fails loudly instead of silently dropping From 6f89f32de14e39f25802828ec5b77443138dfd6d Mon Sep 17 00:00:00 2001 From: fsq5279 <> Date: Sat, 1 Aug 2026 10:45:35 +0200 Subject: [PATCH 14/15] Inherit generation language from the voice profile GenerationRequest.language defaulted to "en", so API callers that omitted it silently bypassed language-specific normalization (e.g. Romanian diacritic and number handling) even on profiles configured for another language. The field is now optional and falls back to the profile's language in both the generate and stream routes; an explicit request value still wins. Co-Authored-By: Claude Fable 5 --- backend/models.py | 4 +++- backend/routes/generations.py | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/models.py b/backend/models.py index 97649a565..2f641f53b 100644 --- a/backend/models.py +++ b/backend/models.py @@ -84,7 +84,9 @@ class GenerationRequest(BaseModel): profile_id: str text: str = Field(..., min_length=1, max_length=50000) - language: str = Field(default="en", pattern=TTS_LANGUAGE_PATTERN) + # None -> inherit the profile's language (falls back to "en" for + # profiles without one); an explicit value always wins. + language: Optional[str] = Field(default=None, pattern=TTS_LANGUAGE_PATTERN) seed: Optional[int] = Field(None, ge=0) model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$") instruct: Optional[str] = Field(None, max_length=500) diff --git a/backend/routes/generations.py b/backend/routes/generations.py index 215c96cb2..f8b285502 100644 --- a/backend/routes/generations.py +++ b/backend/routes/generations.py @@ -75,6 +75,8 @@ async def generate_speech( raise HTTPException(status_code=400, detail=str(e)) model_size = (data.model_size or "1.7B") if engine_has_model_sizes(engine) else None + # No explicit language on the request -> speak in the profile's language. + language = data.language or getattr(profile, "language", None) or "en" text = data.text source = "manual" @@ -91,7 +93,7 @@ async def generate_speech( generation = await history.create_generation( profile_id=data.profile_id, text=text, - language=data.language, + language=language, audio_path="", duration=0, seed=data.seed, @@ -129,7 +131,7 @@ async def generate_speech( generation_id=generation_id, profile_id=data.profile_id, text=text, - language=data.language, + language=language, engine=engine, model_size=model_size, seed=data.seed, @@ -334,6 +336,8 @@ async def stream_speech( raise HTTPException(status_code=400, detail=str(e)) tts_model = get_tts_backend_for_engine(engine) model_size = data.model_size or "1.7B" + # No explicit language on the request -> speak in the profile's language. + language = data.language or getattr(profile, "language", None) or "en" await ensure_model_cached_or_raise(engine, model_size) await load_engine_model(engine, model_size) @@ -356,7 +360,7 @@ async def stream_speech( tts_model, data.text, voice_prompt, - language=data.language, + language=language, seed=data.seed, instruct=data.instruct, max_chunk_chars=data.max_chunk_chars, From b63bf8da8ddc996f9589038a890297dc62fd705d Mon Sep 17 00:00:00 2001 From: fsq5279 <> Date: Sun, 2 Aug 2026 10:24:02 +0200 Subject: [PATCH 15/15] Harden F5 generation: seed crash fix, chunking, and quality knobs - Fix a hard crash on long/chunked F5 generation: F5TTS.infer runs seed_everything(seed), which writes PYTHONHASHSEED; with seed=None F5 drew random.randint(0, sys.maxsize) (~9e18), so the next spawned subprocess aborted with "PYTHONHASHSEED must be in range [0, 4294967295]". Always pass F5 a seed inside that range. - Chunk F5 output to ~one sentence (VOICEBOX_F5 default 140 chars): F5 loses coherence generating long single-shot audio (>~15s came out slurred). max_chunk_chars is now per-engine (None -> engine default). - Add opt-in generation knobs on the F5 engine: * VOICEBOX_F5_NFE (16-128, default 32) - flow-matching steps. * VOICEBOX_F5_BEST_OF (1-8, default 1) - generate N candidates and keep the one an ASR pass transcribes closest to the intended text; the scorer spells numbers in the ASR output so the digit-vs-words format gap can't flatten every candidate to the same score. * VOICEBOX_F5_ONSET_FIX (default off) - F5 garbles a leading t-comma / i-circumflex sound (measured; vowel/common onsets are clean); a throwaway lead-in word absorbs it. Off by default because trimming the lead-in back off via ASR timestamps can clip real speech. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/backends/__init__.py | 14 +++ backend/backends/f5_backend.py | 195 ++++++++++++++++++++++++++++--- backend/models.py | 7 +- backend/routes/generations.py | 11 +- backend/tests/test_f5_backend.py | 89 ++++++++++++++ 5 files changed, 296 insertions(+), 20 deletions(-) diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index 8b7eb8240..7215d3100 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -522,6 +522,20 @@ def engine_needs_trim(engine: str) -> bool: return False +# Per-engine override for the default max_chunk_chars (used when a request +# omits it). F5-TTS handles long text via its own internal batching, so it +# stays on the generic default; app-level re-chunking added sequential-infer +# instability without a quality win once the PYTHONHASHSEED seed bug (which +# was corrupting F5's later internal batches) was fixed. +_ENGINE_DEFAULT_CHUNK_CHARS: dict[str, int] = {"f5": 140} +_GENERIC_DEFAULT_CHUNK_CHARS = 800 + + +def engine_default_chunk_chars(engine: str) -> int: + """Per-engine default for max_chunk_chars when the request omits it.""" + return _ENGINE_DEFAULT_CHUNK_CHARS.get(engine, _GENERIC_DEFAULT_CHUNK_CHARS) + + def engine_has_model_sizes(engine: str) -> bool: """Whether this engine supports multiple model sizes (only Qwen currently).""" configs = [c for c in get_tts_model_configs() if c.engine == engine] diff --git a/backend/backends/f5_backend.py b/backend/backends/f5_backend.py index be8334992..049646ce6 100644 --- a/backend/backends/f5_backend.py +++ b/backend/backends/f5_backend.py @@ -20,6 +20,7 @@ import hashlib import logging import os +import random import re import unicodedata from pathlib import Path @@ -55,6 +56,36 @@ # faster than the desired output pace (1.0 = the model's natural rate, # lower = slower). Personal v3 checkpoint pairs with 0.85. F5_SPEED_ENV = "VOICEBOX_F5_SPEED" +# Flow-matching steps per generation. Higher = cleaner articulation but +# proportionally slower (64 ~= 2x the cost of 32). Default 32 is the +# F5-TTS reference value. +F5_NFE_ENV = "VOICEBOX_F5_NFE" +# Best-of-N: generate this many candidates and keep the one an ASR pass +# transcribes closest to the intended text — trades latency for fewer +# slurred/garbled takes on hard sentences. 1 disables it (default). +F5_BEST_OF_ENV = "VOICEBOX_F5_BEST_OF" + +# Onset fix: F5 garbles the very first word when it starts with a Romanian +# comma-below/circumflex sound (ț, î, â) — measured "Țin"->"Foai/Floi", +# "Țara"->"Foara", "Împreună"->"Om", while vowel/common-consonant onsets are +# clean. A throwaway lead-in word absorbs the unstable onset; it is then +# trimmed off using the ASR word timestamp of the first real word. +F5_HARD_ONSET_CHARS = ("ț", "î", "â") +F5_ONSET_LEAD_IN = "Așa, " +# Default OFF: the lead-in reliably absorbs the garbled onset, but trimming +# it back off via ASR word timestamps proved imprecise and sometimes clips +# real speech — a worse failure than the original garble. Kept as an opt-in +# (VOICEBOX_F5_ONSET_FIX=1) pending a robust trim. Set to "1"/"true" to enable. +F5_ONSET_FIX_ENV = "VOICEBOX_F5_ONSET_FIX" + + +def _f5_onset_fix_enabled() -> bool: + return os.environ.get(F5_ONSET_FIX_ENV, "0").lower() in ("1", "true", "yes", "on") + + +def _has_hard_onset(text: str) -> bool: + stripped = text.lstrip().lower() + return stripped.startswith(F5_HARD_ONSET_CHARS) def _f5_speed() -> float: @@ -71,6 +102,38 @@ def _f5_speed() -> float: return 1.0 return speed + +def _env_int(name: str, default: int, lo: int, hi: int) -> int: + raw = os.environ.get(name) + if not raw: + return default + try: + value = int(raw) + except ValueError: + logger.warning("Invalid %s=%r, using %d", name, raw, default) + return default + if not lo <= value <= hi: + logger.warning("%s=%d outside [%d, %d], using %d", name, value, lo, hi, default) + return default + return value + + +def _f5_nfe_steps() -> int: + return _env_int(F5_NFE_ENV, F5_NFE_STEPS, 16, 128) + + +def _f5_best_of() -> int: + return _env_int(F5_BEST_OF_ENV, 1, 1, 8) + + +def _asr_similarity_key(text: str) -> str: + """Loosely normalize for ASR-vs-intended comparison: lowercase, + strip everything but letters/digits/spaces, collapse whitespace.""" + text = unicodedata.normalize("NFD", text.lower()) + text = "".join(c for c in text if unicodedata.category(c) != "Mn") + text = re.sub(r"[^a-z0-9 ]+", " ", text) + return re.sub(r"\s+", " ", text).strip() + # F5 conditioning degrades with references over ~12s (upstream also hard-clips # at 12s). Trim at the quietest 300ms window found between 8s and 12s so the # cut lands in a natural pause and upstream's cruder clipper never fires. @@ -280,6 +343,68 @@ def __init__(self): self.model_size = "default" self._device: str | None = None self._model_load_lock = asyncio.Lock() + self._scorer = None # lazy Whisper pipeline for best-of-N ranking + + def _score_candidate(self, audio: np.ndarray, sample_rate: int, target: str) -> float: + """Similarity in [0, 1] between an ASR transcription of `audio` and + the intended text. Used only when best-of-N is enabled.""" + from difflib import SequenceMatcher + + if self._scorer is None: + from transformers import pipeline as hf_pipeline + + logger.info("[F5] Loading ASR scorer for best-of-N (whisper-small)...") + self._scorer = hf_pipeline( + "automatic-speech-recognition", model="openai/whisper-small" + ) + heard = self._scorer( + {"array": np.asarray(audio, dtype=np.float32), "sampling_rate": sample_rate}, + generate_kwargs={"language": "romanian", "task": "transcribe"}, + )["text"] + # ASR re-digitizes numbers ("2.487") while the target is already + # spelled out; spell the ASR output too so the format gap doesn't + # swamp the comparison and flatten every candidate to one score. + heard = spell_romanian_numbers(heard) + return SequenceMatcher( + None, _asr_similarity_key(heard), _asr_similarity_key(target) + ).ratio() + + def _trim_lead_in(self, audio: np.ndarray, sample_rate: int, lead_in: str) -> np.ndarray: + """Cut the throwaway onset lead-in off the front of ``audio``. + + Uses word-level ASR timestamps to find where the first non-lead-in + word begins and cuts there (silence-based cutting proved unreliable — + the lead-in's trailing pause isn't cleanly detectable). Runs the ASR + on CPU to avoid contending with F5 on MPS. On any failure it returns + the audio unchanged rather than risk clipping real speech. + """ + lead_words = {w.strip(".,!?").lower() for w in lead_in.split() if w.strip(".,!?")} + try: + if self._scorer is None: + from transformers import pipeline as hf_pipeline + + self._scorer = hf_pipeline( + "automatic-speech-recognition", model="openai/whisper-small", device="cpu" + ) + result = self._scorer( + {"array": np.asarray(audio, dtype=np.float32), "sampling_rate": sample_rate}, + return_timestamps="word", + generate_kwargs={"language": "romanian", "task": "transcribe"}, + ) + for chunk in result.get("chunks", []): + word = chunk["text"].strip().strip(".,!?").lower() + if word and word not in lead_words: + start = chunk["timestamp"][0] + if start is None: + break + # small safety margin so timestamp jitter can't clip the word + cut = max(0, int((start - 0.03) * sample_rate)) + if 0 < cut < len(audio): + return audio[cut:] + break + except Exception as e: # never fail a generation over the cosmetic fix + logger.warning("[F5] onset lead-in trim failed, keeping full audio: %s", e) + return audio def _get_device(self) -> str: # MPS verified stable on this checkpoint with memory free and ~2x @@ -465,21 +590,63 @@ def _generate_sync(): gen_text = unicodedata.normalize("NFC", text) prompt_text = unicodedata.normalize("NFC", trimmed_text) - logger.info("[F5] Generating: lang=%s", language) - - # F5TTS.infer seeds torch/numpy/random itself (seed_everything); - # passing seed=None picks a fresh random seed. - wav, sample_rate, _spec = self.model.infer( - ref_file, - prompt_text, - gen_text, - nfe_step=F5_NFE_STEPS, - seed=seed, - speed=_f5_speed(), - show_info=logger.debug, + nfe_step = _f5_nfe_steps() + speed = _f5_speed() + best_of = _f5_best_of() + # Absorb F5's ț/î/â onset garbling with a throwaway lead-in word + # that gets trimmed back off after generation (Romanian only). + use_onset_fix = ( + language == "ro" and _f5_onset_fix_enabled() and _has_hard_onset(gen_text) + ) + if use_onset_fix: + gen_text = F5_ONSET_LEAD_IN + gen_text + logger.info( + "[F5] Generating: lang=%s nfe=%d speed=%.2f best_of=%d onset_fix=%s", + language, nfe_step, speed, best_of, use_onset_fix, ) - audio = np.asarray(wav, dtype=np.float32) - return audio, int(sample_rate) + def _infer_once(candidate_seed: int | None): + # F5TTS.infer runs seed_everything(seed), which writes + # os.environ["PYTHONHASHSEED"]=str(seed). With seed=None F5 + # draws random.randint(0, sys.maxsize) (~9e18), and the next + # subprocess (e.g. the vocoder worker on a later chunk) then + # aborts with "PYTHONHASHSEED must be in range [0, 4294967295]". + # Always hand F5 a seed inside that range instead. + if candidate_seed is None: + candidate_seed = random.randint(0, 2**32 - 1) + else: + candidate_seed %= 2**32 + wav, sr, _spec = self.model.infer( + ref_file, + prompt_text, + gen_text, + nfe_step=nfe_step, + seed=candidate_seed, + speed=speed, + show_info=logger.debug, + ) + return np.asarray(wav, dtype=np.float32), int(sr) + + if best_of <= 1: + audio, sample_rate = _infer_once(seed) + if use_onset_fix: + audio = self._trim_lead_in(audio, sample_rate, F5_ONSET_LEAD_IN) + return audio, sample_rate + + # Generate N diverse candidates (distinct seeds for reproducibility + # when a base seed is given) and keep the best-transcribed one. + best_audio, best_sr, best_score = None, None, -1.0 + for i in range(best_of): + cand_seed = None if seed is None else seed + i + audio, sr = _infer_once(cand_seed) + score = self._score_candidate(audio, sr, gen_text) + logger.info("[F5] best-of-%d candidate %d/%d score=%.3f", + best_of, i + 1, best_of, score) + if score > best_score: + best_audio, best_sr, best_score = audio, sr, score + logger.info("[F5] best-of-%d selected score=%.3f", best_of, best_score) + if use_onset_fix: + best_audio = self._trim_lead_in(best_audio, best_sr, F5_ONSET_LEAD_IN) + return best_audio, best_sr return await asyncio.to_thread(_generate_sync) diff --git a/backend/models.py b/backend/models.py index 2f641f53b..6e90a1f42 100644 --- a/backend/models.py +++ b/backend/models.py @@ -95,8 +95,11 @@ class GenerationRequest(BaseModel): default=False, description="When true and the profile has a personality prompt, the input text is rewritten in-character before TTS.", ) - max_chunk_chars: int = Field( - default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting" + # None -> per-engine default resolved in the route. F5-TTS degrades on + # long single-shot generations (>~15s), so it chunks far more tightly + # than the generic 800. + max_chunk_chars: Optional[int] = Field( + default=None, ge=100, le=5000, description="Max characters per chunk for long text splitting" ) crossfade_ms: int = Field( default=50, ge=0, le=500, description="Crossfade duration in ms between chunks (0 for hard cut)" diff --git a/backend/routes/generations.py b/backend/routes/generations.py index f8b285502..17f57a56f 100644 --- a/backend/routes/generations.py +++ b/backend/routes/generations.py @@ -66,7 +66,7 @@ async def generate_speech( if not profile: raise HTTPException(status_code=404, detail="Profile not found") - from ..backends import engine_has_model_sizes + from ..backends import engine_has_model_sizes, engine_default_chunk_chars engine = _resolve_generation_engine(data, profile) try: @@ -77,6 +77,8 @@ async def generate_speech( model_size = (data.model_size or "1.7B") if engine_has_model_sizes(engine) else None # No explicit language on the request -> speak in the profile's language. language = data.language or getattr(profile, "language", None) or "en" + # No explicit chunk size -> per-engine default (F5 chunks tightly). + max_chunk_chars = data.max_chunk_chars or engine_default_chunk_chars(engine) text = data.text source = "manual" @@ -139,7 +141,7 @@ async def generate_speech( effects_chain=effects_chain_config, instruct=data.instruct, mode="generate", - max_chunk_chars=data.max_chunk_chars, + max_chunk_chars=max_chunk_chars, crossfade_ms=data.crossfade_ms, ) ) @@ -323,7 +325,7 @@ async def stream_speech( db: Session = Depends(get_db), ): """Generate speech and stream the WAV audio directly without saving to disk.""" - from ..backends import get_tts_backend_for_engine, ensure_model_cached_or_raise, load_engine_model, engine_needs_trim + from ..backends import get_tts_backend_for_engine, ensure_model_cached_or_raise, load_engine_model, engine_needs_trim, engine_default_chunk_chars profile = await profiles.get_profile(data.profile_id, db) if not profile: @@ -338,6 +340,7 @@ async def stream_speech( model_size = data.model_size or "1.7B" # No explicit language on the request -> speak in the profile's language. language = data.language or getattr(profile, "language", None) or "en" + max_chunk_chars = data.max_chunk_chars or engine_default_chunk_chars(engine) await ensure_model_cached_or_raise(engine, model_size) await load_engine_model(engine, model_size) @@ -363,7 +366,7 @@ async def stream_speech( language=language, seed=data.seed, instruct=data.instruct, - max_chunk_chars=data.max_chunk_chars, + max_chunk_chars=max_chunk_chars, crossfade_ms=data.crossfade_ms, trim_fn=trim_fn, ) diff --git a/backend/tests/test_f5_backend.py b/backend/tests/test_f5_backend.py index 3c0c5a556..2cb340943 100644 --- a/backend/tests/test_f5_backend.py +++ b/backend/tests/test_f5_backend.py @@ -125,6 +125,95 @@ def test_invalid_or_out_of_range_falls_back(self, monkeypatch, raw): assert _f5_speed() == 1.0 +class TestEngineDefaultChunkChars: + """F5 chunks to roughly one sentence (long single-shot generations + degrade); other engines keep the generic default.""" + + def test_f5_chunks_per_sentence(self): + from backend.backends import engine_default_chunk_chars + + assert engine_default_chunk_chars("f5") == 140 + + def test_other_engines_generic_default(self): + from backend.backends import engine_default_chunk_chars + + assert engine_default_chunk_chars("qwen") == 800 + assert engine_default_chunk_chars("unknown-engine") == 800 + + +class TestHardOnsetDetection: + """F5 garbles ț/î/â as the first sound; those onsets get the lead-in fix.""" + + @pytest.mark.parametrize("text", ["Țin minte tot.", "Împreună mergem.", "Ântâi plecăm.", + " țara noastră", "ÎN oraș"]) + def test_hard_onsets_detected(self, text): + from backend.backends.f5_backend import _has_hard_onset + + assert _has_hard_onset(text) is True + + @pytest.mark.parametrize("text", ["Astăzi plouă.", "Mașina merge.", "Ora este cinci.", + "Sunt aici.", "El vine."]) + def test_easy_onsets_not_flagged(self, text): + from backend.backends.f5_backend import _has_hard_onset + + assert _has_hard_onset(text) is False + + def test_onset_fix_default_off_and_toggleable(self, monkeypatch): + from backend.backends.f5_backend import F5_ONSET_FIX_ENV, _f5_onset_fix_enabled + + monkeypatch.delenv(F5_ONSET_FIX_ENV, raising=False) + assert _f5_onset_fix_enabled() is False + monkeypatch.setenv(F5_ONSET_FIX_ENV, "1") + assert _f5_onset_fix_enabled() is True + + +class TestF5NfeAndBestOf: + """nfe steps and best-of-N are env-configurable with safe fallbacks.""" + + def test_nfe_default(self, monkeypatch): + from backend.backends.f5_backend import F5_NFE_ENV, _f5_nfe_steps + + monkeypatch.delenv(F5_NFE_ENV, raising=False) + assert _f5_nfe_steps() == 32 + + def test_nfe_valid(self, monkeypatch): + from backend.backends.f5_backend import F5_NFE_ENV, _f5_nfe_steps + + monkeypatch.setenv(F5_NFE_ENV, "64") + assert _f5_nfe_steps() == 64 + + @pytest.mark.parametrize("raw", ["0", "500", "abc", ""]) + def test_nfe_invalid_falls_back(self, monkeypatch, raw): + from backend.backends.f5_backend import F5_NFE_ENV, _f5_nfe_steps + + monkeypatch.setenv(F5_NFE_ENV, raw) + assert _f5_nfe_steps() == 32 + + def test_best_of_default_disabled(self, monkeypatch): + from backend.backends.f5_backend import F5_BEST_OF_ENV, _f5_best_of + + monkeypatch.delenv(F5_BEST_OF_ENV, raising=False) + assert _f5_best_of() == 1 + + def test_best_of_valid(self, monkeypatch): + from backend.backends.f5_backend import F5_BEST_OF_ENV, _f5_best_of + + monkeypatch.setenv(F5_BEST_OF_ENV, "3") + assert _f5_best_of() == 3 + + @pytest.mark.parametrize("raw", ["0", "99", "x"]) + def test_best_of_invalid_falls_back(self, monkeypatch, raw): + from backend.backends.f5_backend import F5_BEST_OF_ENV, _f5_best_of + + monkeypatch.setenv(F5_BEST_OF_ENV, raw) + assert _f5_best_of() == 1 + + def test_similarity_key_ignores_case_punct_diacritics(self): + from backend.backends.f5_backend import _asr_similarity_key + + assert _asr_similarity_key("Șapte porți!") == _asr_similarity_key("sapte porti") + + class TestSpellRomanianNumbers: """Digits are effectively untrained in the fine-tune; they must be spelled out the way the training data wrote them (in letters)."""