From 815dc549ce2038baebbba51f2cfced76ac94c0ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Rodr=C3=ADguez?= Date: Thu, 13 Aug 2026 12:01:54 -0600 Subject: [PATCH 1/3] feat(backend): MP3 export, paragraph pauses, loop/artifact fixes, OpenVoice V2 - GET /audio/{id} and /audio/version/{id} serve MP3 by default (ffmpeg libmp3lame, cached beside the WAV); ?format=wav returns the original. History export-audio now also downloads .mp3. - POST /generate accepts paragraph_pause_ms: inserts exact silence between paragraphs separated by newlines in generate_chunked (default 600ms, 0 disables). - Trim leading burst/decay artifacts from qwen chunks (build_trim_fn) and guard against stochastic 'no no no' loops: max_new_tokens cap, periodic/noise artifact detection and up to 3 retries with seeded generation. - Add OpenVoice V2 backend: Kokoro base TTS + ToneColorConverter voice cloning (EN/ES/FR/ZH/JA/KO). - scripts/clean_leading_artifacts.py: helper to trim artifacts from WAVs. --- backend/backends/__init__.py | 13 + backend/backends/openvoice_backend.py | 375 ++++++++++++++++++++++++++ backend/backends/pytorch_backend.py | 94 ++++++- backend/models.py | 4 + backend/routes/audio.py | 63 ++++- backend/routes/generations.py | 8 +- backend/routes/history.py | 17 +- backend/services/generation.py | 14 +- backend/utils/audio.py | 140 ++++++++++ backend/utils/chunked_tts.py | 40 +++ scripts/clean_leading_artifacts.py | 45 ++++ 11 files changed, 787 insertions(+), 26 deletions(-) create mode 100644 backend/backends/openvoice_backend.py create mode 100755 scripts/clean_leading_artifacts.py diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index 2437a87b3..022009257 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -215,6 +215,7 @@ def is_loaded(self) -> bool: "chatterbox_turbo": "Chatterbox Turbo", "tada": "TADA", "kokoro": "Kokoro", + "openvoice": "OpenVoice V2", } LLM_ENGINES = { @@ -364,6 +365,14 @@ def _get_non_qwen_tts_configs() -> list[ModelConfig]: size_mb=350, languages=["en", "es", "fr", "hi", "it", "pt", "ja", "zh"], ), + ModelConfig( + model_name="openvoice-v2", + display_name="OpenVoice V2 (Multi-lingual Voice Clone)", + engine="openvoice", + hf_repo_id="myshell-ai/OpenVoiceV2", + size_mb=350, + languages=["en", "es", "fr", "zh", "ja", "ko"], + ), ] @@ -704,6 +713,10 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend: from .kokoro_backend import KokoroTTSBackend backend = KokoroTTSBackend() + elif engine == "openvoice": + from .openvoice_backend import OpenVoiceBackend + + backend = OpenVoiceBackend() elif engine == "qwen_custom_voice": from .qwen_custom_voice_backend import QwenCustomVoiceBackend diff --git a/backend/backends/openvoice_backend.py b/backend/backends/openvoice_backend.py new file mode 100644 index 000000000..8b166c1dc --- /dev/null +++ b/backend/backends/openvoice_backend.py @@ -0,0 +1,375 @@ +""" +OpenVoice V2 backend — multi-lingual voice cloning via ToneColorConverter. + +Architecture: + The OpenVoice V2 pipeline separates TTS from voice conversion. + This backend uses Kokoro (lightweight, always available) as the base TTS, + then applies OpenVoice's ToneColorConverter to transfer the target + voice's timbre onto the generated audio. + + Source style embeddings for each language are pre-extracted and stored + on disk — no base-speaker TTS model needed. + +Pipeline (generate): + 1. Generate base audio via Kokoro TTS (lightweight, CPU-friendly) + 2. Load source style embedding for the target language + 3. Load target style embedding (from reference audio via create_voice_prompt) + 4. ToneColorConverter.convert(source_audio, src_se, tgt_se) → cloned audio + +Languages supported: EN, ES, FR, ZH, JP, KR +""" + +import asyncio +import logging +import os +import sys +import tempfile +from typing import Optional + +import numpy as np +import torch + +from . import TTSBackend +from .base import ( + get_torch_device, + empty_device_cache, + manual_seed, + combine_voice_prompts as _combine_voice_prompts, + model_load_progress, +) + +logger = logging.getLogger(__name__) + +# ── Paths ──────────────────────────────────────────────────────────── + +MODELS_DIR = "/mnt/480ssd/voice-models/openvoicev2" +OPENVOICE_REPO = "/mnt/480ssd/voice-models/openvoice-repo" +BASE_SPEAKERS_SES = os.path.join(MODELS_DIR, "base_speakers", "ses") +CONVERTER_DIR = os.path.join(MODELS_DIR, "converter") + +# Language → base speaker filename (style embedding) +LANG_SOURCE_SE = { + "ES": "es", + "EN": "en-default", + "EN-US": "en-us", + "EN-UK": "en-newest", + "EN-IN": "en-india", + "EN-AU": "en-au", + "FR": "fr", + "ZH": "zh", + "JP": "jp", + "KR": "kr", +} + +# Voicebox language codes → OpenVoice source key +LANG_CODE_MAP = { + "es": "ES", + "en": "EN", + "en-us": "EN-US", + "en-uk": "EN-UK", + "en-in": "EN-IN", + "fr": "FR", + "zh": "ZH", + "ja": "JP", + "ko": "KR", +} + +SUPPORTED_LANGUAGES = list(LANG_CODE_MAP.keys()) + +OPENVOICE_SAMPLE_RATE = 22050 + + +class OpenVoiceBackend: + """OpenVoice V2 backend — multi-lingual voice cloning.""" + + def __init__(self): + self._tone_color_converter = None + self._device = None + self._model_load_lock = asyncio.Lock() + self.model_size = "default" + + def _get_device(self) -> str: + return get_torch_device(allow_xpu=True, allow_directml=True) + + def is_loaded(self) -> bool: + return self._tone_color_converter is not None + + def _get_model_path(self, model_size: str = "default") -> str: + return "myshell-ai/OpenVoiceV2" + + def _is_model_cached(self, model_size: str = "default") -> bool: + """Check if ToneColorConverter checkpoint exists.""" + ckpt = os.path.join(CONVERTER_DIR, "checkpoint.pth") + cfg = os.path.join(CONVERTER_DIR, "config.json") + return os.path.isfile(ckpt) and os.path.isfile(cfg) and os.path.isdir(BASE_SPEAKERS_SES) + + async def load_model(self, model_size: str = "default") -> None: + """Load the ToneColorConverter model.""" + if self._tone_color_converter is not None: + return + async with self._model_load_lock: + if self._tone_color_converter is not None: + return + await asyncio.to_thread(self._load_model_sync) + + def _load_model_sync(self): + """Synchronous model loading.""" + model_name = "openvoice-v2" + is_cached = self._is_model_cached() + + with model_load_progress(model_name, is_cached): + device = self._get_device() + self._device = device + logger.info("Loading OpenVoice V2 ToneColorConverter on %s...", device) + + # Add openvoice repo to sys.path if not already there + if os.path.exists(OPENVOICE_REPO) and OPENVOICE_REPO not in sys.path: + sys.path.insert(0, OPENVOICE_REPO) + + from openvoice.api import ToneColorConverter + + config_path = os.path.join(CONVERTER_DIR, "config.json") + ckpt_path = os.path.join(CONVERTER_DIR, "checkpoint.pth") + + self._tone_color_converter = ToneColorConverter( + config_path, device=device + ) + self._tone_color_converter.load_ckpt(ckpt_path) + # Disable watermark to avoid wavmark dependency + self._tone_color_converter.watermark_model = None + self._tone_color_converter.add_watermark = lambda audio, msg: audio + + logger.info("OpenVoice V2 loaded successfully") + + def unload_model(self) -> None: + """Unload model to free memory.""" + if self._tone_color_converter is not None: + device = self._device + del self._tone_color_converter + self._tone_color_converter = None + self._device = None + empty_device_cache(device) + logger.info("OpenVoice V2 unloaded") + + def _load_source_se(self, lang_key: str) -> Optional[torch.Tensor]: + """Load pre-extracted source style embedding for a language.""" + speaker_file = LANG_SOURCE_SE.get(lang_key, "en-default") + path = os.path.join(BASE_SPEAKERS_SES, f"{speaker_file}.pth") + if os.path.exists(path): + return torch.load(path, map_location=self._device) + logger.warning("Source SE not found for %s, falling back to en-default", lang_key) + fallback = os.path.join(BASE_SPEAKERS_SES, "en-default.pth") + if os.path.exists(fallback): + return torch.load(fallback, map_location=self._device) + return None + + async def _generate_base_audio( + self, text: str, language: str, seed: Optional[int] = None + ) -> tuple[np.ndarray, int]: + """ + Generate base audio using Kokoro TTS. + + Kokoro is used because: + - It's lightweight (82M params, always cached) + - It generates directly from text + preset voice (no voice prompt needed) + - It supports multiple languages including Spanish + """ + from ..backends import get_tts_backend_for_engine + + kokoro = get_tts_backend_for_engine("kokoro") + await kokoro.load_model() + + # Use a neutral voice for the base audio + # Kokoro voices: american female "af_heart", american male "am_liam" + # spanish: "ef_dora" (female), "em_alex" (male) + lang_to_voice = { + "es": "ef_dora", + "en": "af_heart", + "fr": "ff_siwis", + "zh": "zf_xiaoxiao", + "ja": "jf_alpha", + "ko": "jf_alpha", # Kokoro doesn't have Korean; fallback to Japanese + } + + voice_id = lang_to_voice.get(language, "af_heart") + kokoro_voice_prompt = { + "voice_type": "preset", + "preset_engine": "kokoro", + "preset_voice_id": voice_id, + } + + audio, sr = await kokoro.generate( + text, kokoro_voice_prompt, language=language, seed=seed + ) + return audio, sr + + async def create_voice_prompt( + self, + audio_path: str, + reference_text: str, + use_cache: bool = True, + ) -> tuple[dict, bool]: + """ + Extract style embedding from reference audio for voice cloning. + + The embedding is stored in the voice_prompt dict for later use + during generation. + + Args: + audio_path: Path to reference audio file + reference_text: Transcript (not used by OpenVoice, kept for protocol) + use_cache: Whether to reuse cached embeddings + + Returns: + Tuple of (voice_prompt_dict, was_cached) + """ + await self.load_model() + + def _extract_se(): + from openvoice import se_extractor + + return se_extractor.get_se( + audio_path, + self._tone_color_converter, + target_dir=os.path.join(MODELS_DIR, "processed_se"), + vad=True, + ) + + try: + target_se, audio_name = await asyncio.to_thread(_extract_se) + except Exception as e: + logger.warning("SE extraction failed with VAD, trying without: %s", e) + try: + def _extract_se_no_vad(): + from openvoice import se_extractor + return se_extractor.get_se( + audio_path, + self._tone_color_converter, + target_dir=os.path.join(MODELS_DIR, "processed_se"), + vad=False, + ) + target_se, audio_name = await asyncio.to_thread(_extract_se_no_vad) + except Exception as e2: + logger.error("SE extraction completely failed: %s", e2) + raise RuntimeError(f"Failed to extract voice embedding: {e2}") from e2 + + voice_prompt = {"target_se": target_se} + return voice_prompt, False + + async def combine_voice_prompts( + self, + audio_paths: list[str], + reference_texts: list[str], + ) -> tuple[np.ndarray, str]: + """Combine multiple reference voice prompts.""" + return await _combine_voice_prompts( + audio_paths, reference_texts, sample_rate=OPENVOICE_SAMPLE_RATE + ) + + async def generate( + self, + text: str, + voice_prompt: dict, + language: str = "en", + seed: Optional[int] = None, + instruct: Optional[str] = None, + ) -> tuple[np.ndarray, int]: + """ + Generate voice-cloned audio using OpenVoice V2 pipeline. + + Pipeline: + 1. Generate base audio via Kokoro TTS (lightweight, CPU-friendly) + 2. Load source SE for the target language + 3. Run ToneColorConverter to transfer voice timbre + + Args: + text: Text to synthesize + voice_prompt: Dict with target_se from create_voice_prompt. + If target_se is absent, falls back to base speaker. + language: Language code (en, es, fr, zh, ja, ko) + seed: Random seed for reproducibility + instruct: Not supported by OpenVoice (ignored) + + Returns: + Tuple of (audio_array, sample_rate) + """ + # Map language code + lang_key = LANG_CODE_MAP.get(language) + if lang_key is None: + for vbox_code, ov_key in LANG_CODE_MAP.items(): + if language.startswith(vbox_code): + lang_key = ov_key + break + if lang_key is None: + logger.warning("Unsupported language %s, falling back to EN", language) + lang_key = "EN" + + await self.load_model() + + # ── Step 1: Generate base audio with Kokoro TTS ── + logger.info("[OpenVoice] Generating base TTS audio via Kokoro...") + base_audio, base_sr = await self._generate_base_audio(text, language, seed) + logger.info( + "[OpenVoice] Base TTS generated: %d samples @ %dHz", + len(base_audio), + base_sr, + ) + + # ── Step 2 & 3 & 4: sync conversion ── + def _convert_sync(): + import soundfile as sf + + if seed is not None: + manual_seed(seed, self._device) + + tmp_path = tempfile.mktemp(suffix=".wav") + out_path = tempfile.mktemp(suffix=".wav") + + try: + sf.write(tmp_path, base_audio, base_sr) + + # Source SE + source_se = self._load_source_se(lang_key) + if source_se is None: + raise RuntimeError( + f"No source style embedding for {lang_key}. " + f"Available: {list(LANG_SOURCE_SE.keys())}" + ) + + # Target SE — if no reference provided, use source = base speaker + target_se = voice_prompt.get("target_se") + if target_se is None: + logger.info("[OpenVoice] No reference voice, using base speaker") + target_se = source_se + + # Convert + logger.info("[OpenVoice] Running ToneColorConverter...") + self._tone_color_converter.convert( + audio_src_path=tmp_path, + src_se=source_se, + tgt_se=target_se, + output_path=out_path, + tau=0.3, + message="voicebox-openvoice", + ) + + # Read result + out_audio, out_sr = sf.read(out_path) + out_audio = out_audio.astype(np.float32) + + logger.info( + "[OpenVoice] Voice clone done: %d samples @ %dHz", + len(out_audio), + out_sr, + ) + return out_audio, out_sr + + finally: + for p in [tmp_path, out_path]: + if os.path.exists(p): + try: + os.unlink(p) + except OSError: + pass + + return await asyncio.to_thread(_convert_sync) diff --git a/backend/backends/pytorch_backend.py b/backend/backends/pytorch_backend.py index f8ae79b86..f07468abe 100644 --- a/backend/backends/pytorch_backend.py +++ b/backend/backends/pytorch_backend.py @@ -223,21 +223,89 @@ async def generate( # Load model await self.load_model_async(None) + def _is_loop_artifact(audio: np.ndarray, sample_rate: int) -> bool: + """Detect the Qwen divergence loop ("no no no" garbage). + + Two signatures are checked: + 1. Periodicity: a clean repeated phrase (syllable-length lags have + a near-perfect autocorrelation peak). + 2. Hum/drone: the loop can also be a low-energy continuous hum — + ~all frames are "active" (no pauses) with an extremely low + zero-crossing rate, unlike real speech which has pauses and + consonant-driven zero crossings. + """ + if audio is None or len(audio) < sample_rate * 5: + return False + + # --- Signature 1: periodicity ------------------------------- + tail_secs = min(3.0, len(audio) / sample_rate * 0.4) + tail = audio[-int(sample_rate * tail_secs):] + step = max(1, sample_rate // 8000) + x = tail[::step].astype(np.float32) + x = x - x.mean() + n = len(x) + if n >= 512: + nfft = 1 << (2 * n - 1).bit_length() + ac = np.fft.irfft( + np.fft.rfft(x, nfft) * np.conj(np.fft.rfft(x, nfft)), nfft + )[:n] + denom = float(np.sum(x * x)) + if denom > 0: + ac = ac / denom + lag_min = int(0.12 * 8000) # 120 ms + lag_max = min(int(0.80 * 8000), n - 1) # 800 ms + if lag_max > lag_min and float(np.max(ac[lag_min:lag_max])) > 0.8: + return True + + # --- Signature 2: continuous low-zcr hum --------------------- + win = int(sample_rate * 0.02) + n_frames = len(audio) // win + if n_frames < 32: + return False + frames = audio[: n_frames * win].reshape(n_frames, win) + frms = np.sqrt(np.mean(frames ** 2, axis=1)) + active = float(np.mean(frms > 10 ** (-45 / 20))) + zcr = float( + np.mean(np.abs(np.diff(np.signbit(audio).astype(np.int8)))) + ) + return active > 0.85 and zcr < 0.06 + def _generate_sync(): """Run synchronous generation in thread pool.""" - # Set seed if provided - if seed is not None: - manual_seed(seed, self.device) - - # See _create_prompt_sync comment — inference runs with the - # process's default HF_HUB_OFFLINE state (issue #462). - wavs, sample_rate = self.model.generate_voice_clone( - text=text, - voice_clone_prompt=voice_prompt, - language=LANGUAGE_CODE_TO_NAME.get(language, "auto"), - instruct=instruct, - ) - return wavs[0], sample_rate + # Cap the worst-case length: a divergence loop must not be able to + # generate 10+ minutes of garbage. ~1.15 tokens/char + 50 headroom + # (≈ 0.35 s/char) is several times what real speech needs. + max_new_tokens = min(2048, int(len(text) * 1.15) + 50) + last_audio, last_sr = None, None + for attempt in range(3): + # Always seed so every attempt samples from a fresh RNG state; + # vary the seed on retries to escape the loop divergence. + attempt_seed = ( + seed if seed is not None else (hash(text) + attempt) & 0xFFFFFFFF + ) + manual_seed(attempt_seed, self.device) + wavs, sample_rate = self.model.generate_voice_clone( + text=text, + voice_clone_prompt=voice_prompt, + language=LANGUAGE_CODE_TO_NAME.get(language, "auto"), + instruct=instruct, + max_new_tokens=max_new_tokens, + ) + audio = wavs[0] + last_audio, last_sr = audio, sample_rate + duration = len(audio) / sample_rate if audio is not None else 0.0 + # A sane upper bound for real speech (~0.6 s/char + 20 s slack); + # the loop blows way past it. The periodicity check catches + # short-but-repetitive loops that fit under the bound. + expected = len(text) * 0.6 + 20.0 + if duration <= expected and not _is_loop_artifact(audio, sample_rate): + return audio, sample_rate + logger.warning( + "Qwen loop detectado (dur=%.1fs, texto=%d chars, intento %d/3) — " + "reintentando con otro seed", + duration, len(text), attempt + 1, + ) + return last_audio, last_sr # Run blocking inference in thread pool to avoid blocking event loop audio, sample_rate = await asyncio.to_thread(_generate_sync) diff --git a/backend/models.py b/backend/models.py index 7970ce41e..104d20d39 100644 --- a/backend/models.py +++ b/backend/models.py @@ -96,6 +96,10 @@ class GenerationRequest(BaseModel): crossfade_ms: int = Field( default=50, ge=0, le=500, description="Crossfade duration in ms between chunks (0 for hard cut)" ) + paragraph_pause_ms: int = Field( + default=600, ge=0, le=10000, + description="Pause in ms inserted between newline-separated paragraphs (0 = disabled)", + ) normalize: bool = Field(default=True, description="Normalize output audio volume") effects_chain: Optional[List["EffectConfig"]] = Field( None, description="Effects chain to apply after generation (overrides profile default)" diff --git a/backend/routes/audio.py b/backend/routes/audio.py index 791755681..eea2b149d 100644 --- a/backend/routes/audio.py +++ b/backend/routes/audio.py @@ -1,6 +1,8 @@ """Audio file serving endpoints.""" +import logging import mimetypes +import subprocess from pathlib import Path from fastapi import APIRouter, Depends, HTTPException @@ -11,8 +13,13 @@ from ..services import history from ..database import get_db +logger = logging.getLogger(__name__) + router = APIRouter() +# MP3 conversion settings +_MP3_BITRATE_K = 192 + def _audio_media_type(path: Path) -> str: """Derive the Content-Type from the file extension. @@ -24,8 +31,37 @@ def _audio_media_type(path: Path) -> str: return guessed or "audio/wav" +def _ensure_mp3(wav_path: Path) -> Path | None: + """Convert *wav_path* to MP3 (cached on disk next to the WAV). + + Returns the path of the MP3 file. The first request for a given WAV + transcodes it with ffmpeg and caches ``.mp3`` beside it, so + subsequent requests are cheap. Returns ``None`` when ffmpeg is + missing or the conversion fails.""" + mp3_path = wav_path.with_suffix(".mp3") + if mp3_path.exists() and mp3_path.stat().st_mtime >= wav_path.stat().st_mtime: + return mp3_path + try: + subprocess.run( + [ + "ffmpeg", "-y", "-v", "error", + "-i", str(wav_path), + "-codec:a", "libmp3lame", + "-b:a", f"{_MP3_BITRATE_K}k", + str(mp3_path), + ], + check=True, + capture_output=True, + timeout=120, + ) + except (subprocess.SubprocessError, FileNotFoundError) as e: + logger.warning("MP3 conversion failed for %s: %s", wav_path, e) + return None + return mp3_path + + @router.get("/audio/version/{version_id}") -async def get_version_audio(version_id: str, db: Session = Depends(get_db)): +async def get_version_audio(version_id: str, db: Session = Depends(get_db), format: str = "mp3"): """Serve audio for a specific version.""" from ..services import versions as versions_mod @@ -37,6 +73,15 @@ async def get_version_audio(version_id: str, db: Session = Depends(get_db)): if audio_path is None or not audio_path.exists(): raise HTTPException(status_code=404, detail="Audio file not found") + if format == "mp3" and audio_path.suffix.lower() in (".wav", ".flac", ".ogg"): + mp3_path = _ensure_mp3(audio_path) + if mp3_path is not None: + return FileResponse( + mp3_path, + media_type="audio/mpeg", + filename=f"generation_{version.generation_id}_{version.label}.mp3", + ) + return FileResponse( audio_path, media_type=_audio_media_type(audio_path), @@ -45,8 +90,11 @@ async def get_version_audio(version_id: str, db: Session = Depends(get_db)): @router.get("/audio/{generation_id}") -async def get_audio(generation_id: str, db: Session = Depends(get_db)): - """Serve generated audio file (serves the default version).""" +async def get_audio(generation_id: str, db: Session = Depends(get_db), format: str = "mp3"): + """Serve generated audio file (serves the default version). + + Defaults to MP3 (converted on demand and cached beside the WAV); pass + ``?format=wav`` to get the original file.""" generation = await history.get_generation(generation_id, db) if not generation: raise HTTPException(status_code=404, detail="Generation not found") @@ -55,6 +103,15 @@ async def get_audio(generation_id: str, db: Session = Depends(get_db)): if audio_path is None or not audio_path.exists(): raise HTTPException(status_code=404, detail="Audio file not found") + if format == "mp3" and audio_path.suffix.lower() in (".wav", ".flac", ".ogg"): + mp3_path = _ensure_mp3(audio_path) + if mp3_path is not None: + return FileResponse( + mp3_path, + media_type="audio/mpeg", + filename=f"generation_{generation_id}.mp3", + ) + return FileResponse( audio_path, media_type=_audio_media_type(audio_path), diff --git a/backend/routes/generations.py b/backend/routes/generations.py index 215c96cb2..340c77925 100644 --- a/backend/routes/generations.py +++ b/backend/routes/generations.py @@ -139,6 +139,7 @@ async def generate_speech( mode="generate", max_chunk_chars=data.max_chunk_chars, crossfade_ms=data.crossfade_ms, + paragraph_pause_ms=data.paragraph_pause_ms, ) ) @@ -346,11 +347,9 @@ async def stream_speech( from ..utils.chunked_tts import generate_chunked - trim_fn = None - if engine_needs_trim(engine): - from ..utils.audio import trim_tts_output + from ..utils.audio import build_trim_fn - trim_fn = trim_tts_output + trim_fn = build_trim_fn(engine) audio, sample_rate = await generate_chunked( tts_model, @@ -361,6 +360,7 @@ async def stream_speech( instruct=data.instruct, max_chunk_chars=data.max_chunk_chars, crossfade_ms=data.crossfade_ms, + paragraph_pause_ms=data.paragraph_pause_ms, trim_fn=trim_fn, ) diff --git a/backend/routes/history.py b/backend/routes/history.py index 1cd7694c1..94df0b3b7 100644 --- a/backend/routes/history.py +++ b/backend/routes/history.py @@ -10,6 +10,7 @@ from ..services import export_import, history from ..app import safe_content_disposition from ..database import Generation as DBGeneration, VoiceProfile as DBVoiceProfile, get_db +from .audio import _audio_media_type router = APIRouter() @@ -180,10 +181,22 @@ async def export_generation_audio( safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip() if not safe_text: safe_text = "generation" - filename = f"{safe_text}.wav" + filename = f"{safe_text}.mp3" + + # Default export is MP3 (converted on demand and cached beside the WAV). + from .audio import _ensure_mp3 + + if audio_path.suffix.lower() in (".wav", ".flac", ".ogg"): + mp3_path = _ensure_mp3(audio_path) + if mp3_path is not None: + return FileResponse( + mp3_path, + media_type="audio/mpeg", + headers={"Content-Disposition": safe_content_disposition("attachment", filename)}, + ) return FileResponse( audio_path, - media_type="audio/wav", + media_type=_audio_media_type(audio_path), headers={"Content-Disposition": safe_content_disposition("attachment", filename)}, ) diff --git a/backend/services/generation.py b/backend/services/generation.py index ce8fe93c9..33d757a1e 100644 --- a/backend/services/generation.py +++ b/backend/services/generation.py @@ -41,6 +41,7 @@ async def run_generation( mode: Literal["generate", "retry", "regenerate"], max_chunk_chars: Optional[int] = None, crossfade_ms: Optional[int] = None, + paragraph_pause_ms: Optional[int] = None, version_id: Optional[str] = None, ) -> None: """Execute TTS inference and persist the result. @@ -50,7 +51,7 @@ async def run_generation( """ from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim from ..utils.chunked_tts import generate_chunked - from ..utils.audio import normalize_audio, save_audio, trim_tts_output + from ..utils.audio import normalize_audio, save_audio, build_trim_fn task_manager = get_task_manager() bg_db = next(get_db()) @@ -71,7 +72,7 @@ async def run_generation( ) await history.update_generation_status(generation_id, "generating", bg_db) - trim_fn = trim_tts_output if engine_needs_trim(engine) else None + trim_fn = build_trim_fn(engine) gen_kwargs: dict = dict( language=language, @@ -83,6 +84,8 @@ async def run_generation( gen_kwargs["max_chunk_chars"] = max_chunk_chars if crossfade_ms is not None: gen_kwargs["crossfade_ms"] = crossfade_ms + if paragraph_pause_ms is not None: + gen_kwargs["paragraph_pause_ms"] = paragraph_pause_ms audio, sample_rate = await generate_chunked(tts_model, text, voice_prompt, **gen_kwargs) @@ -254,6 +257,7 @@ async def generate_audio_sync( normalize: bool = True, max_chunk_chars: Optional[int] = None, crossfade_ms: Optional[int] = None, + paragraph_pause_ms: Optional[int] = None, ) -> bytes: """Run a TTS generation synchronously and return the resulting wav bytes. @@ -269,7 +273,7 @@ async def generate_audio_sync( """ from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim from ..utils.chunked_tts import generate_chunked - from ..utils.audio import normalize_audio, trim_tts_output + from ..utils.audio import normalize_audio, build_trim_fn from . import tts bg_db = next(get_db()) @@ -286,7 +290,7 @@ async def generate_audio_sync( finally: bg_db.close() - trim_fn = trim_tts_output if engine_needs_trim(engine) else None + trim_fn = build_trim_fn(engine) gen_kwargs: dict = dict( language=language, @@ -298,6 +302,8 @@ async def generate_audio_sync( gen_kwargs["max_chunk_chars"] = max_chunk_chars if crossfade_ms is not None: gen_kwargs["crossfade_ms"] = crossfade_ms + if paragraph_pause_ms is not None: + gen_kwargs["paragraph_pause_ms"] = paragraph_pause_ms audio, sample_rate = await generate_chunked( tts_model, text, voice_prompt, **gen_kwargs diff --git a/backend/utils/audio.py b/backend/utils/audio.py index 7e0fd6fd5..f14fd9eb4 100644 --- a/backend/utils/audio.py +++ b/backend/utils/audio.py @@ -199,6 +199,146 @@ def trim_tts_output( return trimmed +def trim_leading_artifact( + audio: np.ndarray, + sample_rate: int = 24000, + frame_ms: int = 10, + burst_threshold_db: float = -38.0, + dip_threshold_db: float = -58.0, + min_dip_ms: int = 30, + max_burst_ms: int = 800, + lookahead_ms: int = 1200, + speech_after_db: float = -45.0, + decay_diff_db: float = 6.0, + margin_ms: int = 20, + fade_ms: int = 10, +) -> np.ndarray: + """ + Remove the leading "beat" artifact some TTS engines emit at the start of + each chunk (e.g. Qwen3-TTS: a short breath/noise burst that decays into + silence before the real speech begins). + + Signature it looks for: [loud burst] -> [monotonic decay] -> [deep + silence dip] -> [real speech]. Only trims when ALL conditions hold, so + audio that starts with real speech is left untouched. + + Args: + audio: Input audio array (mono float32) + sample_rate: Sample rate in Hz + frame_ms: Frame size for RMS energy calculation + burst_threshold_db: First frame above this dB counts as burst start + dip_threshold_db: Frames below this dB count as the silence dip + min_dip_ms: Minimum dip duration to qualify + max_burst_ms: Dip must appear within this many ms of burst start + lookahead_ms: After the dip, real speech must appear within this window + speech_after_db: "Real speech" = frames above this dB in lookahead + decay_diff_db: Mean RMS of burst's first half must be this many dB + louder than the second half (decay signature) + margin_ms: Extra silence kept after the dip before the trim point + fade_ms: Cosine fade-in applied at the new start + + Returns: + Trimmed audio array (unchanged if no artifact is detected) + """ + frame_len = int(sample_rate * frame_ms / 1000) + if frame_len == 0 or len(audio) < frame_len * 4: + return audio + + n_frames = len(audio) // frame_len + rms = np.array( + [ + np.sqrt(np.mean(audio[i * frame_len : (i + 1) * frame_len] ** 2)) + for i in range(n_frames) + ] + ) + db = 20.0 * np.log10(rms + 1e-12) + + burst_threshold = burst_threshold_db + dip_threshold = dip_threshold_db + min_dip_frames = max(1, int(min_dip_ms / frame_ms)) + max_burst_frames = int(max_burst_ms / frame_ms) + lookahead_frames = int(lookahead_ms / frame_ms) + + # 1) Find first "loud" frame (burst start) + loud_idx = np.where(db > burst_threshold)[0] + if len(loud_idx) == 0: + return audio + burst_start = int(loud_idx[0]) + + # 2) Find the first deep-silence dip after the burst (within max_burst_ms) + dip_start = -1 + dip_len = 0 + i = burst_start + 1 + limit = min(n_frames, burst_start + max_burst_frames + min_dip_frames) + while i < limit: + if db[i] < dip_threshold: + j = i + while j < limit and db[j] < dip_threshold: + j += 1 + if j - i >= min_dip_frames: + dip_start = i + dip_len = j - i + break + i = j + else: + i += 1 + + if dip_start == -1: + return audio + + # 3) Decay signature: first half of the burst must be clearly louder + # than the second half (the artifact decays; real speech does not). + seg = db[burst_start:dip_start] + if len(seg) >= 4: + half = len(seg) // 2 + first_half = seg[:half] + second_half = seg[half : half * 2] + if float(np.mean(first_half) - np.mean(second_half)) < decay_diff_db: + return audio + + # 4) Real speech must resume after the dip + after_start = dip_start + dip_len + window_end = min(n_frames, after_start + lookahead_frames) + if window_end <= after_start: + return audio + if float(np.max(db[after_start:window_end])) < speech_after_db: + return audio + + # 5) Trim to just past the dip (+ small margin into the silence) + margin_frames = int(margin_ms / frame_ms) + trim_frame = after_start + margin_frames + start_sample = min(trim_frame * frame_len, len(audio) - 1) + + trimmed = audio[start_sample:].copy() + + # Short cosine fade-in to avoid any click at the new start + fade_samples = int(sample_rate * fade_ms / 1000) + if fade_samples > 0 and len(trimmed) > fade_samples: + fade = np.sin(np.linspace(0, np.pi / 2, fade_samples)) ** 2 + trimmed[:fade_samples] *= fade + + return trimmed + + +def build_trim_fn(engine: str): + """Build the per-chunk trim pipeline for an engine. + + Always removes the leading burst artifact (pattern-detected, safe for + all engines); additionally applies ``trim_tts_output`` for engines that + hallucinate trailing noise (e.g. Chatterbox). + """ + # Local import to avoid any import cycle with backends. + from ..backends import engine_needs_trim + + def _trim(audio: np.ndarray, sample_rate: int) -> np.ndarray: + audio = trim_leading_artifact(audio, sample_rate) + if engine_needs_trim(engine): + audio = trim_tts_output(audio, sample_rate) + return audio + + return _trim + + def preprocess_reference_audio( audio: np.ndarray, sample_rate: int, diff --git a/backend/utils/chunked_tts.py b/backend/utils/chunked_tts.py index 1f43379eb..712eb9453 100644 --- a/backend/utils/chunked_tts.py +++ b/backend/utils/chunked_tts.py @@ -211,6 +211,7 @@ async def generate_chunked( max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS, crossfade_ms: int = 50, trim_fn=None, + paragraph_pause_ms: int = 0, ) -> Tuple[np.ndarray, int]: """Generate audio with automatic chunking for long text. @@ -244,6 +245,45 @@ async def generate_chunked( ------- (audio, sample_rate) : Tuple[np.ndarray, int] """ + # Newline-separated paragraphs: if a pause is configured, generate each + # paragraph independently and join them with a silence gap. This gives + # writers explicit timing control (e.g. "\n" = 1s pause) that the TTS + # models themselves do not support. + if paragraph_pause_ms > 0 and "\n" in text: + paragraphs = [p.strip() for p in text.split("\n") if p.strip()] + if len(paragraphs) > 1: + logger.info( + "Splitting into %d paragraphs with %dms pause between them", + len(paragraphs), + paragraph_pause_ms, + ) + audio_chunks: List[np.ndarray] = [] + sample_rate: int | None = None + silence: np.ndarray | None = None + for para in paragraphs: + if sample_rate is not None and silence is not None: + audio_chunks.append(silence) + para_audio, para_sr = await generate_chunked( + backend, + para, + voice_prompt, + language, + seed, + instruct, + max_chunk_chars, + crossfade_ms, + trim_fn, + paragraph_pause_ms=0, # inner chunks never re-split + ) + if sample_rate is None: + sample_rate = para_sr + silence = np.zeros( + int(sample_rate * paragraph_pause_ms / 1000), + dtype=np.float32, + ) + audio_chunks.append(np.asarray(para_audio, dtype=np.float32)) + return np.concatenate(audio_chunks), sample_rate + chunks = split_text_into_chunks(text, max_chunk_chars) if len(chunks) <= 1: diff --git a/scripts/clean_leading_artifacts.py b/scripts/clean_leading_artifacts.py new file mode 100755 index 000000000..025e5c514 --- /dev/null +++ b/scripts/clean_leading_artifacts.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Limpia el artefacto "beat/noa" del inicio de audios TTS (voicebox). + +Uso: + backend/venv/bin/python scripts/clean_leading_artifacts.py [wav2 ...] + +Cada archivo se limpia IN-PLACE y el original se respalda como _orig.wav. +""" +import sys, os, wave +import numpy as np +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'backend')) +from utils.audio import trim_leading_artifact + +def load(path): + w = wave.open(path, 'rb') + sr = w.getframerate() + data = np.frombuffer(w.readframes(w.getnframes()), dtype=np.int16).astype(np.float32)/32768 + return data, sr + +def save(audio, sr, path): + w = wave.open(path, 'wb') + w.setnchannels(1); w.setsampwidth(2); w.setframerate(sr) + w.writeframes((np.clip(audio, -1, 1) * 32767).astype(np.int16).tobytes()) + w.close() + +def main(): + if len(sys.argv) < 2: + print(__doc__); return 1 + for path in sys.argv[1:]: + if not os.path.exists(path): + print(f"⚠ no existe: {path}"); continue + a, sr = load(path) + before = len(a) / sr + t = trim_leading_artifact(a, sr) + if len(t) == len(a): + print(f"• {os.path.basename(path)}: sin artefacto, sin cambios") + continue + backup = path.replace('.wav', '_orig.wav') + os.rename(path, backup) + save(t, sr, path) + print(f"✓ {os.path.basename(path)}: {before:.2f}s -> {len(t)/sr:.2f}s (cortó {before-len(t)/sr:.3f}s, backup {os.path.basename(backup)})") + return 0 + +if __name__ == '__main__': + sys.exit(main()) From b31b2fe8c0f69cc1ca375e9bb23de534fa57241c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Rodr=C3=ADguez?= Date: Thu, 13 Aug 2026 12:01:59 -0600 Subject: [PATCH 2/3] fix(web): mount PlatformProvider for web build, .mp3 export filenames The web SPA crashed with 'usePlatform must be used within PlatformProvider' after the Tauri platform refactor moved the platform implementation into tauri/ and left app/ without a provider. Add a browser-safe webPlatform (blob-URL download for saveFile, no-ops for desktop-only updater/audio/ lifecycle) and mount it in main.tsx so the web build renders again. Also fix export filenames in useHistory/useStories: the backend now serves MP3, so downloads were saving MP3 data with a .wav extension. --- app/src/lib/hooks/useHistory.ts | 4 +- app/src/lib/hooks/useStories.ts | 4 +- app/src/main.tsx | 8 ++- app/src/platform/webPlatform.ts | 111 ++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 app/src/platform/webPlatform.ts diff --git a/app/src/lib/hooks/useHistory.ts b/app/src/lib/hooks/useHistory.ts index e6f4aa7a6..6b2b1dcca 100644 --- a/app/src/lib/hooks/useHistory.ts +++ b/app/src/lib/hooks/useHistory.ts @@ -78,12 +78,12 @@ export function useExportGenerationAudio() { .substring(0, 30) .replace(/[^a-z0-9]/gi, '-') .toLowerCase(); - const filename = `${safeText}.wav`; + const filename = `${safeText}.mp3`; await platform.filesystem.saveFile(filename, blob, [ { name: 'Audio File', - extensions: ['wav'], + extensions: ['mp3'], }, ]); diff --git a/app/src/lib/hooks/useStories.ts b/app/src/lib/hooks/useStories.ts index 7c7eae6c9..f7b6cebb9 100644 --- a/app/src/lib/hooks/useStories.ts +++ b/app/src/lib/hooks/useStories.ts @@ -240,12 +240,12 @@ export function useExportStoryAudio() { .substring(0, 50) .replace(/[^a-z0-9]/gi, '-') .toLowerCase(); - const filename = `${safeName || 'story'}.wav`; + const filename = `${safeName || 'story'}.mp3`; await platform.filesystem.saveFile(filename, blob, [ { name: 'Audio File', - extensions: ['wav'], + extensions: ['mp3'], }, ]); diff --git a/app/src/main.tsx b/app/src/main.tsx index 52dc90445..34b458c78 100644 --- a/app/src/main.tsx +++ b/app/src/main.tsx @@ -6,12 +6,16 @@ import App from './App'; import './i18n'; import './index.css'; import { queryClient } from './lib/queryClient'; +import { PlatformProvider } from './platform/PlatformContext'; +import { webPlatform } from './platform/webPlatform'; ReactDOM.createRoot(document.getElementById('root')!).render( - - {/* */} + + + {/* */} + , ); diff --git a/app/src/platform/webPlatform.ts b/app/src/platform/webPlatform.ts new file mode 100644 index 000000000..479fe5298 --- /dev/null +++ b/app/src/platform/webPlatform.ts @@ -0,0 +1,111 @@ +/** + * Web platform implementation + * + * Voicebox's primary target is the Tauri desktop app; the web build + * (app/ served as a static SPA by the Python backend) had no platform + * implementation after the Tauri refactor, which crashed the UI with + * "usePlatform must be used within PlatformProvider". + * + * This implementation provides browser-safe no-ops for the desktop-only + * features (updater, system audio capture, server lifecycle) and a + * download-based saveFile so audio export keeps working in the browser. + */ + +import type { + Platform, + PlatformAudio, + PlatformFilesystem, + PlatformLifecycle, + PlatformMetadata, + PlatformUpdater, + UpdateStatus, +} from './types'; + +const noop = () => {}; + +const EMPTY_UPDATE_STATUS: UpdateStatus = { + checking: false, + available: false, + downloading: false, + installing: false, + readyToInstall: false, +}; + +const webFilesystem: PlatformFilesystem = { + async saveFile(filename: string, blob: Blob) { + // Browser download via object URL — file filters are a desktop concept + const url = URL.createObjectURL(blob); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = filename; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + setTimeout(() => URL.revokeObjectURL(url), 10_000); + }, + + async openPath(path: string) { + window.open(path, '_blank', 'noopener'); + }, + + async pickDirectory() { + // Browsers can't return a persistent directory path + return null; + }, +}; + +const webUpdater: PlatformUpdater = { + async checkForUpdates() {}, + async downloadAndInstall() {}, + async restartAndInstall() {}, + getStatus: () => ({ ...EMPTY_UPDATE_STATUS }), + subscribe: () => noop, +}; + +const webAudio: PlatformAudio = { + async isSystemAudioSupported() { + return false; + }, + async startSystemAudioCapture() { + throw new Error('System audio capture is not supported in the browser.'); + }, + async stopSystemAudioCapture() { + throw new Error('System audio capture is not supported in the browser.'); + }, + async listOutputDevices() { + return []; + }, + async playToDevices() { + // Web playback uses the default