diff --git a/CHANGELOG.md b/CHANGELOG.md index ceed10f4..6a104d4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ The format is loosely based on [Keep a Changelog](https://keepachangelog.com/). Versions track the desktop app (`tauri.conf.json` + `frontend/src-tauri/Cargo.toml`). The bundled TTS model package (`pyproject.toml`) is versioned independently. +## [Unreleased] + +### Fixed + +- **8 GB GPUs: voice-clone/dub transcription no longer kills the backend.** + On cards where the TTS model already held most of the VRAM (e.g. RTX + 4060 Ti 8 GB), loading whisper `large-v3` in float16 for a reference-clip + or dub transcription died as a *native* CUDA out-of-memory abort — the + whole backend process vanished with no error logged, and the app showed + "Can't reach the local OmniVoice backend." A new VRAM preflight re-checks + free GPU memory right before the ASR load and steps down float16 → + int8 → CPU instead of attempting a load that can't fit (opt-out: + `OMNIVOICE_ASR_VRAM_PREFLIGHT=0`). (#723) + ## [0.3.8] — 2026-07-01 A stability-focused release that makes first-run and Windows "just work," ships diff --git a/backend/services/asr_backend.py b/backend/services/asr_backend.py index 26724f1d..dbc6c5a5 100644 --- a/backend/services/asr_backend.py +++ b/backend/services/asr_backend.py @@ -316,6 +316,74 @@ class WhisperXBackend(ASRBackend): pass return "cpu", "int8" + # Peak VRAM (GB) to load *and transcribe* whisper large-v3 per CTranslate2 + # compute type (weights + encoder/decoder workspace, with headroom). #723: + # on an 8 GB card with the TTS model resident, loading fp16 large-v3 dies + # as a *native* CUDA OOM abort — the process is killed, no Python + # exception ever fires, and the UI reports "Can't reach the local + # backend". The only defense is to never start that load, so the device + # pick is re-checked against actually-free VRAM right before loading. + _CUDA_VRAM_BUDGET_GB = {"float16": 5.0, "int8_float16": 3.5, "int8": 3.0} + + #: Budget multiplier by model size (budgets above are for large-v3). + _MODEL_VRAM_SCALE = ( + ("large", 1.0), ("turbo", 0.55), ("medium", 0.5), + ("small", 0.25), ("base", 0.15), ("tiny", 0.1), + ) + + @staticmethod + def _free_vram_gb(): + """Device-wide free VRAM in GB (counts other processes), or None.""" + try: + import torch + if torch.cuda.is_available(): + free, _total = torch.cuda.mem_get_info() + return free / 1024**3 + except Exception: # noqa: BLE001 — preflight must never block ASR + pass + return None + + @classmethod + def _model_scale(cls, model_name: str) -> float: + name = (model_name or "").lower() + for key, scale in cls._MODEL_VRAM_SCALE: + if key in name: + return scale + return 1.0 # unknown → assume large + + def _degrade_for_vram(self, device: str, compute_type: str) -> tuple[str, str]: + """Downgrade the CUDA compute type (or fall to CPU) if free VRAM can't + hold the model — preventing the un-catchable native OOM abort (#723). + Opt-out: OMNIVOICE_ASR_VRAM_PREFLIGHT=0.""" + if device != "cuda" or os.environ.get( + "OMNIVOICE_ASR_VRAM_PREFLIGHT", "1" + ).strip().lower() in ("0", "false", "no"): + return device, compute_type + free = self._free_vram_gb() + if free is None: + return device, compute_type + scale = self._model_scale(self._model_name) + candidates = list(self._CUDA_VRAM_BUDGET_GB) + start = candidates.index(compute_type) if compute_type in candidates else 0 + for ct in candidates[start:]: + if free >= self._CUDA_VRAM_BUDGET_GB[ct] * scale: + if ct != compute_type: + logger.warning( + "whisperx VRAM preflight: %.1f GB free < %.1f GB needed " + "for %s %s — degrading to %s (#723)", + free, self._CUDA_VRAM_BUDGET_GB[compute_type] * scale, + self._model_name, compute_type, ct, + ) + return device, ct + logger.warning( + "whisperx VRAM preflight: %.1f GB free is too little for %s on CUDA " + "(needs ≥%.1f GB even at int8) — using CPU int8 instead. Free VRAM " + "(flush the TTS model, or close other GPU apps) for GPU-speed ASR. (#723)", + free, self._model_name, + self._CUDA_VRAM_BUDGET_GB["int8"] * scale, + ) + return "cpu", "int8" + @classmethod def is_available(cls) -> tuple[bool, str]: try: @@ -346,6 +414,13 @@ class WhisperXBackend(ASRBackend): # (#630/#611/#647). No-op on macOS/Linux and when speechbrain is absent. _harden_speechbrain_lazy_imports() import whisperx + # #723: re-check the CUDA pick against *currently free* VRAM — the TTS + # model may have claimed the card since __init__. A too-big load dies + # as a native abort (whole process, no exception), so it must be + # avoided up front rather than caught below. + self._device, self._compute_type = self._degrade_for_vram( + self._device, self._compute_type + ) logger.info( "whisperx loading ASR %s on %s (%s)", self._model_name, self._device, self._compute_type, diff --git a/tests/test_asr_vram_preflight.py b/tests/test_asr_vram_preflight.py new file mode 100644 index 00000000..a7abbf42 --- /dev/null +++ b/tests/test_asr_vram_preflight.py @@ -0,0 +1,93 @@ +"""WhisperX VRAM preflight (#723). + +On an 8 GB card with the TTS model resident, loading whisper large-v3 fp16 +dies as a *native* CUDA OOM abort — the backend process is killed outright, +no Python exception fires, and the UI reports "Can't reach the local +backend". The load-time fp16→int8 / OOM→CPU fallbacks in `_ensure_asr` never +run because nothing is raised. The only defense is a preflight: re-check the +device pick against actually-free VRAM (`torch.cuda.mem_get_info`) right +before loading, degrading fp16 → int8_float16 → int8 → CPU. + +Backend classes are resolved at RUNTIME (see test_asr_gpu_compat.py rationale). +""" +from __future__ import annotations + +import pytest + + +def _backend(): + from services.asr_backend import _REGISTRY + b = _REGISTRY["whisperx"].__new__(_REGISTRY["whisperx"]) # skip __init__ (no torch probe) + b._model_name = "large-v3" + return b + + +def _degrade(b, free_gb, device="cuda", compute="float16"): + b._free_vram_gb = lambda: free_gb + return b._degrade_for_vram(device, compute) + + +# ── The #723 crash scenario: TTS resident, ~2 GB free, fp16 requested ────── + +def test_starved_card_falls_back_to_cpu(): + assert _degrade(_backend(), 2.0) == ("cpu", "int8") + + +def test_mid_vram_degrades_to_int8_on_cuda(): + # 3.5 GB free: can't hold fp16 (5.0) or int8_float16 (3.5 is not > needed + # headroom boundary — equal passes), int8 (3.0) certainly fits. + dev, ct = _degrade(_backend(), 3.2) + assert (dev, ct) == ("cuda", "int8") + + +def test_ample_vram_keeps_fp16(): + assert _degrade(_backend(), 7.0) == ("cuda", "float16") + + +# ── Preflight must never *break* ASR ──────────────────────────────────────── + +def test_unknown_vram_is_left_alone(): + assert _degrade(_backend(), None) == ("cuda", "float16") + + +def test_cpu_pick_is_untouched(): + b = _backend() + b._free_vram_gb = lambda: 0.5 + assert b._degrade_for_vram("cpu", "int8") == ("cpu", "int8") + + +def test_small_models_not_over_evicted(): + # A 2 GB-free card comfortably runs whisper-small fp16 (5.0 * 0.25 budget); + # the large-v3 budgets must not evict smaller models from CUDA. + b = _backend() + b._model_name = "small" + assert _degrade(b, 2.0) == ("cuda", "float16") + + +def test_env_opt_out(monkeypatch): + monkeypatch.setenv("OMNIVOICE_ASR_VRAM_PREFLIGHT", "0") + assert _degrade(_backend(), 0.5) == ("cuda", "float16") + + +# ── Wiring: _ensure_asr must preflight BEFORE whisperx.load_model ────────── + +def test_ensure_asr_applies_preflight_before_load(monkeypatch): + import sys, types + + calls = {} + + fake_whisperx = types.ModuleType("whisperx") + def _load_model(name, device=None, compute_type=None, **kw): + calls["load"] = (device, compute_type) + return object() + fake_whisperx.load_model = _load_model + monkeypatch.setitem(sys.modules, "whisperx", fake_whisperx) + + b = _backend() + b._asr = None + b._device, b._compute_type = "cuda", "float16" + b._free_vram_gb = lambda: 2.0 # the #723 card state + b._allow_vad_pickle_globals = lambda: None + + b._ensure_asr() + assert calls["load"] == ("cpu", "int8") # degraded BEFORE the load call