diff --git a/backend/api/routers/generation.py b/backend/api/routers/generation.py index 859d655e..16795e9d 100644 --- a/backend/api/routers/generation.py +++ b/backend/api/routers/generation.py @@ -216,6 +216,17 @@ async def generate_speech( except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + # #308: a transcript-less reference is transcribed with the active ASR + # backend (whisperx / faster-whisper / mlx-whisper) instead of the model's + # built-in transformers pipeline, which cannot load whisper-large-v3-turbo + # on transformers 5.3. On failure ref_text stays None and the model's + # fallback behaves exactly as before. + if ref_audio_path and not ref_text: + from services.asr_backend import transcribe_reference + ref_text = await asyncio.get_running_loop().run_in_executor( + _gpu_pool, transcribe_reference, ref_audio_path + ) + start_time = time.time() try: loop = asyncio.get_running_loop() diff --git a/backend/services/asr_backend.py b/backend/services/asr_backend.py index 295cf905..f0070005 100644 --- a/backend/services/asr_backend.py +++ b/backend/services/asr_backend.py @@ -1016,6 +1016,44 @@ def get_active_asr_backend(*, asr_pipe=None) -> ASRBackend: return _REGISTRY[bid]() +def transcribe_reference(audio_path: str) -> str | None: + """Transcribe a voice-clone reference clip with the active ASR backend. + + Voice cloning without a user-supplied transcript used to fall through to + ``OmniVoice.load_asr_model()`` — a transformers ``pipeline()`` load of + whisper-large-v3-turbo that fails outright on transformers 5.3 (#308), + even when whisperx / faster-whisper / mlx-whisper are installed and + working. Route the reference transcript through the registry instead, so + the model-attached pipeline is only reached when it is genuinely the last + resort. Returns ``None`` on any failure — callers pass ``ref_text=None`` + through and the model's built-in fallback still gets its chance. + """ + try: + backend = get_active_asr_backend() + except Exception as e: # noqa: BLE001 — never let ASR break generation + logger.warning("transcribe_reference: no ASR backend available (%s)", e) + return None + if isinstance(backend, PyTorchWhisperBackend): + # The registry fell through to the model-attached pipeline; let the + # model load it lazily rather than constructing a second copy here. + return None + try: + result = backend.transcribe(audio_path, word_timestamps=False) + except Exception as e: # noqa: BLE001 — degrade to the model fallback + logger.warning( + "transcribe_reference: %s failed (%s) — deferring to the model's " + "built-in ASR fallback", + backend.id, e, + ) + return None + result = result or {} + text = result.get("text") or " ".join( + (seg.get("text") or "").strip() for seg in result.get("segments", []) + ) + text = (text or "").strip() + return text or None + + _capture_backend: ASRBackend | None = None diff --git a/tests/test_transcribe_reference.py b/tests/test_transcribe_reference.py new file mode 100644 index 00000000..439e683a --- /dev/null +++ b/tests/test_transcribe_reference.py @@ -0,0 +1,78 @@ +"""Voice-clone reference transcription goes through the ASR registry (#308). + +A transcript-less reference used to fall through to OmniVoice's built-in +transformers pipeline (`load_asr_model`), which cannot load +whisper-large-v3-turbo on transformers 5.3 — even when whisperx / +faster-whisper / mlx-whisper were installed and working. `transcribe_reference` +must use the active registry backend, and degrade to None (the model fallback) +rather than raise. +""" +from services import asr_backend as ab + + +class _FakeBackend(ab.ASRBackend): + id = "fake" + display_name = "Fake" + + def __init__(self, result=None, exc=None): + self._result = result + self._exc = exc + + @classmethod + def is_available(cls): + return True, "ready" + + def transcribe(self, audio_path, *, word_timestamps=True): + if self._exc: + raise self._exc + return self._result + + +def test_uses_active_backend_text(monkeypatch): + monkeypatch.setattr( + ab, "get_active_asr_backend", + lambda **kw: _FakeBackend(result={"text": " hello there "}), + ) + assert ab.transcribe_reference("ref.wav") == "hello there" + + +def test_joins_segments_when_no_top_level_text(monkeypatch): + """WhisperX results carry no top-level "text" — only segments.""" + monkeypatch.setattr( + ab, "get_active_asr_backend", + lambda **kw: _FakeBackend(result={ + "segments": [{"text": " hello"}, {"text": "world "}], + }), + ) + assert ab.transcribe_reference("ref.wav") == "hello world" + + +def test_backend_failure_degrades_to_none(monkeypatch): + monkeypatch.setattr( + ab, "get_active_asr_backend", + lambda **kw: _FakeBackend(exc=RuntimeError("model load failed")), + ) + assert ab.transcribe_reference("ref.wav") is None + + +def test_registry_resolution_failure_degrades_to_none(monkeypatch): + def _boom(**kw): + raise ValueError("unknown backend") + monkeypatch.setattr(ab, "get_active_asr_backend", _boom) + assert ab.transcribe_reference("ref.wav") is None + + +def test_pytorch_whisper_defers_to_model_fallback(monkeypatch): + """When the registry itself resolves to pytorch-whisper, defer to the + model's lazy load instead of constructing a second pipeline.""" + be = ab.PyTorchWhisperBackend(asr_pipe=object()) + monkeypatch.setattr(ab, "get_active_asr_backend", lambda **kw: be) + assert ab.transcribe_reference("ref.wav") is None + + +def test_empty_result_degrades_to_none(monkeypatch): + monkeypatch.setattr( + ab, "get_active_asr_backend", + lambda **kw: _FakeBackend(result={"text": " "}), + ) + assert ab.transcribe_reference("ref.wav") is None