fix(asr): clone references transcribe via the ASR registry, not the broken transformers pipeline (#308) (#321)

Voice cloning without a transcript fell through to OmniVoice's built-in
load_asr_model() — a transformers pipeline() load of
whisper-large-v3-turbo that fails outright on transformers 5.3 — even
when whisperx / faster-whisper / mlx-whisper were installed and working.
The dub pipeline already used the registry; the /generate clone path
never did.

- services/asr_backend.py: new transcribe_reference() resolves the
  active registry backend (honoring auto-detect order and the
  OMNIVOICE_ASR_BACKEND override), extracts text from either result
  shape (top-level "text" or whisperx-style segments), and degrades to
  None on any failure so the model fallback behaves exactly as before.
  When the registry itself resolves to pytorch-whisper it defers to the
  model's lazy load instead of building a second pipeline.
- api/routers/generation.py: transcript-less references get transcribed
  in the GPU pool before inference.
- tests/test_transcribe_reference.py: covers both result shapes,
  failure degradation, and the pytorch-whisper deferral.

The remaining half of #308 — pytorch-whisper itself being incompatible
with transformers 5.3 when it truly is the last resort — is tracked in
the issue.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

backend/api/routers/generation.py

@@ -216,6 +216,17 @@ async def generate_speech(
         except Exception as e:
             raise HTTPException(status_code=500, detail=str(e))
 
+    # #308: a transcript-less reference is transcribed with the active ASR
+    # backend (whisperx / faster-whisper / mlx-whisper) instead of the model's
+    # built-in transformers pipeline, which cannot load whisper-large-v3-turbo
+    # on transformers 5.3. On failure ref_text stays None and the model's
+    # fallback behaves exactly as before.
+    if ref_audio_path and not ref_text:
+        from services.asr_backend import transcribe_reference
+        ref_text = await asyncio.get_running_loop().run_in_executor(
+            _gpu_pool, transcribe_reference, ref_audio_path
+        )
+
     start_time = time.time()
     try:
         loop = asyncio.get_running_loop()

backend/services/asr_backend.py

@@ -1016,6 +1016,44 @@ def get_active_asr_backend(*, asr_pipe=None) -> ASRBackend:
     return _REGISTRY[bid]()
 
 
+def transcribe_reference(audio_path: str) -> str | None:
+    """Transcribe a voice-clone reference clip with the active ASR backend.
+
+    Voice cloning without a user-supplied transcript used to fall through to
+    ``OmniVoice.load_asr_model()`` — a transformers ``pipeline()`` load of
+    whisper-large-v3-turbo that fails outright on transformers 5.3 (#308),
+    even when whisperx / faster-whisper / mlx-whisper are installed and
+    working. Route the reference transcript through the registry instead, so
+    the model-attached pipeline is only reached when it is genuinely the last
+    resort. Returns ``None`` on any failure — callers pass ``ref_text=None``
+    through and the model's built-in fallback still gets its chance.
+    """
+    try:
+        backend = get_active_asr_backend()
+    except Exception as e:  # noqa: BLE001 — never let ASR break generation
+        logger.warning("transcribe_reference: no ASR backend available (%s)", e)
+        return None
+    if isinstance(backend, PyTorchWhisperBackend):
+        # The registry fell through to the model-attached pipeline; let the
+        # model load it lazily rather than constructing a second copy here.
+        return None
+    try:
+        result = backend.transcribe(audio_path, word_timestamps=False)
+    except Exception as e:  # noqa: BLE001 — degrade to the model fallback
+        logger.warning(
+            "transcribe_reference: %s failed (%s) — deferring to the model's "
+            "built-in ASR fallback",
+            backend.id, e,
+        )
+        return None
+    result = result or {}
+    text = result.get("text") or " ".join(
+        (seg.get("text") or "").strip() for seg in result.get("segments", [])
+    )
+    text = (text or "").strip()
+    return text or None
+
+
 _capture_backend: ASRBackend | None = None
 
 

tests/test_transcribe_reference.py

@@ -0,0 +1,78 @@
+"""Voice-clone reference transcription goes through the ASR registry (#308).
+
+A transcript-less reference used to fall through to OmniVoice's built-in
+transformers pipeline (`load_asr_model`), which cannot load
+whisper-large-v3-turbo on transformers 5.3 — even when whisperx /
+faster-whisper / mlx-whisper were installed and working. `transcribe_reference`
+must use the active registry backend, and degrade to None (the model fallback)
+rather than raise.
+"""
+from services import asr_backend as ab
+
+
+class _FakeBackend(ab.ASRBackend):
+    id = "fake"
+    display_name = "Fake"
+
+    def __init__(self, result=None, exc=None):
+        self._result = result
+        self._exc = exc
+
+    @classmethod
+    def is_available(cls):
+        return True, "ready"
+
+    def transcribe(self, audio_path, *, word_timestamps=True):
+        if self._exc:
+            raise self._exc
+        return self._result
+
+
+def test_uses_active_backend_text(monkeypatch):
+    monkeypatch.setattr(
+        ab, "get_active_asr_backend",
+        lambda **kw: _FakeBackend(result={"text": " hello there "}),
+    )
+    assert ab.transcribe_reference("ref.wav") == "hello there"
+
+
+def test_joins_segments_when_no_top_level_text(monkeypatch):
+    """WhisperX results carry no top-level "text" — only segments."""
+    monkeypatch.setattr(
+        ab, "get_active_asr_backend",
+        lambda **kw: _FakeBackend(result={
+            "segments": [{"text": " hello"}, {"text": "world "}],
+        }),
+    )
+    assert ab.transcribe_reference("ref.wav") == "hello world"
+
+
+def test_backend_failure_degrades_to_none(monkeypatch):
+    monkeypatch.setattr(
+        ab, "get_active_asr_backend",
+        lambda **kw: _FakeBackend(exc=RuntimeError("model load failed")),
+    )
+    assert ab.transcribe_reference("ref.wav") is None
+
+
+def test_registry_resolution_failure_degrades_to_none(monkeypatch):
+    def _boom(**kw):
+        raise ValueError("unknown backend")
+    monkeypatch.setattr(ab, "get_active_asr_backend", _boom)
+    assert ab.transcribe_reference("ref.wav") is None
+
+
+def test_pytorch_whisper_defers_to_model_fallback(monkeypatch):
+    """When the registry itself resolves to pytorch-whisper, defer to the
+    model's lazy load instead of constructing a second pipeline."""
+    be = ab.PyTorchWhisperBackend(asr_pipe=object())
+    monkeypatch.setattr(ab, "get_active_asr_backend", lambda **kw: be)
+    assert ab.transcribe_reference("ref.wav") is None
+
+
+def test_empty_result_degrades_to_none(monkeypatch):
+    monkeypatch.setattr(
+        ab, "get_active_asr_backend",
+        lambda **kw: _FakeBackend(result={"text": "   "}),
+    )
+    assert ab.transcribe_reference("ref.wav") is None