fix(generation): classify network/download failures — stop mislabeling every unknown error as OOM (#880) (#893)

A kittentts first-use HuggingFace download died with httpx's "Cannot send
a request, as the client has been closed", and the generation error
classifier's catch-all fallback told the user (CPU-only ~80 MB ONNX engine,
12 GB-VRAM box) they were OUT OF MEMORY and to press Flush — the wrong
remedy for a network failure.

Three-part class fix:

- generation.py: new #880 branch (before the OOM hint) classifies
  httpx/requests transport failures — matched over the whole exception
  chain (type names like ConnectError/ReadTimeout plus stringified
  signatures like "client has been closed") — as a download/network
  problem with a retry/check-connection remedy.
- generation.py (the real class bug): the OOM hint is no longer the
  catch-all. It now requires an actual OOM signature (typed
  OutOfMemoryError/MemoryError anywhere in the chain, or CUDA/MPS/CPU
  allocator wording); genuinely unknown errors surface as unrecognized
  with the underlying detail instead of a false "ran out of memory".
- tts_backend.py: KittenTTS's first-use load retries exactly once with a
  fresh HF Hub client (huggingface_hub.utils.close_session()) on the
  specific closed-client failure — hub ≥1.x shares one global httpx
  client, and a closed one is recoverable, so the download self-heals
  instead of failing the generation.

Fail-before/pass-after tests: classifier (closed-client message, wrapped
httpx type names, unknown error, real OOM signatures incl. typed
OutOfMemoryError, WinError 1455) + the retry helper (recovers once,
walks the chain, no retry on unrelated errors, single-shot).

Fixes #880

Co-authored-by: mergetest <test@local>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

backend/api/routers/generation.py

@@ -142,6 +142,97 @@ def _apply_effect_chain(audio_out, sample_rate, effect_preset, *, skip_mastering
     return normalize_audio(audio_out, target_dBFS=-2.0)
 
 
+def _exception_chain(e):
+    """Yield ``e`` plus every ``__cause__``/``__context__`` beneath it
+    (cycle-safe). Engines and hub libraries routinely wrap the original
+    transport/allocator error, so classification must look at the whole
+    chain, not just the outermost message."""
+    seen = set()
+    stack = [e]
+    while stack:
+        exc = stack.pop()
+        if exc is None or id(exc) in seen:
+            continue
+        seen.add(id(exc))
+        yield exc
+        stack.append(exc.__cause__)
+        stack.append(exc.__context__)
+
+
+# #880: transport-level exception type names from httpx (huggingface_hub ≥1.x
+# downloads over it) and requests/urllib3 (older engine deps). Any of these
+# anywhere in the exception chain means the network — not memory — killed the
+# generation.
+_NETWORK_EXC_NAMES = frozenset({
+    # httpx
+    "ConnectError", "ConnectTimeout", "ReadTimeout", "ReadError",
+    "WriteError", "WriteTimeout", "PoolTimeout", "NetworkError",
+    "TransportError", "RemoteProtocolError", "ProxyError", "CloseError",
+    # requests / urllib3
+    "ConnectionError", "ChunkedEncodingError", "MaxRetryError",
+    "NewConnectionError", "ProtocolError",
+    # stdlib socket-level drops mid-download
+    "ConnectionResetError", "ConnectionAbortedError", "ConnectionRefusedError",
+    # huggingface_hub: failed first-use download with nothing in the disk cache
+    "LocalEntryNotFoundError",
+})
+
+# Same class, but the transport error was stringified into a wrapper message
+# (so the type name is gone). All lowercase; matched against .lower().
+_NETWORK_MSG_SIGNATURES = (
+    "client has been closed",    # httpx closed-client lifecycle error (#880)
+    "cannot send a request",     # httpx: same error, message head
+    "connection error",          # requests / huggingface_hub wording
+    "connection reset",          # ECONNRESET mid-download
+    "read timed out",            # requests/urllib3 timeout wording
+    "max retries exceeded",      # urllib3 retry exhaustion
+    "temporary failure in name resolution",  # DNS down (glibc)
+    "name or service not known",             # DNS down (glibc)
+    "getaddrinfo failed",                    # DNS down (Windows)
+)
+
+
+def _is_network_failure(e) -> bool:
+    """True iff the failure (anywhere in its chain) is an HTTP-client
+    lifecycle / network-transport error — e.g. a first-use model download
+    from the HF Hub dying mid-generation (#880)."""
+    for exc in _exception_chain(e):
+        if type(exc).__name__ in _NETWORK_EXC_NAMES:
+            return True
+        low = str(exc).lower()
+        if any(sig in low for sig in _NETWORK_MSG_SIGNATURES):
+            return True
+    return False
+
+
+# Signatures of an *actual* out-of-memory condition. All lowercase.
+_OOM_MSG_SIGNATURES = (
+    "out of memory",               # CUDA / MPS / generic torch wording
+    "not enough memory",           # torch CPU DefaultCPUAllocator
+    "cannot allocate memory",      # OS-level ENOMEM
+    "std::bad_alloc",              # C++ allocator failure
+    "cublas_status_alloc_failed",  # cuBLAS workspace allocation
+    "cuda_error_out_of_memory",    # raw CUDA driver error name
+    "paging file is too small",    # Windows [WinError 1455] mapping DLLs
+)
+
+
+def _is_oom_failure(e) -> bool:
+    """True iff the failure (anywhere in its chain) actually looks like an
+    out-of-memory condition — the only case where the Flush hint is honest."""
+    for exc in _exception_chain(e):
+        if isinstance(exc, MemoryError):
+            return True
+        # torch.cuda.OutOfMemoryError subclasses RuntimeError; match by name
+        # so this needs no torch import (and covers other frameworks' twins).
+        if type(exc).__name__ == "OutOfMemoryError":
+            return True
+        low = str(exc).lower()
+        if any(sig in low for sig in _OOM_MSG_SIGNATURES):
+            return True
+    return False
+
+
 def _oom_friendly_reraise(e):
     """Best-effort cache flush + the user-facing OOM hint shared by both
     inference paths."""
@@ -233,10 +324,38 @@ def _oom_friendly_reraise(e):
             f"Restart the app and try again; the Flush button won't help here. "
             f"Underlying error: {e}"
         ) from e
+    # #880: an httpx/requests transport failure surfacing from generation —
+    # most commonly a first-use model download from the HF Hub dying with
+    # httpx's "Cannot send a request, as the client has been closed" (the
+    # shared client got closed mid-lifecycle), a connect/read timeout, or a
+    # dropped connection — is NOT out of memory. The model never finished
+    # loading, so Flush is the wrong remedy; retrying is. Matched over the
+    # whole exception chain (type names + stringified signatures) because
+    # engines wrap the original transport error.
+    if _is_network_failure(e):
+        raise RuntimeError(
+            f"A model download or network call failed mid-generation (usually "
+            f"the engine fetching its model files on first use). This is a "
+            f"network problem, not a memory problem — flushing VRAM won't "
+            f"help. Retry the generation; if it keeps failing, check your "
+            f"internet connection and any HF_ENDPOINT/mirror setting. "
+            f"Underlying error: {e}"
+        ) from e
+    # #880 (the class bug): the OOM hint used to be the catch-all fallback,
+    # so ANY unrecognized error told the user to press Flush for memory they
+    # never ran out of. Only claim OOM when something in the chain actually
+    # looks like one; everything else surfaces as what it is — unrecognized —
+    # with the real error front and center.
+    if _is_oom_failure(e):
+        raise RuntimeError(
+            f"TTS engine stopped mid-generation. This usually means it ran out of memory. "
+            f"Try the Flush button to reload the model, then regenerate. Underlying error: {e}"
+        ) from e
     raise RuntimeError(
-        f"TTS engine stopped mid-generation. This usually means it ran out of memory. "
-        f"Try the Flush button to reload the model, then regenerate. Underlying error: {e}"
-    )
+        f"TTS engine stopped mid-generation with an error OmniVoice doesn't "
+        f"recognize. Retry once; if it keeps failing, please report it with "
+        f"the full trace. Underlying error: {e}"
+    ) from e
 
 
 def _run_inference(

backend/services/tts_backend.py

@@ -55,6 +55,59 @@ def _mask_hf_tokens(value):
     return _HF_TOKEN_MASK_RE.sub(_HF_TOKEN_MASK, value)
 
 
+# ── HF Hub closed-client recovery (#880) ────────────────────────────────────
+#
+# huggingface_hub ≥1.x shares ONE global httpx client across every download.
+# If anything closes it mid-lifecycle, every later hub call — e.g. an engine's
+# first-use model download inside the generate path — dies with httpx's
+# "Cannot send a request, as the client has been closed". The client is
+# recoverable: ``close_session()`` drops it and the next hub call builds a
+# fresh one, so the correct handling is a single targeted retry, not a
+# user-facing failure.
+
+
+def _is_closed_client_error(e) -> bool:
+    """True iff ``e`` (or anything in its __cause__/__context__ chain) is
+    httpx's closed-client lifecycle error. Cycle-safe."""
+    seen, stack = set(), [e]
+    while stack:
+        exc = stack.pop()
+        if exc is None or id(exc) in seen:
+            continue
+        seen.add(id(exc))
+        low = str(exc).lower()
+        if "client has been closed" in low or "cannot send a request" in low:
+            return True
+        stack.append(exc.__cause__)
+        stack.append(exc.__context__)
+    return False
+
+
+def _retry_once_with_fresh_hf_client(loader, what: str):
+    """Run ``loader()`` — a model constructor that may download from the HF
+    Hub on first use. On the specific closed-client failure above, reset the
+    hub's shared client and retry exactly ONCE. Any other failure (and a
+    repeat closed-client failure) propagates untouched, where the generation
+    error classifier labels it as a network problem (#880)."""
+    try:
+        return loader()
+    except Exception as e:
+        if not _is_closed_client_error(e):
+            raise
+        logger.warning(
+            "%s: HF Hub httpx client was closed mid-download (%s); "
+            "retrying once with a fresh client.", what, e,
+        )
+        try:
+            from huggingface_hub.utils import close_session
+            close_session()
+        except Exception:  # pragma: no cover — hub too old / API renamed
+            logger.warning(
+                "%s: couldn't reset the HF Hub client; retrying anyway.", what,
+            )
+        return loader()
+
+
 # ── Protocol ────────────────────────────────────────────────────────────────
 
 
@@ -587,7 +640,13 @@ class KittenTTSBackend(TTSBackend):
             "OMNIVOICE_KITTENTTS_MODEL", "KittenML/kitten-tts-mini-0.8"
         )
         logger.info("Loading KittenTTS from %s", checkpoint)
-        self._model = KittenTTS(checkpoint)
+        # #880: the first-use load downloads ~80 MB from the HF Hub inside the
+        # generate path; if the hub's shared httpx client was closed
+        # mid-lifecycle, retry once with a fresh client instead of failing
+        # the whole generation.
+        self._model = _retry_once_with_fresh_hf_client(
+            lambda: KittenTTS(checkpoint), what="KittenTTS"
+        )
 
     def generate(self, text: str, **kw) -> torch.Tensor:
         import numpy as np

tests/test_engines.py

@@ -131,3 +131,68 @@ def test_llm_auto_selects_openai_compat_when_configured(monkeypatch):
     except ImportError:
         pytest.skip("openai package not available in this environment")
     assert llm_backend.active_backend_id() == "openai-compat"
+
+
+# ── HF Hub closed-client recovery (#880) ────────────────────────────────────
+#
+# huggingface_hub ≥1.x shares one global httpx client; if it gets closed
+# mid-lifecycle, an engine's first-use model download inside the generate
+# path dies with "Cannot send a request, as the client has been closed".
+# The load must retry exactly once with a fresh client — and must NOT retry
+# unrelated failures.
+
+
+def test_hf_retry_recovers_from_closed_client_once():
+    calls = []
+
+    def loader():
+        calls.append(1)
+        if len(calls) == 1:
+            raise RuntimeError("Cannot send a request, as the client has been closed.")
+        return "model"
+
+    assert tts_backend._retry_once_with_fresh_hf_client(loader, what="test") == "model"
+    assert len(calls) == 2
+
+
+def test_hf_retry_matches_wrapped_closed_client_error():
+    # An engine can wrap the httpx error — detection walks the chain.
+    calls = []
+
+    def loader():
+        calls.append(1)
+        if len(calls) == 1:
+            try:
+                raise RuntimeError("Cannot send a request, as the client has been closed.")
+            except RuntimeError as inner:
+                raise RuntimeError("KittenTTS init failed") from inner
+        return "model"
+
+    assert tts_backend._retry_once_with_fresh_hf_client(loader, what="test") == "model"
+    assert len(calls) == 2
+
+
+def test_hf_retry_does_not_retry_unrelated_errors():
+    calls = []
+
+    def loader():
+        calls.append(1)
+        raise ValueError("bad checkpoint id")
+
+    with pytest.raises(ValueError):
+        tts_backend._retry_once_with_fresh_hf_client(loader, what="test")
+    assert len(calls) == 1
+
+
+def test_hf_retry_is_single_shot():
+    # A second closed-client failure propagates (the generation classifier
+    # then labels it a network problem) — no infinite retry loop.
+    calls = []
+
+    def loader():
+        calls.append(1)
+        raise RuntimeError("Cannot send a request, as the client has been closed.")
+
+    with pytest.raises(RuntimeError):
+        tts_backend._retry_once_with_fresh_hf_client(loader, what="test")
+    assert len(calls) == 2

tests/test_generation_audio_guard.py

@@ -55,6 +55,79 @@ def test_generic_failure_still_uses_oom_hint():
     assert "ran out of memory" in str(ei.value)
 
 
+def test_httpx_closed_client_is_a_download_failure_not_oom():
+    # #880: kittentts's first-use HF download died with httpx's closed-client
+    # lifecycle error, and the OOM catch-all told a user running a CPU-only
+    # ~80 MB ONNX engine on a 12 GB-VRAM box to press Flush. It's a network
+    # failure — say so, and don't send them to the Flush button.
+    err = RuntimeError("Cannot send a request, as the client has been closed.")
+    with pytest.raises(RuntimeError) as ei:
+        _oom_friendly_reraise(err)
+    msg = str(ei.value)
+    assert "network" in msg
+    assert "download" in msg
+    assert "Retry" in msg
+    assert "client has been closed" in msg  # underlying detail preserved
+    assert "ran out of memory" not in msg
+    assert "Try the Flush button" not in msg
+
+
+@pytest.mark.parametrize("exc_name", ["ConnectError", "ReadTimeout"])
+def test_httpx_transport_error_in_chain_is_a_download_failure(exc_name):
+    # #880: engines wrap the original httpx error, so classification must
+    # look at exception TYPE NAMES anywhere in the chain, not just the
+    # outermost message (which here carries no network signature at all).
+    fake_httpx_exc = type(exc_name, (Exception,), {})
+    try:
+        try:
+            raise fake_httpx_exc("")
+        except Exception as inner:
+            raise RuntimeError("model load failed") from inner
+    except RuntimeError as wrapped:
+        err = wrapped
+    with pytest.raises(RuntimeError) as ei:
+        _oom_friendly_reraise(err)
+    msg = str(ei.value)
+    assert "network" in msg
+    assert "ran out of memory" not in msg
+    assert "Try the Flush button" not in msg
+
+
+def test_unknown_error_is_not_labelled_oom():
+    # #880 (the class bug): the OOM hint was the catch-all fallback, so ANY
+    # unrecognized error claimed "ran out of memory" + Flush. A genuinely
+    # unknown error must surface as unknown, detail intact.
+    with pytest.raises(RuntimeError) as ei:
+        _oom_friendly_reraise(RuntimeError("segfault in frobnicator: code 7"))
+    msg = str(ei.value)
+    assert "segfault in frobnicator: code 7" in msg
+    assert "ran out of memory" not in msg
+    assert "Try the Flush button" not in msg
+
+
+@pytest.mark.parametrize("reason", [
+    "CUDA out of memory. Tried to allocate 20.00 MiB",
+    "MPS backend out of memory (MPS allocated: 8.00 GB)",
+    "DefaultCPUAllocator: not enough memory: you tried to allocate 1073741824 bytes",
+    "[enforce fail at alloc_cpu.cpp] posix_memalign. Cannot allocate memory",
+    "[WinError 1455] The paging file is too small for this operation to complete",
+])
+def test_real_oom_signatures_still_classify_as_oom(reason):
+    with pytest.raises(RuntimeError) as ei:
+        _oom_friendly_reraise(RuntimeError(reason))
+    assert "ran out of memory" in str(ei.value)
+    assert "Try the Flush button" in str(ei.value)
+
+
+def test_typed_oom_without_oom_message_still_classifies_as_oom():
+    # torch.cuda.OutOfMemoryError can carry an opaque allocator message; the
+    # tightened OOM branch must also match the exception type name.
+    fake_torch_oom = type("OutOfMemoryError", (RuntimeError,), {})
+    with pytest.raises(RuntimeError) as ei:
+        _oom_friendly_reraise(fake_torch_oom("CUBLAS workspace reservation failed"))
+    assert "ran out of memory" in str(ei.value)
+
+
 def test_unsupported_instruct_is_a_validation_error_not_oom():
     # #664: free-form prose in the instruct field must surface as a 400-mapped
     # ValueError with the instruct guidance — NOT a 500 "ran out of memory".