fix(engine): validate ASR timeouts, reserve outer guard grace, and enhance diagnostics (#2103)

backend/engines/confucius4/__init__.py

@@ -103,6 +103,7 @@ class Confucius4Backend(SubprocessBackend):
 
     @property
     def recv_timeout_s(self) -> float:
+        """Receive timeout in seconds for the Confucius4 sidecar process (#2103)."""
         # Confucius4 is an LLM-based TTS (~17x realtime on CPU); synthesis legitimately
         # outruns the 60s class default. OMNIVOICE_CONFUCIUS4_RECV_TIMEOUT_S tunes it (#2103).
         try:

backend/engines/dots_tts/__init__.py

@@ -125,6 +125,7 @@ class DotsTTSBackend(SubprocessBackend):
 
     @property
     def recv_timeout_s(self) -> float:
+        """Receive timeout in seconds for the dots.tts sidecar process (#2103)."""
         # dots.tts is a 2B autoregressive model; synthesis on CPU legitimately
         # outruns the 60s class default. OMNIVOICE_DOTS_TTS_RECV_TIMEOUT_S tunes it (#2103).
         try:

backend/engines/moss_tts_v15/__init__.py

@@ -131,6 +131,7 @@ class MossTTSV15Backend(SubprocessBackend):
 
     @property
     def recv_timeout_s(self) -> float:
+        """Receive timeout in seconds for the MOSS-TTS-v1.5 sidecar process (#2103)."""
         # MOSS-TTS-v1.5 is an 8B model; synthesis legitimately outruns the
         # 60s class default. OMNIVOICE_MOSS_TTS_V15_RECV_TIMEOUT_S tunes it (#2103).
         try:

backend/engines/supertonic3/backend.py

@@ -102,6 +102,7 @@ class Supertonic3Backend(SubprocessBackend):
 
     @property
     def recv_timeout_s(self) -> float:
+        """Receive timeout in seconds for the Supertonic-3 sidecar process (#2103)."""
         # Supertonic-3 runs ONNX on CPU; cold load downloads ~400MB and long
         # synthesis benefits from more headroom than 60s. OMNIVOICE_SUPERTONIC3_RECV_TIMEOUT_S (#2103).
         try:

backend/services/model_manager.py

@@ -1,11 +1,12 @@
-import os
-import re
-import sys
-import time
 import asyncio
 import logging
+import math
+import os
 import queue
+import re
+import sys
 import threading
+import time
 from concurrent.futures import Executor, Future, ThreadPoolExecutor
 
 from utils.containment import contain_system_exit
@@ -613,10 +614,14 @@ def generate_timeout_s(
 
     # If the engine specifies its own sidecar receive timeout (e.g. SubprocessBackend
     # engines like Confucius, Dots, Moss, Supertonic), the outer execution budget
-    # must not cut the sidecar off early (#2103).
+    # must not cut the sidecar off early (#2103). A bounded 5s grace period ensures
+    # the sidecar's watchdog timer fires and surfaces its actionable timeout error
+    # before the outer pool cancellation cuts it off.
     if engine is not None and hasattr(engine, "recv_timeout_s"):
         try:
-            base = max(base, float(engine.recv_timeout_s))
+            sidecar_timeout = float(engine.recv_timeout_s)
+            if math.isfinite(sidecar_timeout) and sidecar_timeout > 0:
+                base = max(base, sidecar_timeout + 5.0)
         except (TypeError, ValueError):
             pass
 

backend/services/subprocess_asr.py

@@ -38,7 +38,7 @@ logger = logging.getLogger("omnivoice.asr.subprocess")
 # A model load + transcription can take a while on CPU for a long clip; give
 # the transcribe round-trip more headroom than the TTS default. Configurable via
 # OMNIVOICE_ASR_RECV_TIMEOUT_S (#2103).
-ASR_RECV_TIMEOUT_S = float(os.environ.get("OMNIVOICE_ASR_RECV_TIMEOUT_S", "600.0"))
+ASR_RECV_TIMEOUT_S = 600.0
 
 
 class SubprocessASRBackend(SubprocessBackend):
@@ -84,7 +84,13 @@ class SubprocessASRBackend(SubprocessBackend):
     @property
     def recv_timeout_s(self) -> float:
         """Wall-clock timeout in seconds waiting for an ASR sidecar response (#2103)."""
-        return float(os.environ.get("OMNIVOICE_ASR_RECV_TIMEOUT_S", ASR_RECV_TIMEOUT_S))
+        try:
+            v = float(os.environ.get("OMNIVOICE_ASR_RECV_TIMEOUT_S", str(ASR_RECV_TIMEOUT_S)))
+        except (ValueError, TypeError):
+            return ASR_RECV_TIMEOUT_S
+        if not math.isfinite(v):
+            return ASR_RECV_TIMEOUT_S
+        return max(30.0, v)
 
     def transcribe(self, audio_path: str, *, word_timestamps: bool = True) -> dict:
         """Transcribe ``audio_path`` in the sidecar. Returns the engine's
@@ -131,11 +137,12 @@ class SubprocessASRBackend(SubprocessBackend):
                     "decode_options": asr_decode_defaults(),
                 })
                 reply = self._recv_with_timeout(timeout_s)
+                timed_out = self._last_recv_timed_out
             if not reply:
                 # EOF can arrive before Windows updates poll(); retire the
                 # stale handle so an immediate retry respawns the sidecar.
                 self.shutdown()
-                if self._last_recv_timed_out:
+                if timed_out:
                     raise RuntimeError(
                         f"{self.id} ASR sidecar exceeded receive timeout "
                         f"({timeout_s:g}s); killed mid-transcription "

backend/services/subprocess_backend.py

@@ -791,15 +791,18 @@ class SubprocessBackend(TTSBackend):
                         pass  # the heartbeat is best-effort; never fail a synth over it
                     try:
                         reply = self._recv_with_timeout(self.recv_timeout_s)
+                        timed_out = self._last_recv_timed_out
                     except (RuntimeError, OSError):
                         self._reap_unusable_process(proc)
                         raise
                 if not reply:
                     self._reap_unusable_process(proc)
-                    if self._last_recv_timed_out:
+                    if timed_out:
+                        env_key = f"OMNIVOICE_{self.id.upper().replace('-', '_')}_RECV_TIMEOUT_S"
                         raise RuntimeError(
                             f"{self.id} sidecar exceeded receive timeout "
-                            f"({self.recv_timeout_s:g}s); killed mid-generate"
+                            f"({self.recv_timeout_s:g}s); killed mid-generate "
+                            f"— retry or raise {env_key}."
                         )
                     raise RuntimeError(f"{self.id} sidecar closed pipe mid-generate")
             if reply.get("op") == "error":

backend/tests/test_omnivoice_subprocess.py

@@ -454,6 +454,7 @@ def test_subprocess_engine_timeouts_raised():
 
 
 def test_subprocess_engine_timeout_env_overrides(monkeypatch):
+    """Subprocess TTS engines honor their engine-specific receive timeout env overrides (#2103)."""
     from engines.confucius4 import Confucius4Backend
     from engines.dots_tts import DotsTTSBackend
     from engines.moss_tts_v15 import MossTTSV15Backend
@@ -471,6 +472,8 @@ def test_subprocess_engine_timeout_env_overrides(monkeypatch):
 
 
 def test_subprocess_asr_recv_timeout_env_override(monkeypatch):
+    """SubprocessASRBackend.recv_timeout_s honors OMNIVOICE_ASR_RECV_TIMEOUT_S
+    and safely rejects non-finite, malformed, zero, or negative inputs (#2103)."""
     from pathlib import Path
     from services.subprocess_asr import SubprocessASRBackend
 
@@ -487,13 +490,29 @@ def test_subprocess_asr_recv_timeout_env_override(monkeypatch):
         def sidecar_script(cls): return Path("fake")
 
     b = _FakeASR()
-    assert b.recv_timeout_s >= 300.0
+    assert b.recv_timeout_s == 600.0
 
+    # Valid override
     monkeypatch.setenv("OMNIVOICE_ASR_RECV_TIMEOUT_S", "750")
     assert b.recv_timeout_s == 750.0
 
+    # Malformed value falls back to default
+    monkeypatch.setenv("OMNIVOICE_ASR_RECV_TIMEOUT_S", "not-a-number")
+    assert b.recv_timeout_s == 600.0
+
+    # Non-finite values fall back to default
+    for invalid in ("nan", "inf", "-inf"):
+        monkeypatch.setenv("OMNIVOICE_ASR_RECV_TIMEOUT_S", invalid)
+        assert b.recv_timeout_s == 600.0
+
+    # Zero or negative values clamped to 30.0 minimum
+    for low in ("0", "-10", "15"):
+        monkeypatch.setenv("OMNIVOICE_ASR_RECV_TIMEOUT_S", low)
+        assert b.recv_timeout_s == 30.0
+
 
 def test_subprocess_asr_timeout_error_message(monkeypatch):
+    """SubprocessASRBackend.transcribe() raises actionable timeout guidance when watchdog fires (#2103)."""
     from pathlib import Path
     from services.subprocess_asr import SubprocessASRBackend
 
@@ -535,14 +554,15 @@ def test_subprocess_asr_timeout_error_message(monkeypatch):
 
 
 def test_generate_timeout_s_coordinates_with_engine_recv_timeout():
-    """Outer generation timeout must not be shorter than the engine sidecar timeout (#2103)."""
+    """Outer generation timeout must coordinate with engine sidecar timeout with bounded grace (#2103)."""
     from services.model_manager import generate_timeout_s
 
     class _SlowEngine:
         recv_timeout_s = 900.0
 
+    # With 900s sidecar timeout, outer budget must include at least 5s grace (>= 905s).
     budget = generate_timeout_s("short text", engine=_SlowEngine())
-    assert budget >= 900.0
+    assert budget >= 905.0
 
     class _FastEngine:
         recv_timeout_s = 60.0