fix(tts): torch.compile failures fall back to eager — generation never fails on unsupported GPUs (#278) (#327)

* fix(tts): torch.compile failures fall back to eager — generation never fails on unsupported GPUs (#278)

On GPU architectures the bundled Triton doesn't support (e.g. Blackwell
sm_120 / RTX 5060), the compiled model dies mid-generation inside the
Dynamo/Inductor/Triton/cudagraph stack — previously surfaced as a fake
'ran out of memory' error and a dead Archetype preview. Now:

- up-front arch gate: skip compile when the GPU's compute capability is
  not in this torch build's arch list (OMNIVOICE_FORCE_TORCH_COMPILE=1
  overrides for PTX forward-compat setups)
- runtime fallback: model.generate is wrapped once; a compile-stack
  failure (classified by exception chain: module, message, traceback
  paths — the cudagraph case is a bare AssertionError) logs a warning,
  restores the eager module, disables compile for the session, resets
  dynamo state, and retries eagerly. Non-compile errors propagate
  unchanged.
- the /generate OOM handler no longer mislabels compile crashes as OOM
  and points users at the actual remedy.

Fixes #278

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Potential fix for pull request finding 'CodeQL / Empty except'

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* Potential fix for pull request finding 'CodeQL / Empty except'

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* Update backend/api/routers/generation.py

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
Co-authored-by: mergetest <test@local>

backend/api/routers/generation.py

@@ -106,6 +106,16 @@ def _oom_friendly_reraise(e):
         torch.mps.empty_cache()
     elif torch.cuda.is_available():
         torch.cuda.empty_cache()
+    # #278: don't mislabel a torch.compile/Triton/Inductor crash as an
+    # out-of-memory condition. (model_manager's generate wrapper already
+    # retries these eagerly; this only triggers if that retry also died.)
+    from services.model_manager import _is_compile_runtime_failure
+    if _is_compile_runtime_failure(e):
+        raise RuntimeError(
+            f"TTS engine hit a torch.compile/Triton error (not out of memory). "
+            f"Disable torch.compile in Settings → Performance, use the Flush "
+            f"button to reload the model, then regenerate. Underlying error: {e}"
+        ) from e
     raise RuntimeError(
         f"TTS engine stopped mid-generation. This usually means it ran out of memory. "
         f"Try the Flush button to reload the model, then regenerate. Underlying error: {e}"

backend/services/engine_env.py

@@ -24,6 +24,78 @@ logger = logging.getLogger("omnivoice.engine_env")
 
 _TORCH_COMPILE_KEY = "perf.torch_compile_disabled"
 
+# #278: explicit opt-in override — set to 1/true to attempt torch.compile even
+# when the GPU's compute capability is not in this PyTorch build's arch list
+# (e.g. a brand-new architecture running through PTX forward-compat).
+_FORCE_COMPILE_ENV = "OMNIVOICE_FORCE_TORCH_COMPILE"
+
+# #278: set (with a reason) the first time torch.compile — or *running* the
+# compiled model — fails at runtime in this process. Once set, every later
+# load in the same session goes straight to eager instead of re-tripping the
+# same Dynamo/Inductor/Triton failure.
+_compile_runtime_failure: Optional[str] = None
+
+
+def mark_compile_runtime_failure(reason: str) -> None:
+    """Record that torch.compile (or compiled execution) failed at runtime.
+
+    Called by ``services.model_manager`` when compilation raises, or when a
+    generation through the compiled model dies inside the Dynamo / Inductor /
+    Triton stack (#278). Disables compile for the rest of the process — eager
+    mode from here on; the next app restart probes again.
+    """
+    global _compile_runtime_failure
+    _compile_runtime_failure = reason or "unknown torch.compile runtime failure"
+    logger.warning(
+        "torch.compile disabled for this session after a runtime failure: %s",
+        _compile_runtime_failure,
+    )
+
+
+def _force_compile_requested() -> bool:
+    value = os.environ.get(_FORCE_COMPILE_ENV, "")
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _cuda_arch_supported_for_compile() -> "tuple[bool, str]":
+    """Check the GPU's compute capability against this torch build's arch list.
+
+    New GPU architectures (e.g. Blackwell sm_120, issue #278) routinely break
+    torch.compile/Triton before upstream support lands: the eager model runs
+    via PTX forward-compat, but Inductor/Triton kernel compilation targets the
+    new arch directly and fails mid-generation. If the device's ``sm_XY`` tag
+    is absent from ``torch.cuda.get_arch_list()`` we treat compile as
+    unsupported and use eager.
+
+    Returns ``(supported, reason)``. Fails open — any probe error returns
+    ``(True, "")`` so a weird torch build never silently loses the
+    optimization (the runtime fallback in model_manager still protects
+    generation).
+    """
+    try:
+        import torch
+
+        if not torch.cuda.is_available():
+            return True, ""
+        major, minor = torch.cuda.get_device_capability(0)
+        arch_list = list(getattr(torch.cuda, "get_arch_list", lambda: [])() or [])
+        if not arch_list:
+            return True, ""
+        sm_tag = f"sm_{major}{minor}"
+        if sm_tag in arch_list or f"compute_{major}{minor}" in arch_list:
+            return True, ""
+        try:
+            device_name = torch.cuda.get_device_name(0)
+        except Exception:
+            device_name = "GPU"
+        return False, (
+            f"{device_name} (compute capability {major}.{minor} / {sm_tag}) is not "
+            f"in this PyTorch build's supported arch list ({', '.join(arch_list)})"
+        )
+    except Exception:
+        logger.debug("CUDA arch probe for torch.compile failed; assuming supported", exc_info=True)
+        return True, ""
+
 
 def should_torch_compile(device: str) -> bool:
     """Decide whether to apply ``torch.compile`` to an in-process model.
@@ -34,9 +106,14 @@ def should_torch_compile(device: str) -> bool:
       - device == "cuda" (compile only helps the CUDA path here),
       - Triton importable (``find_spec`` — the cross-platform gate that closes
         #65; no Windows wheel ⇒ skip ⇒ eager),
-      - the user has NOT set the ``perf.torch_compile_disabled`` escape hatch.
+      - the user has NOT set the ``perf.torch_compile_disabled`` escape hatch,
+      - compile has NOT already failed at runtime in this process (#278),
+      - the GPU's compute capability is in this torch build's arch list (#278)
+        — overridable via ``OMNIVOICE_FORCE_TORCH_COMPILE=1``.
 
     Returns False (→ eager mode) on any of those, logging the reason at INFO.
+    torch.compile is an optimization, never a requirement — generation must
+    always work without it.
     """
     if device != "cuda":
         return False
@@ -51,6 +128,25 @@ def should_torch_compile(device: str) -> bool:
             return False
     except Exception:
         logger.exception("should_torch_compile: settings read failed; proceeding")
+    if _compile_runtime_failure is not None:
+        logger.info(
+            "torch.compile skipped: failed earlier this session (%s) — using eager mode.",
+            _compile_runtime_failure,
+        )
+        return False
+    supported, reason = _cuda_arch_supported_for_compile()
+    if not supported:
+        if _force_compile_requested():
+            logger.warning(
+                "torch.compile forced via %s=1 despite: %s", _FORCE_COMPILE_ENV, reason,
+            )
+            return True
+        logger.info(
+            "torch.compile skipped: %s — using eager mode. "
+            "(Set %s=1 to attempt compile anyway.)",
+            reason, _FORCE_COMPILE_ENV,
+        )
+        return False
     return True
 
 

backend/services/model_manager.py

@@ -231,6 +231,113 @@ def get_best_device():
 
     return "cpu"
 
+_COMPILE_ERR_MODULE_PREFIXES = ("torch._dynamo", "torch._inductor", "torch.fx", "triton")
+_COMPILE_ERR_TB_MARKERS = ("/_dynamo/", "/_inductor/", "/triton/", "torch/fx/")
+_COMPILE_ERR_MSG_MARKERS = (
+    "dynamo", "inductor", "triton", "cudagraph",
+    "symbolically trace", "torch.compile", "fx graph",
+)
+
+
+def _is_compile_runtime_failure(exc: BaseException) -> bool:
+    """True when an exception originates in the torch.compile stack (Dynamo /
+    Inductor / Triton / FX / CUDA-graph trees) rather than in the model itself.
+
+    #278: on GPU architectures Triton doesn't support yet (e.g. Blackwell
+    sm_120), the compiled model dies mid-generation with errors like
+    "Detected that you are using FX to symbolically trace a dynamo-optimized
+    function" or an AssertionError out of torch/_inductor/cudagraph_trees.py.
+    Walks the exception chain and checks (a) the exception type's module,
+    (b) the message, (c) the traceback file paths — the cudagraph case is a
+    bare AssertionError, so the traceback check is load-bearing.
+    """
+    import traceback as _tb
+
+    seen: set[int] = set()
+    cur: BaseException | None = exc
+    while cur is not None and id(cur) not in seen:
+        seen.add(id(cur))
+        mod = type(cur).__module__ or ""
+        if mod.startswith(_COMPILE_ERR_MODULE_PREFIXES):
+            return True
+        msg = str(cur).lower()
+        if any(marker in msg for marker in _COMPILE_ERR_MSG_MARKERS):
+            return True
+        try:
+            for frame in _tb.extract_tb(cur.__traceback__):
+                filename = (frame.filename or "").replace("\\", "/")
+                if any(marker in filename for marker in _COMPILE_ERR_TB_MARKERS):
+                    return True
+        except Exception as traceback_scan_error:
+            logging.debug(
+                "Skipping traceback marker scan while classifying compile runtime failure: %s",
+                traceback_scan_error,
+            )
+        # Follow the chain, honoring `raise ... from None` (the eager-retry
+        # path suppresses the original compile error so a genuine eager
+        # failure isn't misclassified as a compile failure).
+        if cur.__cause__ is not None:
+            cur = cur.__cause__
+        elif not cur.__suppress_context__:
+            cur = cur.__context__
+        else:
+            cur = None
+    return False
+
+
+def _install_compile_fallback(_model) -> None:
+    """Wrap ``model.generate`` so a torch.compile failure at inference time
+    falls back to the eager (uncompiled) model instead of failing the
+    generation (#278).
+
+    All TTS paths (generate, archetype previews, dub, stream, batch) funnel
+    through ``model.generate``, so this is the single choke point. On a
+    compile-stack failure we: log a clear warning, restore the eager module
+    (``OptimizedModule._orig_mod``), disable compile for the rest of the
+    session via ``engine_env.mark_compile_runtime_failure``, reset dynamo
+    state, and retry the call once eagerly. Non-compile errors (real OOM,
+    validation, …) propagate unchanged — fully backward compatible for users
+    whose torch.compile works.
+    """
+    orig_generate = _model.generate
+
+    def _generate_with_compile_fallback(*args, **kwargs):
+        try:
+            return orig_generate(*args, **kwargs)
+        except Exception as exc:
+            compiled = getattr(_model, "llm", None)
+            eager = getattr(compiled, "_orig_mod", None)
+            if eager is None or not _is_compile_runtime_failure(exc):
+                raise
+            logger.warning(
+                "torch.compile runtime failure during generation (%s: %s) — "
+                "falling back to the eager model and disabling torch.compile "
+                "for this session. Generation is being retried without it.",
+                type(exc).__name__, exc,
+            )
+            from services import engine_env
+            engine_env.mark_compile_runtime_failure(f"{type(exc).__name__}: {exc}")
+            _model.llm = eager
+            try:
+                torch = _lazy_torch()
+                torch._dynamo.reset()
+            except Exception as reset_exc:
+                logger.debug(
+                    "Non-fatal: failed to reset torch._dynamo state after compile failure (%s: %s). "
+                    "Continuing with eager fallback.",
+                    type(reset_exc).__name__,
+                    reset_exc,
+                )
+            try:
+                return orig_generate(*args, **kwargs)
+            except Exception as eager_exc:
+                # `from None` so a genuine eager failure (e.g. a real OOM)
+                # isn't chained to — and misclassified as — the compile error.
+                raise eager_exc from None
+
+    _model.generate = _generate_with_compile_fallback
+
+
 def _set_loading(sub_stage: str, detail: str = "", error: str | None = None, progress: float | None = None):
     """Update the loading detail dict atomically."""
     _loading_detail["sub_stage"] = sub_stage
@@ -303,8 +410,25 @@ def _load_model_sync():
 
             if should_torch_compile(device):
                 _set_loading("compiling", "Compiling model (torch.compile)…")
-                _model.llm = torch.compile(_model.llm, mode="reduce-overhead")
-                logger.info("torch.compile applied.")
+                try:
+                    _model.llm = torch.compile(_model.llm, mode="reduce-overhead")
+                except Exception as compile_exc:
+                    # #278: compile is an optimization, never a point of
+                    # failure — keep the eager model and remember the failure
+                    # so later loads this session skip compile up front.
+                    from services.engine_env import mark_compile_runtime_failure
+                    mark_compile_runtime_failure(f"{type(compile_exc).__name__}: {compile_exc}")
+                    logger.warning(
+                        "torch.compile failed (%s) — continuing with the eager model.",
+                        compile_exc,
+                    )
+                else:
+                    # Compilation is lazy: Dynamo/Inductor/Triton can still
+                    # blow up on the first *forward* (e.g. unsupported new GPU
+                    # archs, #278). Wrap generate so that falls back to eager
+                    # instead of failing the generation.
+                    _install_compile_fallback(_model)
+                    logger.info("torch.compile applied.")
         except Exception as e:
             logger.info("torch.compile skipped: %s", e)
 

tests/test_compile_fallback.py

@@ -0,0 +1,201 @@
+"""#278 — torch.compile failures must fall back to eager, never fail generation.
+
+On GPU architectures Triton/Inductor doesn't support yet (e.g. RTX 50-series
+Blackwell, sm_120), `torch.compile` succeeds at load time but the *first
+generation* dies inside the Dynamo/FX/Inductor stack ("Detected that you are
+using FX to symbolically trace a dynamo-optimized function", AssertionError in
+torch/_inductor/cudagraph_trees.py) and was mislabeled as an OOM.
+
+These tests pin the contract: compile is an optimization, never a point of
+failure — a compile-stack error during generation triggers a one-shot eager
+retry, disables compile for the session, and genuine model errors (real OOM,
+validation) still propagate unchanged.
+"""
+from __future__ import annotations
+
+import importlib
+
+import pytest
+
+
+@pytest.fixture
+def engine_env(monkeypatch):
+    """The *live* services.engine_env, with the session flag isolated.
+
+    Resolved at test time (not module import time): other tests (e.g.
+    tests/backend/test_perf_settings.py) delete and re-import the whole
+    ``services`` package mid-session, and the production fallback wrapper's
+    runtime ``from services import engine_env`` always resolves the fresh
+    module — a module-level import here would assert against a stale one.
+    """
+    mod = importlib.import_module("services.engine_env")
+    monkeypatch.setattr(mod, "_compile_runtime_failure", None)
+    return mod
+
+
+@pytest.fixture
+def model_manager(engine_env):
+    """The *live* services.model_manager (same rationale as engine_env)."""
+    return importlib.import_module("services.model_manager")
+
+
+# ── helpers ─────────────────────────────────────────────────────────────────
+
+
+def _dynamo_exc() -> Exception:
+    """An exception whose type lives in the torch._dynamo namespace."""
+
+    class TorchRuntimeError(RuntimeError):
+        pass
+
+    TorchRuntimeError.__module__ = "torch._dynamo.exc"
+    return TorchRuntimeError("backend='inductor' raised")
+
+
+def _fx_trace_exc() -> Exception:
+    """The exact failure mode from issue #278's logs (message-based)."""
+    return RuntimeError(
+        "Detected that you are using FX to symbolically trace "
+        "a dynamo-optimized function. This is not supported at the moment."
+    )
+
+
+def _cudagraph_assertion() -> BaseException:
+    """A bare AssertionError raised from torch/_inductor/cudagraph_trees.py.
+
+    Compiles a snippet under that filename so the traceback frame carries the
+    inductor path — exactly what the real cudagraph_trees failure looks like
+    (no message, builtin type; only the traceback identifies it).
+    """
+    src = "def boom():\n    raise AssertionError\n"
+    ns: dict = {}
+    exec(compile(src, "/x/site-packages/torch/_inductor/cudagraph_trees.py", "exec"), ns)
+    try:
+        ns["boom"]()
+    except AssertionError as e:
+        return e
+    raise RuntimeError("unreachable")
+
+
+class _FakeCompiledLLM:
+    """Stands in for torch.compile's OptimizedModule (has ``_orig_mod``)."""
+
+    def __init__(self, orig):
+        self._orig_mod = orig
+
+
+class _FakeModel:
+    """Model whose ``generate`` raises the given exceptions, in order, then
+    succeeds."""
+
+    def __init__(self, failures):
+        self.eager_llm = object()
+        self.llm = _FakeCompiledLLM(self.eager_llm)
+        self.calls = 0
+        self._failures = list(failures)
+
+    def generate(self, *args, **kwargs):
+        self.calls += 1
+        if self._failures:
+            raise self._failures.pop(0)
+        return ["audio-tensor"]
+
+
+# ── _is_compile_runtime_failure classification ──────────────────────────────
+
+
+def test_detects_dynamo_module_exception(model_manager):
+    assert model_manager._is_compile_runtime_failure(_dynamo_exc()) is True
+
+
+def test_detects_fx_symbolic_trace_message(model_manager):
+    assert model_manager._is_compile_runtime_failure(_fx_trace_exc()) is True
+
+
+def test_detects_inductor_traceback_frames(model_manager):
+    # Bare AssertionError — only the traceback file path identifies it.
+    assert model_manager._is_compile_runtime_failure(_cudagraph_assertion()) is True
+
+
+def test_detects_compile_error_wrapped_in_chain(model_manager):
+    try:
+        try:
+            raise _fx_trace_exc()
+        except RuntimeError as inner:
+            raise RuntimeError("TTS engine stopped mid-generation") from inner
+    except RuntimeError as outer:
+        assert model_manager._is_compile_runtime_failure(outer) is True
+
+
+def test_real_oom_is_not_classified_as_compile_failure(model_manager):
+    exc = RuntimeError("CUDA out of memory. Tried to allocate 2.50 GiB")
+    assert model_manager._is_compile_runtime_failure(exc) is False
+
+
+def test_validation_error_is_not_classified(model_manager):
+    assert model_manager._is_compile_runtime_failure(ValueError("bad preset")) is False
+
+
+# ── generate() fallback wrapper ─────────────────────────────────────────────
+
+
+def test_compile_failure_falls_back_to_eager_and_succeeds(engine_env, model_manager):
+    model = _FakeModel(failures=[_fx_trace_exc()])
+    model_manager._install_compile_fallback(model)
+
+    result = model.generate(text="hello")
+
+    assert result == ["audio-tensor"]
+    assert model.calls == 2  # compiled attempt + eager retry
+    assert model.llm is model.eager_llm  # compiled module swapped out
+    # Compile is disabled for the rest of the session...
+    assert engine_env._compile_runtime_failure is not None
+    # ...so the next load goes straight to eager.
+    assert engine_env.should_torch_compile("cuda") is False
+
+
+def test_cudagraph_assertion_falls_back_to_eager(engine_env, model_manager):
+    model = _FakeModel(failures=[_cudagraph_assertion()])
+    model_manager._install_compile_fallback(model)
+
+    assert model.generate(text="hello") == ["audio-tensor"]
+    assert model.calls == 2
+    assert model.llm is model.eager_llm
+
+
+def test_non_compile_error_propagates_unchanged(engine_env, model_manager):
+    model = _FakeModel(failures=[ValueError("bad input")])
+    model_manager._install_compile_fallback(model)
+
+    with pytest.raises(ValueError, match="bad input"):
+        model.generate(text="hello")
+
+    assert model.calls == 1  # no retry
+    assert isinstance(model.llm, _FakeCompiledLLM)  # compiled module kept
+    assert engine_env._compile_runtime_failure is None  # compile stays enabled
+
+
+def test_no_fallback_when_already_eager(engine_env, model_manager):
+    """If llm has no ``_orig_mod`` (already eager) the error propagates."""
+    model = _FakeModel(failures=[_fx_trace_exc()])
+    model.llm = object()  # no _orig_mod
+    model_manager._install_compile_fallback(model)
+
+    with pytest.raises(RuntimeError):
+        model.generate(text="hello")
+    assert model.calls == 1
+
+
+def test_eager_retry_failure_is_not_misclassified(engine_env, model_manager):
+    """If the eager retry then hits a *real* error (e.g. OOM), the propagated
+    exception must not be classified as a compile failure via the chained
+    original compile error."""
+    real_oom = RuntimeError("CUDA out of memory. Tried to allocate 2.50 GiB")
+    model = _FakeModel(failures=[_dynamo_exc(), real_oom])
+    model_manager._install_compile_fallback(model)
+
+    with pytest.raises(RuntimeError) as excinfo:
+        model.generate(text="hello")
+
+    assert excinfo.value is real_oom
+    assert model_manager._is_compile_runtime_failure(excinfo.value) is False

tests/test_torch_compile_gate.py

@@ -4,14 +4,55 @@
 Windows build, so on Windows+CUDA the compile path failed and surfaced as a
 confusing "OOM". The gate skips compile (→ eager) when Triton is absent or the
 user disabled it. Tests force find_spec / the setting and assert the decision.
+
+#278 adds two more gates: the GPU's compute capability must be in the torch
+build's arch list (new archs like Blackwell sm_120 break Triton/Inductor
+before upstream support lands — overridable via OMNIVOICE_FORCE_TORCH_COMPILE),
+and a compile failure earlier in the session disables compile for the rest of
+the process.
 """
 from __future__ import annotations
 
 import importlib.util
+import sys
+from types import SimpleNamespace
+
+import pytest
 
 from services import engine_env
 
 
+class _FakeCuda:
+    def __init__(self, capability=(9, 0), arch_list=("sm_80", "sm_86", "sm_90")):
+        self._cap = tuple(capability)
+        self._arch = list(arch_list)
+
+    def is_available(self):
+        return True
+
+    def get_device_capability(self, idx=0):
+        return self._cap
+
+    def get_arch_list(self):
+        return self._arch
+
+    def get_device_name(self, idx=0):
+        return "NVIDIA GeForce RTX (fake)"
+
+
+@pytest.fixture
+def compile_friendly_env(monkeypatch):
+    """Triton present, setting off, no prior session failure, no force env."""
+    monkeypatch.setattr(importlib.util, "find_spec", lambda name: object())
+    monkeypatch.setattr("services.settings_store.get_text", lambda key, default="0": "0")
+    monkeypatch.setattr(engine_env, "_compile_runtime_failure", None)
+    monkeypatch.delenv("OMNIVOICE_FORCE_TORCH_COMPILE", raising=False)
+
+
+def _fake_torch(monkeypatch, cuda):
+    monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=cuda))
+
+
 def test_skips_when_device_not_cuda():
     assert engine_env.should_torch_compile("cpu") is False
     assert engine_env.should_torch_compile("mps") is False
@@ -25,13 +66,56 @@ def test_skips_when_triton_missing(monkeypatch):
     assert engine_env.should_torch_compile("cuda") is False
 
 
-def test_enabled_when_triton_present_and_not_disabled(monkeypatch):
-    monkeypatch.setattr(importlib.util, "find_spec", lambda name: object())
-    monkeypatch.setattr("services.settings_store.get_text", lambda key, default="0": "0")
+def test_enabled_when_triton_present_and_not_disabled(monkeypatch, compile_friendly_env):
+    _fake_torch(monkeypatch, _FakeCuda(capability=(9, 0)))
     assert engine_env.should_torch_compile("cuda") is True
 
 
-def test_skips_when_disabled_in_settings(monkeypatch):
-    monkeypatch.setattr(importlib.util, "find_spec", lambda name: object())
+def test_skips_when_disabled_in_settings(monkeypatch, compile_friendly_env):
     monkeypatch.setattr("services.settings_store.get_text", lambda key, default="0": "1")
     assert engine_env.should_torch_compile("cuda") is False
+
+
+# ── #278: unsupported / unknown GPU architecture ─────────────────────────────
+
+
+def test_skips_when_gpu_arch_not_in_torch_build(monkeypatch, compile_friendly_env):
+    # RTX 5060 (Blackwell, sm_120) on a torch build that only knows ≤ sm_90.
+    _fake_torch(monkeypatch, _FakeCuda(capability=(12, 0)))
+    assert engine_env.should_torch_compile("cuda") is False
+
+
+def test_force_env_overrides_arch_gate(monkeypatch, compile_friendly_env):
+    _fake_torch(monkeypatch, _FakeCuda(capability=(12, 0)))
+    monkeypatch.setenv("OMNIVOICE_FORCE_TORCH_COMPILE", "1")
+    assert engine_env.should_torch_compile("cuda") is True
+
+
+def test_supported_arch_still_compiles(monkeypatch, compile_friendly_env):
+    # Backward compat: users whose torch.compile works keep it.
+    _fake_torch(monkeypatch, _FakeCuda(capability=(8, 6)))
+    assert engine_env.should_torch_compile("cuda") is True
+
+
+def test_empty_arch_list_fails_open(monkeypatch, compile_friendly_env):
+    _fake_torch(monkeypatch, _FakeCuda(capability=(12, 0), arch_list=()))
+    assert engine_env.should_torch_compile("cuda") is True
+
+
+def test_arch_probe_error_fails_open(monkeypatch, compile_friendly_env):
+    class _BrokenCuda(_FakeCuda):
+        def get_device_capability(self, idx=0):
+            raise RuntimeError("driver error")
+
+    _fake_torch(monkeypatch, _BrokenCuda())
+    assert engine_env.should_torch_compile("cuda") is True
+
+
+# ── #278: session-wide disable after a runtime failure ──────────────────────
+
+
+def test_skips_after_runtime_failure_marked(monkeypatch, compile_friendly_env):
+    _fake_torch(monkeypatch, _FakeCuda(capability=(9, 0)))
+    assert engine_env.should_torch_compile("cuda") is True
+    engine_env.mark_compile_runtime_failure("AssertionError: cudagraph_trees")
+    assert engine_env.should_torch_compile("cuda") is False