On a Tesla T4 the backend exited during the first /generate with no traceback and no HTTP response, leaving the client with RemoteDisconnected and every later call with ConnectionRefused. Three separate defects combined, which is why none of the reporter's workarounds helped. 1. torch.compile(mode="reduce-overhead") captures CUDA graphs. T4 (sm_75) passed the existing arch gate, so capture was attempted and aborted the process from inside the native CUDA library — below the interpreter, where neither the #278 eager-fallback wrapper nor any except clause can see it. The compile mode is now resolved per GPU: Ampere (sm_80) and newer keep the cudagraph mode, older cards drop to the non-cudagraph "default" mode and keep their compiled Inductor kernels. Fails open on any probe error, so no GPU that works today loses the optimization. OMNIVOICE_FORCE_CUDAGRAPH=1 restores it. 2. should_torch_compile() never read TORCH_COMPILE_DISABLE. main.py sets it on win32, build_engine_env injected it into subprocesses, and docs/install/windows.md tells users to export it — but the in-process gate ignored it, so the reporter exported the documented variable and still got "torch.compile applied". The gate now honours TORCH_COMPILE_DISABLE / TORCHDYNAMO_DISABLE / TORCHINDUCTOR_DISABLE on every platform, and an env opt-out on the parent propagates to engine subprocesses. The settings DB path is logged alongside the toggle: the reporter had three omnivoice.db files and edited one the backend never opened. 3. Settings -> Performance -> "Disable torch.compile" was rendered disabled outside Windows in both the Tauri and Electron UIs, so the one control that would have stopped this was unreachable for the affected Linux user. The toggle is now live on every platform, and build_engine_env honours it everywhere rather than only on win32. Also arms faulthandler before torch is imported, so a fatal native signal writes the faulting thread's Python stack to backend_err.log instead of the process vanishing silently. This does not prevent a crash; it makes one diagnosable. OMNIVOICE_DISABLE_FAULTHANDLER=1 skips it. Tests fail before / pass after, verified by stashing the source and running the new tests against unfixed code. The crash test kills a real child interpreter with a real SIGSEGV and requires a named Python frame in the output. test_torch_compile_path_gate's fixture now clears the compile-disable env vars: main.py setdefaults them on win32, so on a Windows runner they leaked into os.environ and decided those tests. Not verified on real hardware — no Turing GPU available. The sm_80 floor is inferred from the crash report and from docs/hardware-notes-tesla-t4.md, which already flagged cudagraphs on T4 as attempted by default and never evaluated. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
128 lines
5.4 KiB
Python
128 lines
5.4 KiB
Python
"""#1266: a torch lib path with whitespace can never survive inductor's linker.
|
|
|
|
Inductor passes the torch library directory to clang++/g++ as an unquoted
|
|
``-L`` flag, so a path containing a space splits into two arguments and the
|
|
compile dies with ``no such file or directory: 'Support/...'``. The bug is
|
|
inside PyTorch; what we control is not paying for a compile attempt that cannot
|
|
succeed and not flooding the log tail with its failure — which is how it turned
|
|
up in #1259, consuming a chunk of the captured crash output while not being the
|
|
actual fault.
|
|
|
|
Not hypothetical anywhere: macOS keeps app data under
|
|
``~/Library/Application Support/`` and a Windows profile is routinely
|
|
``C:/Users/First Last``.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import types
|
|
|
|
import pytest
|
|
|
|
|
|
def _env(monkeypatch, *, torch_file, triton=True):
|
|
"""Bind should_torch_compile's dependencies to a controlled fake."""
|
|
from services import engine_env
|
|
|
|
monkeypatch.setattr(
|
|
engine_env.importlib.util,
|
|
"find_spec",
|
|
lambda name: object() if (triton and name == "triton") else None,
|
|
)
|
|
monkeypatch.setattr(engine_env, "_compile_runtime_failure", None, raising=False)
|
|
# #2135: should_torch_compile() now also honours the TORCH_COMPILE_DISABLE
|
|
# family. `backend/main.py` setdefaults those on win32, so on a Windows
|
|
# runner they leak into os.environ as soon as any test imports main — and
|
|
# would then decide these tests instead of the path logic under test.
|
|
for _name in engine_env._COMPILE_DISABLE_ENVS:
|
|
monkeypatch.delenv(_name, raising=False)
|
|
# Isolate the Settings gate: should_torch_compile() otherwise reads the real
|
|
# settings_store, so a persisted perf.torch_compile_disabled=1 (or a missing
|
|
# settings table) would decide these tests instead of the path logic.
|
|
monkeypatch.setattr(
|
|
engine_env, "settings_store", None, raising=False
|
|
)
|
|
# monkeypatch.setitem, never a raw assignment: a bare object dropped into
|
|
# sys.modules leaks process-wide out of collection and breaks every later
|
|
# import of the real module in a mixed run. backend/tests/
|
|
# test_no_module_stubs.py exists to catch exactly that, and caught this.
|
|
import sys as _sys
|
|
import types as _types
|
|
|
|
monkeypatch.setitem(
|
|
_sys.modules,
|
|
"services.settings_store",
|
|
_types.SimpleNamespace(get_text=lambda *a, **k: "0"),
|
|
)
|
|
monkeypatch.setattr(
|
|
engine_env, "_cuda_arch_supported_for_compile", lambda: (True, "")
|
|
)
|
|
monkeypatch.setitem(
|
|
__import__("sys").modules, "torch", types.SimpleNamespace(__file__=torch_file)
|
|
)
|
|
return engine_env
|
|
|
|
|
|
CLEAN = "/opt/venv/lib/python3.11/site-packages/torch/__init__.py"
|
|
SPACED = "/Users/x/Library/Application Support/omnivoice/.venv/lib/torch/__init__.py"
|
|
|
|
|
|
def test_compile_allowed_on_a_clean_path(monkeypatch):
|
|
ee = _env(monkeypatch, torch_file=CLEAN)
|
|
assert ee.should_torch_compile("cuda") is True
|
|
|
|
|
|
def test_compile_skipped_when_the_torch_path_has_a_space(monkeypatch, caplog):
|
|
"""The regression: this used to attempt, fail in clang++, and log the wreck."""
|
|
ee = _env(monkeypatch, torch_file=SPACED)
|
|
with caplog.at_level("INFO", logger="omnivoice.engine_env"):
|
|
assert ee.should_torch_compile("cuda") is False
|
|
assert any("whitespace" in r.getMessage() for r in caplog.records), (
|
|
"the skip must say WHY, or it is indistinguishable from the other skips"
|
|
)
|
|
|
|
|
|
def test_force_env_still_overrides(monkeypatch, caplog):
|
|
"""Consistent with the arch gate: the user can insist — and is told."""
|
|
ee = _env(monkeypatch, torch_file=SPACED)
|
|
monkeypatch.setenv(ee._FORCE_COMPILE_ENV, "1")
|
|
with caplog.at_level("WARNING", logger="omnivoice.engine_env"):
|
|
assert ee.should_torch_compile("cuda") is True
|
|
assert any(
|
|
"forced" in r.getMessage().lower() for r in caplog.records
|
|
), "forcing past a known-broken path must be recorded, not silent"
|
|
|
|
|
|
def test_skip_message_names_the_escape_hatch(monkeypatch, caplog):
|
|
"""A skip the user cannot override is a dead end; the hint must be there."""
|
|
ee = _env(monkeypatch, torch_file=SPACED)
|
|
with caplog.at_level("INFO", logger="omnivoice.engine_env"):
|
|
assert ee.should_torch_compile("cuda") is False
|
|
assert any(ee._FORCE_COMPILE_ENV in r.getMessage() for r in caplog.records)
|
|
|
|
|
|
def test_home_directory_is_not_logged_verbatim(monkeypatch, caplog):
|
|
"""The reason string is logged and lands in pasted bug reports, so the path
|
|
goes through the same redaction as every other user-facing failure text."""
|
|
import pathlib as _pl
|
|
|
|
home = str(_pl.Path.home())
|
|
ee = _env(monkeypatch, torch_file=f"{home}/Library/Application Support/x/torch/__init__.py")
|
|
with caplog.at_level("INFO", logger="omnivoice.engine_env"):
|
|
ee.should_torch_compile("cuda")
|
|
joined = " ".join(r.getMessage() for r in caplog.records)
|
|
assert home not in joined, f"home directory leaked into the log: {joined}"
|
|
|
|
|
|
def test_unreadable_torch_path_is_not_a_reason_to_skip(monkeypatch):
|
|
"""No torch metadata means no evidence of a problem — fail open, matching
|
|
the module's other probes."""
|
|
ee = _env(monkeypatch, torch_file=None)
|
|
assert ee.should_torch_compile("cuda") is True
|
|
|
|
|
|
@pytest.mark.parametrize("device", ["mps", "cpu", "xpu"])
|
|
def test_non_cuda_devices_are_unaffected(monkeypatch, device):
|
|
"""The device gate runs first and must keep short-circuiting."""
|
|
ee = _env(monkeypatch, torch_file=SPACED)
|
|
assert ee.should_torch_compile(device) is False
|