From 09a260bb26e7d666d2fe26869e139323dd218b38 Mon Sep 17 00:00:00 2001 From: Palash Debnath Date: Wed, 9 Sep 2026 11:39:50 -0700 Subject: [PATCH 1/6] fix(backend): stop Windows desktop launches freezing at "Loading ML runtime" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every backend spawned by the Windows desktop shell hung forever in the startup worker's `import torch`, inside the loader for numpy's OpenBLAS DLL. The desktop parent-liveness watchdog (0a20aeb0) parks a synchronous read on the stdin pipe the shell hands the backend, and that pending read deadlocks the DLL initializer. The identical command from a terminal, with no stdin pipe and no watchdog, starts in seconds — which is why it only reproduced under the app. Bisected outside the app by spawning the backend with the shell's exact env, pipes, creation flags and job object: a watchdog thread that merely sleeps is harmless; a pending ReadFile, via the C runtime or straight to the kernel, hangs it every time. Native stacks (py-spy --native) show the watchdog in NtReadFile and the importer waiting on a critical section from inside the OpenBLAS initializer. Fix: on Windows the watchdog polls PeekNamedPipe and reads only bytes that are already buffered, so no I/O is ever outstanding on the pipe. It still exits the instant the desktop closes its end (ERROR_BROKEN_PIPE), and a non-pipe stdin keeps the shared blocking reader. Verified: the app-style spawn goes from an indefinite hang to ready in ~3 s, and the desktop-prod build boots and loads the model. Not in v0.5.1; the watchdog landed 2026-08-30 on main. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HCDZcBpP6QQa4dzUa8z6rh --- backend/core/parent_liveness.py | 80 +++++++++++++- tests/backend/core/test_parent_liveness.py | 123 +++++++++++++++++++++ 2 files changed, 200 insertions(+), 3 deletions(-) diff --git a/backend/core/parent_liveness.py b/backend/core/parent_liveness.py index e6707752..523e894a 100644 --- a/backend/core/parent_liveness.py +++ b/backend/core/parent_liveness.py @@ -4,7 +4,14 @@ from __future__ import annotations import os import sys import threading -from typing import BinaryIO, Callable +import time +from typing import Any, BinaryIO, Callable, Optional + +# Poll cadence for the Windows pipe watcher. Exit latency after the desktop +# closes its end is bounded by this; the desktop's own kill-on-close Job is the +# hard backstop, so a quarter second is plenty and costs nothing measurable. +WINDOWS_PIPE_POLL_INTERVAL_S = 0.25 +_FILE_TYPE_PIPE = 3 # winbase.h FILE_TYPE_PIPE def _watch_parent_pipe(reader: BinaryIO, exit_process: Callable[[int], None]) -> None: @@ -18,6 +25,64 @@ def _watch_parent_pipe(reader: BinaryIO, exit_process: Callable[[int], None]) -> exit_process(0) +def _watch_parent_pipe_handle( + handle: int, + exit_process: Callable[[int], None], + *, + peek: Optional[Callable[[int], Any]] = None, + read_file: Optional[Callable[[int, int], Any]] = None, + sleep: Callable[[float], None] = time.sleep, + interval: float = WINDOWS_PIPE_POLL_INTERVAL_S, +) -> None: + """Windows twin of :func:`_watch_parent_pipe` that never leaves a read + pending on the pipe. + + A synchronous ``ReadFile`` parked on the stdin pipe — whether issued through + the C runtime's ``read()`` or straight to the kernel — deadlocks the + OpenBLAS DLL initializer that ``import torch`` reaches (numpy's + ``_multiarray_umath``) in the startup worker: every desktop-spawned backend + on Windows froze at "Loading ML runtime (PyTorch)" while the identical + command from a terminal, with no stdin pipe and no watchdog, started in + seconds. A thread that merely sleeps does not trigger it; only the pending + read on that pipe does. So instead of blocking in a read, poll with + ``PeekNamedPipe``: it returns immediately, holds no I/O on the file object, + drains any keepalive bytes the desktop might write, and fails with + ``ERROR_BROKEN_PIPE`` the moment the desktop closes its end — which is the + same EOF signal the POSIX reader gets. + """ + if peek is None or read_file is None: + import _winapi # Windows-only stdlib module; the caller gates on the platform + + peek = peek or _winapi.PeekNamedPipe + read_file = read_file or _winapi.ReadFile + try: + while True: + available, _ = peek(handle) + if available: + # Bytes are already buffered, so this read cannot block. + read_file(handle, available) + else: + sleep(interval) + except OSError: + # ERROR_BROKEN_PIPE (109) is how the closed parent end surfaces here. + pass + exit_process(0) + + +def _windows_pipe_handle(reader: Any) -> Optional[int]: + """The OS handle behind ``reader`` when it is a pipe, else None.""" + try: + import msvcrt + import _winapi + + handle = msvcrt.get_osfhandle(reader.fileno()) + if _winapi.GetFileType(handle) != _FILE_TYPE_PIPE: + return None + return handle + except (OSError, ValueError, AttributeError, ImportError): + return None + + def arm_desktop_parent_watchdog() -> bool: """Use stdin EOF as an unforgeable parent-liveness signal for desktop runs.""" if os.environ.get("OMNIVOICE_DESKTOP_CONTAINED") != "1": @@ -25,9 +90,18 @@ def arm_desktop_parent_watchdog() -> bool: reader = getattr(sys.stdin, "buffer", None) if reader is None: return False + target: Callable[..., None] = _watch_parent_pipe + args: tuple = (reader, os._exit) + if os.name == "nt": + handle = _windows_pipe_handle(reader) + if handle is not None: + target = _watch_parent_pipe_handle + args = (handle, os._exit) + # A non-pipe stdin (file, NUL) cannot have a read pending against a + # pipe file object, so the blocking reader stays correct there. threading.Thread( - target=_watch_parent_pipe, - args=(reader, os._exit), + target=target, + args=args, name="desktop-parent-watchdog", daemon=True, ).start() diff --git a/tests/backend/core/test_parent_liveness.py b/tests/backend/core/test_parent_liveness.py index f33eeb2c..5f11edb3 100644 --- a/tests/backend/core/test_parent_liveness.py +++ b/tests/backend/core/test_parent_liveness.py @@ -53,3 +53,126 @@ def test_desktop_child_exits_when_parent_closes_stdin(): if child.poll() is None: child.kill() child.wait() + + +# ── Windows: the watchdog must never leave a read pending on the stdin pipe ── +# A synchronous ReadFile parked on the desktop-owned stdin pipe (via the C +# runtime's read() or straight to the kernel) deadlocked the OpenBLAS DLL +# initializer that `import torch` reaches in the startup worker: every +# desktop-spawned backend on Windows froze at "Loading ML runtime" while the +# same command from a terminal (no stdin pipe, no watchdog) started in seconds. +# A thread that only sleeps is harmless; the pending read is the trigger. The +# Windows watcher therefore polls PeekNamedPipe and reads only bytes that are +# already buffered, so no I/O is ever outstanding on that file object. + + +def _peek_sequence(events): + """`events` items: int → bytes available; OSError → broken pipe.""" + it = iter(events) + + def peek(handle): + ev = next(it) + if isinstance(ev, BaseException): + raise ev + return (ev, 0) + + return peek + + +def test_windows_watcher_exits_on_broken_pipe_without_blocking_reads(): + from core.parent_liveness import _watch_parent_pipe_handle + + reads, sleeps, exits = [], [], [] + _watch_parent_pipe_handle( + 7, + exits.append, + peek=_peek_sequence([0, 0, OSError(109, "The pipe has been ended")]), + read_file=lambda h, n: reads.append((h, n)), + sleep=sleeps.append, + interval=0.01, + ) + assert exits == [0] + assert reads == [] # nothing buffered → never a read, let alone a pending one + assert sleeps == [0.01, 0.01] + + +def test_windows_watcher_drains_only_buffered_bytes_until_eof(): + from core.parent_liveness import _watch_parent_pipe_handle + + reads, exits = [], [] + _watch_parent_pipe_handle( + 7, + exits.append, + peek=_peek_sequence([3, 0, 1, OSError(109, "The pipe has been ended")]), + read_file=lambda h, n: reads.append((h, n)) or (b"x" * n, 0), + sleep=lambda s: None, + ) + assert exits == [0] + # Reads are sized to exactly what PeekNamedPipe reported, so they return + # immediately instead of parking on the pipe. + assert reads == [(7, 3), (7, 1)] + + +def test_windows_watchdog_arms_pipe_poller_on_pipe_stdin(monkeypatch): + import types + import core.parent_liveness as pl + + monkeypatch.setenv("OMNIVOICE_DESKTOP_CONTAINED", "1") + monkeypatch.setattr(pl.os, "name", "nt") + + class FakeBuffer: + def fileno(self): + return 0 + + monkeypatch.setattr(pl.sys, "stdin", types.SimpleNamespace(buffer=FakeBuffer())) + monkeypatch.setitem(sys.modules, "msvcrt", types.SimpleNamespace(get_osfhandle=lambda fd: 0xABC)) + monkeypatch.setitem(sys.modules, "_winapi", types.SimpleNamespace(GetFileType=lambda h: 3)) + + started = {} + + class FakeThread: + def __init__(self, *, target, args, name, daemon): + started.update(target=target, args=args, daemon=daemon) + + def start(self): + started["started"] = True + + monkeypatch.setattr(pl.threading, "Thread", FakeThread) + assert pl.arm_desktop_parent_watchdog() is True + assert started["started"] is True + assert started["target"] is pl._watch_parent_pipe_handle + assert started["args"] == (0xABC, os._exit) + assert started["daemon"] is True + + +def test_windows_watchdog_keeps_blocking_reader_for_non_pipe_stdin(monkeypatch): + """A file/NUL stdin has no pipe file object to deadlock against, and a + blocking read on it returns EOF promptly — keep the shared reader there.""" + import types + import core.parent_liveness as pl + + monkeypatch.setenv("OMNIVOICE_DESKTOP_CONTAINED", "1") + monkeypatch.setattr(pl.os, "name", "nt") + + class FakeBuffer: + def fileno(self): + return 0 + + fake_stdin = types.SimpleNamespace(buffer=FakeBuffer()) + monkeypatch.setattr(pl.sys, "stdin", fake_stdin) + monkeypatch.setitem(sys.modules, "msvcrt", types.SimpleNamespace(get_osfhandle=lambda fd: 0xABC)) + monkeypatch.setitem(sys.modules, "_winapi", types.SimpleNamespace(GetFileType=lambda h: 1)) # FILE_TYPE_DISK + + started = {} + + class FakeThread: + def __init__(self, *, target, args, name, daemon): + started.update(target=target, args=args) + + def start(self): + started["started"] = True + + monkeypatch.setattr(pl.threading, "Thread", FakeThread) + assert pl.arm_desktop_parent_watchdog() is True + assert started["target"] is pl._watch_parent_pipe + assert started["args"] == (fake_stdin.buffer, os._exit) From 6a6dd83efabc9b971701ba57d1b0a5944e6c4b88 Mon Sep 17 00:00:00 2001 From: Palash Debnath Date: Wed, 9 Sep 2026 11:40:13 -0700 Subject: [PATCH 2/6] docs(changelog): note the Windows backend startup hang fix (#1955) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HCDZcBpP6QQa4dzUa8z6rh --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a95f61b0..ef859945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ the frozen-backend fallback mirror it for their toolchains. ### Fixed +- Windows desktop launches no longer freeze at "Loading ML runtime (PyTorch)": the parent-liveness watchdog polls the stdin pipe instead of leaving a read pending, which deadlocked numpy's OpenBLAS initializer (#1955) - Install documentation help now prints correctly on Windows consoles using legacy encodings (#1815) — thanks @dajiaohuang! - Saved transcriptions with missing or invalid timestamps now remain readable (#1799) — thanks @yunaremaia and @tvbht! - Copying a saved transcription now uses the shared clipboard helper and reports failed copies accurately (#1803) — thanks @tvbht! From 5c7f8a8e7aa0e3c0adac6ecba7b5a76b779f2cff Mon Sep 17 00:00:00 2001 From: Palash Debnath Date: Wed, 9 Sep 2026 11:51:17 -0700 Subject: [PATCH 3/6] test(backend): Windows integration regression for the desktop stdin watchdog hang Spawns the real backend the way the desktop shell does (containment marker plus a piped stdin) and asserts startup gets past the ML import. On the pre-fix watchdog it times out after 180 s; on the fix it passes in ~4 s. Windows-only, since the deadlock is a Windows loader-lock interaction and CI's backend job runs on Linux. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HCDZcBpP6QQa4dzUa8z6rh --- tests/backend/core/test_parent_liveness.py | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/backend/core/test_parent_liveness.py b/tests/backend/core/test_parent_liveness.py index 5f11edb3..60bc7d46 100644 --- a/tests/backend/core/test_parent_liveness.py +++ b/tests/backend/core/test_parent_liveness.py @@ -4,6 +4,8 @@ import subprocess import sys from pathlib import Path +import pytest + def test_parent_pipe_eof_exits_cleanly(): from core.parent_liveness import _watch_parent_pipe @@ -176,3 +178,75 @@ def test_windows_watchdog_keeps_blocking_reader_for_non_pipe_stdin(monkeypatch): assert pl.arm_desktop_parent_watchdog() is True assert started["target"] is pl._watch_parent_pipe assert started["args"] == (fake_stdin.buffer, os._exit) + + +# ── Windows integration: the real backend must get past the ML import ──────── +# The deadlock needs the real startup: numpy's OpenBLAS DLL loaded through +# `import torch` in the startup worker while the desktop watchdog is armed on a +# piped stdin. Smaller reproductions (a pending read + `import torch` in a bare +# child) do not trigger it, so this spawns the actual backend exactly as the +# desktop shell does. It fails on the pre-fix watchdog by timing out in the +# "ml_imports" step and passes in well under a minute on the fix. Windows-only: +# CI's backend job runs on Linux, so this is exercised on Windows dev machines. + + +@pytest.mark.skipif(sys.platform != "win32", reason="desktop stdin-pipe watchdog deadlock is Windows-only") +def test_desktop_spawned_backend_gets_past_ml_imports_on_windows(tmp_path): + pytest.importorskip("torch") + pytest.importorskip("uvicorn") + import json + import socket + import threading + import time + import urllib.request + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + + root = Path(__file__).parents[3] + env = os.environ.copy() + env.update( + { + "OMNIVOICE_DESKTOP_CONTAINED": "1", # arms the watchdog on the stdin pipe + "OMNIVOICE_PORT": str(port), + "OMNIVOICE_DATA_DIR": str(tmp_path / "data"), + "OMNIVOICE_CACHE_DIR": str(tmp_path / "hf_cache"), + "HF_HUB_OFFLINE": "1", + "PYTHONUNBUFFERED": "1", + "PYTHONUTF8": "1", + } + ) + env.pop("PYTHONPATH", None) + child = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "main:app", "--app-dir", "backend", + "--host", "127.0.0.1", "--port", str(port)], + cwd=root, + env=env, + stdin=subprocess.PIPE, # the desktop shell keeps this open and never writes + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for stream in (child.stdout, child.stderr): + threading.Thread(target=lambda s=stream: s.read(), daemon=True).start() + seen = [] + try: + deadline = time.time() + 180 + while time.time() < deadline: + if child.poll() is not None: + pytest.fail(f"backend exited early with {child.returncode}; progress seen: {seen[-3:]}") + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/startup/progress", timeout=2) as resp: + progress = json.load(resp) + except (OSError, ValueError): + time.sleep(1) + continue + seen.append((progress.get("status"), progress.get("step"))) + done = {s["id"] for s in progress.get("steps", []) if s.get("state") == "done"} + if progress.get("status") != "starting" or "ml_imports" in done: + return + time.sleep(1) + pytest.fail(f"backend never got past ml_imports in 180s (deadlocked watchdog?); last progress: {seen[-3:]}") + finally: + child.kill() + child.wait() From 4f4d21ff5bc1104feec90395071efd1d1fb48039 Mon Sep 17 00:00:00 2001 From: Palash Debnath Date: Wed, 9 Sep 2026 12:06:40 -0700 Subject: [PATCH 4/6] fix(errors): name the backend error class on an unclassified streaming failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #1800. Every engine failure the taxonomy cannot classify renders one floor message, "Generation failed. Check the selected engine and try again." The auto bug reporter puts that message and a stack of minified bundle frames into the issue, so unrelated faults arrive as byte-identical reports — roughly a dozen of the open issues are that same report filed again, and none of them can be told apart, let alone triaged. The streaming error frame now carries the exception's TYPE NAME, the frontend keeps it on StreamingPreviewError, and the report prints it as "Backend error class: …". A MemoryError and a FileNotFoundError stop being the same issue. Only the class name — no substring of the exception message is copied, so the response-safety contract still holds and a test pins that a path in the exception never reaches the payload. This is the same datum the dub routes already put on the wire as error_class and the analytics allowlist already treats as content-free. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017ypcgSsh5j2PEonSJiAU1S --- CHANGELOG.md | 1 + backend/core/public_errors.py | 10 +++++ frontend/src/test/streamingTts.test.js | 25 ++++++++++++ frontend/src/utils/bugReport.js | 5 +++ frontend/src/utils/bugReport.test.js | 12 ++++++ frontend/src/utils/streamingTts.js | 6 +++ tests/test_stream_error_class_1800.py | 53 ++++++++++++++++++++++++++ 7 files changed, 112 insertions(+) create mode 100644 tests/test_stream_error_class_1800.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a95f61b0..a252bfdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ the frozen-backend fallback mirror it for their toolchains. ## [Unreleased] **Highlights** +- A generation failure that the app cannot classify now names the backend error class, so two unrelated faults stop arriving as the same untriageable report (#1800) - Validate current-user Windows installers under a standard account on hosted runners (#1883) diff --git a/backend/core/public_errors.py b/backend/core/public_errors.py index e5ad1499..4941a039 100644 --- a/backend/core/public_errors.py +++ b/backend/core/public_errors.py @@ -94,6 +94,16 @@ def stream_generation_failure(error: BaseException | object) -> dict[str, object replace the failure being diagnosed. """ payload = stream_failure("generation_failed") + if isinstance(error, BaseException): + # The exception's TYPE NAME, never its message. Two failures that both + # render the floor message "Generation failed. Check the selected + # engine and try again." are indistinguishable in an auto-filed report, + # so every unclassified streaming failure arrives as the same issue and + # none of them can be triaged (#1800). A class name is VoiceStudio-safe + # by the same reasoning that already puts it on the wire as + # `error_class` in the dub routes and on the analytics allowlist: it is + # a Python type, not user text, and no substring of `error` is copied. + payload["error_class"] = type(error).__name__ try: enriched = public_exception_response(error, fallback=str(payload["detail"])) except Exception: diff --git a/frontend/src/test/streamingTts.test.js b/frontend/src/test/streamingTts.test.js index e09748b8..6931e73a 100644 --- a/frontend/src/test/streamingTts.test.js +++ b/frontend/src/test/streamingTts.test.js @@ -329,6 +329,31 @@ describe('streamGenerateSpeech', () => { expect(FakeAudioContext.instances[0].state).toBe('closed'); }); + it('carries the backend error class from the error frame (#1800)', async () => { + // Every unclassified engine failure renders the same floor message, so + // the auto-filed reports were byte-identical and none could be triaged. + // The class name is the only thing that separates them. + apiFetch.mockResolvedValue( + ndjsonResponse([ + startEvent(3), + chunkEvent(0), + { type: 'error', detail: 'Generation failed.', error_class: 'MemoryError' }, + ]), + ); + await expect(streamGenerateSpeech(new FormData(), {})).rejects.toMatchObject({ + errorClass: 'MemoryError', + }); + }); + + it('leaves the error class null when the frame omits it', async () => { + apiFetch.mockResolvedValue( + ndjsonResponse([startEvent(3), chunkEvent(0), { type: 'error', detail: 'boom' }]), + ); + await expect(streamGenerateSpeech(new FormData(), {})).rejects.toMatchObject({ + errorClass: null, + }); + }); + it('carries the retryable marker from a GPU-timeout error frame (#1190)', async () => { // A retryable failure means the backend already spent the full budget on // this text and the abandoned job still holds the device — useTTS uses diff --git a/frontend/src/utils/bugReport.js b/frontend/src/utils/bugReport.js index 1251cc84..058ea49b 100644 --- a/frontend/src/utils/bugReport.js +++ b/frontend/src/utils/bugReport.js @@ -362,11 +362,16 @@ export async function buildBugReportUrl({ title = '[Bug] ', error } = {}) { msg.length > MAX_MSG_CHARS ? `${msg.slice(0, MAX_MSG_CHARS)}\n… (truncated)` : msg; let stack = error?.stack ? scrubText(error.stack) : ''; if (stack.length > MAX_STACK_CHARS) stack = `${stack.slice(0, MAX_STACK_CHARS)}\n… (truncated)`; + // A generic failure message plus a stack of minified bundle frames is + // the same report every time; the backend class name is what separates + // one unclassified engine failure from another (#1800). + const klass = typeof error?.errorClass === 'string' ? scrubText(error.errorClass) : ''; errorSection.push( '## Error', '', '```', msgForBody, + ...(klass ? [`Backend error class: ${klass}`] : []), ...(stack && stack !== msgForBody ? [stack] : []), '```', '', diff --git a/frontend/src/utils/bugReport.test.js b/frontend/src/utils/bugReport.test.js index 96246b0a..209d2b90 100644 --- a/frontend/src/utils/bugReport.test.js +++ b/frontend/src/utils/bugReport.test.js @@ -117,6 +117,18 @@ describe('buildBugReportUrl', () => { expect(body).not.toContain('/Users/alice'); }); + it('records the backend error class so identical messages differ (#1800)', async () => { + const err = new Error('Generation failed. Check the selected engine and try again.'); + err.errorClass = 'MemoryError'; + const body = decodeURIComponent(await buildBugReportUrl({ error: err })); + expect(body).toContain('Backend error class: MemoryError'); + }); + + it('omits the class line when the failure carries none', async () => { + const body = decodeURIComponent(await buildBugReportUrl({ error: new Error('plain failure') })); + expect(body).not.toContain('Backend error class'); + }); + it('seeds the title with the error message', async () => { const url = await buildBugReportUrl({ error: new Error('synthesis exploded') }); expect(decodeURIComponent(url)).toContain('[Bug] synthesis exploded'); diff --git a/frontend/src/utils/streamingTts.js b/frontend/src/utils/streamingTts.js index 7f0d6da7..832df5ae 100644 --- a/frontend/src/utils/streamingTts.js +++ b/frontend/src/utils/streamingTts.js @@ -80,6 +80,11 @@ export class StreamingPreviewError extends Error { this.retryable = opts?.retryable === true; this.retryAfter = opts?.retryAfter ?? null; this.terminal = opts?.terminal === true; + // The backend exception TYPE behind an otherwise generic failure. The + // floor message is identical for every unclassified engine failure, so + // without this an auto-filed report cannot be told apart from any + // other (#1800). Never the exception message — only its class name. + this.errorClass = opts?.errorClass || null; } } @@ -404,6 +409,7 @@ async function _streamGenerateSpeech( retryable: ev.retryable === true, retryAfter: ev.retry_after ?? null, terminal, + errorClass: ev.error_class || null, }); } }; diff --git a/tests/test_stream_error_class_1800.py b/tests/test_stream_error_class_1800.py new file mode 100644 index 00000000..1e35023b --- /dev/null +++ b/tests/test_stream_error_class_1800.py @@ -0,0 +1,53 @@ +"""#1800 — an unclassified streaming failure must carry its exception class. + +Every engine failure the taxonomy cannot classify renders the same floor +message, "Generation failed. Check the selected engine and try again." The +auto bug reporter puts that message and a stack of minified bundle frames into +the issue, so roughly a dozen separate faults arrived as byte-identical, +untriageable reports. The exception's TYPE NAME is what separates them. + +It is the type name only. No substring of the exception message is copied, so +the response-safety contract in tests/test_response_safety.py still holds. +""" +import pytest + +from core.public_errors import stream_failure, stream_generation_failure + + +def test_unclassified_failure_carries_its_exception_class(): + payload = stream_generation_failure(RuntimeError("CUDA error: device-side assert")) + assert payload["code"] == "generation_failed" + assert payload["error_class"] == "RuntimeError" + + +def test_two_unrelated_failures_are_distinguishable(): + # The whole point: these used to be the same report. + a = stream_generation_failure(RuntimeError("boom")) + b = stream_generation_failure(MemoryError("out of memory")) + assert a["detail"] == b["detail"] + assert a["error_class"] != b["error_class"] + + +def test_the_exception_message_is_never_copied(): + secret = "C:/Users/someone/private-voice-sample.wav" + payload = stream_generation_failure(FileNotFoundError(secret)) + assert payload["error_class"] == "FileNotFoundError" + for value in payload.values(): + assert secret not in str(value) + assert "someone" not in str(value) + + +def test_a_non_exception_gets_no_class(): + # dict(...) of the plain floor payload, unchanged — nothing to name. + payload = stream_generation_failure("not an exception") + assert "error_class" not in payload + assert payload["detail"] == stream_failure("generation_failed")["detail"] + + +@pytest.mark.parametrize("exc", [RuntimeError("x"), ValueError("y"), OSError("z")]) +def test_class_is_present_alongside_a_classified_hint(exc): + # Enrichment and the class name are independent: adding one must not drop + # the other, whichever branch the taxonomy takes. + payload = stream_generation_failure(exc) + assert payload["error_class"] == type(exc).__name__ + assert payload["detail"] From f49d31d5b388e825dea0d4c9bfc677f0dc39a7e3 Mon Sep 17 00:00:00 2001 From: Palash Debnath Date: Wed, 9 Sep 2026 12:06:46 -0700 Subject: [PATCH 5/6] test(backend): only accept positive startup evidence in the Windows watchdog test The poll loop treated any status other than "starting" as success, so a backend that stayed alive but reported a failed startup would pass the very test meant to catch a broken start (CodeRabbit + Greptile on #1955). Succeed only when the ML import step is done or status is ready; fail loudly on any other terminal status or error. Also give the child an empty HF_HUB_CACHE so it never reads the developer's populated cache. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HCDZcBpP6QQa4dzUa8z6rh --- tests/backend/core/test_parent_liveness.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/backend/core/test_parent_liveness.py b/tests/backend/core/test_parent_liveness.py index 60bc7d46..343a6f3f 100644 --- a/tests/backend/core/test_parent_liveness.py +++ b/tests/backend/core/test_parent_liveness.py @@ -212,6 +212,7 @@ def test_desktop_spawned_backend_gets_past_ml_imports_on_windows(tmp_path): "OMNIVOICE_PORT": str(port), "OMNIVOICE_DATA_DIR": str(tmp_path / "data"), "OMNIVOICE_CACHE_DIR": str(tmp_path / "hf_cache"), + "HF_HUB_CACHE": str(tmp_path / "hf_cache"), # never the developer's populated cache "HF_HUB_OFFLINE": "1", "PYTHONUNBUFFERED": "1", "PYTHONUTF8": "1", @@ -241,10 +242,18 @@ def test_desktop_spawned_backend_gets_past_ml_imports_on_windows(tmp_path): except (OSError, ValueError): time.sleep(1) continue - seen.append((progress.get("status"), progress.get("step"))) + status = progress.get("status") + seen.append((status, progress.get("step"))) done = {s["id"] for s in progress.get("steps", []) if s.get("state") == "done"} - if progress.get("status") != "starting" or "ml_imports" in done: + # Positive evidence only: the ML import step finished, or startup is + # ready. Any other terminal state is the failure this test exists for. + if "ml_imports" in done or status == "ready": return + if status != "starting" or progress.get("error"): + pytest.fail( + f"backend startup reported status={status!r} error={progress.get('error')!r}; " + f"progress seen: {seen[-3:]}" + ) time.sleep(1) pytest.fail(f"backend never got past ml_imports in 180s (deadlocked watchdog?); last progress: {seen[-3:]}") finally: From d630db29f52c995696956982417771fad97ab439 Mon Sep 17 00:00:00 2001 From: Palash Debnath Date: Wed, 9 Sep 2026 12:13:25 -0700 Subject: [PATCH 6/6] fix(errors): stop a context-free failure borrowing another stage's remediation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #1943. A macOS mlx-audio text-to-speech failure returned a 500 advising the user that "the connection to the video server dropped mid-download". No video was involved. VIDEO_DOWNLOAD_NETWORK triggers on bare phrases — "timed out", "connection reset", "broken pipe" — so any unrelated failure carrying one is handed a confidently wrong next step, which is worse than no hint at all. failure._CONTEXT_FREE_HINT_CLASSES already existed for exactly this, and its own comment names VIDEO_DOWNLOAD_NETWORK as the class that must never appear on a stageless surface. Only append_hint honoured it; public_exception_response took over the 500 path without carrying the rule across, and the streaming error frame then inherited the same gap through it. The filter now lives in public_exception_response, so every context-free caller gets it. MODEL_CACHE_CORRUPT joins the allowlist — its trigger is a VoiceStudio-authored sentence, no library can produce it, and the 500 handler is the surface a corrupt cache actually reaches. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017ypcgSsh5j2PEonSJiAU1S --- CHANGELOG.md | 1 + backend/core/failure.py | 7 +++ backend/core/public_errors.py | 23 ++++++++- tests/test_context_free_hints_1943.py | 69 +++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 tests/test_context_free_hints_1943.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a252bfdf..48e42d64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ the frozen-backend fallback mirror it for their toolchains. ## [Unreleased] **Highlights** +- A failure with no stage attached no longer borrows another stage's advice, so a text-to-speech error stops telling you the video server dropped the download (#1943) - A generation failure that the app cannot classify now names the backend error class, so two unrelated faults stop arriving as the same untriageable report (#1800) - Validate current-user Windows installers under a standard account on hosted runners (#1883) diff --git a/backend/core/failure.py b/backend/core/failure.py index 2930ad20..21fd815f 100644 --- a/backend/core/failure.py +++ b/backend/core/failure.py @@ -308,6 +308,13 @@ _CONTEXT_FREE_HINT_CLASSES = frozenset({ # a Windows virtual-memory setting rather than a connectivity problem, and # the detailed hint we already had for it never reached them. "WINDOWS_PAGING_FILE_TOO_SMALL", + # Its trigger is a VoiceStudio-authored sentence — "the TTS model cache + # for … is incomplete" plus "could not be auto-repaired" / "weights + # missing" — so it cannot be produced by an unrelated library. The 500 + # handler is the surface a corrupt cache actually reaches, and dropping + # its hint there would leave the user with no way to know a redownload + # is the fix. + "MODEL_CACHE_CORRUPT", }) diff --git a/backend/core/public_errors.py b/backend/core/public_errors.py index 4941a039..91501c08 100644 --- a/backend/core/public_errors.py +++ b/backend/core/public_errors.py @@ -159,12 +159,31 @@ def public_exception_response(error: BaseException, *, fallback: str) -> dict[st Classification may inspect the private diagnostic locally, but response values come exclusively from VoiceStudio-owned constants. No substring of ``error`` is copied into the payload. + + Every caller is a CONTEXT-FREE surface — the global 500 handler, the + streaming generate error frame, the dub GPU-OOM 503 — so the topic is + filtered through ``failure._CONTEXT_FREE_HINT_CLASSES`` before its hint is + attached. Without that filter a topic whose trigger is a generic phrase + stamps a confidently wrong remediation on an unrelated failure: #1943 is a + macOS mlx-audio TTS 500 that came back advising the user that "the + connection to the video server dropped mid-download", because + VIDEO_DOWNLOAD_NETWORK triggers on a bare "timed out" / "connection + reset". The allowlist already existed and already named that class as the + example of what must not appear here; only :func:`failure.append_hint` + honoured it, and this helper replaced ``append_hint`` on the 500 path + without carrying the rule across. + + HF_MIRROR_UNREACHABLE is allowed alongside it: its hint is dynamic (it + names the configured mirror) and its trigger requires that a mirror is + configured at all, so it cannot fire on an unrelated failure (#874). """ - from core.failure import classify, public_hint_for_topic + from core.failure import _CONTEXT_FREE_HINT_CLASSES, classify, public_hint_for_topic try: topic = classify(str(error)) - hint = public_hint_for_topic(topic) + if topic and topic not in _CONTEXT_FREE_HINT_CLASSES and topic != "HF_MIRROR_UNREACHABLE": + topic = "" + hint = public_hint_for_topic(topic) if topic else "" except Exception: topic = "" hint = "" diff --git a/tests/test_context_free_hints_1943.py b/tests/test_context_free_hints_1943.py new file mode 100644 index 00000000..967a06d6 --- /dev/null +++ b/tests/test_context_free_hints_1943.py @@ -0,0 +1,69 @@ +"""#1943 — a context-free surface must not attach a stage-specific hint. + +A macOS mlx-audio text-to-speech failure came back as a 500 reading "… The +connection to the video server dropped mid-download (often a transient +CDN/network blip …)". No video was involved. VIDEO_DOWNLOAD_NETWORK triggers +on bare phrases like "timed out" and "connection reset", so any unrelated +failure whose message contains one is handed a confidently wrong remediation. + +failure._CONTEXT_FREE_HINT_CLASSES already existed for this, and its own +comment names VIDEO_DOWNLOAD_NETWORK as the class that must never appear on a +context-free surface. Only append_hint honoured it; public_exception_response +replaced append_hint on the 500 path without carrying the rule across. +""" +import pytest + +from core.failure import _CONTEXT_FREE_HINT_CLASSES, classify +from core.public_errors import public_exception_response, stream_generation_failure + +# Phrases that really do classify as the video-download class but say nothing +# about a video when they arrive from a model load or a synthesis run. +_GENERIC_NETWORK_WORDING = [ + "operation timed out", + "connection reset by peer", + "broken pipe", + "remote end closed connection without response", +] + + +@pytest.mark.parametrize("message", _GENERIC_NETWORK_WORDING) +def test_no_video_hint_on_a_context_free_failure(message): + assert classify(message) == "VIDEO_DOWNLOAD_NETWORK" # still classified... + payload = public_exception_response(RuntimeError(message), fallback="Internal error.") + # ...but its hint must not reach a surface that does not know the stage. + assert payload["detail"] == "Internal error." + assert "video server" not in payload["detail"] + assert "docs_topic" not in payload + + +@pytest.mark.parametrize("message", _GENERIC_NETWORK_WORDING) +def test_no_video_hint_on_the_streaming_generate_frame(message): + payload = stream_generation_failure(RuntimeError(message)) + assert "video" not in str(payload["detail"]).lower() + assert "docs_topic" not in payload + + +def test_allowlisted_classes_keep_their_hint(): + payload = public_exception_response( + RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB"), + fallback="Internal error.", + ) + assert payload.get("docs_topic") == "GPU_OOM" + assert payload["detail"] != "Internal error." + + +def test_windows_paging_file_keeps_its_hint(): + # An allowlisted class whose trigger is unmistakable — the regression guard + # for over-filtering, which would silently strip every useful remediation. + payload = public_exception_response( + OSError("[WinError 1455] The paging file is too small for this operation to complete"), + fallback="Internal error.", + ) + assert payload.get("docs_topic") == "WINDOWS_PAGING_FILE_TOO_SMALL" + + +def test_the_filter_matches_the_documented_allowlist(): + # Pins the contract itself: the class the allowlist's own comment calls out + # as unsafe must not be a member. + assert "VIDEO_DOWNLOAD_NETWORK" not in _CONTEXT_FREE_HINT_CLASSES + assert "GPU_OOM" in _CONTEXT_FREE_HINT_CLASSES