Merge remote-tracking branch 'origin/main' into land/queue1
This commit is contained in:
@@ -9,6 +9,8 @@ the frozen-backend fallback mirror it for their toolchains.
|
||||
## [Unreleased]
|
||||
|
||||
**Highlights**
|
||||
- A failure with no stage attached no longer borrows another stage's advice, so a text-to-speech error stops telling you the video server dropped the download (#1943)
|
||||
- A generation failure that the app cannot classify now names the backend error class, so two unrelated faults stop arriving as the same untriageable report (#1800)
|
||||
|
||||
- Transcriptions dictation wakes the desktop recorder, presents one contextual start action, and centers its microphone icon with the label (#1902)
|
||||
- Apple Silicon now shows one canonical OmniVoice choice in the engine picker while retaining its automatic crash-isolated sidecar runtime (#1913)
|
||||
@@ -58,6 +60,7 @@ the frozen-backend fallback mirror it for their toolchains.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Windows desktop launches no longer freeze at "Loading ML runtime (PyTorch)": the parent-liveness watchdog polls the stdin pipe instead of leaving a read pending, which deadlocked numpy's OpenBLAS initializer (#1955)
|
||||
- Voice synthesis progress no longer races to a fabricated 95%; it stays indeterminate until the active generation path reports real progress (#1907) — thanks @psiberfunk!
|
||||
- Long audiobook chapters now use the same device- and text-length-aware synthesis timeout as other TTS routes (#1910) — thanks @psiberfunk!
|
||||
|
||||
|
||||
@@ -308,6 +308,13 @@ _CONTEXT_FREE_HINT_CLASSES = frozenset({
|
||||
# a Windows virtual-memory setting rather than a connectivity problem, and
|
||||
# the detailed hint we already had for it never reached them.
|
||||
"WINDOWS_PAGING_FILE_TOO_SMALL",
|
||||
# Its trigger is a VoiceStudio-authored sentence — "the TTS model cache
|
||||
# for … is incomplete" plus "could not be auto-repaired" / "weights
|
||||
# missing" — so it cannot be produced by an unrelated library. The 500
|
||||
# handler is the surface a corrupt cache actually reaches, and dropping
|
||||
# its hint there would leave the user with no way to know a redownload
|
||||
# is the fix.
|
||||
"MODEL_CACHE_CORRUPT",
|
||||
})
|
||||
|
||||
|
||||
|
||||
@@ -4,7 +4,14 @@ from __future__ import annotations
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from typing import BinaryIO, Callable
|
||||
import time
|
||||
from typing import Any, BinaryIO, Callable, Optional
|
||||
|
||||
# Poll cadence for the Windows pipe watcher. Exit latency after the desktop
|
||||
# closes its end is bounded by this; the desktop's own kill-on-close Job is the
|
||||
# hard backstop, so a quarter second is plenty and costs nothing measurable.
|
||||
WINDOWS_PIPE_POLL_INTERVAL_S = 0.25
|
||||
_FILE_TYPE_PIPE = 3 # winbase.h FILE_TYPE_PIPE
|
||||
|
||||
|
||||
def _watch_parent_pipe(reader: BinaryIO, exit_process: Callable[[int], None]) -> None:
|
||||
@@ -18,6 +25,64 @@ def _watch_parent_pipe(reader: BinaryIO, exit_process: Callable[[int], None]) ->
|
||||
exit_process(0)
|
||||
|
||||
|
||||
def _watch_parent_pipe_handle(
|
||||
handle: int,
|
||||
exit_process: Callable[[int], None],
|
||||
*,
|
||||
peek: Optional[Callable[[int], Any]] = None,
|
||||
read_file: Optional[Callable[[int, int], Any]] = None,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
interval: float = WINDOWS_PIPE_POLL_INTERVAL_S,
|
||||
) -> None:
|
||||
"""Windows twin of :func:`_watch_parent_pipe` that never leaves a read
|
||||
pending on the pipe.
|
||||
|
||||
A synchronous ``ReadFile`` parked on the stdin pipe — whether issued through
|
||||
the C runtime's ``read()`` or straight to the kernel — deadlocks the
|
||||
OpenBLAS DLL initializer that ``import torch`` reaches (numpy's
|
||||
``_multiarray_umath``) in the startup worker: every desktop-spawned backend
|
||||
on Windows froze at "Loading ML runtime (PyTorch)" while the identical
|
||||
command from a terminal, with no stdin pipe and no watchdog, started in
|
||||
seconds. A thread that merely sleeps does not trigger it; only the pending
|
||||
read on that pipe does. So instead of blocking in a read, poll with
|
||||
``PeekNamedPipe``: it returns immediately, holds no I/O on the file object,
|
||||
drains any keepalive bytes the desktop might write, and fails with
|
||||
``ERROR_BROKEN_PIPE`` the moment the desktop closes its end — which is the
|
||||
same EOF signal the POSIX reader gets.
|
||||
"""
|
||||
if peek is None or read_file is None:
|
||||
import _winapi # Windows-only stdlib module; the caller gates on the platform
|
||||
|
||||
peek = peek or _winapi.PeekNamedPipe
|
||||
read_file = read_file or _winapi.ReadFile
|
||||
try:
|
||||
while True:
|
||||
available, _ = peek(handle)
|
||||
if available:
|
||||
# Bytes are already buffered, so this read cannot block.
|
||||
read_file(handle, available)
|
||||
else:
|
||||
sleep(interval)
|
||||
except OSError:
|
||||
# ERROR_BROKEN_PIPE (109) is how the closed parent end surfaces here.
|
||||
pass
|
||||
exit_process(0)
|
||||
|
||||
|
||||
def _windows_pipe_handle(reader: Any) -> Optional[int]:
|
||||
"""The OS handle behind ``reader`` when it is a pipe, else None."""
|
||||
try:
|
||||
import msvcrt
|
||||
import _winapi
|
||||
|
||||
handle = msvcrt.get_osfhandle(reader.fileno())
|
||||
if _winapi.GetFileType(handle) != _FILE_TYPE_PIPE:
|
||||
return None
|
||||
return handle
|
||||
except (OSError, ValueError, AttributeError, ImportError):
|
||||
return None
|
||||
|
||||
|
||||
def arm_desktop_parent_watchdog() -> bool:
|
||||
"""Use stdin EOF as an unforgeable parent-liveness signal for desktop runs."""
|
||||
if os.environ.get("OMNIVOICE_DESKTOP_CONTAINED") != "1":
|
||||
@@ -25,9 +90,18 @@ def arm_desktop_parent_watchdog() -> bool:
|
||||
reader = getattr(sys.stdin, "buffer", None)
|
||||
if reader is None:
|
||||
return False
|
||||
target: Callable[..., None] = _watch_parent_pipe
|
||||
args: tuple = (reader, os._exit)
|
||||
if os.name == "nt":
|
||||
handle = _windows_pipe_handle(reader)
|
||||
if handle is not None:
|
||||
target = _watch_parent_pipe_handle
|
||||
args = (handle, os._exit)
|
||||
# A non-pipe stdin (file, NUL) cannot have a read pending against a
|
||||
# pipe file object, so the blocking reader stays correct there.
|
||||
threading.Thread(
|
||||
target=_watch_parent_pipe,
|
||||
args=(reader, os._exit),
|
||||
target=target,
|
||||
args=args,
|
||||
name="desktop-parent-watchdog",
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
@@ -94,6 +94,16 @@ def stream_generation_failure(error: BaseException | object) -> dict[str, object
|
||||
replace the failure being diagnosed.
|
||||
"""
|
||||
payload = stream_failure("generation_failed")
|
||||
if isinstance(error, BaseException):
|
||||
# The exception's TYPE NAME, never its message. Two failures that both
|
||||
# render the floor message "Generation failed. Check the selected
|
||||
# engine and try again." are indistinguishable in an auto-filed report,
|
||||
# so every unclassified streaming failure arrives as the same issue and
|
||||
# none of them can be triaged (#1800). A class name is VoiceStudio-safe
|
||||
# by the same reasoning that already puts it on the wire as
|
||||
# `error_class` in the dub routes and on the analytics allowlist: it is
|
||||
# a Python type, not user text, and no substring of `error` is copied.
|
||||
payload["error_class"] = type(error).__name__
|
||||
try:
|
||||
enriched = public_exception_response(error, fallback=str(payload["detail"]))
|
||||
except Exception:
|
||||
@@ -149,12 +159,31 @@ def public_exception_response(error: BaseException, *, fallback: str) -> dict[st
|
||||
Classification may inspect the private diagnostic locally, but response
|
||||
values come exclusively from VoiceStudio-owned constants. No substring of
|
||||
``error`` is copied into the payload.
|
||||
|
||||
Every caller is a CONTEXT-FREE surface — the global 500 handler, the
|
||||
streaming generate error frame, the dub GPU-OOM 503 — so the topic is
|
||||
filtered through ``failure._CONTEXT_FREE_HINT_CLASSES`` before its hint is
|
||||
attached. Without that filter a topic whose trigger is a generic phrase
|
||||
stamps a confidently wrong remediation on an unrelated failure: #1943 is a
|
||||
macOS mlx-audio TTS 500 that came back advising the user that "the
|
||||
connection to the video server dropped mid-download", because
|
||||
VIDEO_DOWNLOAD_NETWORK triggers on a bare "timed out" / "connection
|
||||
reset". The allowlist already existed and already named that class as the
|
||||
example of what must not appear here; only :func:`failure.append_hint`
|
||||
honoured it, and this helper replaced ``append_hint`` on the 500 path
|
||||
without carrying the rule across.
|
||||
|
||||
HF_MIRROR_UNREACHABLE is allowed alongside it: its hint is dynamic (it
|
||||
names the configured mirror) and its trigger requires that a mirror is
|
||||
configured at all, so it cannot fire on an unrelated failure (#874).
|
||||
"""
|
||||
from core.failure import classify, public_hint_for_topic
|
||||
from core.failure import _CONTEXT_FREE_HINT_CLASSES, classify, public_hint_for_topic
|
||||
|
||||
try:
|
||||
topic = classify(str(error))
|
||||
hint = public_hint_for_topic(topic)
|
||||
if topic and topic not in _CONTEXT_FREE_HINT_CLASSES and topic != "HF_MIRROR_UNREACHABLE":
|
||||
topic = ""
|
||||
hint = public_hint_for_topic(topic) if topic else ""
|
||||
except Exception:
|
||||
topic = ""
|
||||
hint = ""
|
||||
|
||||
@@ -329,6 +329,31 @@ describe('streamGenerateSpeech', () => {
|
||||
expect(FakeAudioContext.instances[0].state).toBe('closed');
|
||||
});
|
||||
|
||||
it('carries the backend error class from the error frame (#1800)', async () => {
|
||||
// Every unclassified engine failure renders the same floor message, so
|
||||
// the auto-filed reports were byte-identical and none could be triaged.
|
||||
// The class name is the only thing that separates them.
|
||||
apiFetch.mockResolvedValue(
|
||||
ndjsonResponse([
|
||||
startEvent(3),
|
||||
chunkEvent(0),
|
||||
{ type: 'error', detail: 'Generation failed.', error_class: 'MemoryError' },
|
||||
]),
|
||||
);
|
||||
await expect(streamGenerateSpeech(new FormData(), {})).rejects.toMatchObject({
|
||||
errorClass: 'MemoryError',
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves the error class null when the frame omits it', async () => {
|
||||
apiFetch.mockResolvedValue(
|
||||
ndjsonResponse([startEvent(3), chunkEvent(0), { type: 'error', detail: 'boom' }]),
|
||||
);
|
||||
await expect(streamGenerateSpeech(new FormData(), {})).rejects.toMatchObject({
|
||||
errorClass: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('carries the retryable marker from a GPU-timeout error frame (#1190)', async () => {
|
||||
// A retryable failure means the backend already spent the full budget on
|
||||
// this text and the abandoned job still holds the device — useTTS uses
|
||||
|
||||
@@ -362,11 +362,16 @@ export async function buildBugReportUrl({ title = '[Bug] ', error } = {}) {
|
||||
msg.length > MAX_MSG_CHARS ? `${msg.slice(0, MAX_MSG_CHARS)}\n… (truncated)` : msg;
|
||||
let stack = error?.stack ? scrubText(error.stack) : '';
|
||||
if (stack.length > MAX_STACK_CHARS) stack = `${stack.slice(0, MAX_STACK_CHARS)}\n… (truncated)`;
|
||||
// A generic failure message plus a stack of minified bundle frames is
|
||||
// the same report every time; the backend class name is what separates
|
||||
// one unclassified engine failure from another (#1800).
|
||||
const klass = typeof error?.errorClass === 'string' ? scrubText(error.errorClass) : '';
|
||||
errorSection.push(
|
||||
'## Error',
|
||||
'',
|
||||
'```',
|
||||
msgForBody,
|
||||
...(klass ? [`Backend error class: ${klass}`] : []),
|
||||
...(stack && stack !== msgForBody ? [stack] : []),
|
||||
'```',
|
||||
'',
|
||||
|
||||
@@ -117,6 +117,18 @@ describe('buildBugReportUrl', () => {
|
||||
expect(body).not.toContain('/Users/alice');
|
||||
});
|
||||
|
||||
it('records the backend error class so identical messages differ (#1800)', async () => {
|
||||
const err = new Error('Generation failed. Check the selected engine and try again.');
|
||||
err.errorClass = 'MemoryError';
|
||||
const body = decodeURIComponent(await buildBugReportUrl({ error: err }));
|
||||
expect(body).toContain('Backend error class: MemoryError');
|
||||
});
|
||||
|
||||
it('omits the class line when the failure carries none', async () => {
|
||||
const body = decodeURIComponent(await buildBugReportUrl({ error: new Error('plain failure') }));
|
||||
expect(body).not.toContain('Backend error class');
|
||||
});
|
||||
|
||||
it('seeds the title with the error message', async () => {
|
||||
const url = await buildBugReportUrl({ error: new Error('synthesis exploded') });
|
||||
expect(decodeURIComponent(url)).toContain('[Bug] synthesis exploded');
|
||||
|
||||
@@ -80,6 +80,11 @@ export class StreamingPreviewError extends Error {
|
||||
this.retryable = opts?.retryable === true;
|
||||
this.retryAfter = opts?.retryAfter ?? null;
|
||||
this.terminal = opts?.terminal === true;
|
||||
// The backend exception TYPE behind an otherwise generic failure. The
|
||||
// floor message is identical for every unclassified engine failure, so
|
||||
// without this an auto-filed report cannot be told apart from any
|
||||
// other (#1800). Never the exception message — only its class name.
|
||||
this.errorClass = opts?.errorClass || null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -404,6 +409,7 @@ async function _streamGenerateSpeech(
|
||||
retryable: ev.retryable === true,
|
||||
retryAfter: ev.retry_after ?? null,
|
||||
terminal,
|
||||
errorClass: ev.error_class || null,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -4,6 +4,8 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
def test_parent_pipe_eof_exits_cleanly():
|
||||
from core.parent_liveness import _watch_parent_pipe
|
||||
|
||||
@@ -53,3 +55,207 @@ def test_desktop_child_exits_when_parent_closes_stdin():
|
||||
if child.poll() is None:
|
||||
child.kill()
|
||||
child.wait()
|
||||
|
||||
|
||||
# ── Windows: the watchdog must never leave a read pending on the stdin pipe ──
|
||||
# A synchronous ReadFile parked on the desktop-owned stdin pipe (via the C
|
||||
# runtime's read() or straight to the kernel) deadlocked the OpenBLAS DLL
|
||||
# initializer that `import torch` reaches in the startup worker: every
|
||||
# desktop-spawned backend on Windows froze at "Loading ML runtime" while the
|
||||
# same command from a terminal (no stdin pipe, no watchdog) started in seconds.
|
||||
# A thread that only sleeps is harmless; the pending read is the trigger. The
|
||||
# Windows watcher therefore polls PeekNamedPipe and reads only bytes that are
|
||||
# already buffered, so no I/O is ever outstanding on that file object.
|
||||
|
||||
|
||||
def _peek_sequence(events):
|
||||
"""`events` items: int → bytes available; OSError → broken pipe."""
|
||||
it = iter(events)
|
||||
|
||||
def peek(handle):
|
||||
ev = next(it)
|
||||
if isinstance(ev, BaseException):
|
||||
raise ev
|
||||
return (ev, 0)
|
||||
|
||||
return peek
|
||||
|
||||
|
||||
def test_windows_watcher_exits_on_broken_pipe_without_blocking_reads():
|
||||
from core.parent_liveness import _watch_parent_pipe_handle
|
||||
|
||||
reads, sleeps, exits = [], [], []
|
||||
_watch_parent_pipe_handle(
|
||||
7,
|
||||
exits.append,
|
||||
peek=_peek_sequence([0, 0, OSError(109, "The pipe has been ended")]),
|
||||
read_file=lambda h, n: reads.append((h, n)),
|
||||
sleep=sleeps.append,
|
||||
interval=0.01,
|
||||
)
|
||||
assert exits == [0]
|
||||
assert reads == [] # nothing buffered → never a read, let alone a pending one
|
||||
assert sleeps == [0.01, 0.01]
|
||||
|
||||
|
||||
def test_windows_watcher_drains_only_buffered_bytes_until_eof():
|
||||
from core.parent_liveness import _watch_parent_pipe_handle
|
||||
|
||||
reads, exits = [], []
|
||||
_watch_parent_pipe_handle(
|
||||
7,
|
||||
exits.append,
|
||||
peek=_peek_sequence([3, 0, 1, OSError(109, "The pipe has been ended")]),
|
||||
read_file=lambda h, n: reads.append((h, n)) or (b"x" * n, 0),
|
||||
sleep=lambda s: None,
|
||||
)
|
||||
assert exits == [0]
|
||||
# Reads are sized to exactly what PeekNamedPipe reported, so they return
|
||||
# immediately instead of parking on the pipe.
|
||||
assert reads == [(7, 3), (7, 1)]
|
||||
|
||||
|
||||
def test_windows_watchdog_arms_pipe_poller_on_pipe_stdin(monkeypatch):
|
||||
import types
|
||||
import core.parent_liveness as pl
|
||||
|
||||
monkeypatch.setenv("OMNIVOICE_DESKTOP_CONTAINED", "1")
|
||||
monkeypatch.setattr(pl.os, "name", "nt")
|
||||
|
||||
class FakeBuffer:
|
||||
def fileno(self):
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(pl.sys, "stdin", types.SimpleNamespace(buffer=FakeBuffer()))
|
||||
monkeypatch.setitem(sys.modules, "msvcrt", types.SimpleNamespace(get_osfhandle=lambda fd: 0xABC))
|
||||
monkeypatch.setitem(sys.modules, "_winapi", types.SimpleNamespace(GetFileType=lambda h: 3))
|
||||
|
||||
started = {}
|
||||
|
||||
class FakeThread:
|
||||
def __init__(self, *, target, args, name, daemon):
|
||||
started.update(target=target, args=args, daemon=daemon)
|
||||
|
||||
def start(self):
|
||||
started["started"] = True
|
||||
|
||||
monkeypatch.setattr(pl.threading, "Thread", FakeThread)
|
||||
assert pl.arm_desktop_parent_watchdog() is True
|
||||
assert started["started"] is True
|
||||
assert started["target"] is pl._watch_parent_pipe_handle
|
||||
assert started["args"] == (0xABC, os._exit)
|
||||
assert started["daemon"] is True
|
||||
|
||||
|
||||
def test_windows_watchdog_keeps_blocking_reader_for_non_pipe_stdin(monkeypatch):
|
||||
"""A file/NUL stdin has no pipe file object to deadlock against, and a
|
||||
blocking read on it returns EOF promptly — keep the shared reader there."""
|
||||
import types
|
||||
import core.parent_liveness as pl
|
||||
|
||||
monkeypatch.setenv("OMNIVOICE_DESKTOP_CONTAINED", "1")
|
||||
monkeypatch.setattr(pl.os, "name", "nt")
|
||||
|
||||
class FakeBuffer:
|
||||
def fileno(self):
|
||||
return 0
|
||||
|
||||
fake_stdin = types.SimpleNamespace(buffer=FakeBuffer())
|
||||
monkeypatch.setattr(pl.sys, "stdin", fake_stdin)
|
||||
monkeypatch.setitem(sys.modules, "msvcrt", types.SimpleNamespace(get_osfhandle=lambda fd: 0xABC))
|
||||
monkeypatch.setitem(sys.modules, "_winapi", types.SimpleNamespace(GetFileType=lambda h: 1)) # FILE_TYPE_DISK
|
||||
|
||||
started = {}
|
||||
|
||||
class FakeThread:
|
||||
def __init__(self, *, target, args, name, daemon):
|
||||
started.update(target=target, args=args)
|
||||
|
||||
def start(self):
|
||||
started["started"] = True
|
||||
|
||||
monkeypatch.setattr(pl.threading, "Thread", FakeThread)
|
||||
assert pl.arm_desktop_parent_watchdog() is True
|
||||
assert started["target"] is pl._watch_parent_pipe
|
||||
assert started["args"] == (fake_stdin.buffer, os._exit)
|
||||
|
||||
|
||||
# ── Windows integration: the real backend must get past the ML import ────────
|
||||
# The deadlock needs the real startup: numpy's OpenBLAS DLL loaded through
|
||||
# `import torch` in the startup worker while the desktop watchdog is armed on a
|
||||
# piped stdin. Smaller reproductions (a pending read + `import torch` in a bare
|
||||
# child) do not trigger it, so this spawns the actual backend exactly as the
|
||||
# desktop shell does. It fails on the pre-fix watchdog by timing out in the
|
||||
# "ml_imports" step and passes in well under a minute on the fix. Windows-only:
|
||||
# CI's backend job runs on Linux, so this is exercised on Windows dev machines.
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="desktop stdin-pipe watchdog deadlock is Windows-only")
|
||||
def test_desktop_spawned_backend_gets_past_ml_imports_on_windows(tmp_path):
|
||||
pytest.importorskip("torch")
|
||||
pytest.importorskip("uvicorn")
|
||||
import json
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
port = probe.getsockname()[1]
|
||||
|
||||
root = Path(__file__).parents[3]
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"OMNIVOICE_DESKTOP_CONTAINED": "1", # arms the watchdog on the stdin pipe
|
||||
"OMNIVOICE_PORT": str(port),
|
||||
"OMNIVOICE_DATA_DIR": str(tmp_path / "data"),
|
||||
"OMNIVOICE_CACHE_DIR": str(tmp_path / "hf_cache"),
|
||||
"HF_HUB_CACHE": str(tmp_path / "hf_cache"), # never the developer's populated cache
|
||||
"HF_HUB_OFFLINE": "1",
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PYTHONUTF8": "1",
|
||||
}
|
||||
)
|
||||
env.pop("PYTHONPATH", None)
|
||||
child = subprocess.Popen(
|
||||
[sys.executable, "-m", "uvicorn", "main:app", "--app-dir", "backend",
|
||||
"--host", "127.0.0.1", "--port", str(port)],
|
||||
cwd=root,
|
||||
env=env,
|
||||
stdin=subprocess.PIPE, # the desktop shell keeps this open and never writes
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
for stream in (child.stdout, child.stderr):
|
||||
threading.Thread(target=lambda s=stream: s.read(), daemon=True).start()
|
||||
seen = []
|
||||
try:
|
||||
deadline = time.time() + 180
|
||||
while time.time() < deadline:
|
||||
if child.poll() is not None:
|
||||
pytest.fail(f"backend exited early with {child.returncode}; progress seen: {seen[-3:]}")
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/startup/progress", timeout=2) as resp:
|
||||
progress = json.load(resp)
|
||||
except (OSError, ValueError):
|
||||
time.sleep(1)
|
||||
continue
|
||||
status = progress.get("status")
|
||||
seen.append((status, progress.get("step")))
|
||||
done = {s["id"] for s in progress.get("steps", []) if s.get("state") == "done"}
|
||||
# Positive evidence only: the ML import step finished, or startup is
|
||||
# ready. Any other terminal state is the failure this test exists for.
|
||||
if "ml_imports" in done or status == "ready":
|
||||
return
|
||||
if status != "starting" or progress.get("error"):
|
||||
pytest.fail(
|
||||
f"backend startup reported status={status!r} error={progress.get('error')!r}; "
|
||||
f"progress seen: {seen[-3:]}"
|
||||
)
|
||||
time.sleep(1)
|
||||
pytest.fail(f"backend never got past ml_imports in 180s (deadlocked watchdog?); last progress: {seen[-3:]}")
|
||||
finally:
|
||||
child.kill()
|
||||
child.wait()
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
"""#1943 — a context-free surface must not attach a stage-specific hint.
|
||||
|
||||
A macOS mlx-audio text-to-speech failure came back as a 500 reading "… The
|
||||
connection to the video server dropped mid-download (often a transient
|
||||
CDN/network blip …)". No video was involved. VIDEO_DOWNLOAD_NETWORK triggers
|
||||
on bare phrases like "timed out" and "connection reset", so any unrelated
|
||||
failure whose message contains one is handed a confidently wrong remediation.
|
||||
|
||||
failure._CONTEXT_FREE_HINT_CLASSES already existed for this, and its own
|
||||
comment names VIDEO_DOWNLOAD_NETWORK as the class that must never appear on a
|
||||
context-free surface. Only append_hint honoured it; public_exception_response
|
||||
replaced append_hint on the 500 path without carrying the rule across.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from core.failure import _CONTEXT_FREE_HINT_CLASSES, classify
|
||||
from core.public_errors import public_exception_response, stream_generation_failure
|
||||
|
||||
# Phrases that really do classify as the video-download class but say nothing
|
||||
# about a video when they arrive from a model load or a synthesis run.
|
||||
_GENERIC_NETWORK_WORDING = [
|
||||
"operation timed out",
|
||||
"connection reset by peer",
|
||||
"broken pipe",
|
||||
"remote end closed connection without response",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("message", _GENERIC_NETWORK_WORDING)
|
||||
def test_no_video_hint_on_a_context_free_failure(message):
|
||||
assert classify(message) == "VIDEO_DOWNLOAD_NETWORK" # still classified...
|
||||
payload = public_exception_response(RuntimeError(message), fallback="Internal error.")
|
||||
# ...but its hint must not reach a surface that does not know the stage.
|
||||
assert payload["detail"] == "Internal error."
|
||||
assert "video server" not in payload["detail"]
|
||||
assert "docs_topic" not in payload
|
||||
|
||||
|
||||
@pytest.mark.parametrize("message", _GENERIC_NETWORK_WORDING)
|
||||
def test_no_video_hint_on_the_streaming_generate_frame(message):
|
||||
payload = stream_generation_failure(RuntimeError(message))
|
||||
assert "video" not in str(payload["detail"]).lower()
|
||||
assert "docs_topic" not in payload
|
||||
|
||||
|
||||
def test_allowlisted_classes_keep_their_hint():
|
||||
payload = public_exception_response(
|
||||
RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB"),
|
||||
fallback="Internal error.",
|
||||
)
|
||||
assert payload.get("docs_topic") == "GPU_OOM"
|
||||
assert payload["detail"] != "Internal error."
|
||||
|
||||
|
||||
def test_windows_paging_file_keeps_its_hint():
|
||||
# An allowlisted class whose trigger is unmistakable — the regression guard
|
||||
# for over-filtering, which would silently strip every useful remediation.
|
||||
payload = public_exception_response(
|
||||
OSError("[WinError 1455] The paging file is too small for this operation to complete"),
|
||||
fallback="Internal error.",
|
||||
)
|
||||
assert payload.get("docs_topic") == "WINDOWS_PAGING_FILE_TOO_SMALL"
|
||||
|
||||
|
||||
def test_the_filter_matches_the_documented_allowlist():
|
||||
# Pins the contract itself: the class the allowlist's own comment calls out
|
||||
# as unsafe must not be a member.
|
||||
assert "VIDEO_DOWNLOAD_NETWORK" not in _CONTEXT_FREE_HINT_CLASSES
|
||||
assert "GPU_OOM" in _CONTEXT_FREE_HINT_CLASSES
|
||||
@@ -0,0 +1,53 @@
|
||||
"""#1800 — an unclassified streaming failure must carry its exception class.
|
||||
|
||||
Every engine failure the taxonomy cannot classify renders the same floor
|
||||
message, "Generation failed. Check the selected engine and try again." The
|
||||
auto bug reporter puts that message and a stack of minified bundle frames into
|
||||
the issue, so roughly a dozen separate faults arrived as byte-identical,
|
||||
untriageable reports. The exception's TYPE NAME is what separates them.
|
||||
|
||||
It is the type name only. No substring of the exception message is copied, so
|
||||
the response-safety contract in tests/test_response_safety.py still holds.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from core.public_errors import stream_failure, stream_generation_failure
|
||||
|
||||
|
||||
def test_unclassified_failure_carries_its_exception_class():
|
||||
payload = stream_generation_failure(RuntimeError("CUDA error: device-side assert"))
|
||||
assert payload["code"] == "generation_failed"
|
||||
assert payload["error_class"] == "RuntimeError"
|
||||
|
||||
|
||||
def test_two_unrelated_failures_are_distinguishable():
|
||||
# The whole point: these used to be the same report.
|
||||
a = stream_generation_failure(RuntimeError("boom"))
|
||||
b = stream_generation_failure(MemoryError("out of memory"))
|
||||
assert a["detail"] == b["detail"]
|
||||
assert a["error_class"] != b["error_class"]
|
||||
|
||||
|
||||
def test_the_exception_message_is_never_copied():
|
||||
secret = "C:/Users/someone/private-voice-sample.wav"
|
||||
payload = stream_generation_failure(FileNotFoundError(secret))
|
||||
assert payload["error_class"] == "FileNotFoundError"
|
||||
for value in payload.values():
|
||||
assert secret not in str(value)
|
||||
assert "someone" not in str(value)
|
||||
|
||||
|
||||
def test_a_non_exception_gets_no_class():
|
||||
# dict(...) of the plain floor payload, unchanged — nothing to name.
|
||||
payload = stream_generation_failure("not an exception")
|
||||
assert "error_class" not in payload
|
||||
assert payload["detail"] == stream_failure("generation_failed")["detail"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exc", [RuntimeError("x"), ValueError("y"), OSError("z")])
|
||||
def test_class_is_present_alongside_a_classified_hint(exc):
|
||||
# Enrichment and the class name are independent: adding one must not drop
|
||||
# the other, whichever branch the taxonomy takes.
|
||||
payload = stream_generation_failure(exc)
|
||||
assert payload["error_class"] == type(exc).__name__
|
||||
assert payload["detail"]
|
||||
Reference in New Issue
Block a user