Merge remote-tracking branch 'origin/pr/1942' into land/queue1

# Conflicts:
#	CHANGELOG.md
This commit is contained in:
Palash Debnath
2026-09-09 12:25:35 -07:00
10 changed files with 506 additions and 29 deletions
+8
View File
@@ -13,6 +13,8 @@ the frozen-backend fallback mirror it for their toolchains.
- Transcriptions dictation wakes the desktop recorder, presents one contextual start action, and centers its microphone icon with the label (#1902)
- Apple Silicon now shows one canonical OmniVoice choice in the engine picker while retaining its automatic crash-isolated sidecar runtime (#1913)
- Validate current-user Windows installers under a standard account on hosted runners (#1883)
- Model downloads survive a flaky connection instead of restarting from zero (#1940)
- `bun run dev` recovers on Windows instead of demanding Task Manager (#1941)
- The desktop app builds and opens from a fresh clone again (#1818) — thanks @flutterkage2k!
- GPUs with less VRAM than the engine needs no longer get half the compute-time budget a CPU gets (#1806) — thanks @VishvakR!
@@ -66,6 +68,12 @@ the frozen-backend fallback mirror it for their toolchains.
- The first-run Activity log counts every line instead of freezing at 200 while the install is still running, and Copy now hands back the whole run rather than the last 200 lines (#1847) — thanks @psiberfunk!
- A first-run failure that happened early in a long install keeps its specific advice, instead of falling back to the generic retry hint once the log scrolled past 200 lines (#1847) — thanks @psiberfunk!
- Opening the log panel no longer clips the Launchpad's heading and slides the feature cards up over it — the page scrolls instead of squashing itself (#1859) — thanks @psiberfunk!
- Segmented model downloads split files into 16 MB ranges instead of one range per connection, so a dropped connection refetches one range rather than restarting the file (#1940)
- The download accelerator is kept across retries after a transient network failure and resumes from its manifest, instead of falling back to a from-zero `snapshot_download` (#1940)
- `dev-backend.mjs` stops the backend by process tree on Windows, so an orphaned uvicorn no longer holds port 3900 and turns a source reload into three phantom crashes (#1941)
- `clear-dev-ports.mjs` can free a stuck development port on Windows again, bound to the inspected process instance so a recycled pid is never terminated (#1941)
- Checkout-ownership matching no longer resolves POSIX paths with the host's separator, which made the guard's own test fail on Windows (#1941)
- Install documentation help now prints correctly on Windows consoles using legacy encodings (#1815) — thanks @dajiaohuang!
- Saved transcriptions with missing or invalid timestamps now remain readable (#1799) — thanks @yunaremaia and @tvbht!
- Copying a saved transcription now uses the shared clipboard helper and reports failed copies accurately (#1803) — thanks @tvbht!
+64 -7
View File
@@ -381,6 +381,51 @@ def _is_retryable_download_error(exc: BaseException) -> bool:
return is_hf_connectivity_error(str(exc))
def _segmented_retry_plan(
exc: BaseException, attempt: int, max_attempts: int
) -> tuple[bool, bool]:
"""What to do after the segmented accelerator failed on ``attempt``.
Returns ``(disable_accelerator, reraise)``.
A dropped connection is not the accelerator's fault, so the error is
re-raised for the outer retry: the next attempt re-enters
:func:`_segmented_snapshot`, which resumes from the ``.part`` manifest.
Falling straight through to ``snapshot_download`` instead would finish the
install from a separate ``.incomplete`` file and strand that manifest — the
restart-from-zero this exists to prevent.
The final attempt is always reserved for the plain path, so the accelerator
can never be the reason an install fails outright. The two flags are
decoupled for that handover: the attempt that exhausts the accelerator still
re-raises, so the plain path starts on the LAST attempt rather than the
second-to-last. Disabling and falling through in the same attempt would
abandon the resumable manifest one attempt early and restart through a
separate file — which is the failure this whole helper exists to avoid.
"""
if not _is_retryable_download_error(exc):
return True, False # the accelerator cannot work here at all
if attempt >= max_attempts:
# Nothing left to hand over to: take the plain path now rather than
# re-raising out of the loop with no fallback ever tried.
return True, False
return attempt >= max_attempts - 1, True
def _segmented_retry_note(disable: bool, reraise: bool) -> str:
"""How to describe the outcome of :func:`_segmented_retry_plan` in the log.
Three distinct states, and reading only ``disable`` conflates two of them:
the attempt that exhausts the accelerator is disabled AND re-raises, so the
fallback starts on the NEXT attempt, not this one.
"""
if not disable:
return "kept for the next attempt (resumes from its manifest)"
if reraise:
return "exhausted — retrying once more, then snapshot_download takes over"
return "disabled for this install — falling back to snapshot_download now"
@router.post("/models/install")
async def install_model(req: InstallModelRequest):
"""Download one HF repo snapshot; progress goes through the shared
@@ -557,6 +602,11 @@ async def install_model(req: InstallModelRequest):
_max_attempts = 5
_attempt = 0
# The accelerator is retried across attempts so its manifest-based
# resume actually gets used; it is disabled for the rest of the
# install only when it fails for a reason that is NOT transient
# network trouble (i.e. the accelerator itself is unusable here).
_segmented_off = False
while True:
if req.repo_id in _cancelled:
raise _InstallCancelled()
@@ -564,12 +614,13 @@ async def install_model(req: InstallModelRequest):
try:
# Segmented accelerator (FDL-09, default ON): parallel
# byte-range fetch with real live progress, for the
# legacy-LFS path. Any failure falls through to
# snapshot_download — the accelerator can never compromise a
# correct install.
# legacy-LFS path. A failure that is not transient network
# trouble falls through to snapshot_download, and so does the
# install's last attempt — the accelerator can never
# compromise a correct install (see _segmented_retry_plan).
_snapshot_path = None
if (
_attempt == 1
not _segmented_off
and not allow_patterns
and _segmented_enabled()
and not _xet_active()
@@ -583,10 +634,16 @@ async def install_model(req: InstallModelRequest):
except _InstallCancelled:
raise
except Exception as _seg_err:
logger.info(
"segmented download for %s failed (%s); falling back to snapshot_download",
req.repo_id, _seg_err,
_segmented_off, _seg_reraise = _segmented_retry_plan(
_seg_err, _attempt, _max_attempts
)
logger.info(
"segmented download for %s failed (%s); accelerator %s",
req.repo_id, _seg_err,
_segmented_retry_note(_segmented_off, _seg_reraise),
)
if _seg_reraise:
raise
_snapshot_path = None
if _snapshot_path is None:
_snapshot_path = snapshot_download(**dl_kwargs) # nosec B615 -- immutable revision_for pin
+25 -1
View File
@@ -29,6 +29,14 @@ import httpx
_HF_AUTH_HOSTS = ("huggingface.co", "hf.co")
_DEFAULT_CONNECTIONS = 8
_MIN_SEGMENT_BYTES = 4 * 1024 * 1024 # don't split below this — overhead > gain
# Cap on a single segment. Progress is committed to the manifest only when a
# whole segment lands, so the segment size is also the MOST bytes a dropped
# connection can throw away. Sizing segments as size/num_connections made that
# ~100 MB on an 800 MB blob: on a link that drops every ~50 MB no segment ever
# completed, the manifest was never written, and every retry restarted from
# zero (#1224 follow-up). Bounded segments turn the same flaky link into steady
# forward progress.
_MAX_SEGMENT_BYTES = 16 * 1024 * 1024
_READ_CHUNK = 1024 * 1024
@@ -67,8 +75,16 @@ async def _resolve(client: httpx.AsyncClient, url: str, token: Optional[str], ma
def _plan_segments(size: int, num_connections: int) -> list[tuple[int, int]]:
"""Byte ranges to fetch, each at most ``_MAX_SEGMENT_BYTES``.
``num_connections`` controls how many run at once (see the semaphore in
:func:`segmented_download`), NOT how many segments exist — a large file is
split into many bounded segments so each one commits to the manifest
quickly and a dropped connection costs at most one segment.
"""
n = max(1, min(num_connections, max(1, size // _MIN_SEGMENT_BYTES)))
step = -(-size // n) # ceil
step = max(_MIN_SEGMENT_BYTES, min(step, _MAX_SEGMENT_BYTES))
segs = []
start = 0
while start < size:
@@ -143,6 +159,10 @@ async def segmented_download(
_preallocate(part, size)
segments = [s for s in _plan_segments(size, num_connections) if s not in done]
lock = asyncio.Lock()
# Segments are bounded, so a big file yields many more of them than
# there are connections. The semaphore — not the segment count — is
# what keeps concurrency at num_connections.
sem = asyncio.Semaphore(max(1, num_connections))
async def _fetch(seg: tuple[int, int]):
start, end = seg
@@ -169,8 +189,12 @@ async def segmented_download(
done.add(seg)
_save_done(part, size, done)
async def _fetch_limited(seg: tuple[int, int]):
async with sem:
await _fetch(seg)
if segments:
await asyncio.gather(*(_fetch(s) for s in segments))
await asyncio.gather(*(_fetch_limited(s) for s in segments))
# ── verify ──────────────────────────────────────────────────────
actual = os.path.getsize(part)
+13 -1
View File
@@ -38,7 +38,19 @@ To keep that path **fast** despite Xet being off, the app runs a built-in
**multi-connection (segmented) downloader on by default** — it fetches each file
over parallel byte-ranges (IDM/uGet style), so the legacy-LFS path is no longer
single-stream. It reports real live speed/ETA and **falls back to the normal
download on any error**, so it can never compromise a correct install. Adding a
download**, so it can never compromise a correct install.
Ranges are capped at 16 MB each and eight of them are in flight at a time, so a
completed range is committed to a resume manifest every few seconds. On a
connection that drops mid-transfer, only the ranges in flight are refetched: the
attempt is retried and the accelerator resumes from its manifest rather than
starting the file over. An origin that does not serve ranges at all is handled
inside the accelerator as a single stream, not as a failure.
The accelerator is disabled for the rest of the install — handing over to the
plain `snapshot_download` path — when it fails for a reason that is not
transient network trouble, and on the install's final attempt, so it can never
be the reason an install fails outright. Adding a
free Hugging Face token (first-run setup, or Settings → Credentials) makes this
faster still — authenticated downloads get higher rate limits and fewer stalls.
To force the old single-stream path, set `OMNIVOICE_SEGMENTED_DOWNLOAD=0`.
+55 -10
View File
@@ -2,7 +2,7 @@
import { spawnSync } from "node:child_process";
import { readFileSync, readlinkSync } from "node:fs";
import { dirname, resolve, sep } from "node:path";
import path, { dirname, resolve, sep } from "node:path";
import { fileURLToPath } from "node:url";
const DEFAULT_PORTS = [3900, 3901];
@@ -69,7 +69,11 @@ function normalized(value, windows = process.platform === "win32") {
return String(value || "")
.replaceAll("/", "\\")
.toLowerCase();
return resolve(String(value || ""));
// path.posix, not the host resolver: `windows` is an explicit parameter, so
// POSIX normalisation must stay POSIX even when this runs on Windows. The
// host resolver turned "/work/VoiceStudio" into "C:\work\VoiceStudio",
// which then matched nothing in a POSIX command line.
return path.posix.resolve(String(value || ""));
}
export function belongsToCheckout(
@@ -80,7 +84,7 @@ export function belongsToCheckout(
checkoutRoot = CHECKOUT_ROOT,
) {
const root = normalized(checkoutRoot, windows);
const prefix = `${root}${windows ? "\\" : sep}`;
const prefix = `${root}${windows ? "\\" : path.posix.sep}`;
const ownedPath = (value) => {
if (!value) return false;
const path = normalized(value, windows);
@@ -160,7 +164,9 @@ function inspectWindows(pid) {
const script = [
`$p = Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}'`,
"if ($null -ne $p) {",
" $p | Select-Object ProcessId,ExecutablePath,CommandLine,CreationDate | ConvertTo-Json -Compress",
// Started is formatted explicitly (round-trip 'o') so the identity string
// is byte-stable and can be re-compared inside the stop script below.
" $p | Select-Object ProcessId,ExecutablePath,CommandLine,@{n='Started';e={$_.CreationDate.ToUniversalTime().ToString('o')}} | ConvertTo-Json -Compress",
"}",
].join("; ");
const result = spawnSync(
@@ -175,24 +181,63 @@ function inspectWindows(pid) {
if (!result.stdout.trim()) return null;
const info = JSON.parse(result.stdout);
return {
identity: `windows:${info.CreationDate}`,
identity: `windows:${info.Started}`,
owned: belongsToCheckout("", info.CommandLine, info.ExecutablePath, true),
};
}
/**
* Terminate a Windows listener, bound to the process INSTANCE.
*
* `taskkill /pid` targets a reusable PID, so a pid recycled between inspect and
* kill would take an unrelated process down — which is why auto-stop used to be
* refused outright on Windows, leaving `bun run dev` permanently stuck behind
* "stop it in Task Manager and retry" whenever a backend was orphaned. Fetching
* the CIM instance, re-checking its creation timestamp, and terminating THAT
* instance in one PowerShell pass closes the race: the terminate acts on the
* object the check validated, not on a pid looked up again afterwards.
*/
export function stopWindowsProcess(pid, _force, identity, run = spawnSync) {
const expected = String(identity || "").replace(/^windows:/, "");
const script = [
`$p = Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}'`,
"if ($null -eq $p) { exit 0 }",
`if ($p.CreationDate.ToUniversalTime().ToString('o') -ne '${expected}') { exit 3 }`,
// Terminate reports failure through ReturnValue, not through a thrown
// error: discarding it would report success on an access-denied kill.
"$r = Invoke-CimMethod -InputObject $p -MethodName Terminate",
"if ($r.ReturnValue -ne 0) { Write-Output $r.ReturnValue; exit 4 }",
].join("; ");
const result = run(
"powershell.exe",
["-NoProfile", "-NonInteractive", "-Command", script],
{ encoding: "utf8" },
);
if (result.error) throw result.error;
// exit 3 == the pid now belongs to a different process; leave it alone.
if (result.status === 3) return;
if (result.status === 4) {
throw new Error(
`Could not stop process ${pid}: Terminate returned ${String(result.stdout || "").trim()}`,
);
}
if (result.status !== 0) {
throw new Error(`Could not stop process ${pid}`);
}
}
function systemOperations() {
const windows = process.platform === "win32";
return {
// Windows taskkill targets a reusable PID, not the inspected process
// instance. Refuse automatic termination until it can be handle-bound.
canStop: !windows,
canStop: true,
// macOS exposes process start time to ps at one-second resolution. That is
// sufficient for a graceful stop, but not safe proof for SIGKILL escalation.
canForce: process.platform !== "darwin",
listeners: windows ? windowsListeners : unixListeners,
inspect: windows ? inspectWindows : process.platform === "darwin" ? inspectMac : inspectLinux,
stop(pid, force) {
stopUnixProcess(pid, force);
stop(pid, force, identity) {
if (windows) stopWindowsProcess(pid, force, identity);
else stopUnixProcess(pid, force);
},
sleep(ms) {
return new Promise((done) => setTimeout(done, ms));
+34 -4
View File
@@ -21,7 +21,7 @@
// the Windows CreateProcess PATH search — no shell needed).
// ──────────────────────────────────────────────────────────────────────────
import { spawn } from "node:child_process";
import { spawn, spawnSync } from "node:child_process";
import { existsSync, readFileSync, watch } from "node:fs";
import { homedir } from "node:os";
import path from "node:path";
@@ -47,6 +47,29 @@ export const CRASH_RESTART_LIMIT = 3;
export const CRASH_RESTART_WINDOW_MS = 60_000;
export const SOURCE_RELOAD_DEBOUNCE_MS = 250;
/**
* Stop a spawned backend and everything it started.
*
* We spawn `uv`, which spawns uvicorn as its own child. Windows has no signals:
* `child.kill()` maps to TerminateProcess on the DIRECT child only, so killing
* `uv` orphaned the uvicorn grandchild — which kept holding port 3900. The next
* spawn then failed to bind ([Errno 10048]), which the supervisor counted as a
* crash, and three of those tore the whole dev stack down. `taskkill /T` walks
* the tree; POSIX keeps plain signal delivery.
*/
export function killProcessTree(child, sig, { platform = process.platform, run = spawnSync } = {}) {
if (platform !== "win32") {
child.kill(sig);
return "signal";
}
const result = run("taskkill", ["/pid", String(child.pid), "/T", "/F"], { stdio: "ignore" });
if (!result || result.error || result.status !== 0) {
child.kill(sig); // taskkill unavailable or the process already exited
return "fallback";
}
return "taskkill";
}
export function isBackendSourceChange(filename) {
return typeof filename === "string" && filename.toLowerCase().endsWith(".py");
}
@@ -123,6 +146,7 @@ export function createBackendSupervisor({
spawnBackend = () => spawn("uv", uvRunArgs(), { stdio: "inherit" }),
watchBackend = (onChange) =>
watch("backend", { recursive: true }, (_event, filename) => onChange(filename?.toString())),
killBackend = (proc, sig) => killProcessTree(proc, sig),
schedule = setTimeout,
cancelSchedule = clearTimeout,
now = Date.now,
@@ -135,6 +159,10 @@ export function createBackendSupervisor({
let restartTimer = null;
let reloadTimer = null;
let reloadRequested = false;
// True when the reload we asked for was served by a forced tree-kill, which
// reports a non-zero exit and no signal. Only then may such an exit be read
// as "the reload we asked for" instead of "it crashed mid-reload".
let reloadKillForced = false;
let watcher = null;
let crashTimes = [];
let requestedExitCode = null;
@@ -165,7 +193,7 @@ export function createBackendSupervisor({
return;
}
try {
child.kill(sig);
killBackend(child, sig);
} catch {
exit(requestedExitCode ?? 0);
}
@@ -180,7 +208,7 @@ export function createBackendSupervisor({
reloadRequested = true;
report(`[dev-backend] ${filename} changed; reloading backend…`);
try {
child.kill("SIGTERM");
reloadKillForced = killBackend(child, "SIGTERM") === "taskkill";
} catch {
reloadRequested = false;
}
@@ -236,7 +264,9 @@ export function createBackendSupervisor({
if (reloadRequested) {
reloadRequested = false;
const expectedReloadExit = code === 0 || signal === "SIGTERM";
const expectedReloadExit =
code === 0 || signal === "SIGTERM" || reloadKillForced;
reloadKillForced = false;
if (expectedReloadExit) {
start();
return;
+45
View File
@@ -8,6 +8,7 @@ import {
parseSsListeners,
parseWindowsListeners,
stopUnixProcess,
stopWindowsProcess,
} from "../../scripts/clear-dev-ports.mjs";
test("parses only requested Windows TCP listeners", () => {
@@ -154,3 +155,47 @@ test("waits for the listener to disappear after force stop", async () => {
]);
assert.equal(sleeps, 12);
});
// Windows auto-stop is allowed again, but only bound to the process INSTANCE:
// a pid recycled between inspect and kill must never be terminated.
test("windows stop re-checks the process identity before terminating", () => {
const calls = [];
const run = (_exe, args) => {
calls.push(args.at(-1));
return { status: 0 };
};
stopWindowsProcess(4242, false, "windows:2026-09-08T21:25:58.5000000Z", run);
assert.equal(calls.length, 1);
assert.match(calls[0], /ProcessId = 4242/);
assert.match(calls[0], /2026-09-08T21:25:58\.5000000Z/);
assert.match(calls[0], /Invoke-CimMethod -InputObject \$p -MethodName Terminate/);
// The identity must be compared before the terminate, never after.
assert.ok(calls[0].indexOf("-ne '2026") < calls[0].indexOf("Invoke-CimMethod"));
});
test("windows stop leaves a recycled pid alone instead of failing the run", () => {
assert.doesNotThrow(() =>
stopWindowsProcess(4242, false, "windows:whatever", () => ({ status: 3 })),
);
});
test("windows stop surfaces a real termination failure", () => {
assert.throws(
() => stopWindowsProcess(4242, false, "windows:whatever", () => ({ status: 1 })),
/Could not stop process 4242/,
);
});
// Win32_Process.Terminate reports failure through ReturnValue, not by throwing:
// discarding it would report success on an access-denied kill, and the port
// would still be held.
test("windows stop fails when Terminate reports a non-zero ReturnValue", () => {
const run = (_exe, args) => {
assert.match(args.at(-1), /ReturnValue -ne 0/);
return { status: 4, stdout: "2" };
};
assert.throws(
() => stopWindowsProcess(4242, false, "windows:whatever", run),
/Could not stop process 4242: Terminate returned 2/,
);
});
+45 -1
View File
@@ -20,11 +20,12 @@ import {
buildExitBanner,
createBackendSupervisor,
isBackendSourceChange,
killProcessTree,
resolveDataDir,
tailFile,
} from '../../scripts/dev-backend.mjs';
function supervisorHarness() {
function supervisorHarness(overrides = {}) {
const children = [];
const timers = [];
const exits = [];
@@ -64,6 +65,7 @@ function supervisorHarness() {
exit: (code) => exits.push(code),
report: (message) => reports.push(message),
dataDir: () => '/missing-test-data',
...overrides,
});
return {
@@ -282,3 +284,45 @@ test('failure to spawn uv exits immediately instead of looping', () => {
assert.deepEqual(exits, [1]);
assert.match(reports[0], /could not start uv: uv is missing/);
});
// Windows has no signals: killing the spawned `uv` left the uvicorn grandchild
// alive on port 3900, so every restart failed to bind and the supervisor tore
// the stack down after three "crashes".
test('a backend is stopped by process tree on windows and by signal elsewhere', () => {
const child = { pid: 4242, killedWith: null, kill(sig) { this.killedWith = sig; } };
const calls = [];
const run = (exe, args) => {
calls.push([exe, ...args].join(' '));
return { status: 0 };
};
assert.equal(killProcessTree(child, 'SIGTERM', { platform: 'win32', run }), 'taskkill');
assert.deepEqual(calls, ['taskkill /pid 4242 /T /F']);
assert.equal(child.killedWith, null, 'must not also signal the direct child');
assert.equal(killProcessTree(child, 'SIGTERM', { platform: 'linux', run }), 'signal');
assert.equal(calls.length, 1, 'POSIX must not shell out');
assert.equal(child.killedWith, 'SIGTERM');
});
test('a failed taskkill falls back to signalling the direct child', () => {
const child = { pid: 4242, killedWith: null, kill(sig) { this.killedWith = sig; } };
const run = () => ({ status: 128 });
assert.equal(killProcessTree(child, 'SIGTERM', { platform: 'win32', run }), 'fallback');
assert.equal(child.killedWith, 'SIGTERM');
});
test('a forced tree-kill reload restarts instead of reporting a crash', () => {
const harness = supervisorHarness({ killBackend: () => 'taskkill' });
harness.supervisor.start();
harness.watchers[0].onChange('main.py');
harness.runNextTimer();
// taskkill /F reports a non-zero exit and no signal — that is the reload we
// asked for, not a crash.
harness.children[0].emit('exit', 1, null);
assert.doesNotMatch(harness.reports.join('\n'), /BACKEND DIED/);
assert.equal(harness.children.length, 2);
assert.deepEqual(harness.exits, []);
});
+123 -1
View File
@@ -13,7 +13,13 @@ import os
import httpx
import pytest
from services.segmented_download import segmented_download, DownloadCancelled
import services.segmented_download as sd
from services.segmented_download import (
segmented_download,
DownloadCancelled,
_plan_segments,
_MAX_SEGMENT_BYTES,
)
PAYLOAD = bytes((i % 256) for i in range(1_000_000)) # 1 MB deterministic body
@@ -108,3 +114,119 @@ def test_on_bytes_reports_total(tmp_path):
seen = []
_download(_ranged_handler(), dest, expected_size=len(PAYLOAD), on_bytes=lambda d: seen.append(d))
assert sum(seen) == len(PAYLOAD)
# ── #1224 follow-up: bounded segments so a flaky link makes progress ────────
#
# Progress is committed to the manifest only when a WHOLE segment lands, so the
# segment size is also the most bytes a dropped connection can throw away.
# Sizing segments as size/num_connections made that ~100 MB on an 800 MB blob:
# on a link dropping every ~50 MB no segment ever completed, the manifest was
# never written, and every retry restarted from zero.
def test_large_file_is_split_into_bounded_segments():
"""Fail-before: the old plan returned 8 segments of ~100 MB for this size."""
size = 805_665_628 # the k2-fsa/OmniVoice blob that reproduced the stall
segs = _plan_segments(size, 8)
assert max(e - s + 1 for s, e in segs) <= _MAX_SEGMENT_BYTES
assert len(segs) > 8, "segment count must not be capped by num_connections"
# Exact cover: no gap, no overlap, no byte past the end.
assert segs[0][0] == 0 and segs[-1][1] == size - 1
assert all(segs[i][1] + 1 == segs[i + 1][0] for i in range(len(segs) - 1))
def test_small_file_still_single_segment():
"""The cap must not shard tiny files into per-request overhead."""
assert len(_plan_segments(1_000_000, 8)) == 1
def test_concurrency_stays_at_num_connections(tmp_path, monkeypatch):
"""Many bounded segments must not all fire at once.
The handler has to HOLD requests open: a synchronous mock returns before any
other task is scheduled, so nothing ever overlaps and the assertion passes
without exercising the semaphore at all.
"""
monkeypatch.setattr(sd, "_MIN_SEGMENT_BYTES", 16 * 1024)
monkeypatch.setattr(sd, "_MAX_SEGMENT_BYTES", 32 * 1024)
connections = 4
assert len(_plan_segments(len(PAYLOAD), connections)) > connections, (
"test needs more segments than connections"
)
dest = str(tmp_path / "m.bin")
state = {"inflight": 0, "peak": 0}
async def _run():
saturated = asyncio.Event()
async def handler(request: httpx.Request) -> httpx.Response:
if request.method == "HEAD":
return httpx.Response(200, headers={
"content-length": str(len(PAYLOAD)), "accept-ranges": "bytes"})
state["inflight"] += 1
state["peak"] = max(state["peak"], state["inflight"])
try:
if state["inflight"] >= connections:
saturated.set()
# Hold the range open until the pool fills, so overlap is
# observable without sleeping. The timeout keeps an
# over-restrictive semaphore a failure instead of a hang.
try:
await asyncio.wait_for(saturated.wait(), timeout=1)
except asyncio.TimeoutError:
pass
lo, hi = request.headers["range"].replace("bytes=", "").split("-")
return httpx.Response(206, content=PAYLOAD[int(lo):int(hi) + 1])
finally:
state["inflight"] -= 1
async with httpx.AsyncClient(
transport=httpx.MockTransport(handler), follow_redirects=False
) as client:
await segmented_download(
"https://cdn.example.com/f.bin", dest, client=client,
expected_size=len(PAYLOAD), num_connections=connections,
)
asyncio.run(_run())
assert state["peak"] == connections, (
f"expected exactly {connections} ranges in flight, saw {state['peak']}"
)
with open(dest, "rb") as f:
assert f.read() == PAYLOAD
def test_dropped_connection_resumes_from_manifest(tmp_path, monkeypatch):
"""A drop must cost one segment, not the whole file."""
monkeypatch.setattr(sd, "_MIN_SEGMENT_BYTES", 16 * 1024)
monkeypatch.setattr(sd, "_MAX_SEGMENT_BYTES", 32 * 1024)
dest = str(tmp_path / "m.bin")
served = []
fail_after = {"n": 3}
def handler(request: httpx.Request) -> httpx.Response:
if request.method == "HEAD":
return httpx.Response(200, headers={
"content-length": str(len(PAYLOAD)), "accept-ranges": "bytes"})
if fail_after["n"] > 0:
fail_after["n"] -= 1
if fail_after["n"] == 0:
raise httpx.RemoteProtocolError("peer closed connection", request=request)
lo, hi = request.headers["range"].replace("bytes=", "").split("-")
served.append(int(hi) - int(lo) + 1)
return httpx.Response(206, content=PAYLOAD[int(lo):int(hi) + 1])
with pytest.raises(httpx.RemoteProtocolError):
_download(handler, dest, expected_size=len(PAYLOAD), num_connections=2)
assert os.path.exists(dest + ".part.done"), "completed segments must be committed"
before = sum(served)
assert before > 0
_download(handler, dest, expected_size=len(PAYLOAD), num_connections=2)
with open(dest, "rb") as f:
assert f.read() == PAYLOAD
# Resumed, not restarted: total bytes served stay below two full copies.
assert sum(served) < 2 * len(PAYLOAD)
assert sum(served) >= len(PAYLOAD)
+94 -4
View File
@@ -11,20 +11,110 @@ import sys
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "backend"))
from api.routers.setup.download import _segmented_enabled # noqa: E402
def _enabled():
"""Resolve the app module at run time, not at collection."""
from api.routers.setup.download import _segmented_enabled
return _segmented_enabled()
def test_segmented_is_on_by_default(monkeypatch):
monkeypatch.delenv("OMNIVOICE_SEGMENTED_DOWNLOAD", raising=False)
assert _segmented_enabled() is True
assert _enabled() is True
def test_env_override_can_disable(monkeypatch):
monkeypatch.setenv("OMNIVOICE_SEGMENTED_DOWNLOAD", "0")
assert _segmented_enabled() is False
assert _enabled() is False
def test_env_override_truthy_keeps_it_on(monkeypatch):
for val in ("1", "true", "on", "yes"):
monkeypatch.setenv("OMNIVOICE_SEGMENTED_DOWNLOAD", val)
assert _segmented_enabled() is True
assert _enabled() is True
# The accelerator must be re-entered on the NEXT attempt after a dropped
# connection, so it resumes from its .part manifest. Falling straight through to
# snapshot_download in the same attempt finishes the install from a separate
# .incomplete file and strands the manifest — restart-from-zero all over again.
import httpx # noqa: E402
_MAX = 5
def _plan(exc, attempt, max_attempts=_MAX):
"""Resolve the app module at run time, not at collection.
A module-level import of an app module goes stale when an earlier test
pollutes ``sys.modules``.
"""
from api.routers.setup.download import _segmented_retry_plan
return _segmented_retry_plan(exc, attempt, max_attempts)
def _dropped():
return httpx.RemoteProtocolError(
"peer closed connection without sending complete message body"
)
def test_dropped_connection_reraises_so_the_next_attempt_resumes():
for attempt in (1, 2, 3):
disable, reraise = _plan(_dropped(), attempt)
assert reraise is True, f"attempt {attempt} must reach the outer retry"
assert disable is False, f"attempt {attempt} must keep the accelerator"
def test_the_accelerator_keeps_the_second_to_last_attempt():
"""Handover must not start early.
Disabling AND falling through in the same attempt would abandon the
resumable manifest one attempt sooner than needed and restart the file
through a separate `.incomplete`.
"""
disable, reraise = _plan(_dropped(), _MAX - 1)
assert (disable, reraise) == (True, True), (
"the attempt that exhausts the accelerator still re-raises, so the "
"plain path starts on the last attempt, not the second-to-last"
)
def test_final_attempt_takes_the_plain_path_without_reraising():
"""The accelerator can never be the reason an install fails outright."""
assert _plan(_dropped(), _MAX) == (True, False)
assert _plan(_dropped(), 1, 1) == (True, False), "single-attempt install"
def test_a_non_network_failure_disables_the_accelerator_at_once():
"""An accelerator that cannot work here must not burn every retry."""
assert _plan(ValueError("sha256 mismatch"), 1) == (True, False)
def _note(disable, reraise):
from api.routers.setup.download import _segmented_retry_note
return _segmented_retry_note(disable, reraise)
def test_the_log_does_not_promise_a_fallback_that_has_not_happened_yet():
"""The exhausting attempt re-raises, so its fallback is next — not now."""
assert _note(*_plan(_dropped(), _MAX - 1)) == (
"exhausted — retrying once more, then snapshot_download takes over"
)
assert "now" not in _note(*_plan(_dropped(), _MAX - 1))
# The branch that really does hand over says so.
assert _note(*_plan(_dropped(), _MAX)).endswith("snapshot_download now")
assert _note(*_plan(ValueError("sha256 mismatch"), 1)).endswith(
"snapshot_download now"
)
# Still accelerating.
assert _note(*_plan(_dropped(), 1)) == (
"kept for the next attempt (resumes from its manifest)"
)