Merge branch 'codex/consolidate-1809' into codex/pr-queue-integration

# Conflicts:
#	CHANGELOG.md

CHANGELOG.md

@@ -28,6 +28,7 @@ the frozen-backend fallback mirror it for their toolchains.
 - Voice reference preparation reclaims allocator memory before one bounded retry, then reports persistent GPU out-of-memory failures (#1811)
 - `bun run desktop` now opens on a fresh clone: the Vite alias for `@tauri-apps/plugin-dialog` no longer assumes a nested `frontend/node_modules`, which bun's workspace hoisting leaves empty (#1818) — thanks @flutterkage2k!
 
+- Slow backend startups remain running with progress updates, and Retry can interrupt startup reliably (#1809)
 
 ## [0.5.2] — 2026-09-02
 

frontend/src-tauri/src/bootstrap.rs

@@ -265,6 +265,9 @@ pub fn respawn_backend<R: tauri::Runtime>(
     stage: Arc<Mutex<BootstrapStage>>,
     logs: Arc<Mutex<Vec<LogPayload>>>,
 ) {
+    // Before anything reaches for lifecycle ownership: a readiness wait may be
+    // holding it while a slow backend starts (#1791).
+    preempt_backend_wait();
     if let Ok(mut guard) = stage.lock() {
         *guard = BootstrapStage::Checking;
     }
@@ -308,6 +311,7 @@ pub fn with_backend_stopped<R: tauri::Runtime, T>(
     app: &tauri::AppHandle<R>,
     action: impl FnOnce() -> T,
 ) -> Result<T, String> {
+    preempt_backend_wait();
     let state = app.state::<BackendState>();
     let _lifecycle = state.lifecycle.lock().unwrap_or_else(|e| e.into_inner());
     if let Err(error) = stop_backend_locked(app) {
@@ -615,11 +619,18 @@ fn launch_backend_and_wait<R: tauri::Runtime>(
     stage_handle: &Arc<Mutex<BootstrapStage>>,
     first_run_gate: bool,
 ) {
+    // Preserve cancellation arriving while waiting for ownership or preparing launch.
+    let wait_generation = backend_wait_generation();
     let outcome = {
         let state = app.state::<BackendState>();
         let _lifecycle = state.lifecycle.lock().unwrap_or_else(|e| e.into_inner());
-        if backend_stop_requested(app) {
-            log::info!("App is quitting — backend launch cancelled");
+        #[cfg(debug_assertions)]
+        wait_for_tracking_test_gate(
+            "OMNIVOICE_TEST_LAUNCH_LOCKED_ENTERED",
+            "OMNIVOICE_TEST_LAUNCH_LOCKED_RELEASE",
+        );
+        if backend_stop_requested(app) || backend_wait_generation() != wait_generation {
+            log::info!("Backend launch cancelled before preparation");
             LaunchOutcome::Done
         } else {
             match prepare_backend_launch(app, stage_handle) {
@@ -638,10 +649,10 @@ fn launch_backend_and_wait<R: tauri::Runtime>(
                             set_stage(stage_handle, BootstrapStage::AwaitingSetup);
                             LaunchOutcome::Done
                         } else {
-                            spawn_with_supervisor_owner(app, stage_handle)
+                            spawn_with_supervisor_owner(app, stage_handle, wait_generation)
                         }
                     } else {
-                        spawn_with_supervisor_owner(app, stage_handle)
+                        spawn_with_supervisor_owner(app, stage_handle, wait_generation)
                     }
                 }
             }
@@ -659,9 +670,10 @@ fn launch_backend_and_wait<R: tauri::Runtime>(
 fn spawn_with_supervisor_owner<R: tauri::Runtime>(
     app: &tauri::AppHandle<R>,
     stage_handle: &Arc<Mutex<BootstrapStage>>,
+    wait_generation: u64,
 ) -> LaunchOutcome {
     let supervisor_owner = SUPERVISOR_OWNER.fetch_add(1, Ordering::SeqCst) + 1;
-    if spawn_backend_until_ready(app, stage_handle) {
+    if spawn_backend_until_ready(app, stage_handle, wait_generation) {
         LaunchOutcome::SupervisedReady {
             owner: supervisor_owner,
         }
@@ -680,25 +692,39 @@ fn spawn_with_supervisor_owner<R: tauri::Runtime>(
 fn spawn_backend_until_ready<R: tauri::Runtime>(
     app: &tauri::AppHandle<R>,
     stage_handle: &Arc<Mutex<BootstrapStage>>,
+    wait_generation: u64,
 ) -> bool {
     let mut venv_heal_attempted = false;
     'bootstrap: loop {
-        if backend_stop_requested(app) {
+        if backend_stop_requested(app) || backend_wait_generation() != wait_generation {
             return false;
         }
         spawn_and_track_backend(app, stage_handle);
         let start = std::time::Instant::now();
+        // Latest `/startup/progress` status, or None while nothing answers.
+        let mut last_status: Option<String> = None;
         // Early-bind narration: the backend answers /startup/progress within
         // ~1s of spawn, long before it is Ready — surface each step change
         // as a log line so the splash shows "Loading ML runtime (PyTorch)…"
         // instead of a silent 300s wait. An old backend (no endpoint) yields
         // None and the wait looks exactly as it did before.
         let mut last_step = String::new();
-        while start.elapsed() < startup_budget() {
+        // #1791: the clock measures time since the last sign of life, not time
+        // since spawn — see `keep_waiting_for_backend`.
+        let mut last_progress = start;
+        while keep_waiting_for_backend(last_status.as_deref(), last_progress.elapsed(), startup_budget())
+        {
             if backend_stop_requested(app) {
                 log::info!("App is quitting — backend startup poll cancelled");
                 return false;
             }
+            if backend_wait_generation() != wait_generation {
+                log::info!(
+                    "Retry/reset is taking the backend lifecycle — standing down from \
+                     the startup wait so it can proceed"
+                );
+                return false;
+            }
             if crate::backend::backend_ready(backend_port()) {
                 set_stage(stage_handle, BootstrapStage::Ready);
                 return true;
@@ -838,13 +864,20 @@ fn spawn_backend_until_ready<R: tauri::Runtime>(
                 set_stage(stage_handle, BootstrapStage::Failed { message: msg });
                 return false;
             }
-            if let Some((status, step, label)) =
-                crate::backend::startup_progress(backend_port())
-            {
-                if status == "starting" && !step.is_empty() && step != last_step {
-                    last_step = step;
-                    emit_log(app, "starting_backend", &format!("Startup: {label}"));
+            match crate::backend::startup_progress(backend_port()) {
+                Some((status, step, label)) => {
+                    if status == "starting" {
+                        // Alive, serving, and naming the step it is on — that is
+                        // the evidence the wait is keyed on (#1791).
+                        last_progress = std::time::Instant::now();
+                        if !step.is_empty() && step != last_step {
+                            last_step = step;
+                            emit_log(app, "starting_backend", &format!("Startup: {label}"));
+                        }
+                    }
+                    last_status = Some(status);
                 }
+                None => last_status = None,
             }
             std::thread::sleep(Duration::from_millis(500));
         }
@@ -889,6 +922,30 @@ static SUPERVISOR_OWNER: AtomicU64 = AtomicU64::new(0);
 /// moment a fresh child is spawned and tracked (`track_backend_child`).
 static BACKEND_KILL_INTENDED: AtomicBool = AtomicBool::new(false);
 
+/// Bumped by any flow about to take backend lifecycle ownership for a
+/// deliberate replacement — Retry, Clean & Retry, reset, uninstall.
+///
+/// #1791: the readiness wait keeps waiting for as long as the backend answers
+/// `/startup/progress`, and it holds `BackendState::lifecycle` the whole time
+/// (`launch_backend_and_wait` takes it around the entire launch). Without a way
+/// to interrupt that wait, the user's own escape hatch would deadlock behind
+/// it: Retry and Clean & Retry both need the same lock, so pressing either on
+/// a slow start would hang instead of restarting anything — trading a backend
+/// killed too early for an app with no way out, which is worse. Every such
+/// flow bumps this BEFORE reaching for the lock; the waiting loop sees the
+/// change within one poll, returns, and releases it (Greptile, #1809).
+static BACKEND_WAIT_GENERATION: AtomicU64 = AtomicU64::new(0);
+
+/// Ask any in-flight readiness wait to stand down, so this caller can take
+/// lifecycle ownership. Call before locking, never while holding the lock.
+pub fn preempt_backend_wait() {
+    BACKEND_WAIT_GENERATION.fetch_add(1, Ordering::SeqCst);
+}
+
+fn backend_wait_generation() -> u64 {
+    BACKEND_WAIT_GENERATION.load(Ordering::SeqCst)
+}
+
 /// Bumped every time `track_backend_child` installs a new child. The
 /// supervisor snapshots it when it observes a death; a change during its
 /// backoff pause means ANOTHER flow (Retry / Clean & Retry) spawned and
@@ -1124,6 +1181,42 @@ fn startup_budget() -> Duration {
         .unwrap_or(Duration::from_secs(300))
 }
 
+/// Should the readiness poll keep waiting for a backend that is not Ready yet?
+///
+/// `status` is the `status` field of the latest `/startup/progress` reply, or
+/// `None` while nothing answers on the port.
+///
+/// #1791: this used to be a flat `elapsed < startup_budget()` from spawn, so a
+/// host where the cold start genuinely takes longer than five minutes — a
+/// project on a mapped network drive, a cold `import torch` off a spinning
+/// disk, a first CUDA DLL load — had its backend killed *while it was still
+/// importing*, reported as "the backend never reported ready". The respawn
+/// then threw away the warm work and raced the same clock again, so the app
+/// could never start even though launching the same backend by hand reached
+/// ready in under a minute.
+///
+/// A backend answering `status: "starting"` is not a backend we have to guess
+/// about: it bound its socket, it is serving HTTP, and it is telling us which
+/// step it is on. Killing it cannot make the next attempt faster, and the
+/// launcher has no information the user lacks — so keep waiting and keep
+/// narrating. The user's escape hatch is deliberate rather than clock-driven:
+/// the splash's own stall budget surfaces Retry and the logs, and its
+/// `/health` recovery poll walks straight into the app if the slow start does
+/// finish. The budget still governs *silence* — nothing answering, or a
+/// `failed`/unknown status — because there we truly cannot tell a slow
+/// backend from a wedged one, and the existing failure path (stderr tail +
+/// Retry) is the right answer.
+fn keep_waiting_for_backend(
+    status: Option<&str>,
+    since_progress: Duration,
+    budget: Duration,
+) -> bool {
+    match status {
+        Some("starting") => true,
+        _ => since_progress < budget,
+    }
+}
+
 /// The supervisor's death-detection poll interval. 2s in production;
 /// `OMNIVOICE_SUPERVISOR_POLL_MS` shrinks it for the harness only.
 fn supervisor_poll() -> Duration {
@@ -3396,6 +3489,58 @@ mod tests {
         std::env::remove_var("OMNIVOICE_SUPERVISOR_POLL_MS");
     }
 
+    #[test]
+    fn a_backend_that_is_still_starting_is_never_timed_out() {
+        let budget = Duration::from_secs(300);
+        // #1791: the whole point — a slow cold start on a network drive or a
+        // cold disk keeps waiting no matter how long it has already taken,
+        // because it is demonstrably alive and naming its current step.
+        assert!(keep_waiting_for_backend(
+            Some("starting"),
+            Duration::from_secs(3_600),
+            budget
+        ));
+        // Silence is the case we cannot read, so the budget still governs it:
+        // wait up to the budget, then fail with the stderr tail as before.
+        assert!(keep_waiting_for_backend(None, Duration::from_secs(299), budget));
+        assert!(!keep_waiting_for_backend(None, budget, budget));
+        // A backend that reports its own startup failure gets no extension —
+        // it is not making progress and never will.
+        assert!(!keep_waiting_for_backend(
+            Some("failed"),
+            Duration::from_secs(301),
+            budget
+        ));
+        // An unrecognised status is treated as silence, not as liveness.
+        assert!(!keep_waiting_for_backend(
+            Some("wat"),
+            Duration::from_secs(301),
+            budget
+        ));
+    }
+
+    #[test]
+    fn a_retry_preempts_an_in_flight_readiness_wait() {
+        // #1791 + Greptile on #1809: the wait holds lifecycle ownership, which
+        // Retry and Clean & Retry both need. A waiter that cannot be asked to
+        // stand down turns a slow start into an app with no way out — strictly
+        // worse than the early kill this fix removed. The waiter snapshots the
+        // generation before ownership is acquired; a later bump means someone else is
+        // taking over.
+        let snapshot = backend_wait_generation();
+        assert_eq!(backend_wait_generation(), snapshot, "nothing changed yet");
+        preempt_backend_wait();
+        assert_ne!(
+            backend_wait_generation(),
+            snapshot,
+            "Retry must be visible to the waiting loop"
+        );
+        // And the flow that preempted then takes its OWN snapshot, so it does
+        // not immediately cancel itself.
+        let theirs = backend_wait_generation();
+        assert_eq!(backend_wait_generation(), theirs);
+    }
+
     #[test]
     fn restart_backoff_escalates_but_first_respawn_is_immediate() {
         // A one-off crash self-heals with zero added latency; repeat deaths

frontend/src-tauri/tests/backend_lifecycle.rs

@@ -345,6 +345,8 @@ const SCENARIO_ENV: &[&str] = &[
     "OMNIVOICE_TEST_BEFORE_TRACK_RELEASE",
     "OMNIVOICE_TEST_AFTER_TRACK_ENTERED",
     "OMNIVOICE_TEST_AFTER_TRACK_RELEASE",
+    "OMNIVOICE_TEST_LAUNCH_LOCKED_ENTERED",
+    "OMNIVOICE_TEST_LAUNCH_LOCKED_RELEASE",
     "OMNIVOICE_BACKEND_CMD",
     "OMNIVOICE_LOG_DIR",
     "OMNIVOICE_PORT",
@@ -1760,3 +1762,29 @@ fn deferred_startup_failure_names_the_step() {
         logs.iter().map(|l| &l.line).collect::<Vec<_>>()
     );
 }
+
+
+#[test]
+fn retry_preempts_launch_before_the_readiness_wait_starts() {
+    let t = TestApp::new(&Scenario { serve_ms: Some(0), ..Default::default() });
+    std::env::set_var("OMNIVOICE_SCENARIO_PROGRESS_ONLY", "1");
+    let entered = t._logdir.path().join("launch-locked");
+    let release = t._logdir.path().join("release-launch");
+    std::env::set_var("OMNIVOICE_TEST_LAUNCH_LOCKED_ENTERED", &entered);
+    std::env::set_var("OMNIVOICE_TEST_LAUNCH_LOCKED_RELEASE", &release);
+    let bootstrap = t.run_bootstrap();
+    assert!(wait_until(Duration::from_secs(5), || entered.exists()));
+    app_lib::bootstrap::preempt_backend_wait();
+    let handle = t.handle();
+    let retry = std::thread::spawn(move || {
+        let state = handle.state::<BackendState>();
+        let _ownership = state.lifecycle.lock().unwrap();
+    });
+    std::fs::write(&release, b"release").unwrap();
+    let acquired = wait_until(Duration::from_secs(3), || retry.is_finished());
+    t.quit();
+    t.kill_tracked_child();
+    join_with_timeout(bootstrap, Duration::from_secs(10), "preempted launch");
+    join_with_timeout(retry, Duration::from_secs(10), "retry ownership");
+    assert!(acquired, "old launch swallowed Retry's generation and held lifecycle");
+}

frontend/src/components/BootstrapSplash.jsx

@@ -365,6 +365,27 @@ function IpcLostRecovery({ t }) {
 }
 
 /** Mono error block. */
+/** When the backend last produced ANY line of bootstrap output.
+ *
+ * #1791: the stall watchdog in `useBootstrapStage` keys on `bootstrap_status`,
+ * which sits on a single stage for the whole of a slow start — a cold `import
+ * torch` off a mapped network drive can hold `starting_backend` for many
+ * minutes. The narration that proves it is alive ("Loading ML runtime
+ * (PyTorch)…") arrives on the separate `bootstrap-log` event stream instead,
+ * so the watchdog never saw it and called a live launch stuck. A splash that
+ * is still printing new lines is by definition not the info-less spinner the
+ * watchdog exists to break.
+ *
+ * Module-scoped rather than a ref because the two halves live in different
+ * components — `BootstrapSplash` owns the event subscription, `useBootstrapStage`
+ * owns the watchdog — and they are always mounted as a pair.
+ */
+let lastBootstrapLogTs = 0;
+
+export function noteBootstrapLogActivity(ts = Date.now()) {
+  lastBootstrapLogTs = ts;
+}
+
 function ErrorBox({ children }) {
   return (
     <pre className="m-0 overflow-x-auto whitespace-pre-wrap break-words rounded-md bg-danger/10 px-3 py-2 font-mono text-[0.66rem] leading-relaxed text-danger shadow-[inset_2px_0_0_var(--color-danger)]">
@@ -523,6 +544,7 @@ export function BootstrapSplash({ stage, message }) {
         unlistenLog = await listen('bootstrap-log', (e) => {
           const { stage: s, line } = e.payload || {};
           if (!line) return;
+          noteBootstrapLogActivity();
           setLogs((prev) => {
             // Deduplicate against backfill by checking the last few lines.
             const lastFew = prev.slice(-5);
@@ -982,7 +1004,8 @@ export function useBootstrapStage(pollMs = 1000) {
           }
           // Rust returns { stage: 'ready' } or { stage: 'failed', message: '…' } etc.
           if (stage !== 'ready' && stage !== 'failed') {
-            if (Date.now() - lastChangeTs > stallBudgetMs(stage)) {
+            const lastActivity = Math.max(lastChangeTs, lastBootstrapLogTs);
+            if (Date.now() - lastActivity > stallBudgetMs(stage)) {
               // Stuck — surface it as a failure so Retry/logs/hints appear.
               setState({
                 stage: 'failed',

frontend/src/test/BootstrapSplashSlowBackendStall.test.jsx

@@ -0,0 +1,108 @@
+/**
+ * #1791 — a backend that is still narrating its startup is not stuck.
+ *
+ * The reporter's project lived on a mapped network drive, where the cold
+ * `import torch` took far longer than the shell's five-minute readiness
+ * budget. The shell killed the still-importing backend and respawned it, the
+ * respawn raced the same clock, and the app reported "the backend never
+ * reported ready" — while launching the very same backend by hand reached
+ * ready in under a minute once the file cache was warm.
+ *
+ * Rust now keeps waiting for as long as `/startup/progress` answers (see
+ * `keep_waiting_for_backend` in bootstrap.rs). This is the splash's half of
+ * that contract: the stall watchdog keys on `bootstrap_status`, which sits on
+ * `starting_backend` for the entire slow start, so on its own it would still
+ * declare a false failure at six minutes. The proof of life arrives on the
+ * `bootstrap-log` stream instead, and that has to count as activity.
+ */
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { renderHook, act } from '@testing-library/react';
+import { useBootstrapStage, noteBootstrapLogActivity } from '../components/BootstrapSplash';
+
+const invokeMock = vi.fn();
+
+vi.mock('@tauri-apps/api/core', () => ({
+  invoke: (...args) => invokeMock(...args),
+}));
+vi.mock('@tauri-apps/api/event', () => ({
+  listen: vi.fn(async () => () => {}),
+}));
+vi.mock('@tauri-apps/plugin-opener', () => ({
+  revealItemInDir: vi.fn(),
+}));
+
+let warnSpy;
+
+beforeEach(() => {
+  warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+  invokeMock.mockReset();
+  vi.useFakeTimers();
+  window.__TAURI_INTERNALS__ = {};
+  vi.stubEnv('DEV', false);
+  // The backend has not bound its port yet, so nothing answers /health — the
+  // same condition that keeps the #879 IPC watchdog out of the way.
+  vi.stubGlobal(
+    'fetch',
+    vi.fn(async () => {
+      throw new Error('ECONNREFUSED');
+    }),
+  );
+});
+
+afterEach(() => {
+  vi.useRealTimers();
+  vi.unstubAllEnvs();
+  vi.unstubAllGlobals();
+  warnSpy.mockRestore();
+  delete window.__TAURI_INTERNALS__;
+  // Module-scoped signal — clear it so a later test's silent backend really is
+  // silent.
+  noteBootstrapLogActivity(0);
+});
+
+describe('useBootstrapStage — a narrating backend is not stuck (#1791)', () => {
+  it('keeps waiting past the stall budget while startup steps still arrive', async () => {
+    invokeMock.mockImplementation(async (cmd) =>
+      cmd === 'bootstrap_status' ? { stage: 'starting_backend' } : undefined,
+    );
+
+    const { result } = renderHook(() => useBootstrapStage());
+    await act(async () => {});
+    expect(result.current.stage).toBe('starting_backend');
+
+    // Twenty minutes of a genuinely slow cold start, with the backend
+    // reporting a new step every four minutes — well inside the six-minute
+    // budget each time, but far past it in total.
+    for (let i = 0; i < 5; i += 1) {
+      await act(async () => {
+        await vi.advanceTimersByTimeAsync(4 * 60 * 1000);
+      });
+      expect(result.current.stage).toBe('starting_backend');
+      expect(result.current.message ?? '').not.toMatch(/stuck/i);
+      act(() => {
+        noteBootstrapLogActivity();
+      });
+    }
+
+    expect(result.current.stage).toBe('starting_backend');
+    expect(result.current.message ?? '').not.toMatch(/stuck/i);
+  });
+
+  it('still calls a genuinely silent backend stuck', async () => {
+    // The other half: output is the evidence, and with none the watchdog has
+    // to keep breaking the info-less spinner it exists for (#879).
+    invokeMock.mockImplementation(async (cmd) =>
+      cmd === 'bootstrap_status' ? { stage: 'starting_backend' } : undefined,
+    );
+
+    const { result } = renderHook(() => useBootstrapStage());
+    await act(async () => {});
+
+    await act(async () => {
+      await vi.advanceTimersByTimeAsync(6 * 60 * 1000 + 2_000);
+    });
+
+    expect(result.current.stage).toBe('failed');
+    expect(result.current.message).toMatch(/stuck/i);
+  });
+});