diff --git a/CHANGELOG.md b/CHANGELOG.md index fe3212f3..2ea29e91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is loosely based on [Keep a Changelog](https://keepachangelog.com/). Versions track the desktop app (`tauri.conf.json` + `frontend/src-tauri/Cargo.toml`). The bundled TTS model package (`pyproject.toml`) is versioned independently. +## [Unreleased] + +### Added + +- **The memory panel now tells the whole truth.** `Settings → Models` (and `GET /model/loaded`) used to report only the OmniVoice core model — a resident second engine like MLX-Audio, or the warm dictation model, was invisible, so the memory picture looked ~2 GB lighter than reality. It now lists every resident model (in-process engines and the dictation ASR included) and adds a system block with free/total RAM (and free VRAM on a dedicated GPU) plus a low-memory warning. On top of that, a load that starts while memory is already low leaves a breadcrumb in the backend log, so a subsequent out-of-memory kill points at the load that tipped it instead of dying silently. Advisory only — nothing is blocked (the OS can reclaim memory, and refusing a load on an estimate would brick machines that would actually cope). Tune the threshold with `OMNIVOICE_LOW_MEMORY_HEADROOM_GB` (default 2). + ## [0.3.21] — 2026-07-12 The memory release. The reason the app kept saying "Can't reach the local backend" on 16 GB machines was never really the network — the backend was quietly running out of memory and getting killed. This release fixes that at the source: the models it holds now get out of each other's way. Plus the uninstaller and factory reset grew into a proper Settings → Storage pair. diff --git a/backend/api/routers/generation.py b/backend/api/routers/generation.py index bf247579..a8785ba0 100644 --- a/backend/api/routers/generation.py +++ b/backend/api/routers/generation.py @@ -792,6 +792,18 @@ async def generate_speech( from services.engine_memory import evict_other_tts_engines await evict_other_tts_engines(engine_id) + # Non-blocking breadcrumb: if free memory is already low before this load, + # log it. A later OOM kill (the 16 GB-Mac class) then has a trail pointing + # at the load that tipped it, instead of a silent process death. Never + # blocks — the OS can reclaim cache, and a hard refuse would brick + # legitimate loads. + try: + from services.memory_budget import log_if_low + + log_if_low(f"TTS load ({engine_id})") + except Exception: + pass + _model = None _backend = None if backend_cls is OmniVoiceBackend: diff --git a/backend/services/memory_budget.py b/backend/services/memory_budget.py new file mode 100644 index 00000000..8f91bc3f --- /dev/null +++ b/backend/services/memory_budget.py @@ -0,0 +1,94 @@ +"""Free-memory probe + a non-blocking low-memory advisory. + +The device-caps probe (core.device_caps) reports *total* memory, resolved once +per process. Load decisions need *free* memory at the moment of loading — and on +Apple Silicon the number that matters is free **system RAM**, because MPS uses +unified memory (there is no separate VRAM pool). This module fills that gap. + +Deliberately advisory, never blocking: a hard "refuse to load" on an estimate +would brick legitimate loads on machines that would actually cope (the estimate +can't know a model's true resident size ahead of time, and the OS can reclaim +cache under pressure). Instead it surfaces a warning so the UI and logs can say +"you're low on memory" — and the single-active-engine eviction +(services.engine_memory) is what actually reclaims room before a load. + +Stdlib + psutil (already a runtime dep). Never raises. +""" +from __future__ import annotations + +import logging +import os +from typing import Optional + +logger = logging.getLogger("omnivoice.memory_budget") + +# Below this much free RAM, a heavy model load is at real risk of tipping the +# machine into the OOM-kill territory behind the 16 GB-Mac "Can't reach the +# backend" reports. Tunable for smaller/larger boxes. +_LOW_RAM_HEADROOM_GB = float(os.environ.get("OMNIVOICE_LOW_MEMORY_HEADROOM_GB", "2.0")) + + +def available_memory() -> dict: + """Free/total memory right now. Never raises; fields absent when unknown. + + Always includes system RAM (``ram_available_gb`` / ``ram_total_gb``). On a + CUDA/ROCm host also includes GPU VRAM (``vram_free_gb`` / ``vram_total_gb``) + from ``torch.cuda.mem_get_info``. On MPS the relevant figure is system RAM + (unified memory), so no separate VRAM fields are reported.""" + out: dict = {} + try: + import psutil + + vm = psutil.virtual_memory() + out["ram_available_gb"] = round(vm.available / (1024 ** 3), 2) + out["ram_total_gb"] = round(vm.total / (1024 ** 3), 2) + except Exception: # noqa: BLE001 — psutil missing/failed: RAM unknown, not fatal + pass + try: + torch = __import__("torch") + if torch.cuda.is_available(): + free, total = torch.cuda.mem_get_info() + out["vram_free_gb"] = round(free / (1024 ** 3), 2) + out["vram_total_gb"] = round(total / (1024 ** 3), 2) + except Exception: # noqa: BLE001 — no CUDA / probe failed + pass + return out + + +def low_memory_warning(headroom_gb: float = _LOW_RAM_HEADROOM_GB) -> Optional[str]: + """A one-line advisory when free memory is below ``headroom_gb``, else None. + + Checks free VRAM on a dedicated-GPU host, otherwise free system RAM (the + figure that matters on MPS/CPU). Pure given ``available_memory`` output — + ``_format`` does the wording — so the threshold logic is unit-testable.""" + return _format(available_memory(), headroom_gb) + + +def _format(mem: dict, headroom_gb: float) -> Optional[str]: + vram = mem.get("vram_free_gb") + if vram is not None: + if vram < headroom_gb: + return ( + f"Low GPU memory: {vram:.1f} GB free. Loading another model may " + "run out of VRAM — unload one you're not using (Settings → " + "Models), or switch to a smaller engine." + ) + return None + ram = mem.get("ram_available_gb") + if ram is not None and ram < headroom_gb: + return ( + f"Low memory: {ram:.1f} GB free. Loading a large model here risks the " + "backend being killed by the OS — close some apps, or unload a model " + "you're not using (Settings → Models)." + ) + return None + + +def log_if_low(context: str, headroom_gb: float = _LOW_RAM_HEADROOM_GB) -> Optional[str]: + """Log (once, at WARNING) and return the advisory when memory is low before + a heavy operation named by ``context``. Non-blocking — the caller proceeds + regardless; this is forensics, so a later OOM death has a breadcrumb.""" + msg = low_memory_warning(headroom_gb) + if msg: + logger.warning("%s: %s", context, msg) + return msg diff --git a/backend/services/model_lifecycle.py b/backend/services/model_lifecycle.py index c597a13c..e48f9b74 100644 --- a/backend/services/model_lifecycle.py +++ b/backend/services/model_lifecycle.py @@ -133,7 +133,70 @@ def list_loaded() -> dict: except Exception: pass - return {"models": models, "count": len(models)} + # 5. In-process engine instances that hold a model (mlx-audio, cosyvoice, + # voxcpm2, kittentts, …). These live in the generate path's instance + # cache, separate from the OmniVoice core above — and were INVISIBLE here + # until now, so a resident non-OmniVoice engine (up to a few GB) didn't + # show in the panel at all. Report each that currently holds a model. + # VRAM isn't self-reported by these engines → 0 (unmeasured), same + # convention as a CPU/uninstrumented sidecar. Enumeration is best-effort. + try: + from api.routers.engines import _ENGINE_INSTANCES + from services.tts_backend import OmniVoiceBackend + + for cls, inst in list(_ENGINE_INSTANCES.items()): + if cls is OmniVoiceBackend: + continue # the shared core is already section 1 (mm.model) + if not any(getattr(inst, a, None) is not None + for a in getattr(inst, "_MODEL_ATTRS", ("_model", "_tts"))): + continue # instance exists but hasn't loaded its weights + eid = getattr(cls, "id", cls.__name__) + models.append({ + "id": f"engine:{eid}", + "name": getattr(inst, "display_name", None) or f"{eid} (engine)", + "checkpoint": eid, + "device": get_best_device(), + "vram_mb": 0, # not self-reported by in-process engines + "unloadable": True, + **_tts_attribution(eid, active_tts), + }) + except Exception: + pass + + # 6. The warm capture/dictation ASR singleton — resident until idle-released + # (#1101 class). Held separately from the co-loaded WhisperX ASR above. + try: + import services.asr_backend as ab + + cap = getattr(ab, "_capture_backend", None) + if cap is not None: + models.append({ + "id": "capture-asr", + "name": f"{type(cap).__name__} (dictation)", + "checkpoint": getattr(ab, "_capture_backend_key", None) or type(cap).__name__, + "device": get_best_device(), + "vram_mb": 0, + "unloadable": True, + "note": "released after the idle timeout", + }) + except Exception: + pass + + # System memory snapshot — free/total RAM (and VRAM on a dedicated GPU) plus + # a low-memory advisory, so the panel can show pressure instead of leaving + # the 16 GB-Mac OOM class invisible until the backend dies. + system: dict = {} + try: + from services.memory_budget import available_memory, low_memory_warning + + system = available_memory() + warn = low_memory_warning() + if warn: + system["warning"] = warn + except Exception: + pass + + return {"models": models, "count": len(models), "system": system} async def unload(model_id: str) -> dict: diff --git a/frontend/src/api/system.ts b/frontend/src/api/system.ts index 3e561d2f..ab86af56 100644 --- a/frontend/src/api/system.ts +++ b/frontend/src/api/system.ts @@ -94,7 +94,7 @@ export async function modelStatus(): Promise { * `engine_id`/`is_active_engine` attribute TTS-family entries to an engine * (a model can stay resident after the user switches engines). */ export interface LoadedModel { - id: string; // 'tts' | 'asr' | 'diarization' | 'sidecar:' + id: string; // 'tts' | 'asr' | 'diarization' | 'sidecar:' | 'engine:' | 'capture-asr' name: string; checkpoint: string; device: string; @@ -105,9 +105,21 @@ export interface LoadedModel { is_active_engine?: boolean | null; } +/** Free/total memory snapshot from GET /model/loaded. RAM is always present; + * VRAM fields appear only on a dedicated-GPU host; `warning` is a low-memory + * advisory string when free memory is below the headroom threshold. */ +export interface SystemMemory { + ram_available_gb?: number; + ram_total_gb?: number; + vram_free_gb?: number; + vram_total_gb?: number; + warning?: string; +} + export interface LoadedModelsResponse { models: LoadedModel[]; count: number; + system?: SystemMemory; } export async function listLoadedModels(): Promise { diff --git a/tests/test_memory_budget.py b/tests/test_memory_budget.py new file mode 100644 index 00000000..fe112b5a --- /dev/null +++ b/tests/test_memory_budget.py @@ -0,0 +1,68 @@ +"""Free-memory probe + the non-blocking low-memory advisory. + +The device-caps probe reports *total* memory once per process; a load decision +needs *free* memory now — and on MPS that's free system RAM (unified memory). +These pin the threshold logic and the never-raises contract. +""" +from __future__ import annotations + +import os + +os.environ.setdefault("OMNIVOICE_MODEL", "test") +os.environ.setdefault("OMNIVOICE_DISABLE_FILE_LOG", "1") + +from services import memory_budget as mb + + +def test_available_memory_reports_system_ram(): + m = mb.available_memory() + # psutil is a runtime dep, so RAM must be present and sane. + assert m["ram_total_gb"] > 0 + assert 0 <= m["ram_available_gb"] <= m["ram_total_gb"] + + +def test_warning_fires_on_low_ram_and_is_silent_with_headroom(): + # Below headroom → advisory; comfortably above → None. Uses the pure + # formatter so the test doesn't depend on the box's real free memory. + low = mb._format({"ram_available_gb": 1.2, "ram_total_gb": 16.0}, headroom_gb=2.0) + assert low and "Low memory" in low and "1.2 GB" in low + + ok = mb._format({"ram_available_gb": 9.0, "ram_total_gb": 16.0}, headroom_gb=2.0) + assert ok is None + + +def test_vram_takes_precedence_on_a_dedicated_gpu(): + # When a CUDA host reports free VRAM, THAT is the figure checked (not RAM). + warn = mb._format( + {"vram_free_gb": 0.8, "vram_total_gb": 8.0, "ram_available_gb": 40.0}, + headroom_gb=2.0, + ) + assert warn and "GPU memory" in warn and "0.8 GB" in warn + + ok = mb._format( + {"vram_free_gb": 6.0, "vram_total_gb": 8.0, "ram_available_gb": 1.0}, + headroom_gb=2.0, + ) + assert ok is None # plenty of VRAM → no warning, even though RAM is low + + +def test_format_is_silent_when_memory_is_unknown(): + assert mb._format({}, headroom_gb=2.0) is None + + +def test_log_if_low_never_raises_and_returns_the_message(monkeypatch, caplog): + monkeypatch.setattr(mb, "available_memory", lambda: {"ram_available_gb": 0.5, "ram_total_gb": 16.0}) + msg = mb.log_if_low("TTS load (omnivoice)", headroom_gb=2.0) + assert msg and "Low memory" in msg + + # No memory info → no message, no raise. + monkeypatch.setattr(mb, "available_memory", lambda: {}) + assert mb.log_if_low("TTS load", headroom_gb=2.0) is None + + +def test_available_memory_never_raises(monkeypatch): + # psutil blowing up must not propagate — a broken probe returns {}, not a 500. + import sys + + monkeypatch.setitem(sys.modules, "psutil", None) # force ImportError path + assert isinstance(mb.available_memory(), dict)