fix: repair incomplete Sherpa model caches (#1753)

Repairs missing and zero-byte Sherpa ONNX cache assets before model loading, with offline regression coverage. Closes #1733.

CHANGELOG.md

@@ -27,6 +27,7 @@ the frozen-backend fallback mirror it for their toolchains.
 
 ### Fixed
 
+- Incomplete Sherpa-ONNX model snapshots now self-repair before recognizer startup instead of failing on a missing ONNX file (#1733)
 - OmniVoice subprocess startup now allows slow packaged Windows Python runtimes to signal readiness before termination (#1711)
 - SRT files selected during source analysis now wait for speaker cloning, then replace transcript text without losing voices (#1709)
 - Windows MSI deployments can now prohibit WebView2 bootstrap with `DISABLEWEBVIEW2BOOTSTRAP=1`, and `AUTOLAUNCHAPP=0` reliably suppresses first launch (#1714)

backend/services/sherpa_dictation.py

@@ -273,6 +273,13 @@ def sherpa_available() -> tuple[bool, str]:
         return False, f"sherpa-onnx unavailable ({type(e).__name__}): {e}"
 
 
+def _usable_model_file(path: str) -> bool:
+    try:
+        return os.path.isfile(path) and os.path.getsize(path) > 0
+    except OSError:
+        return False
+
+
 def _resolve_model_dir(spec: SherpaModelSpec, *, download: bool = True) -> str:
     """Return the local directory containing this model's ONNX assets.
 
@@ -285,6 +292,7 @@ def _resolve_model_dir(spec: SherpaModelSpec, *, download: bool = True) -> str:
     from services.hf_revisions import revision_for
 
     wanted = list(spec.files.values())
+    cache_dir = _live_hub_cache_dir()
     # Probe the revision an existing installation actually resolved.  Older
     # releases followed ``main`` and may therefore have a different snapshot;
     # retaining it preserves offline upgrades.  Any network fetch still uses
@@ -294,12 +302,57 @@ def _resolve_model_dir(spec: SherpaModelSpec, *, download: bool = True) -> str:
         return installed
     if not download:
         raise FileNotFoundError(f"No complete cached snapshot for {spec.repo_id}")
+
+    # A Windows cache can retain a snapshot entry whose target blob vanished,
+    # or a zero-byte ONNX placeholder left by an interrupted download.  Hub may
+    # then treat that entry as already materialized and return the same broken
+    # snapshot.  Repair those entries before asking for another download so the
+    # recognizer never receives a path to a file that does not resolve (#1733).
+    from services.hf_cache_repair import (
+        find_dangling_entries,
+        repair_repo_cache,
+        repo_cache_dir,
+    )
+
+    if find_dangling_entries(repo_cache_dir(spec.repo_id, cache_dir)):
+        repair = repair_repo_cache(spec.repo_id, cache_dir)
+        installed = _installed_snapshot(spec)
+        if installed:
+            return installed
+        if not repair.get("ok"):
+            logger.warning(
+                "sherpa dictation: cache repair for %s failed: %s",
+                spec.repo_id,
+                repair.get("error") or repair.get("outcome") or "unknown error",
+            )
+
     logger.info("sherpa dictation: downloading %s on first use", spec.repo_id)
-    return snapshot_download(
+    snapshot = snapshot_download(
         repo_id=spec.repo_id,
         revision=revision_for(spec.repo_id),
         allow_patterns=wanted,
-        cache_dir=_live_hub_cache_dir(),
+        cache_dir=cache_dir,
+    )
+    missing = [
+        name for name in wanted
+        if not _usable_model_file(os.path.join(snapshot, name))
+    ]
+    if not missing:
+        return snapshot
+
+    # Verify after the Hub reports success.  This catches hosts where a broken
+    # snapshot entry short-circuits snapshot_download.  The generic repair
+    # removes only broken entries, preserves blobs, and retries the immutable
+    # installed revision.
+    repair = repair_repo_cache(spec.repo_id, cache_dir)
+    installed = _installed_snapshot(spec)
+    if installed:
+        return installed
+    detail = repair.get("error") or repair.get("outcome") or "repair did not restore them"
+    raise FileNotFoundError(
+        f"Sherpa model cache is incomplete for {spec.repo_id}; missing "
+        f"{', '.join(missing)}. Cache repair failed: {detail}. Reinstall this "
+        "model from Model Catalogue."
     )
 
 
@@ -324,8 +377,10 @@ def _installed_snapshot(spec: SherpaModelSpec) -> str | None:
         "snapshots",
         revision,
     )
-    if all(os.path.isfile(os.path.join(snapshot, filename))
-           for filename in spec.files.values()):
+    if all(
+        _usable_model_file(os.path.join(snapshot, filename))
+        for filename in spec.files.values()
+    ):
         return snapshot
     return None
 

tests/test_sherpa_dictation.py

@@ -208,12 +208,17 @@ def test_model_resolution_pins_offline_probe_and_download(monkeypatch, tmp_path)
     monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
     calls = []
 
+    downloaded = tmp_path / "downloaded"
+
     def fake_snapshot(**kwargs):
         calls.append(kwargs)
-        return "/cache/pinned"
+        downloaded.mkdir()
+        for filename in spec.files.values():
+            (downloaded / filename).write_bytes(b"model")
+        return str(downloaded)
 
     monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot)
-    assert sd._resolve_model_dir(spec) == "/cache/pinned"
+    assert sd._resolve_model_dir(spec) == str(downloaded)
     assert calls == [{
         "repo_id": spec.repo_id,
         "revision": hf_revisions.revision_for(spec.repo_id),
@@ -222,6 +227,44 @@ def test_model_resolution_pins_offline_probe_and_download(monkeypatch, tmp_path)
     }]
 
 
+def test_model_resolution_repairs_broken_snapshot_before_loading(monkeypatch, tmp_path):
+    """A zero-byte ONNX entry must be repaired before sherpa receives it (#1733)."""
+    from services import hf_cache_repair, sherpa_dictation as sd
+    import huggingface_hub
+
+    spec = sd.get_spec("sherpa-whisper-tiny")
+    revision = "6" * 40
+    repo = tmp_path / "models--csukuangfj--sherpa-onnx-whisper-tiny"
+    ref = repo / "refs" / "main"
+    ref.parent.mkdir(parents=True)
+    ref.write_text(revision + "\n", encoding="ascii")
+    snapshot = repo / "snapshots" / revision
+    snapshot.mkdir(parents=True)
+    broken = snapshot / spec.files["encoder"]
+    broken.write_bytes(b"")
+    for role in ("decoder", "tokens"):
+        (snapshot / spec.files[role]).write_bytes(b"model")
+    monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
+
+    repairs = []
+
+    def fake_repair(repo_id, cache_dir):
+        repairs.append((repo_id, cache_dir))
+        broken.write_bytes(b"restored model")
+        return {"ok": True, "outcome": "healed_with_copies", "error": ""}
+
+    monkeypatch.setattr(hf_cache_repair, "repair_repo_cache", fake_repair)
+    monkeypatch.setattr(
+        huggingface_hub,
+        "snapshot_download",
+        lambda **_kwargs: pytest.fail("a repaired snapshot must be reused"),
+    )
+
+    assert sd._resolve_model_dir(spec) == str(snapshot)
+    assert repairs == [(spec.repo_id, str(tmp_path))]
+    assert broken.read_bytes() == b"restored model"
+
+
 def test_model_resolution_probes_preserved_legacy_snapshot(monkeypatch, tmp_path):
     from services import hf_revisions, sherpa_dictation as sd
     import huggingface_hub