fix(models): repair corrupt configs and isolate ASR preload (#1437)

CHANGELOG.md

@@ -40,7 +40,7 @@ The bundled TTS model package (`pyproject.toml`) is versioned independently.
 
 ### Fixed
 
-- A model file damaged by an interrupted download now repairs itself instead of failing every generation with "Error while deserializing header: header too large". Only *missing* files were repaired before; one that arrived corrupt dead-ended as a raw 500. — thanks @overrunau! (#1406)
+- Model files damaged by an interrupted download now repair themselves instead of failing every generation, including invalid `config.json` files and corrupt weight headers. — thanks @overrunau and @zherunh! (#1406, #1437)
 - A slow machine is no longer told its IndexTTS-2 install isn't there. The check that confirms an engine's virtualenv gave up after 10 seconds and counted that as a broken install, so a cold first run 500'd; it now waits longer and treats slow as unproven, not broken. — thanks @OracleNightmare! (#1414)
 - A generation abandoned while stuck on an internal lock now says so, instead of blaming your hardware and suggesting shorter text. Nothing had been computed, so none of that advice applied. (#1416, #1419)
 - A machine with a GPU that ends up on CPU now says why — a missing device node, a permissions problem, a card newer than the installed ROCm, an `HSA_OVERRIDE_GFX_VERSION` that is doing more harm than good, or an NVIDIA driver the container can't reach each read differently. Before, all of them looked identical to having no GPU at all. (#1274, #1228)

backend/core/failure.py

@@ -730,7 +730,7 @@ def is_incomplete_cache_message(text: str) -> bool:
 #: ``SafetensorError`` from a Rust extension, torch raises ``UnpicklingError``
 #: or a bare ``RuntimeError`` for the same condition in a ``.bin``, and none of
 #: them are ``OSError`` — the type is the least stable thing about this class.
-_CORRUPT_WEIGHTS_PHRASES = (
+_CORRUPT_MODEL_FILE_PHRASES = (
     # safetensors (Rust): header length prefix is larger than the file, or the
     # declared metadata runs past the end of the buffer.
     "error while deserializing header",
@@ -749,11 +749,14 @@ _CORRUPT_WEIGHTS_PHRASES = (
     ("unexpected end of file", "pytorch_model"),
     ("unexpected end of file", "checkpoint"),
     ("failed to load", "checkpoint", "corrupt"),
+    # transformers wraps a JSONDecodeError from a truncated/HTML config file
+    # with this stable, path-bearing message (#1437).
+    ("config file", "not a valid json file"),
 )
 
 
-def is_corrupt_weights_message(text: str) -> bool:
-    """True when *text* is a tensor library refusing to parse a weight file.
+def is_corrupt_model_file_message(text: str) -> bool:
+    """True when a downloaded model weight or config file cannot be parsed.
 
     Distinct from :func:`is_incomplete_cache_message`, which means the file is
     absent. Here it exists and its bytes are wrong — a different repair (force
@@ -762,7 +765,7 @@ def is_corrupt_weights_message(text: str) -> bool:
     the healer and the message can never disagree.
     """
     low = str(text).lower()
-    for phrase in _CORRUPT_WEIGHTS_PHRASES:
+    for phrase in _CORRUPT_MODEL_FILE_PHRASES:
         if isinstance(phrase, tuple):
             if all(part in low for part in phrase):
                 return True
@@ -771,6 +774,11 @@ def is_corrupt_weights_message(text: str) -> bool:
     return False
 
 
+def is_corrupt_weights_message(text: str) -> bool:
+    """Backward-compatible name for :func:`is_corrupt_model_file_message`."""
+    return is_corrupt_model_file_message(text)
+
+
 def is_os_write_refusal(reason: Optional[str]) -> bool:
     """True when *reason* looks like the OS refusing a file operation (a full
     or removed drive, a read-only folder, an antivirus/cloud-sync lock) rather

backend/services/model_manager.py

@@ -1491,8 +1491,8 @@ def _is_incomplete_cache_error(exc: BaseException) -> bool:
     return is_incomplete_cache_message(str(exc))
 
 
-def _is_corrupt_weights_error(exc: BaseException) -> bool:
-    """True when the weight file is present but its bytes cannot be parsed.
+def _is_corrupt_model_file_error(exc: BaseException) -> bool:
+    """True when a model weight or config file cannot be parsed.
 
     The other half of the interrupted-download class (#1406). transformers
     only raises the "does not appear to have a file named …" signature when
@@ -1510,9 +1510,14 @@ def _is_corrupt_weights_error(exc: BaseException) -> bool:
     The whole exception chain is checked, not just the outermost message:
     transformers wraps the tensor library's error in its own before it gets
     here, and matching only the surface would miss every wrapped case."""
-    from core.failure import is_corrupt_weights_message
+    from core.failure import is_corrupt_model_file_message
 
-    return any(is_corrupt_weights_message(str(e)) for e in _exception_chain(exc))
+    return any(is_corrupt_model_file_message(str(e)) for e in _exception_chain(exc))
+
+
+def _is_corrupt_weights_error(exc: BaseException) -> bool:
+    """Backward-compatible wrapper for the original #1406 helper name."""
+    return _is_corrupt_model_file_error(exc)
 
 
 def _hf_offline() -> bool:
@@ -1910,12 +1915,12 @@ def _load_model_sync():
         logger.info("Loading VoiceStudio model on device: %s", device)
         preload_asr = should_preload_tts_asr()
         if preload_asr:
-            logger.info("Preloading PyTorch Whisper with TTS model.")
+            logger.info("Preloading PyTorch Whisper after TTS model load.")
         else:
             logger.info("Skipping PyTorch Whisper preload; ASR will load on demand.")
         def _load():
             return VoiceStudio.from_pretrained(
-                checkpoint, device_map=device, dtype=torch.float16, load_asr=preload_asr,
+                checkpoint, device_map=device, dtype=torch.float16, load_asr=False,
             )
 
         def _recover_corrupt_weights(exc: BaseException):
@@ -1925,28 +1930,6 @@ def _load_model_sync():
             below: a resume trusts a blob that is already the expected size
             and would never re-fetch the one that is actually wrong.
             """
-            if preload_asr:
-                # `_load()` also pulls the Whisper ASR checkpoint — a DIFFERENT
-                # repo. An unparseable shard there would arrive here looking
-                # identical, and re-downloading `checkpoint` would pull
-                # gigabytes, fix nothing, and blame the wrong model
-                # (CodeRabbit). One local load without ASR settles it: if the
-                # TTS weights read fine, they were never the problem.
-                try:
-                    VoiceStudio.from_pretrained(
-                        checkpoint, device_map=device, dtype=torch.float16,
-                        load_asr=False,
-                    )
-                except Exception:
-                    pass  # TTS weights are bad too — fall through and repair
-                else:
-                    raise RuntimeError(
-                        "The transcription model's files are damaged — the "
-                        "TTS model itself reads fine. Open Settings → Models, "
-                        "delete the transcription (ASR) model, and install it "
-                        "again; or set OMNIVOICE_PRELOAD_TTS_ASR=0 to stop "
-                        "loading it alongside TTS."
-                    ) from exc
             if checkpoint in _FORCED_REDOWNLOAD_ATTEMPTED:
                 # Already re-fetched this repo once this process and it is
                 # still unparseable. Re-downloading again would be the same
@@ -2087,6 +2070,22 @@ def _load_model_sync():
                 raise
             _model = _recover_corrupt_weights(e_corrupt)
 
+        if preload_asr:
+            # Keep ASR outside `from_pretrained`: if its separate HF cache is
+            # corrupt, it must never be mistaken for the TTS checkpoint and
+            # trigger a second multi-GB TTS load/re-download (CodeRabbit).
+            try:
+                _model.load_asr_model()
+            except Exception as asr_exc:
+                if not _is_corrupt_model_file_error(asr_exc):
+                    raise
+                raise RuntimeError(
+                    "The transcription model's files are damaged. Open "
+                    "Settings → Models, delete the transcription (ASR) model, "
+                    "and install it again; or set OMNIVOICE_PRELOAD_TTS_ASR=0 "
+                    "to stop preloading it alongside TTS."
+                ) from asr_exc
+
         try:
             # plan-02 (#65): gate on Triton availability (+ user setting), not
             # just device==cuda. Triton has no Windows wheel, so the old

tests/test_corrupt_weights_recovery_1406.py

@@ -24,6 +24,7 @@ from __future__ import annotations
 
 import pytest
 
+
 @pytest.fixture(autouse=True)
 def failure():
     """Resolved at run time: other suites reset `sys.modules` for app modules,
@@ -44,6 +45,8 @@ CORRUPT_WORDINGS = [
     "InvalidHeaderDeserialization",
     "UnpicklingError: invalid load key, '<'.",
     "RuntimeError: unexpected end of file while loading model.safetensors",
+    "It looks like the config file at 'models/snapshots/rev/config.json' "
+    "is not a valid JSON file.",
 ]
 
 
@@ -211,15 +214,35 @@ def test_a_damaged_asr_shard_does_not_re_download_the_tts_model(mm, monkeypatch)
     calls = _drive_load(mm, monkeypatch, _SafetensorError(REPORTED))
     monkeypatch.setattr(mm, "should_preload_tts_asr", lambda: True)
 
-    def _fails_only_with_asr(*a, **kw):
-        calls["load"] += 1
-        if kw.get("load_asr"):
+    loaded = object()
+
+    class _Model:
+        llm = loaded
+
+        def load_asr_model(self):
             raise _SafetensorError(REPORTED)
-        return object()
+
+    def _load_tts_once(*a, **kw):
+        calls["load"] += 1
+        assert kw.get("load_asr") is False
+        return _Model()
 
     monkeypatch.setattr(mm, "_lazy_omnivoice", lambda: type(
-        "C", (), {"from_pretrained": staticmethod(_fails_only_with_asr)}
+        "C", (), {"from_pretrained": staticmethod(_load_tts_once)}
     ))
     with pytest.raises(RuntimeError, match="transcription model"):
         mm._load_model_sync()
+    assert calls["load"] == 1, "ASR diagnosis loaded the multi-GB TTS model twice"
     assert calls["repair"] == [], "the TTS checkpoint was re-downloaded for an ASR fault"
+
+
+def test_a_corrupt_config_is_force_repaired_and_retried(mm, monkeypatch):
+    """#1437: a truncated config.json is the same corrupt-cache class."""
+    error = OSError(
+        "It looks like the config file at 'models/snapshots/rev/config.json' "
+        "is not a valid JSON file."
+    )
+    calls = _drive_load(mm, monkeypatch, error)
+    mm._load_model_sync()
+    assert calls["load"] == 2
+    assert calls["repair"] == [True]

tests/test_model_manager_preload.py

@@ -61,12 +61,19 @@ def test_load_model_skips_pytorch_whisper_by_default(model_manager, monkeypatch)
 
 def test_load_model_can_preload_pytorch_whisper_when_requested(model_manager, monkeypatch):
     calls = []
+    asr_loads = []
+
+    class DummyModel:
+        llm = object()
+
+        def load_asr_model(self):
+            asr_loads.append(True)
 
     class DummyOmniVoice:
         @staticmethod
         def from_pretrained(*args, **kwargs):
             calls.append((args, kwargs))
-            return SimpleNamespace(llm=object())
+            return DummyModel()
 
     monkeypatch.setenv("OMNIVOICE_PRELOAD_TTS_ASR", "1")
     monkeypatch.setattr(model_manager, "_lazy_torch", lambda: SimpleNamespace(float16="float16"))
@@ -75,7 +82,8 @@ def test_load_model_can_preload_pytorch_whisper_when_requested(model_manager, mo
 
     model_manager._load_model_sync()
 
-    assert calls[0][1]["load_asr"] is True
+    assert calls[0][1]["load_asr"] is False
+    assert asr_loads == [True]
 
 
 def test_resolve_checkpoint_honors_test_sentinel(model_manager, monkeypatch):