diff --git a/.gitleaks.toml b/.gitleaks.toml index edbe4872..ce79385e 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -1,7 +1,8 @@ # Gitleaks config — extends the default ruleset. # -# The ONLY sanctioned allowlist entry is PostHog's publishable project token -# (owner decision 2026-07-20, #1193). Per PostHog's docs the `phc_` project +# Every entry below is an exact, anchored non-secret value. PostHog's +# publishable project token is public by design (owner decision 2026-07-20, +# #1193). Per PostHog's docs the `phc_` project # token is a write-only client key with "no access to your private data" — # it ships in every release binary and every official PostHog SDK snippet. # It is NOT a credential. Personal keys (`phx_`) remain fully banned. @@ -12,7 +13,16 @@ useDefault = true [allowlist] -description = "PostHog publishable write-only project token (public by design; #1193)" +description = "Exact public/test literals misclassified as generic API keys" regexes = [ - '''phc_v5wMjnYMPMaEcRNLRKQsTYCzPaYWh7wcHPhXNkNajVf9''', + # Public PostHog project token; personal `phx_` keys remain banned. + '''^phc_v5wMjnYMPMaEcRNLRKQsTYCzPaYWh7wcHPhXNkNajVf9$''', + # Reviewed immutable Hugging Face commit for the Higgs tokenizer. + '''^528e871c2a26c4f0f7773b9754e2e1acae20899d$''', + # Deliberately synthetic fixtures that exercise HF-token redaction/storage. + '''^hf_abcdefghijklmnopqrstuvwxyz01234567890abcd$''', + '''^hf_abcdefghijklmnopqrstuvwxyz0123456789ABCDEF$''', + '''^hf_QWERTYUIOPasdfghjklZXCVBNM0123456789xyzAB$''', + # NLLB generation length argument, not the value of a credential. + '''^max_length=400$''', ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 83087000..baa11170 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The bundled TTS model package (`pyproject.toml`) is versioned independently. ### Fixed - Patched dependency releases now cover 35 Python and Rust security advisories without weakening VoiceStudio's GPU or offline-runtime compatibility. (#1456, #1472, #1473, #1474, #1475, #1476, #1477) +- Curated models now install and repair from reviewed, immutable revisions; custom MOSS remote code requires an explicit safety opt-in. (#1453) - YouTube imports that require a signed-in session can now use an explicitly selected `cookies.txt` export for one import; VoiceStudio never reads browser cookies silently and makes two best-effort attempts to delete its temporary copy. (#1429, #1432) — thanks @dongqing1968-sudo and @phamvandu9595-tech! - First-run source builds no longer stop after uv was successfully downloaded just because its installer failed during a later shell-profile step; app-private uv installs no longer touch shell profiles at all. (#1438) — thanks @AdrianoCahete! - Model files damaged by an interrupted download now repair themselves instead of failing every generation, including invalid `config.json` files and corrupt weight headers. — thanks @overrunau and @zherunh! (#1406, #1437) diff --git a/backend/api/routers/dub_translate.py b/backend/api/routers/dub_translate.py index e297b2e6..a632f61e 100644 --- a/backend/api/routers/dub_translate.py +++ b/backend/api/routers/dub_translate.py @@ -8,12 +8,23 @@ from fastapi.responses import JSONResponse from schemas.requests import TranslateRequest from services.model_manager import _cpu_pool, _gpu_pool +from services.hf_revisions import revision_for from services.translator import cinematic_available, cinematic_refine_many, _cinematic_budget from api.routers.dub_core import _get_job, _save_job router = APIRouter() logger = logging.getLogger("omnivoice.api") +_NLLB_REPO_ID = "facebook/nllb-200-distilled-600M" + + +def _load_nllb_component(factory): + """Load a curated NLLB component from its reviewed immutable revision.""" + return factory.from_pretrained( + _NLLB_REPO_ID, + revision=revision_for(_NLLB_REPO_ID), + ) + TRANSLATE_CODES = { "en": "en", "es": "es", "fr": "fr", "de": "de", "it": "it", "pt": "pt", "ru": "ru", "ja": "ja", "ko": "ko", "zh": "zh-CN", "cmn-Hans": "zh-CN", @@ -287,9 +298,9 @@ async def dub_translate(req: TranslateRequest): try: if _nllb_tokenizer is None: - _nllb_tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M") + _nllb_tokenizer = _load_nllb_component(AutoTokenizer) if _nllb_model is None: - _nllb_model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M") + _nllb_model = _load_nllb_component(AutoModelForSeq2SeqLM) if target_device != "cpu": try: _nllb_model = _nllb_model.to(target_device) diff --git a/backend/api/routers/setup/download.py b/backend/api/routers/setup/download.py index 34fe2c57..0870965b 100644 --- a/backend/api/routers/setup/download.py +++ b/backend/api/routers/setup/download.py @@ -20,6 +20,7 @@ from pydantic import BaseModel from core import prefs from core.failure import is_hf_connectivity_error +from services.hf_revisions import revision_for from utils import hf_progress from utils import download_aggregator # Weight-floor scan (MM2-07 / #352) lives in ``models.py`` — the lowest module in @@ -170,7 +171,7 @@ def _repo_cancelled(repo_id: str) -> bool: return repo_id in _cancelled -def _segmented_snapshot(repo_id: str, *, endpoint: "str | None") -> str: +def _segmented_snapshot(repo_id: str, *, endpoint: "str | None", revision: str) -> str: """Fetch every file of a repo via the segmented downloader into the HF cache, mirroring hf_hub_download's blob+snapshot+refs layout so the result is indistinguishable from snapshot_download (FDL-09) — keeping /models @@ -187,10 +188,10 @@ def _segmented_snapshot(repo_id: str, *, endpoint: "str | None") -> str: token = _resolve_token() api = HfApi(endpoint=endpoint, token=token) - info = api.repo_info(repo_id, repo_type="model") + info = api.repo_info(repo_id, repo_type="model", revision=revision) commit = info.sha files = [s.rfilename for s in (info.siblings or [])] - if not commit or not files: + if commit != revision or not files: raise RuntimeError("repo_info returned no commit/siblings") repo_dir = os.path.join(_C.HF_HUB_CACHE, repo_folder_name(repo_id=repo_id, repo_type="model")) @@ -407,6 +408,7 @@ async def install_model(req: InstallModelRequest): # parallel-files worker count, and honour an optional mirror endpoint. dl_kwargs: dict = { "repo_id": req.repo_id, + "revision": revision_for(req.repo_id), "max_workers": _download_max_workers(), } _tqdm_cls = hf_progress.tracked_tqdm_class() @@ -447,11 +449,15 @@ async def install_model(req: InstallModelRequest): # bytes that will actually download — BEFORE any byte flows. Seeds # the overall aggregator so its bar/ETA are correct from the first # event. Degrades gracefully (totals=None) on older/gated repos. - _preflight_kwargs = {"repo_id": req.repo_id, "dry_run": True} + _preflight_kwargs = { + "repo_id": req.repo_id, + "revision": dl_kwargs["revision"], + "dry_run": True, + } if _endpoint: _preflight_kwargs["endpoint"] = _endpoint try: - _plan = snapshot_download(**_preflight_kwargs) + _plan = snapshot_download(**_preflight_kwargs) # nosec B615 -- immutable revision_for pin _summary = compute_plan(_plan) # Disk-space guard (before a single byte flows): the preflight # gives an exact "to download" size, so reject an install that @@ -515,7 +521,11 @@ async def install_model(req: InstallModelRequest): _snapshot_path = None if _attempt == 1 and _segmented_enabled() and not _xet_active(): try: - _snapshot_path = _segmented_snapshot(req.repo_id, endpoint=_endpoint) + _snapshot_path = _segmented_snapshot( + req.repo_id, + endpoint=_endpoint, + revision=dl_kwargs["revision"], + ) except _InstallCancelled: raise except Exception as _seg_err: @@ -525,8 +535,11 @@ async def install_model(req: InstallModelRequest): ) _snapshot_path = None if _snapshot_path is None: - _snapshot_path = snapshot_download(**dl_kwargs) + _snapshot_path = snapshot_download(**dl_kwargs) # nosec B615 -- immutable revision_for pin _validate_snapshot_has_weights(req.repo_id, _snapshot_path) + from huggingface_hub.constants import HF_HUB_CACHE + from services.hf_revisions import remember_revision + remember_revision(req.repo_id, dl_kwargs["revision"], HF_HUB_CACHE) break except Exception as net_err: # #1224: a truncated body ("peer closed connection without diff --git a/backend/engines/moss_tts_v15/main.py b/backend/engines/moss_tts_v15/main.py index fa5172f5..d9daaff5 100644 --- a/backend/engines/moss_tts_v15/main.py +++ b/backend/engines/moss_tts_v15/main.py @@ -36,6 +36,7 @@ from __future__ import annotations import base64 import json import os +import re import struct import sys import traceback @@ -50,8 +51,30 @@ MAX_FRAME_BYTES = 64 * 1024 * 1024 #: the loaded model at synthesize time; this is the handshake default). MOSS_SAMPLE_RATE = 24000 -#: HF repo id for the weights, overridable for air-gapped / mirror installs. +#: Reviewed HF repo and immutable revision for the default remote-code model. _DEFAULT_REPO = "OpenMOSS-Team/MOSS-TTS-v1.5" +_DEFAULT_REVISION = "cdd3b911b1585e3f2dbc7775ef10f9926f58850a" +_SHA = re.compile(r"[0-9a-f]{40}\Z") + + +def _model_source() -> tuple[str, str]: + """Return a pinned model source; custom remote code requires two opt-ins.""" + repo = os.environ.get("OMNIVOICE_MOSS_TTS_V15_MODEL", _DEFAULT_REPO) + if repo == _DEFAULT_REPO: + return repo, _DEFAULT_REVISION + unsafe = os.environ.get("OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE", "").lower() + revision = os.environ.get("OMNIVOICE_MOSS_TTS_V15_REVISION", "") + if unsafe not in {"1", "true", "yes", "on"}: + raise RuntimeError( + "A custom MOSS model contains executable remote code. Set " + "OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE=1 only after auditing it." + ) + if not _SHA.fullmatch(revision): + raise RuntimeError( + "A custom MOSS model requires OMNIVOICE_MOSS_TTS_V15_REVISION=" + "<40-character commit SHA>; branches and tags are mutable." + ) + return repo, revision #: ISO-639-1 → MOSS language name. MOSS's ``build_user_message`` takes a #: language *name* ("French"), not a code. Unknown codes are omitted so the @@ -130,14 +153,16 @@ def _load_model(stdout): import torch from transformers import AutoModel, AutoProcessor - repo = os.environ.get("OMNIVOICE_MOSS_TTS_V15_MODEL", _DEFAULT_REPO) + repo, revision = _model_source() device = "cuda" if torch.cuda.is_available() else "cpu" dtype = torch.bfloat16 if device == "cuda" else torch.float32 # "sdpa" works on CUDA + CPU and needs no extra dep. flash_attention_2 # (Ampere+ CUDA, optional flash-attn) is opt-in via env. attn = os.environ.get("OMNIVOICE_MOSS_TTS_V15_ATTN", "sdpa") - processor = AutoProcessor.from_pretrained(repo, trust_remote_code=True) + processor = AutoProcessor.from_pretrained( + repo, revision=revision, trust_remote_code=True, + ) # The audio tokenizer is a separate sub-module that must be moved to the # device independently (easy to miss — see upstream README). processor.audio_tokenizer = processor.audio_tokenizer.to(device) @@ -146,6 +171,7 @@ def _load_model(stdout): model = AutoModel.from_pretrained( repo, + revision=revision, trust_remote_code=True, attn_implementation=attn, torch_dtype=dtype, diff --git a/backend/services/hf_cache_repair.py b/backend/services/hf_cache_repair.py index 1b251d4d..9fe30b17 100644 --- a/backend/services/hf_cache_repair.py +++ b/backend/services/hf_cache_repair.py @@ -209,6 +209,12 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict: ) return summary + # Resolve the exact revision before deleting even a broken pointer. An + # unreviewed repository can never turn cache repair into a mutable- + # branch network fetch. + from services.hf_revisions import installed_revision + revision = installed_revision(repo_id, cache_root) + def _remove(paths: list[str]) -> int: n = 0 for path in paths: @@ -232,13 +238,15 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict: from huggingface_hub import snapshot_download - dl_kwargs: dict = {"repo_id": repo_id} - if cache_dir: - dl_kwargs["cache_dir"] = cache_dir + dl_kwargs: dict = { + "repo_id": repo_id, + "revision": revision, + "cache_dir": cache_root, + } endpoint = os.environ.get("HF_ENDPOINT") if endpoint: dl_kwargs["endpoint"] = endpoint - snapshot_download(**dl_kwargs) + snapshot_download(**dl_kwargs) # nosec B615 -- installed immutable revision summary["restored"] = True # Verify-after-repair: hub's memoized symlink probe can claim support @@ -267,7 +275,7 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict: ) return summary summary["removed"] += _remove(still_broken) - snapshot_download(**dl_kwargs) + snapshot_download(**dl_kwargs) # nosec B615 -- installed immutable revision remaining = find_dangling_entries(repo_dir) if remaining: summary["error"] = ( diff --git a/backend/services/hf_revisions.py b/backend/services/hf_revisions.py new file mode 100644 index 00000000..1e13be37 --- /dev/null +++ b/backend/services/hf_revisions.py @@ -0,0 +1,96 @@ +"""Immutable Hugging Face revisions for VoiceStudio's curated repositories. + +Branch names are mutable supply-chain inputs. Every repo the product offers is +resolved here to a reviewed commit SHA; download, preflight, and repair paths +must call :func:`revision_for` instead of following ``main``. +""" +from __future__ import annotations + +import os +import re +from pathlib import Path + +_SHA = re.compile(r"[0-9a-f]{40}\Z") + +CURATED_REVISIONS: dict[str, str] = { + "facebook/nllb-200-distilled-600M": "f8d333a098d19b4fd9a8b18f94170487ad3f821d", + "k2-fsa/OmniVoice": "c5fdb5ccb189668d56333f77ba2629f4cd7535f4", + "Systran/faster-whisper-large-v3": "edaa852ec7e145841d8ffdb056a99866b5f0a478", + "mlx-community/whisper-large-v3-mlx": "49e6aa286ad60c14352c404340ded53710378a11", + "mlx-community/whisper-large-v3-turbo": "a4aaeec0636e6fef84abdcbe3544cb2bf7e9f6fb", + "openai/whisper-large-v3": "06f233fe06e710322aca913c1bc4249a0d71fce1", + "mlx-community/whisper-tiny-mlx": "6caf9c55601caafbe6508a8b0d216bdf4783c4e8", + "deepdml/faster-whisper-large-v3-turbo-ct2": "4df90f75321148c3a29a9e2351b7ddf8f5b115a8", + "Systran/faster-distil-whisper-large-v3": "c3058b475261292e64a0412df1d2681c06260fab", + "Systran/faster-whisper-medium": "08e178d48790749d25932bbc082711ddcfdfbc4f", + "Systran/faster-whisper-small": "536b0662742c02347bc0e980a01041f333bce120", + "Systran/faster-whisper-base": "ebe41f70d5b6dfa9166e2c581c45c9c0cfc57b66", + "nvidia/parakeet-tdt-0.6b-v3": "541d1f99c6b0c3cd0b11a95167540bb8edefd82b", + "nvidia/parakeet-tdt-0.6b-v2": "ae9ad07059c7c739ffaf932226a8fe64ae2620b0", + "mlx-community/parakeet-tdt-0.6b-v3": "ed2b7e8c15f9aaa0b5772e2efb986255eaef7e15", + "UsefulSensors/moonshine-base": "7a73d8d55ac0ba2ef3ae761593f6784b51f96dcf", + "UsefulSensors/moonshine-tiny": "390624ed33d594443aa4aa221f5b9f283b545b5a", + "csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8": "2bda32ec70b097a55adaa07d9a7173915b43cc78", + "csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8": "1ab9323565ddb038682214b292f588070a538ce2", + "csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20": "98590b7ed6443e77b714204da2757d75e1a642f4", + "csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en": "8e40c43232a1c5c66c82111efc5820d3accca11b", + "csukuangfj/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17": "d42f2d9f7ca24806fb667456a18a9f1b60f70d16", + "csukuangfj/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23": "204ad334e2e683fd295359930cc16fc0432a23ac", + "csukuangfj/sherpa-onnx-whisper-tiny": "65176e2deb88badc814a94058666cadccc29b61c", + "pyannote/speaker-diarization-3.1": "84fd25912480287da0247647c3d2b4853cb3ee5d", + "OpenMOSS-Team/MOSS-TTS-Nano-100M": "44502f80dbf9743528fa921cc544d662c685ebec", + "KittenML/kitten-tts-mini-0.8": "c02725660cea441db4c383af69f1f26f5cd00947", + "mlx-community/Kokoro-82M-bf16": "a71e4d38b236d968966a2002c4c895dbd12b1c3c", + "mlx-community/csm-1b-8bit": "fcf0cc857eade3615a60f30722cf5197d4f88406", + "mlx-community/Qwen3-TTS-12Hz-1.7B-VoiceDesign-4bit": "5c390979e4b93af5f2932f90742ca99c7dd04687", + "mlx-community/Dia-1.6B": "de4fa8c178ca5cc4e9d884b55b03fcfaa0995162", + "mlx-community/Llama-OuteTTS-1.0-1B-4bit": "3ac2cff406f7de16a3216c60d0108571a916acc0", + "mlx-community/Chatterbox-TTS-4bit": "a3c8ded2d711d6395410d645b3a97c79fd563a13", + "mlx-community/MeloTTS-English-v3-MLX": "837d15fd72bc35a15033234ce5ea242367ca1960", + "OpenMOSS-Team/MOSS-TTS-v1.5": "cdd3b911b1585e3f2dbc7775ef10f9926f58850a", + "eustlb/higgs-audio-v2-tokenizer": "528e871c2a26c4f0f7773b9754e2e1acae20899d", +} + + +def revision_for(repo_id: str) -> str: + """Return the immutable revision for a curated repo, or raise.""" + try: + return CURATED_REVISIONS[repo_id] + except KeyError as exc: + raise ValueError(f"No reviewed revision is pinned for {repo_id!r}") from exc + + +def _repo_dir(repo_id: str, cache_dir: str) -> Path: + return Path(cache_dir) / ("models--" + repo_id.replace("/", "--")) + + +def remember_revision(repo_id: str, revision: str, cache_dir: str) -> None: + """Persist the exact installed revision for later in-place repair.""" + if not _SHA.fullmatch(revision): + raise ValueError("Hugging Face revision must be a 40-character commit SHA") + repo_dir = _repo_dir(repo_id, cache_dir) + repo_dir.mkdir(parents=True, exist_ok=True) + marker = repo_dir / "voicestudio-revision" + temporary = marker.with_suffix(f".tmp-{os.getpid()}") + temporary.write_text(revision + "\n", encoding="ascii") + os.replace(temporary, marker) + + +def installed_revision(repo_id: str, cache_dir: str) -> str: + """Return VoiceStudio's recorded revision, falling back to the curated pin.""" + # Authenticate the repository before consulting attacker-writable cache + # metadata. A syntactically valid marker must never authorize repair of a + # repository outside VoiceStudio's reviewed catalog. + curated_revision = revision_for(repo_id) + repo_dir = _repo_dir(repo_id, cache_dir) + # New installs write the first marker. ``refs/main`` preserves the commit + # resolved by older VoiceStudio/huggingface_hub installs, so upgrades repair + # the bytes the user actually installed rather than silently changing them. + for marker in (repo_dir / "voicestudio-revision", repo_dir / "refs" / "main"): + try: + revision = marker.read_text(encoding="ascii").strip() + except OSError: + continue + if _SHA.fullmatch(revision): + return revision + return curated_revision diff --git a/backend/services/model_manager.py b/backend/services/model_manager.py index 74f03f19..2c432b6a 100644 --- a/backend/services/model_manager.py +++ b/backend/services/model_manager.py @@ -1662,7 +1662,20 @@ def _repair_model_cache(checkpoint: str, *, force: bool = False) -> bool: logger.warning("Cannot import snapshot_download to repair cache: %s", imp_err) _last_repair_error = f"{type(imp_err).__name__}: {imp_err}" return False - dl_kwargs: dict = {"repo_id": checkpoint} + try: + from services.hf_cache_repair import hf_cache_home + from services.hf_revisions import installed_revision + cache_root = hf_cache_home() + revision = installed_revision(checkpoint, cache_root) + except (OSError, ValueError) as revision_err: + _last_repair_error = str(revision_err) + logger.warning("Refusing unpinned model repair for %s: %s", checkpoint, revision_err) + return False + dl_kwargs: dict = { + "repo_id": checkpoint, + "revision": revision, + "cache_dir": cache_root, + } # Explicit endpoint (HF_ENDPOINT / pref) wins; otherwise the automatic # endpoint selection's cached pick applies (services.endpoint_race). try: @@ -1683,12 +1696,12 @@ def _repair_model_cache(checkpoint: str, *, force: bool = False) -> bool: """One snapshot_download, tolerating an hf_hub that rejects the optional symlink knob. Lets real failures (network, gated repo, disk) propagate.""" try: - snapshot_download(**dl_kwargs) + snapshot_download(**dl_kwargs) # nosec B615 -- installed immutable revision except TypeError: # Older/newer huggingface_hub may not accept local_dir_use_symlinks # on a cache-only call — retry without the optional knob. dl_kwargs.pop("local_dir_use_symlinks", None) - snapshot_download(**dl_kwargs) + snapshot_download(**dl_kwargs) # nosec B615 -- installed immutable revision # Bounded retries (#739): an incomplete cache *is* an interrupted download, so # a single transient blip mid-repair shouldn't drop the user back to a manual diff --git a/backend/services/sherpa_dictation.py b/backend/services/sherpa_dictation.py index 81d3583e..693475bb 100644 --- a/backend/services/sherpa_dictation.py +++ b/backend/services/sherpa_dictation.py @@ -271,12 +271,20 @@ def _resolve_model_dir(spec: SherpaModelSpec, *, download: bool = True) -> str: Restricts the fetch to the exact int8 assets we pin via ``allow_patterns`` so we never pull the bundled fp32 weights or test wavs. """ + from huggingface_hub import constants as hf_constants from huggingface_hub import snapshot_download + from services.hf_revisions import installed_revision, revision_for wanted = list(spec.files.values()) + # Probe the revision an existing installation actually resolved. Older + # releases followed ``main`` and may therefore have a different snapshot; + # retaining it preserves offline upgrades. Any network fetch still uses + # the reviewed immutable pin. + installed = installed_revision(spec.repo_id, hf_constants.HF_HUB_CACHE) try: return snapshot_download( repo_id=spec.repo_id, + revision=installed, local_files_only=True, allow_patterns=wanted, ) @@ -284,7 +292,11 @@ def _resolve_model_dir(spec: SherpaModelSpec, *, download: bool = True) -> str: if not download: raise logger.info("sherpa dictation: downloading %s on first use", spec.repo_id) - return snapshot_download(repo_id=spec.repo_id, allow_patterns=wanted) + return snapshot_download( + repo_id=spec.repo_id, + revision=revision_for(spec.repo_id), + allow_patterns=wanted, + ) def is_installed(spec: SherpaModelSpec) -> bool: diff --git a/docs/engines/moss-tts-v15.md b/docs/engines/moss-tts-v15.md index c931b241..2cf80848 100644 --- a/docs/engines/moss-tts-v15.md +++ b/docs/engines/moss-tts-v15.md @@ -104,7 +104,9 @@ default voice. `duration` (seconds) maps to MOSS's `tokens` argument at | Variable | Default | Purpose | |----------|---------|---------| | `OMNIVOICE_MOSS_TTS_V15_DIR` | — | Path to the MOSS-TTS clone (required). | -| `OMNIVOICE_MOSS_TTS_V15_MODEL` | `OpenMOSS-Team/MOSS-TTS-v1.5` | HF repo id override (mirror / air-gapped). | +| `OMNIVOICE_MOSS_TTS_V15_MODEL` | `OpenMOSS-Team/MOSS-TTS-v1.5` | Advanced HF repo override. Custom repositories execute their modelling code and are rejected unless both safeguards below are set. Configure `HF_ENDPOINT` for a mirror instead. | +| `OMNIVOICE_MOSS_TTS_V15_REVISION` | — | Immutable 40-character commit SHA required with a custom model repository. | +| `OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE` | — | Set to `1` only after auditing a custom repository's Python code. The reviewed built-in repository needs no opt-in. | | `OMNIVOICE_MOSS_TTS_V15_ATTN` | `sdpa` | Attention impl; set `flash_attention_2` on Ampere+ CUDA with `flash-attn` installed. | ## Common errors diff --git a/tests/test_gitleaks_allowlist.py b/tests/test_gitleaks_allowlist.py new file mode 100644 index 00000000..e57cd993 --- /dev/null +++ b/tests/test_gitleaks_allowlist.py @@ -0,0 +1,33 @@ +"""The secret-scan allowlist must remain exact and value-scoped.""" +from pathlib import Path +import re +import tomllib + + +ROOT = Path(__file__).resolve().parents[1] +_POSTHOG_TOKEN = re.search( + r"phc_[A-Za-z0-9]{20,}", + (ROOT / "backend/core/analytics.py").read_text(encoding="utf-8"), +) +assert _POSTHOG_TOKEN is not None +EXPECTED_EXACT_REGEXES = { + f"^{_POSTHOG_TOKEN.group(0)}$", + "^528e871c2a26c4f0f7773b9754e2e1acae20899d$", + "^hf_abcdefghijklmnopqrstuvwxyz01234567890abcd$", + "^hf_abcdefghijklmnopqrstuvwxyz0123456789ABCDEF$", + "^hf_QWERTYUIOPasdfghjklZXCVBNM0123456789xyzAB$", + "^max_length=400$", +} + + +def test_gitleaks_allowlist_contains_only_reviewed_exact_values(): + config = tomllib.loads((ROOT / ".gitleaks.toml").read_text(encoding="utf-8")) + allowlist = config["allowlist"] + + assert set(allowlist) == {"description", "regexes"} + assert set(allowlist["regexes"]) == EXPECTED_EXACT_REGEXES + assert all( + regex.startswith("^") and regex.endswith("$") + for regex in allowlist["regexes"] + ) + assert "rules" not in config diff --git a/tests/test_hf_cache_repair.py b/tests/test_hf_cache_repair.py index a72f8e5a..2e271852 100644 --- a/tests/test_hf_cache_repair.py +++ b/tests/test_hf_cache_repair.py @@ -49,6 +49,12 @@ def _no_ambient_offline_mode(monkeypatch): explicitly via monkeypatch.setenv.""" monkeypatch.delenv("HF_HUB_OFFLINE", raising=False) monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False) + from services import hf_revisions + monkeypatch.setitem( + hf_revisions.CURATED_REVISIONS, + "test/checkpoint", + "a" * 40, + ) def _mk_repo_cache(tmp_path, repo_id: str = "test/checkpoint"): @@ -142,11 +148,11 @@ def test_repair_removes_only_broken_and_redownloads(tmp_path, monkeypatch): _symlink_or_skip(os.path.join("..", "..", "blobs", "MISSING"), str(snap / "model.safetensors")) calls = [] + monkeypatch.setenv("HF_HUB_CACHE", str(cache)) monkeypatch.setattr(huggingface_hub, "snapshot_download", lambda **k: calls.append(k)) - summary = hf_cache_repair.repair_repo_cache("test/checkpoint", - cache_dir=str(cache)) + summary = hf_cache_repair.repair_repo_cache("test/checkpoint") assert summary["found"] == 1 assert summary["removed"] == 1 assert summary["restored"] is True @@ -155,7 +161,11 @@ def test_repair_removes_only_broken_and_redownloads(tmp_path, monkeypatch): assert summary["error"] == "" # The broken entry is gone; snapshot_download was asked to restore it. assert not os.path.lexists(snap / "model.safetensors") - assert calls == [{"repo_id": "test/checkpoint", "cache_dir": str(cache)}] + assert calls == [{ + "repo_id": "test/checkpoint", + "revision": "a" * 40, + "cache_dir": str(cache), + }] # Healthy entries and blobs are untouched. assert (snap / "config.json").read_bytes() == b'{"ok": true}' assert (snap / "tokenizer.json").read_bytes() == b'{"tok": 1}' diff --git a/tests/test_hf_mirror_error_class.py b/tests/test_hf_mirror_error_class.py index 242cd147..329f116a 100644 --- a/tests/test_hf_mirror_error_class.py +++ b/tests/test_hf_mirror_error_class.py @@ -158,6 +158,13 @@ _TRUNCATED = OSError( def test_repair_records_why_it_failed(model_manager, monkeypatch): import huggingface_hub + from services import hf_revisions + + monkeypatch.setitem( + hf_revisions.CURATED_REVISIONS, + "test/checkpoint", + "a" * 40, + ) def boom(**kwargs): raise OSError(_TRANSFORMERS_874) diff --git a/tests/test_hf_revisions.py b/tests/test_hf_revisions.py new file mode 100644 index 00000000..7ecfe4e8 --- /dev/null +++ b/tests/test_hf_revisions.py @@ -0,0 +1,75 @@ +"""Curated Hugging Face inputs are immutable and repair preserves installs.""" +from pathlib import Path + +import yaml + +from services import hf_revisions + + +def test_every_catalog_repo_has_an_immutable_revision(): + catalog = yaml.safe_load(Path("backend/config/models.yaml").read_text(encoding="utf-8")) + missing = { + model["repo_id"] + for model in catalog["models"] + if model["repo_id"] not in hf_revisions.CURATED_REVISIONS + } + assert missing == set() + assert all(len(revision) == 40 for revision in hf_revisions.CURATED_REVISIONS.values()) + assert all(int(revision, 16) >= 0 for revision in hf_revisions.CURATED_REVISIONS.values()) + + +def test_nllb_components_use_the_reviewed_revision(): + from api.routers import dub_translate + + calls = [] + + class FakeFactory: + @classmethod + def from_pretrained(cls, repo_id, **kwargs): + calls.append((repo_id, kwargs)) + return object() + + dub_translate._load_nllb_component(FakeFactory) + + repo_id = "facebook/nllb-200-distilled-600M" + assert calls == [ + (repo_id, {"revision": hf_revisions.revision_for(repo_id)}) + ] + + +def test_installed_revision_round_trips_for_repair(tmp_path): + repo_id = "k2-fsa/OmniVoice" + installed = "f" * 40 + hf_revisions.remember_revision(repo_id, installed, str(tmp_path)) + assert hf_revisions.installed_revision(repo_id, str(tmp_path)) == installed + + +def test_missing_or_invalid_marker_falls_back_to_reviewed_pin(tmp_path): + repo_id = "k2-fsa/OmniVoice" + assert hf_revisions.installed_revision(repo_id, str(tmp_path)) == hf_revisions.revision_for(repo_id) + marker = tmp_path / "models--k2-fsa--OmniVoice" / "voicestudio-revision" + marker.parent.mkdir(parents=True) + marker.write_text("main\n", encoding="ascii") + assert hf_revisions.installed_revision(repo_id, str(tmp_path)) == hf_revisions.revision_for(repo_id) + + +def test_existing_hub_main_ref_is_preserved_for_upgrade_repair(tmp_path): + repo_id = "k2-fsa/OmniVoice" + existing = "e" * 40 + ref = tmp_path / "models--k2-fsa--OmniVoice" / "refs" / "main" + ref.parent.mkdir(parents=True) + ref.write_text(existing + "\n", encoding="ascii") + assert hf_revisions.installed_revision(repo_id, str(tmp_path)) == existing + + +def test_unknown_repo_cannot_start_a_network_repair(tmp_path): + repo_dir = tmp_path / "models--attacker--unreviewed" + for marker in (repo_dir / "voicestudio-revision", repo_dir / "refs" / "main"): + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("a" * 40 + "\n", encoding="ascii") + try: + hf_revisions.installed_revision("attacker/unreviewed", str(tmp_path)) + except ValueError as exc: + assert "No reviewed revision" in str(exc) + else: # pragma: no cover - assertion message is clearer than pytest.raises here + raise AssertionError("unreviewed repository unexpectedly received a revision") diff --git a/tests/test_install_disk_space.py b/tests/test_install_disk_space.py index 0a00d28a..af957fd4 100644 --- a/tests/test_install_disk_space.py +++ b/tests/test_install_disk_space.py @@ -90,7 +90,10 @@ def test_install_worker_emits_install_error_and_skips_download(models_mod, monke # call must never happen (the guard returns first). Fail loudly if it does. import huggingface_hub + calls = [] + def _fake_snapshot(**kwargs): + calls.append(kwargs) if kwargs.get("dry_run"): return [] # preflight plan input (compute_plan is stubbed anyway) raise AssertionError("snapshot_download called for a real download despite disk-full reject") @@ -116,5 +119,46 @@ def test_install_worker_emits_install_error_and_skips_download(models_mod, monke errs = [e for e in events if e.get("phase") == "install_error"] assert errs, f"expected an install_error event, got phases: {[e.get('phase') for e in events]}" assert "disk space" in errs[0]["error"].lower() + from services.hf_revisions import revision_for + assert calls[0]["revision"] == revision_for(repo_id) # And the resolving heartbeat must not have leaked — no infinite 'resolving' # stream after the bail (the worker set the stop event before returning). + + +def test_install_preflight_download_and_repair_marker_share_revision(models_mod, monkeypatch): + download = importlib.import_module("api.routers.setup.download") + import asyncio + import huggingface_hub + from services import hf_revisions + + repo_id = download.KNOWN_MODELS[0]["repo_id"] + expected = hf_revisions.revision_for(repo_id) + calls = [] + remembered = [] + + def fake_snapshot(**kwargs): + calls.append(kwargs) + return [] if kwargs.get("dry_run") else "/cache/snapshots/" + expected + + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot) + monkeypatch.setattr(download, "compute_plan", lambda _plan: { + "total_bytes": 1, "cached_bytes": 0, "to_download_bytes": 1, + "n_files": 1, "n_cached": 0, + }) + monkeypatch.setattr(download, "disk_space_error", lambda *_a, **_k: None) + monkeypatch.setattr(download, "_segmented_enabled", lambda: False) + monkeypatch.setattr(download, "_validate_snapshot_has_weights", lambda *_a: None) + monkeypatch.setattr(hf_revisions, "remember_revision", lambda *args: remembered.append(args)) + + async def run_install(): + await download.install_model(download.InstallModelRequest(repo_id=repo_id)) + pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] + if pending: + await asyncio.gather(*pending) + + asyncio.run(run_install()) + + assert [call["revision"] for call in calls] == [expected, expected] + assert calls[0]["dry_run"] is True + assert "dry_run" not in calls[1] + assert remembered and remembered[0][0:2] == (repo_id, expected) diff --git a/tests/test_model_cache_repair.py b/tests/test_model_cache_repair.py index 25d0e699..08605370 100644 --- a/tests/test_model_cache_repair.py +++ b/tests/test_model_cache_repair.py @@ -23,6 +23,7 @@ def model_manager(monkeypatch): sys.modules.pop(mod_name, None) import services.model_manager as mm + from services import hf_revisions monkeypatch.setattr(mm, "_torch", None) monkeypatch.setattr(mm, "_OmniVoice", None) @@ -33,6 +34,7 @@ def model_manager(monkeypatch): monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False) monkeypatch.setattr(mm, "_lazy_torch", lambda: SimpleNamespace(float16="float16")) monkeypatch.setattr(mm, "get_best_device", lambda: "cpu") + monkeypatch.setitem(hf_revisions.CURATED_REVISIONS, "test/checkpoint", "a" * 40) return mm @@ -189,7 +191,7 @@ def test_repair_skipped_in_offline_mode(model_manager, monkeypatch): assert called == [] # no download attempted offline -def test_repair_invokes_snapshot_download(model_manager, monkeypatch): +def test_repair_invokes_snapshot_download(model_manager, monkeypatch, tmp_path): """Repair re-fetches the repo via snapshot_download (resume/fill missing).""" calls = [] @@ -198,10 +200,13 @@ def test_repair_invokes_snapshot_download(model_manager, monkeypatch): return "/cache/test/checkpoint" import huggingface_hub + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path)) monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) assert model_manager._repair_model_cache("test/checkpoint") is True assert calls and calls[0]["repo_id"] == "test/checkpoint" + assert calls[0]["revision"] == "a" * 40 + assert calls[0]["cache_dir"] == str(tmp_path) def test_repair_returns_false_when_download_fails(model_manager, monkeypatch): diff --git a/tests/test_moss_tts_v15.py b/tests/test_moss_tts_v15.py index d78fd107..7592c4b9 100644 --- a/tests/test_moss_tts_v15.py +++ b/tests/test_moss_tts_v15.py @@ -63,6 +63,46 @@ def test_sidecar_script_ships(): assert MOSS_TTS_V15_SIDECAR_SCRIPT.is_file() +def test_default_model_source_is_pinned(monkeypatch): + from engines.moss_tts_v15 import main + monkeypatch.delenv("OMNIVOICE_MOSS_TTS_V15_MODEL", raising=False) + assert main._model_source() == (main._DEFAULT_REPO, main._DEFAULT_REVISION) + + +def test_default_model_revision_matches_the_central_reviewed_pin(): + from engines.moss_tts_v15 import main + from services.hf_revisions import revision_for + + assert main._DEFAULT_REVISION == revision_for(main._DEFAULT_REPO) + + +def test_custom_remote_code_is_rejected_without_explicit_opt_in(monkeypatch): + from engines.moss_tts_v15 import main + monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_MODEL", "someone/custom-model") + monkeypatch.delenv("OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE", raising=False) + monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_REVISION", "a" * 40) + with pytest.raises(RuntimeError, match="after auditing"): + main._model_source() + + +def test_custom_remote_code_requires_immutable_revision(monkeypatch): + from engines.moss_tts_v15 import main + monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_MODEL", "someone/custom-model") + monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE", "1") + monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_REVISION", "main") + with pytest.raises(RuntimeError, match="40-character commit SHA"): + main._model_source() + + +def test_audited_custom_remote_code_requires_both_opt_ins(monkeypatch): + from engines.moss_tts_v15 import main + revision = "b" * 40 + monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_MODEL", "someone/custom-model") + monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE", "true") + monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_REVISION", revision) + assert main._model_source() == ("someone/custom-model", revision) + + # ── hardware honesty (cross-platform rule) ───────────────────────────────── diff --git a/tests/test_sherpa_dictation.py b/tests/test_sherpa_dictation.py index 85bbc875..2790a7f0 100644 --- a/tests/test_sherpa_dictation.py +++ b/tests/test_sherpa_dictation.py @@ -174,6 +174,60 @@ def test_get_spec_accepts_repo_id(): assert not sd.is_sherpa_model(None) +def test_model_resolution_pins_offline_probe_and_download(monkeypatch, tmp_path): + from services import hf_revisions, sherpa_dictation as sd + import huggingface_hub + from huggingface_hub import constants as hf_constants + + spec = sd.get_spec("sherpa-whisper-tiny") + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + calls = [] + + def fake_snapshot(**kwargs): + calls.append(kwargs) + if kwargs.get("local_files_only"): + raise FileNotFoundError("not cached") + return "/cache/pinned" + + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot) + assert sd._resolve_model_dir(spec) == "/cache/pinned" + assert len(calls) == 2 + assert all(call["revision"] == hf_revisions.revision_for(spec.repo_id) for call in calls) + + +def test_model_resolution_probes_preserved_legacy_snapshot(monkeypatch, tmp_path): + from services import hf_revisions, sherpa_dictation as sd + import huggingface_hub + from huggingface_hub import constants as hf_constants + + spec = sd.get_spec("sherpa-whisper-tiny") + legacy_revision = "e" * 40 + ref = ( + tmp_path + / "models--csukuangfj--sherpa-onnx-whisper-tiny" + / "refs" + / "main" + ) + ref.parent.mkdir(parents=True) + ref.write_text(legacy_revision + "\n", encoding="ascii") + monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path)) + calls = [] + + def fake_snapshot(**kwargs): + calls.append(kwargs) + return "/cache/legacy" + + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot) + assert sd._resolve_model_dir(spec) == "/cache/legacy" + assert calls == [{ + "repo_id": spec.repo_id, + "revision": legacy_revision, + "local_files_only": True, + "allow_patterns": list(spec.files.values()), + }] + assert calls[0]["revision"] != hf_revisions.revision_for(spec.repo_id) + + # ── The 4 recognizer kinds construct + transcribe ───────────────────────────