Merge remote-tracking branch 'origin/main' into fix/ghas-path-boundary

# Conflicts:
#	CHANGELOG.md
This commit is contained in:
debpalash
2026-08-10 08:10:23 +00:00
20 changed files with 513 additions and 31 deletions
+14 -4
View File
@@ -1,7 +1,8 @@
# Gitleaks config — extends the default ruleset.
#
# The ONLY sanctioned allowlist entry is PostHog's publishable project token
# (owner decision 2026-07-20, #1193). Per PostHog's docs the `phc_` project
# Every entry below is an exact, anchored non-secret value. PostHog's
# publishable project token is public by design (owner decision 2026-07-20,
# #1193). Per PostHog's docs the `phc_` project
# token is a write-only client key with "no access to your private data" —
# it ships in every release binary and every official PostHog SDK snippet.
# It is NOT a credential. Personal keys (`phx_`) remain fully banned.
@@ -12,7 +13,16 @@
useDefault = true
[allowlist]
description = "PostHog publishable write-only project token (public by design; #1193)"
description = "Exact public/test literals misclassified as generic API keys"
regexes = [
'''phc_v5wMjnYMPMaEcRNLRKQsTYCzPaYWh7wcHPhXNkNajVf9''',
# Public PostHog project token; personal `phx_` keys remain banned.
'''^phc_v5wMjnYMPMaEcRNLRKQsTYCzPaYWh7wcHPhXNkNajVf9$''',
# Reviewed immutable Hugging Face commit for the Higgs tokenizer.
'''^528e871c2a26c4f0f7773b9754e2e1acae20899d$''',
# Deliberately synthetic fixtures that exercise HF-token redaction/storage.
'''^hf_abcdefghijklmnopqrstuvwxyz01234567890abcd$''',
'''^hf_abcdefghijklmnopqrstuvwxyz0123456789ABCDEF$''',
'''^hf_QWERTYUIOPasdfghjklZXCVBNM0123456789xyzAB$''',
# NLLB generation length argument, not the value of a credential.
'''^max_length=400$''',
]
+1
View File
@@ -42,6 +42,7 @@ The bundled TTS model package (`pyproject.toml`) is versioned independently.
### Fixed
- Remote backends can no longer probe or overwrite arbitrary host files through native-only tools, and imported or persisted paths cannot escape their VoiceStudio data folders. (#1455)
- Curated models now install and repair from reviewed, immutable revisions; custom MOSS remote code requires an explicit safety opt-in. (#1453)
- YouTube imports that require a signed-in session can now use an explicitly selected `cookies.txt` export for one import; VoiceStudio never reads browser cookies silently and makes two best-effort attempts to delete its temporary copy. (#1429, #1432) — thanks @dongqing1968-sudo and @phamvandu9595-tech!
- First-run source builds no longer stop after uv was successfully downloaded just because its installer failed during a later shell-profile step; app-private uv installs no longer touch shell profiles at all. (#1438) — thanks @AdrianoCahete!
- Model files damaged by an interrupted download now repair themselves instead of failing every generation, including invalid `config.json` files and corrupt weight headers. — thanks @overrunau and @zherunh! (#1406, #1437)
+13 -2
View File
@@ -8,12 +8,23 @@ from fastapi.responses import JSONResponse
from schemas.requests import TranslateRequest
from services.model_manager import _cpu_pool, _gpu_pool
from services.hf_revisions import revision_for
from services.translator import cinematic_available, cinematic_refine_many, _cinematic_budget
from api.routers.dub_core import _get_job, _save_job
router = APIRouter()
logger = logging.getLogger("omnivoice.api")
_NLLB_REPO_ID = "facebook/nllb-200-distilled-600M"
def _load_nllb_component(factory):
"""Load a curated NLLB component from its reviewed immutable revision."""
return factory.from_pretrained(
_NLLB_REPO_ID,
revision=revision_for(_NLLB_REPO_ID),
)
TRANSLATE_CODES = {
"en": "en", "es": "es", "fr": "fr", "de": "de", "it": "it", "pt": "pt",
"ru": "ru", "ja": "ja", "ko": "ko", "zh": "zh-CN", "cmn-Hans": "zh-CN",
@@ -287,9 +298,9 @@ async def dub_translate(req: TranslateRequest):
try:
if _nllb_tokenizer is None:
_nllb_tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")
_nllb_tokenizer = _load_nllb_component(AutoTokenizer)
if _nllb_model is None:
_nllb_model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M")
_nllb_model = _load_nllb_component(AutoModelForSeq2SeqLM)
if target_device != "cpu":
try:
_nllb_model = _nllb_model.to(target_device)
+20 -7
View File
@@ -20,6 +20,7 @@ from pydantic import BaseModel
from core import prefs
from core.failure import is_hf_connectivity_error
from services.hf_revisions import revision_for
from utils import hf_progress
from utils import download_aggregator
# Weight-floor scan (MM2-07 / #352) lives in ``models.py`` — the lowest module in
@@ -170,7 +171,7 @@ def _repo_cancelled(repo_id: str) -> bool:
return repo_id in _cancelled
def _segmented_snapshot(repo_id: str, *, endpoint: "str | None") -> str:
def _segmented_snapshot(repo_id: str, *, endpoint: "str | None", revision: str) -> str:
"""Fetch every file of a repo via the segmented downloader into the HF
cache, mirroring hf_hub_download's blob+snapshot+refs layout so the result
is indistinguishable from snapshot_download (FDL-09) keeping /models
@@ -187,10 +188,10 @@ def _segmented_snapshot(repo_id: str, *, endpoint: "str | None") -> str:
token = _resolve_token()
api = HfApi(endpoint=endpoint, token=token)
info = api.repo_info(repo_id, repo_type="model")
info = api.repo_info(repo_id, repo_type="model", revision=revision)
commit = info.sha
files = [s.rfilename for s in (info.siblings or [])]
if not commit or not files:
if commit != revision or not files:
raise RuntimeError("repo_info returned no commit/siblings")
repo_dir = os.path.join(_C.HF_HUB_CACHE, repo_folder_name(repo_id=repo_id, repo_type="model"))
@@ -407,6 +408,7 @@ async def install_model(req: InstallModelRequest):
# parallel-files worker count, and honour an optional mirror endpoint.
dl_kwargs: dict = {
"repo_id": req.repo_id,
"revision": revision_for(req.repo_id),
"max_workers": _download_max_workers(),
}
_tqdm_cls = hf_progress.tracked_tqdm_class()
@@ -447,11 +449,15 @@ async def install_model(req: InstallModelRequest):
# bytes that will actually download — BEFORE any byte flows. Seeds
# the overall aggregator so its bar/ETA are correct from the first
# event. Degrades gracefully (totals=None) on older/gated repos.
_preflight_kwargs = {"repo_id": req.repo_id, "dry_run": True}
_preflight_kwargs = {
"repo_id": req.repo_id,
"revision": dl_kwargs["revision"],
"dry_run": True,
}
if _endpoint:
_preflight_kwargs["endpoint"] = _endpoint
try:
_plan = snapshot_download(**_preflight_kwargs)
_plan = snapshot_download(**_preflight_kwargs) # nosec B615 -- immutable revision_for pin
_summary = compute_plan(_plan)
# Disk-space guard (before a single byte flows): the preflight
# gives an exact "to download" size, so reject an install that
@@ -515,7 +521,11 @@ async def install_model(req: InstallModelRequest):
_snapshot_path = None
if _attempt == 1 and _segmented_enabled() and not _xet_active():
try:
_snapshot_path = _segmented_snapshot(req.repo_id, endpoint=_endpoint)
_snapshot_path = _segmented_snapshot(
req.repo_id,
endpoint=_endpoint,
revision=dl_kwargs["revision"],
)
except _InstallCancelled:
raise
except Exception as _seg_err:
@@ -525,8 +535,11 @@ async def install_model(req: InstallModelRequest):
)
_snapshot_path = None
if _snapshot_path is None:
_snapshot_path = snapshot_download(**dl_kwargs)
_snapshot_path = snapshot_download(**dl_kwargs) # nosec B615 -- immutable revision_for pin
_validate_snapshot_has_weights(req.repo_id, _snapshot_path)
from huggingface_hub.constants import HF_HUB_CACHE
from services.hf_revisions import remember_revision
remember_revision(req.repo_id, dl_kwargs["revision"], HF_HUB_CACHE)
break
except Exception as net_err:
# #1224: a truncated body ("peer closed connection without
+29 -3
View File
@@ -36,6 +36,7 @@ from __future__ import annotations
import base64
import json
import os
import re
import struct
import sys
import traceback
@@ -50,8 +51,30 @@ MAX_FRAME_BYTES = 64 * 1024 * 1024
#: the loaded model at synthesize time; this is the handshake default).
MOSS_SAMPLE_RATE = 24000
#: HF repo id for the weights, overridable for air-gapped / mirror installs.
#: Reviewed HF repo and immutable revision for the default remote-code model.
_DEFAULT_REPO = "OpenMOSS-Team/MOSS-TTS-v1.5"
_DEFAULT_REVISION = "cdd3b911b1585e3f2dbc7775ef10f9926f58850a"
_SHA = re.compile(r"[0-9a-f]{40}\Z")
def _model_source() -> tuple[str, str]:
"""Return a pinned model source; custom remote code requires two opt-ins."""
repo = os.environ.get("OMNIVOICE_MOSS_TTS_V15_MODEL", _DEFAULT_REPO)
if repo == _DEFAULT_REPO:
return repo, _DEFAULT_REVISION
unsafe = os.environ.get("OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE", "").lower()
revision = os.environ.get("OMNIVOICE_MOSS_TTS_V15_REVISION", "")
if unsafe not in {"1", "true", "yes", "on"}:
raise RuntimeError(
"A custom MOSS model contains executable remote code. Set "
"OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE=1 only after auditing it."
)
if not _SHA.fullmatch(revision):
raise RuntimeError(
"A custom MOSS model requires OMNIVOICE_MOSS_TTS_V15_REVISION="
"<40-character commit SHA>; branches and tags are mutable."
)
return repo, revision
#: ISO-639-1 → MOSS language name. MOSS's ``build_user_message`` takes a
#: language *name* ("French"), not a code. Unknown codes are omitted so the
@@ -130,14 +153,16 @@ def _load_model(stdout):
import torch
from transformers import AutoModel, AutoProcessor
repo = os.environ.get("OMNIVOICE_MOSS_TTS_V15_MODEL", _DEFAULT_REPO)
repo, revision = _model_source()
device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if device == "cuda" else torch.float32
# "sdpa" works on CUDA + CPU and needs no extra dep. flash_attention_2
# (Ampere+ CUDA, optional flash-attn) is opt-in via env.
attn = os.environ.get("OMNIVOICE_MOSS_TTS_V15_ATTN", "sdpa")
processor = AutoProcessor.from_pretrained(repo, trust_remote_code=True)
processor = AutoProcessor.from_pretrained(
repo, revision=revision, trust_remote_code=True,
)
# The audio tokenizer is a separate sub-module that must be moved to the
# device independently (easy to miss — see upstream README).
processor.audio_tokenizer = processor.audio_tokenizer.to(device)
@@ -146,6 +171,7 @@ def _load_model(stdout):
model = AutoModel.from_pretrained(
repo,
revision=revision,
trust_remote_code=True,
attn_implementation=attn,
torch_dtype=dtype,
+13 -5
View File
@@ -209,6 +209,12 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict:
)
return summary
# Resolve the exact revision before deleting even a broken pointer. An
# unreviewed repository can never turn cache repair into a mutable-
# branch network fetch.
from services.hf_revisions import installed_revision
revision = installed_revision(repo_id, cache_root)
def _remove(paths: list[str]) -> int:
n = 0
for path in paths:
@@ -232,13 +238,15 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict:
from huggingface_hub import snapshot_download
dl_kwargs: dict = {"repo_id": repo_id}
if cache_dir:
dl_kwargs["cache_dir"] = cache_dir
dl_kwargs: dict = {
"repo_id": repo_id,
"revision": revision,
"cache_dir": cache_root,
}
endpoint = os.environ.get("HF_ENDPOINT")
if endpoint:
dl_kwargs["endpoint"] = endpoint
snapshot_download(**dl_kwargs)
snapshot_download(**dl_kwargs) # nosec B615 -- installed immutable revision
summary["restored"] = True
# Verify-after-repair: hub's memoized symlink probe can claim support
@@ -267,7 +275,7 @@ def repair_repo_cache(repo_id: str, cache_dir: str | None = None) -> dict:
)
return summary
summary["removed"] += _remove(still_broken)
snapshot_download(**dl_kwargs)
snapshot_download(**dl_kwargs) # nosec B615 -- installed immutable revision
remaining = find_dangling_entries(repo_dir)
if remaining:
summary["error"] = (
+96
View File
@@ -0,0 +1,96 @@
"""Immutable Hugging Face revisions for VoiceStudio's curated repositories.
Branch names are mutable supply-chain inputs. Every repo the product offers is
resolved here to a reviewed commit SHA; download, preflight, and repair paths
must call :func:`revision_for` instead of following ``main``.
"""
from __future__ import annotations
import os
import re
from pathlib import Path
_SHA = re.compile(r"[0-9a-f]{40}\Z")
CURATED_REVISIONS: dict[str, str] = {
"facebook/nllb-200-distilled-600M": "f8d333a098d19b4fd9a8b18f94170487ad3f821d",
"k2-fsa/OmniVoice": "c5fdb5ccb189668d56333f77ba2629f4cd7535f4",
"Systran/faster-whisper-large-v3": "edaa852ec7e145841d8ffdb056a99866b5f0a478",
"mlx-community/whisper-large-v3-mlx": "49e6aa286ad60c14352c404340ded53710378a11",
"mlx-community/whisper-large-v3-turbo": "a4aaeec0636e6fef84abdcbe3544cb2bf7e9f6fb",
"openai/whisper-large-v3": "06f233fe06e710322aca913c1bc4249a0d71fce1",
"mlx-community/whisper-tiny-mlx": "6caf9c55601caafbe6508a8b0d216bdf4783c4e8",
"deepdml/faster-whisper-large-v3-turbo-ct2": "4df90f75321148c3a29a9e2351b7ddf8f5b115a8",
"Systran/faster-distil-whisper-large-v3": "c3058b475261292e64a0412df1d2681c06260fab",
"Systran/faster-whisper-medium": "08e178d48790749d25932bbc082711ddcfdfbc4f",
"Systran/faster-whisper-small": "536b0662742c02347bc0e980a01041f333bce120",
"Systran/faster-whisper-base": "ebe41f70d5b6dfa9166e2c581c45c9c0cfc57b66",
"nvidia/parakeet-tdt-0.6b-v3": "541d1f99c6b0c3cd0b11a95167540bb8edefd82b",
"nvidia/parakeet-tdt-0.6b-v2": "ae9ad07059c7c739ffaf932226a8fe64ae2620b0",
"mlx-community/parakeet-tdt-0.6b-v3": "ed2b7e8c15f9aaa0b5772e2efb986255eaef7e15",
"UsefulSensors/moonshine-base": "7a73d8d55ac0ba2ef3ae761593f6784b51f96dcf",
"UsefulSensors/moonshine-tiny": "390624ed33d594443aa4aa221f5b9f283b545b5a",
"csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8": "2bda32ec70b097a55adaa07d9a7173915b43cc78",
"csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8": "1ab9323565ddb038682214b292f588070a538ce2",
"csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20": "98590b7ed6443e77b714204da2757d75e1a642f4",
"csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en": "8e40c43232a1c5c66c82111efc5820d3accca11b",
"csukuangfj/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17": "d42f2d9f7ca24806fb667456a18a9f1b60f70d16",
"csukuangfj/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23": "204ad334e2e683fd295359930cc16fc0432a23ac",
"csukuangfj/sherpa-onnx-whisper-tiny": "65176e2deb88badc814a94058666cadccc29b61c",
"pyannote/speaker-diarization-3.1": "84fd25912480287da0247647c3d2b4853cb3ee5d",
"OpenMOSS-Team/MOSS-TTS-Nano-100M": "44502f80dbf9743528fa921cc544d662c685ebec",
"KittenML/kitten-tts-mini-0.8": "c02725660cea441db4c383af69f1f26f5cd00947",
"mlx-community/Kokoro-82M-bf16": "a71e4d38b236d968966a2002c4c895dbd12b1c3c",
"mlx-community/csm-1b-8bit": "fcf0cc857eade3615a60f30722cf5197d4f88406",
"mlx-community/Qwen3-TTS-12Hz-1.7B-VoiceDesign-4bit": "5c390979e4b93af5f2932f90742ca99c7dd04687",
"mlx-community/Dia-1.6B": "de4fa8c178ca5cc4e9d884b55b03fcfaa0995162",
"mlx-community/Llama-OuteTTS-1.0-1B-4bit": "3ac2cff406f7de16a3216c60d0108571a916acc0",
"mlx-community/Chatterbox-TTS-4bit": "a3c8ded2d711d6395410d645b3a97c79fd563a13",
"mlx-community/MeloTTS-English-v3-MLX": "837d15fd72bc35a15033234ce5ea242367ca1960",
"OpenMOSS-Team/MOSS-TTS-v1.5": "cdd3b911b1585e3f2dbc7775ef10f9926f58850a",
"eustlb/higgs-audio-v2-tokenizer": "528e871c2a26c4f0f7773b9754e2e1acae20899d",
}
def revision_for(repo_id: str) -> str:
"""Return the immutable revision for a curated repo, or raise."""
try:
return CURATED_REVISIONS[repo_id]
except KeyError as exc:
raise ValueError(f"No reviewed revision is pinned for {repo_id!r}") from exc
def _repo_dir(repo_id: str, cache_dir: str) -> Path:
return Path(cache_dir) / ("models--" + repo_id.replace("/", "--"))
def remember_revision(repo_id: str, revision: str, cache_dir: str) -> None:
"""Persist the exact installed revision for later in-place repair."""
if not _SHA.fullmatch(revision):
raise ValueError("Hugging Face revision must be a 40-character commit SHA")
repo_dir = _repo_dir(repo_id, cache_dir)
repo_dir.mkdir(parents=True, exist_ok=True)
marker = repo_dir / "voicestudio-revision"
temporary = marker.with_suffix(f".tmp-{os.getpid()}")
temporary.write_text(revision + "\n", encoding="ascii")
os.replace(temporary, marker)
def installed_revision(repo_id: str, cache_dir: str) -> str:
"""Return VoiceStudio's recorded revision, falling back to the curated pin."""
# Authenticate the repository before consulting attacker-writable cache
# metadata. A syntactically valid marker must never authorize repair of a
# repository outside VoiceStudio's reviewed catalog.
curated_revision = revision_for(repo_id)
repo_dir = _repo_dir(repo_id, cache_dir)
# New installs write the first marker. ``refs/main`` preserves the commit
# resolved by older VoiceStudio/huggingface_hub installs, so upgrades repair
# the bytes the user actually installed rather than silently changing them.
for marker in (repo_dir / "voicestudio-revision", repo_dir / "refs" / "main"):
try:
revision = marker.read_text(encoding="ascii").strip()
except OSError:
continue
if _SHA.fullmatch(revision):
return revision
return curated_revision
+16 -3
View File
@@ -1662,7 +1662,20 @@ def _repair_model_cache(checkpoint: str, *, force: bool = False) -> bool:
logger.warning("Cannot import snapshot_download to repair cache: %s", imp_err)
_last_repair_error = f"{type(imp_err).__name__}: {imp_err}"
return False
dl_kwargs: dict = {"repo_id": checkpoint}
try:
from services.hf_cache_repair import hf_cache_home
from services.hf_revisions import installed_revision
cache_root = hf_cache_home()
revision = installed_revision(checkpoint, cache_root)
except (OSError, ValueError) as revision_err:
_last_repair_error = str(revision_err)
logger.warning("Refusing unpinned model repair for %s: %s", checkpoint, revision_err)
return False
dl_kwargs: dict = {
"repo_id": checkpoint,
"revision": revision,
"cache_dir": cache_root,
}
# Explicit endpoint (HF_ENDPOINT / pref) wins; otherwise the automatic
# endpoint selection's cached pick applies (services.endpoint_race).
try:
@@ -1683,12 +1696,12 @@ def _repair_model_cache(checkpoint: str, *, force: bool = False) -> bool:
"""One snapshot_download, tolerating an hf_hub that rejects the optional
symlink knob. Lets real failures (network, gated repo, disk) propagate."""
try:
snapshot_download(**dl_kwargs)
snapshot_download(**dl_kwargs) # nosec B615 -- installed immutable revision
except TypeError:
# Older/newer huggingface_hub may not accept local_dir_use_symlinks
# on a cache-only call — retry without the optional knob.
dl_kwargs.pop("local_dir_use_symlinks", None)
snapshot_download(**dl_kwargs)
snapshot_download(**dl_kwargs) # nosec B615 -- installed immutable revision
# Bounded retries (#739): an incomplete cache *is* an interrupted download, so
# a single transient blip mid-repair shouldn't drop the user back to a manual
+13 -1
View File
@@ -271,12 +271,20 @@ def _resolve_model_dir(spec: SherpaModelSpec, *, download: bool = True) -> str:
Restricts the fetch to the exact int8 assets we pin via ``allow_patterns``
so we never pull the bundled fp32 weights or test wavs.
"""
from huggingface_hub import constants as hf_constants
from huggingface_hub import snapshot_download
from services.hf_revisions import installed_revision, revision_for
wanted = list(spec.files.values())
# Probe the revision an existing installation actually resolved. Older
# releases followed ``main`` and may therefore have a different snapshot;
# retaining it preserves offline upgrades. Any network fetch still uses
# the reviewed immutable pin.
installed = installed_revision(spec.repo_id, hf_constants.HF_HUB_CACHE)
try:
return snapshot_download(
repo_id=spec.repo_id,
revision=installed,
local_files_only=True,
allow_patterns=wanted,
)
@@ -284,7 +292,11 @@ def _resolve_model_dir(spec: SherpaModelSpec, *, download: bool = True) -> str:
if not download:
raise
logger.info("sherpa dictation: downloading %s on first use", spec.repo_id)
return snapshot_download(repo_id=spec.repo_id, allow_patterns=wanted)
return snapshot_download(
repo_id=spec.repo_id,
revision=revision_for(spec.repo_id),
allow_patterns=wanted,
)
def is_installed(spec: SherpaModelSpec) -> bool:
+3 -1
View File
@@ -104,7 +104,9 @@ default voice. `duration` (seconds) maps to MOSS's `tokens` argument at
| Variable | Default | Purpose |
|----------|---------|---------|
| `OMNIVOICE_MOSS_TTS_V15_DIR` | — | Path to the MOSS-TTS clone (required). |
| `OMNIVOICE_MOSS_TTS_V15_MODEL` | `OpenMOSS-Team/MOSS-TTS-v1.5` | HF repo id override (mirror / air-gapped). |
| `OMNIVOICE_MOSS_TTS_V15_MODEL` | `OpenMOSS-Team/MOSS-TTS-v1.5` | Advanced HF repo override. Custom repositories execute their modelling code and are rejected unless both safeguards below are set. Configure `HF_ENDPOINT` for a mirror instead. |
| `OMNIVOICE_MOSS_TTS_V15_REVISION` | — | Immutable 40-character commit SHA required with a custom model repository. |
| `OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE` | — | Set to `1` only after auditing a custom repository's Python code. The reviewed built-in repository needs no opt-in. |
| `OMNIVOICE_MOSS_TTS_V15_ATTN` | `sdpa` | Attention impl; set `flash_attention_2` on Ampere+ CUDA with `flash-attn` installed. |
## Common errors
+4 -1
View File
@@ -961,9 +961,12 @@ pub fn reveal_host_path(path: String) -> Result<(), String> {
command.arg(&folder);
command
};
crate::tools::no_window(&mut command)
let mut child = crate::tools::no_window(&mut command)
.spawn()
.map_err(|e| format!("Could not open the containing folder: {e}"))?;
std::thread::spawn(move || {
let _ = child.wait();
});
Ok(())
}
+19
View File
@@ -6,6 +6,7 @@ import asyncio
import inspect
import json
from contextlib import contextmanager
from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -89,6 +90,13 @@ def test_duration_probe_rejects_absolute_traversal_and_symlink_escapes(tmp_path,
secret = outside / "secret.mp4"
secret.write_bytes(b"not media")
monkeypatch.setattr(ffmpeg_utils, "find_ffprobe", lambda: "/should/not/run")
spawned = []
async def forbidden_spawn(*args, **kwargs):
spawned.append((args, kwargs))
raise AssertionError("rejected media path reached ffprobe")
monkeypatch.setattr(ffmpeg_utils, "spawn_subprocess", forbidden_spawn)
for value in (secret, "../outside/secret.mp4", r"..\outside\secret.mp4"):
assert asyncio.run(ffmpeg_utils.probe_duration(str(value), allowed_root=str(root))) is None
try:
@@ -98,6 +106,17 @@ def test_duration_probe_rejects_absolute_traversal_and_symlink_escapes(tmp_path,
assert asyncio.run(
ffmpeg_utils.probe_duration(str(root / "link" / "secret.mp4"), allowed_root=str(root))
) is None
assert spawned == []
def test_native_reveal_reaps_the_opener_child():
source = Path("frontend/src-tauri/src/commands.rs").read_text(encoding="utf-8")
reveal = source.split("pub fn reveal_host_path", 1)[1].split(
"// ── WebView cache repair", 1
)[0]
assert "let mut child = crate::tools::no_window(&mut command)" in reveal
assert "std::thread::spawn(move ||" in reveal
assert "child.wait()" in reveal
def test_marketplace_filename_cannot_escape_store(tmp_path, monkeypatch):
+33
View File
@@ -0,0 +1,33 @@
"""The secret-scan allowlist must remain exact and value-scoped."""
from pathlib import Path
import re
import tomllib
ROOT = Path(__file__).resolve().parents[1]
_POSTHOG_TOKEN = re.search(
r"phc_[A-Za-z0-9]{20,}",
(ROOT / "backend/core/analytics.py").read_text(encoding="utf-8"),
)
assert _POSTHOG_TOKEN is not None
EXPECTED_EXACT_REGEXES = {
f"^{_POSTHOG_TOKEN.group(0)}$",
"^528e871c2a26c4f0f7773b9754e2e1acae20899d$",
"^hf_abcdefghijklmnopqrstuvwxyz01234567890abcd$",
"^hf_abcdefghijklmnopqrstuvwxyz0123456789ABCDEF$",
"^hf_QWERTYUIOPasdfghjklZXCVBNM0123456789xyzAB$",
"^max_length=400$",
}
def test_gitleaks_allowlist_contains_only_reviewed_exact_values():
config = tomllib.loads((ROOT / ".gitleaks.toml").read_text(encoding="utf-8"))
allowlist = config["allowlist"]
assert set(allowlist) == {"description", "regexes"}
assert set(allowlist["regexes"]) == EXPECTED_EXACT_REGEXES
assert all(
regex.startswith("^") and regex.endswith("$")
for regex in allowlist["regexes"]
)
assert "rules" not in config
+13 -3
View File
@@ -49,6 +49,12 @@ def _no_ambient_offline_mode(monkeypatch):
explicitly via monkeypatch.setenv."""
monkeypatch.delenv("HF_HUB_OFFLINE", raising=False)
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)
from services import hf_revisions
monkeypatch.setitem(
hf_revisions.CURATED_REVISIONS,
"test/checkpoint",
"a" * 40,
)
def _mk_repo_cache(tmp_path, repo_id: str = "test/checkpoint"):
@@ -142,11 +148,11 @@ def test_repair_removes_only_broken_and_redownloads(tmp_path, monkeypatch):
_symlink_or_skip(os.path.join("..", "..", "blobs", "MISSING"),
str(snap / "model.safetensors"))
calls = []
monkeypatch.setenv("HF_HUB_CACHE", str(cache))
monkeypatch.setattr(huggingface_hub, "snapshot_download",
lambda **k: calls.append(k))
summary = hf_cache_repair.repair_repo_cache("test/checkpoint",
cache_dir=str(cache))
summary = hf_cache_repair.repair_repo_cache("test/checkpoint")
assert summary["found"] == 1
assert summary["removed"] == 1
assert summary["restored"] is True
@@ -155,7 +161,11 @@ def test_repair_removes_only_broken_and_redownloads(tmp_path, monkeypatch):
assert summary["error"] == ""
# The broken entry is gone; snapshot_download was asked to restore it.
assert not os.path.lexists(snap / "model.safetensors")
assert calls == [{"repo_id": "test/checkpoint", "cache_dir": str(cache)}]
assert calls == [{
"repo_id": "test/checkpoint",
"revision": "a" * 40,
"cache_dir": str(cache),
}]
# Healthy entries and blobs are untouched.
assert (snap / "config.json").read_bytes() == b'{"ok": true}'
assert (snap / "tokenizer.json").read_bytes() == b'{"tok": 1}'
+7
View File
@@ -158,6 +158,13 @@ _TRUNCATED = OSError(
def test_repair_records_why_it_failed(model_manager, monkeypatch):
import huggingface_hub
from services import hf_revisions
monkeypatch.setitem(
hf_revisions.CURATED_REVISIONS,
"test/checkpoint",
"a" * 40,
)
def boom(**kwargs):
raise OSError(_TRANSFORMERS_874)
+75
View File
@@ -0,0 +1,75 @@
"""Curated Hugging Face inputs are immutable and repair preserves installs."""
from pathlib import Path
import yaml
from services import hf_revisions
def test_every_catalog_repo_has_an_immutable_revision():
catalog = yaml.safe_load(Path("backend/config/models.yaml").read_text(encoding="utf-8"))
missing = {
model["repo_id"]
for model in catalog["models"]
if model["repo_id"] not in hf_revisions.CURATED_REVISIONS
}
assert missing == set()
assert all(len(revision) == 40 for revision in hf_revisions.CURATED_REVISIONS.values())
assert all(int(revision, 16) >= 0 for revision in hf_revisions.CURATED_REVISIONS.values())
def test_nllb_components_use_the_reviewed_revision():
from api.routers import dub_translate
calls = []
class FakeFactory:
@classmethod
def from_pretrained(cls, repo_id, **kwargs):
calls.append((repo_id, kwargs))
return object()
dub_translate._load_nllb_component(FakeFactory)
repo_id = "facebook/nllb-200-distilled-600M"
assert calls == [
(repo_id, {"revision": hf_revisions.revision_for(repo_id)})
]
def test_installed_revision_round_trips_for_repair(tmp_path):
repo_id = "k2-fsa/OmniVoice"
installed = "f" * 40
hf_revisions.remember_revision(repo_id, installed, str(tmp_path))
assert hf_revisions.installed_revision(repo_id, str(tmp_path)) == installed
def test_missing_or_invalid_marker_falls_back_to_reviewed_pin(tmp_path):
repo_id = "k2-fsa/OmniVoice"
assert hf_revisions.installed_revision(repo_id, str(tmp_path)) == hf_revisions.revision_for(repo_id)
marker = tmp_path / "models--k2-fsa--OmniVoice" / "voicestudio-revision"
marker.parent.mkdir(parents=True)
marker.write_text("main\n", encoding="ascii")
assert hf_revisions.installed_revision(repo_id, str(tmp_path)) == hf_revisions.revision_for(repo_id)
def test_existing_hub_main_ref_is_preserved_for_upgrade_repair(tmp_path):
repo_id = "k2-fsa/OmniVoice"
existing = "e" * 40
ref = tmp_path / "models--k2-fsa--OmniVoice" / "refs" / "main"
ref.parent.mkdir(parents=True)
ref.write_text(existing + "\n", encoding="ascii")
assert hf_revisions.installed_revision(repo_id, str(tmp_path)) == existing
def test_unknown_repo_cannot_start_a_network_repair(tmp_path):
repo_dir = tmp_path / "models--attacker--unreviewed"
for marker in (repo_dir / "voicestudio-revision", repo_dir / "refs" / "main"):
marker.parent.mkdir(parents=True, exist_ok=True)
marker.write_text("a" * 40 + "\n", encoding="ascii")
try:
hf_revisions.installed_revision("attacker/unreviewed", str(tmp_path))
except ValueError as exc:
assert "No reviewed revision" in str(exc)
else: # pragma: no cover - assertion message is clearer than pytest.raises here
raise AssertionError("unreviewed repository unexpectedly received a revision")
+44
View File
@@ -90,7 +90,10 @@ def test_install_worker_emits_install_error_and_skips_download(models_mod, monke
# call must never happen (the guard returns first). Fail loudly if it does.
import huggingface_hub
calls = []
def _fake_snapshot(**kwargs):
calls.append(kwargs)
if kwargs.get("dry_run"):
return [] # preflight plan input (compute_plan is stubbed anyway)
raise AssertionError("snapshot_download called for a real download despite disk-full reject")
@@ -116,5 +119,46 @@ def test_install_worker_emits_install_error_and_skips_download(models_mod, monke
errs = [e for e in events if e.get("phase") == "install_error"]
assert errs, f"expected an install_error event, got phases: {[e.get('phase') for e in events]}"
assert "disk space" in errs[0]["error"].lower()
from services.hf_revisions import revision_for
assert calls[0]["revision"] == revision_for(repo_id)
# And the resolving heartbeat must not have leaked — no infinite 'resolving'
# stream after the bail (the worker set the stop event before returning).
def test_install_preflight_download_and_repair_marker_share_revision(models_mod, monkeypatch):
download = importlib.import_module("api.routers.setup.download")
import asyncio
import huggingface_hub
from services import hf_revisions
repo_id = download.KNOWN_MODELS[0]["repo_id"]
expected = hf_revisions.revision_for(repo_id)
calls = []
remembered = []
def fake_snapshot(**kwargs):
calls.append(kwargs)
return [] if kwargs.get("dry_run") else "/cache/snapshots/" + expected
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot)
monkeypatch.setattr(download, "compute_plan", lambda _plan: {
"total_bytes": 1, "cached_bytes": 0, "to_download_bytes": 1,
"n_files": 1, "n_cached": 0,
})
monkeypatch.setattr(download, "disk_space_error", lambda *_a, **_k: None)
monkeypatch.setattr(download, "_segmented_enabled", lambda: False)
monkeypatch.setattr(download, "_validate_snapshot_has_weights", lambda *_a: None)
monkeypatch.setattr(hf_revisions, "remember_revision", lambda *args: remembered.append(args))
async def run_install():
await download.install_model(download.InstallModelRequest(repo_id=repo_id))
pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
if pending:
await asyncio.gather(*pending)
asyncio.run(run_install())
assert [call["revision"] for call in calls] == [expected, expected]
assert calls[0]["dry_run"] is True
assert "dry_run" not in calls[1]
assert remembered and remembered[0][0:2] == (repo_id, expected)
+6 -1
View File
@@ -23,6 +23,7 @@ def model_manager(monkeypatch):
sys.modules.pop(mod_name, None)
import services.model_manager as mm
from services import hf_revisions
monkeypatch.setattr(mm, "_torch", None)
monkeypatch.setattr(mm, "_OmniVoice", None)
@@ -33,6 +34,7 @@ def model_manager(monkeypatch):
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)
monkeypatch.setattr(mm, "_lazy_torch", lambda: SimpleNamespace(float16="float16"))
monkeypatch.setattr(mm, "get_best_device", lambda: "cpu")
monkeypatch.setitem(hf_revisions.CURATED_REVISIONS, "test/checkpoint", "a" * 40)
return mm
@@ -189,7 +191,7 @@ def test_repair_skipped_in_offline_mode(model_manager, monkeypatch):
assert called == [] # no download attempted offline
def test_repair_invokes_snapshot_download(model_manager, monkeypatch):
def test_repair_invokes_snapshot_download(model_manager, monkeypatch, tmp_path):
"""Repair re-fetches the repo via snapshot_download (resume/fill missing)."""
calls = []
@@ -198,10 +200,13 @@ def test_repair_invokes_snapshot_download(model_manager, monkeypatch):
return "/cache/test/checkpoint"
import huggingface_hub
monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path))
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download)
assert model_manager._repair_model_cache("test/checkpoint") is True
assert calls and calls[0]["repo_id"] == "test/checkpoint"
assert calls[0]["revision"] == "a" * 40
assert calls[0]["cache_dir"] == str(tmp_path)
def test_repair_returns_false_when_download_fails(model_manager, monkeypatch):
+40
View File
@@ -63,6 +63,46 @@ def test_sidecar_script_ships():
assert MOSS_TTS_V15_SIDECAR_SCRIPT.is_file()
def test_default_model_source_is_pinned(monkeypatch):
from engines.moss_tts_v15 import main
monkeypatch.delenv("OMNIVOICE_MOSS_TTS_V15_MODEL", raising=False)
assert main._model_source() == (main._DEFAULT_REPO, main._DEFAULT_REVISION)
def test_default_model_revision_matches_the_central_reviewed_pin():
from engines.moss_tts_v15 import main
from services.hf_revisions import revision_for
assert main._DEFAULT_REVISION == revision_for(main._DEFAULT_REPO)
def test_custom_remote_code_is_rejected_without_explicit_opt_in(monkeypatch):
from engines.moss_tts_v15 import main
monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_MODEL", "someone/custom-model")
monkeypatch.delenv("OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE", raising=False)
monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_REVISION", "a" * 40)
with pytest.raises(RuntimeError, match="after auditing"):
main._model_source()
def test_custom_remote_code_requires_immutable_revision(monkeypatch):
from engines.moss_tts_v15 import main
monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_MODEL", "someone/custom-model")
monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE", "1")
monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_REVISION", "main")
with pytest.raises(RuntimeError, match="40-character commit SHA"):
main._model_source()
def test_audited_custom_remote_code_requires_both_opt_ins(monkeypatch):
from engines.moss_tts_v15 import main
revision = "b" * 40
monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_MODEL", "someone/custom-model")
monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_TRUST_REMOTE_CODE", "true")
monkeypatch.setenv("OMNIVOICE_MOSS_TTS_V15_REVISION", revision)
assert main._model_source() == ("someone/custom-model", revision)
# ── hardware honesty (cross-platform rule) ─────────────────────────────────
+54
View File
@@ -174,6 +174,60 @@ def test_get_spec_accepts_repo_id():
assert not sd.is_sherpa_model(None)
def test_model_resolution_pins_offline_probe_and_download(monkeypatch, tmp_path):
from services import hf_revisions, sherpa_dictation as sd
import huggingface_hub
from huggingface_hub import constants as hf_constants
spec = sd.get_spec("sherpa-whisper-tiny")
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
calls = []
def fake_snapshot(**kwargs):
calls.append(kwargs)
if kwargs.get("local_files_only"):
raise FileNotFoundError("not cached")
return "/cache/pinned"
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot)
assert sd._resolve_model_dir(spec) == "/cache/pinned"
assert len(calls) == 2
assert all(call["revision"] == hf_revisions.revision_for(spec.repo_id) for call in calls)
def test_model_resolution_probes_preserved_legacy_snapshot(monkeypatch, tmp_path):
from services import hf_revisions, sherpa_dictation as sd
import huggingface_hub
from huggingface_hub import constants as hf_constants
spec = sd.get_spec("sherpa-whisper-tiny")
legacy_revision = "e" * 40
ref = (
tmp_path
/ "models--csukuangfj--sherpa-onnx-whisper-tiny"
/ "refs"
/ "main"
)
ref.parent.mkdir(parents=True)
ref.write_text(legacy_revision + "\n", encoding="ascii")
monkeypatch.setattr(hf_constants, "HF_HUB_CACHE", str(tmp_path))
calls = []
def fake_snapshot(**kwargs):
calls.append(kwargs)
return "/cache/legacy"
monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot)
assert sd._resolve_model_dir(spec) == "/cache/legacy"
assert calls == [{
"repo_id": spec.repo_id,
"revision": legacy_revision,
"local_files_only": True,
"allow_patterns": list(spec.files.values()),
}]
assert calls[0]["revision"] != hf_revisions.revision_for(spec.repo_id)
# ── The 4 recognizer kinds construct + transcribe ───────────────────────────