diff --git a/CHANGELOG.md b/CHANGELOG.md index 54ad6ec3..0237aefe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,10 @@ The bundled TTS model package (`pyproject.toml`) is versioned independently. ### Fixed - Curated models now install and repair from reviewed, immutable revisions; custom MOSS remote code requires an explicit safety opt-in. (#1453) +- YouTube imports that require a signed-in session can now use an explicitly selected `cookies.txt` export for one import; VoiceStudio never reads browser cookies silently and makes two best-effort attempts to delete its temporary copy. (#1429, #1432) — thanks @dongqing1968-sudo and @phamvandu9595-tech! +- First-run source builds no longer stop after uv was successfully downloaded just because its installer failed during a later shell-profile step; app-private uv installs no longer touch shell profiles at all. (#1438) — thanks @AdrianoCahete! +- Model files damaged by an interrupted download now repair themselves instead of failing every generation, including invalid `config.json` files and corrupt weight headers. — thanks @overrunau and @zherunh! (#1406, #1437) +- ROCm Docker now installs and starts the backend with the same Python whose AMD torch build was validated, instead of launching a second CUDA-only environment and silently running on CPU. (#1274) — thanks @simmessa and @spicchio72! - An error whose text merely contained the digits 401 — a file path, a byte count, a job id — no longer tells you to fix your Hugging Face token. (#1427) - Custom MLX model IDs and saved voice instructions are now validated in bounded time, so malformed input cannot stall the backend. (#1446) - Streaming and provider failures now return stable recovery guidance without exposing exception details. (#1462) diff --git a/backend/api/routers/dub_core.py b/backend/api/routers/dub_core.py index 1821ca8f..576d5e24 100644 --- a/backend/api/routers/dub_core.py +++ b/backend/api/routers/dub_core.py @@ -4,9 +4,12 @@ import asyncio import logging import shutil import subprocess +import tempfile +from urllib.parse import urlsplit import soundfile as sf import torch from typing import Optional +from fastapi import Request from fastapi import APIRouter, File, Form, UploadFile, HTTPException from fastapi.responses import FileResponse, StreamingResponse, JSONResponse @@ -40,6 +43,67 @@ from services import dub_pipeline router = APIRouter() logger = logging.getLogger("omnivoice.api") +_MAX_COOKIE_EXPORT_BYTES = 1024 * 1024 + + +def _cookie_transport_allowed( + scheme: str, client_host: str | None, origin: str | None +) -> bool: + """Credentials may cross HTTP only from a local UI to a loopback peer.""" + from api.dependencies import is_local_host + + if scheme == "https": + return True + try: + origin_host = urlsplit(origin or "").hostname or "" + except ValueError: + return False + return is_local_host(client_host or "") and ( + is_local_host(origin_host) or origin_host == "tauri.localhost" + ) + + +def _stage_cookie_export(contents: str | None) -> str | None: + """Write an explicitly supplied cookies.txt export to a private temp file.""" + if contents is None: + return None + cookie_bytes = contents.encode("utf-8") + if len(cookie_bytes) > _MAX_COOKIE_EXPORT_BYTES: + raise HTTPException( + status_code=400, + detail=( + "Cookie file is too large (maximum 1 MB). Export cookies in " + "Netscape cookies.txt format and try again." + ), + ) + first_line = contents.lstrip("\ufeff\r\n ").splitlines()[0] if contents.strip() else "" + if not first_line.startswith(("# Netscape HTTP Cookie File", "# HTTP Cookie File")): + raise HTTPException( + status_code=400, + detail=( + "This is not a Netscape cookies.txt export. Export cookies as " + "cookies.txt from your browser, then choose that file." + ), + ) + fd, cookie_path = tempfile.mkstemp( + prefix="voicestudio-ytdlp-", suffix=".cookies.txt", + ) + try: + os.chmod(cookie_path, 0o600) + with os.fdopen(fd, "wb") as cookie_handle: + cookie_handle.write(cookie_bytes) + except Exception: + try: + os.close(fd) + except OSError: + pass # Best effort: fdopen may already have consumed/closed the descriptor. + try: + os.unlink(cookie_path) + except OSError: + pass # Best effort: preserve the original staging error. + raise + return cookie_path + # ── Legacy-name aliases to services/dub_pipeline.py ──────────────────────── # Phase 2.4 moved the business logic into a service. Other routers @@ -395,7 +459,7 @@ async def dub_upload( @router.post("/dub/ingest-url") -async def dub_ingest_url(req: DubIngestUrlRequest): +async def dub_ingest_url(req: DubIngestUrlRequest, request: Request): """Ingest a remote video URL via yt-dlp. Queues background prep task. Returns 202 immediately with {job_id, task_id}. All work (download, @@ -424,7 +488,17 @@ async def dub_ingest_url(req: DubIngestUrlRequest): status_code=400, detail="Invalid job_id. Must be alphanumeric + hyphens/underscores only, ≤64 chars. Generate a fresh job_id or omit it to auto-create one.", ) + if req.cookie_file and not _cookie_transport_allowed( + request.url.scheme, + request.client.host if request.client else None, + request.headers.get("origin"), + ): + raise HTTPException( + status_code=403, + detail="Cookie exports require HTTPS or the local desktop app.", + ) os.makedirs(job_dir, exist_ok=True) + cookie_path = _stage_cookie_export(req.cookie_file) task_id = f"prep_{job_id}" source = { @@ -432,12 +506,21 @@ async def dub_ingest_url(req: DubIngestUrlRequest): "url": url, "fetch_subs": bool(req.fetch_subs), "sub_langs": req.sub_langs or None, + "cookie_file": cookie_path, } - await task_manager.add_task( - task_id, "prep", - _ingest_gen, job_id, job_dir, - source, None, - ) + try: + await task_manager.add_task( + task_id, "prep", + _ingest_gen, job_id, job_dir, + source, None, + ) + except Exception: + if cookie_path: + try: + os.unlink(cookie_path) + except OSError: + pass # Best effort: do not hide the task-enqueue failure. + raise return JSONResponse( status_code=202, content={"job_id": job_id, "task_id": task_id, "filename": ""}, diff --git a/backend/core/failure.py b/backend/core/failure.py index 1d7a343b..c0c0b851 100644 --- a/backend/core/failure.py +++ b/backend/core/failure.py @@ -90,7 +90,7 @@ _HINTS: dict[str, str] = { # wins over the symptom. "MODEL_DOWNLOAD_INTERRUPTED": "A model download was cut off mid-request, and the component it was fetching then failed to load. Nothing is wrong with your install — reinstalling won't help, and the partial download is resumed rather than restarted. Just retry. If it keeps happening, check your connection (and any VPN, proxy or HF mirror setting); if only transcription is affected, switching ASR to faster-whisper in Settings → Models avoids the pipeline that downloads this component.", "BROKEN_VENV": "The Python backend environment was moved or damaged. VoiceStudio rebuilds it automatically on the next launch; if it keeps failing, use Clean & Retry on the setup screen.", - "MODEL_CACHE_CORRUPT": "The model cache had broken file links — snapshot entries that no longer point at their downloaded data (interrupted renames or antivirus interference can cause this). VoiceStudio repairs this automatically and retries the load once. If the error persists, quit VoiceStudio, delete the model's models---- folder inside the Hugging Face cache, and restart — the model re-downloads automatically.", + "MODEL_CACHE_CORRUPT": "A model file is missing or damaged — a download that stopped part-way, a broken link to downloaded data, or a file changed on disk after it arrived (interrupted renames and antivirus interference both cause this). VoiceStudio repairs it automatically and retries the load once, re-downloading the damaged file where a resume would not have replaced it. If the error persists, quit VoiceStudio, delete the model's models---- folder inside the Hugging Face cache, and restart — the model re-downloads automatically.", # HF_MIRROR_UNREACHABLE has a DYNAMIC hint (it names the configured mirror) # — see hf_mirror_hint(); build_failure special-cases it. } @@ -351,7 +351,16 @@ def classify(reason: str) -> str: # load surface can leak them) and VoiceStudio's own repair messages, so the # user-facing error and the auto bug report name the class and its # automatic repair. - if is_incomplete_cache_message(low) or "broken file link" in low: + # Same class, both halves of it: a shard that is MISSING and a shard that + # is PRESENT but unparseable are the same problem (an interrupted or + # mangled download) with the same remedy, and only the first half was ever + # matched — so "Error while deserializing header: header too large" fell + # through to "" and shipped as a raw 500 with no repair (#1406). + if ( + is_incomplete_cache_message(low) + or is_corrupt_weights_message(low) + or "broken file link" in low + ): return "MODEL_CACHE_CORRUPT" # #1347: an import that failed because its DOWNLOAD died is a network # problem wearing an import problem's clothes. The reporter's message named @@ -731,6 +740,72 @@ def is_incomplete_cache_message(text: str) -> bool: return False +#: The weight file is PRESENT but its bytes are not a valid tensor file. +#: +#: The sibling of ``_INCOMPLETE_CACHE_PHRASES`` above, and the half that had no +#: handling at all (#1406). An interrupted download that stops mid-file, an +#: antivirus that truncates a shard, or a proxy that saved an HTML error page +#: under the shard's name all leave a file transformers is happy to open — so +#: the "does not appear to have a file named …" check passes — and safetensors +#: then fails parsing its 8-byte header-length prefix. The user got a raw 500 +#: ("Error while deserializing header: header too large") on every generation, +#: from voice design and gallery previews alike, with nothing actionable in it +#: and no repair attempted, because the recovery ladder is reached only through +#: the *missing*-weights signature. +#: +#: Matched on wording rather than exception type on purpose: safetensors raises +#: ``SafetensorError`` from a Rust extension, torch raises ``UnpicklingError`` +#: or a bare ``RuntimeError`` for the same condition in a ``.bin``, and none of +#: them are ``OSError`` — the type is the least stable thing about this class. +_CORRUPT_MODEL_FILE_PHRASES = ( + # safetensors (Rust): header length prefix is larger than the file, or the + # declared metadata runs past the end of the buffer. + "error while deserializing header", + "headertoolarge", + "metadataincompletebuffer", + "invalidheaderdeserialization", + "deserializing header", + # torch.load on a truncated / non-pickle .bin shard. "invalid load key" is + # unambiguous — only the pickle reader says it. "unexpected end of file" is + # not: zipfile, tarfile, gzip and several parsers share the wording, and any + # of them can surface inside a model-load chain, where a false positive + # would force a multi-GB re-download of an undamaged cache (CodeRabbit). It + # therefore needs a weight-file co-marker, like the entry below. + "invalid load key", + ("unexpected end of file", "safetensors"), + ("unexpected end of file", "pytorch_model"), + ("unexpected end of file", "checkpoint"), + ("failed to load", "checkpoint", "corrupt"), + # transformers wraps a JSONDecodeError from a truncated/HTML config file + # with this stable, path-bearing message (#1437). + ("config file", "not a valid json file"), +) + + +def is_corrupt_model_file_message(text: str) -> bool: + """True when a downloaded model weight or config file cannot be parsed. + + Distinct from :func:`is_incomplete_cache_message`, which means the file is + absent. Here it exists and its bytes are wrong — a different repair (force + a re-download; a resume would trust the bad blob) and a different thing to + tell the user. Shared by :func:`classify` and model_manager's self-heal so + the healer and the message can never disagree. + """ + low = str(text).lower() + for phrase in _CORRUPT_MODEL_FILE_PHRASES: + if isinstance(phrase, tuple): + if all(part in low for part in phrase): + return True + elif phrase in low: + return True + return False + + +def is_corrupt_weights_message(text: str) -> bool: + """Backward-compatible name for :func:`is_corrupt_model_file_message`.""" + return is_corrupt_model_file_message(text) + + def is_os_write_refusal(reason: Optional[str]) -> bool: """True when *reason* looks like the OS refusing a file operation (a full or removed drive, a read-only folder, an antivirus/cloud-sync lock) rather diff --git a/backend/schemas/requests.py b/backend/schemas/requests.py index c7d606f4..8d501b8a 100644 --- a/backend/schemas/requests.py +++ b/backend/schemas/requests.py @@ -197,6 +197,10 @@ class DubIngestUrlRequest(BaseModel): # YouTube auto-translates for us. fetch_subs: Optional[bool] = False sub_langs: Optional[List[str]] = None + # Explicit, per-import Netscape cookie export. This is never populated + # automatically: browser cookie stores contain unrelated login secrets and + # VoiceStudio must not inspect them without a deliberate user action. + cookie_file: Optional[str] = None class ProjectSaveRequest(BaseModel): name: str diff --git a/backend/services/dub_pipeline.py b/backend/services/dub_pipeline.py index 954a6055..a1df9a31 100644 --- a/backend/services/dub_pipeline.py +++ b/backend/services/dub_pipeline.py @@ -840,6 +840,18 @@ def _cleanup_partial_download(job_dir: str) -> None: pass +def _delete_cookie_export(cookie_file: str | None) -> bool: + """Best-effort removal of the per-import authentication export.""" + if not cookie_file: + return True + try: + os.unlink(cookie_file) + except OSError: + # Best effort: cleanup must never replace the download result. + return False + return True + + def yt_download_sync( url: str, job_dir: str, @@ -847,6 +859,7 @@ def yt_download_sync( fetch_subs: bool = False, sub_langs: list[str] | None = None, progress_hook=None, + cookie_file: str | None = None, ) -> tuple[str, str, list[str]]: """Blocking yt-dlp download into `job_dir`. @@ -918,6 +931,8 @@ def yt_download_sync( "extractor_retries": 5, "skip_unavailable_fragments": True, } + if cookie_file: + ydl_opts["cookiefile"] = cookie_file # #712: the format selector above pulls separate video+audio streams, so # yt-dlp muxes them via ffmpeg (merge_output_format=mp4). yt-dlp only looks # for ffmpeg on PATH and aborts with "you have requested merging of multiple @@ -1108,6 +1123,7 @@ async def ingest_pipeline( url = source["url"] fetch_subs = bool(source.get("fetch_subs")) sub_langs = source.get("sub_langs") or None + cookie_file = source.get("cookie_file") or None yield prep_event("download_start", url=url) # Bridge yt-dlp's per-fragment progress callback (fires inside # the worker thread) into the async generator via a threadsafe @@ -1137,6 +1153,7 @@ async def ingest_pipeline( yt_download_sync, url, job_dir, fetch_subs=fetch_subs, sub_langs=sub_langs, progress_hook=_yt_progress, + cookie_file=cookie_file, )) try: while not dl_task.done(): @@ -1157,6 +1174,10 @@ async def ingest_pipeline( yield prep_event("error", **failure.build_failure(e, stage="download")) shutil.rmtree(job_dir, ignore_errors=True) return + # yt-dlp (including its optional subtitle pass) is finished. Drop + # the login credential before the much longer audio-prep stages. + if _delete_cookie_export(cookie_file): + source["cookie_file"] = None filename = title or os.path.basename(video_path) try: size = os.path.getsize(video_path) @@ -1439,6 +1460,11 @@ async def ingest_pipeline( yield prep_event("error", **failure.build_failure(e, stage="ingest")) return finally: + # Cookie exports are login credentials. Keep an explicitly selected + # export only for this download, then remove it on success, failure or + # cancellation; never copy it into the project/job directory. + cookie_file = source.get("cookie_file") + _delete_cookie_export(cookie_file) end_ingest(job_id) with _active_procs_lock: _active_procs.pop(job_id, None) diff --git a/backend/services/model_manager.py b/backend/services/model_manager.py index c2ebb4f6..82ff30f3 100644 --- a/backend/services/model_manager.py +++ b/backend/services/model_manager.py @@ -1491,6 +1491,35 @@ def _is_incomplete_cache_error(exc: BaseException) -> bool: return is_incomplete_cache_message(str(exc)) +def _is_corrupt_model_file_error(exc: BaseException) -> bool: + """True when a model weight or config file cannot be parsed. + + The other half of the interrupted-download class (#1406). transformers + only raises the "does not appear to have a file named …" signature when + the shard is *absent*; a shard that stops mid-file, gets truncated by + antivirus, or is actually a saved HTML error page opens fine and then + fails inside safetensors: + + Error while deserializing header: header too large + + That is a ``SafetensorError`` from a Rust extension — not an ``OSError``, + so it never reached the recovery ladder and surfaced as a raw 500 on every + generation (the reporter hit it from voice design *and* from a gallery + preview, which is what a shared broken shard looks like). + + The whole exception chain is checked, not just the outermost message: + transformers wraps the tensor library's error in its own before it gets + here, and matching only the surface would miss every wrapped case.""" + from core.failure import is_corrupt_model_file_message + + return any(is_corrupt_model_file_message(str(e)) for e in _exception_chain(exc)) + + +def _is_corrupt_weights_error(exc: BaseException) -> bool: + """Backward-compatible wrapper for the original #1406 helper name.""" + return _is_corrupt_model_file_error(exc) + + def _hf_offline() -> bool: """Respect HF's offline switches so repair never makes a network call the user opted out of. `snapshot_download` would itself raise offline, but @@ -1516,6 +1545,13 @@ def _hf_offline() -> bool: # stays broken can't loop repair↔retry. _LINK_REPAIR_ATTEMPTED: set[str] = set() +#: Repos whose weights we have already force-re-downloaded this process +#: (#1406). Without it, a shard that stays unparseable after a full re-fetch +#: would pull the whole model again on EVERY generate request — one bad file +#: turning into unbounded traffic. Same once-per-repo-per-process contract as +#: the snapshot-link repair above (CodeRabbit). +_FORCED_REDOWNLOAD_ATTEMPTED: set[str] = set() + def _selfheal_broken_snapshot_links(checkpoint: str) -> bool: """Rung 0 of cache recovery: delete-and-restore broken snapshot entries. @@ -1887,14 +1923,74 @@ def _load_model_sync(): logger.info("Loading VoiceStudio model on device: %s", device) preload_asr = should_preload_tts_asr() if preload_asr: - logger.info("Preloading PyTorch Whisper with TTS model.") + logger.info("Preloading PyTorch Whisper after TTS model load.") else: logger.info("Skipping PyTorch Whisper preload; ASR will load on demand.") def _load(): return VoiceStudio.from_pretrained( - checkpoint, device_map=device, dtype=torch.float16, load_asr=preload_asr, + checkpoint, device_map=device, dtype=torch.float16, load_asr=False, ) + def _recover_corrupt_weights(exc: BaseException): + """Re-fetch weights that are on disk but unparseable (#1406). + + Deliberately a FORCED re-download rather than the resume ladder + below: a resume trusts a blob that is already the expected size + and would never re-fetch the one that is actually wrong. + """ + repair_checkpoint = checkpoint + for nested_exc in _exception_chain(exc): + repository_id = getattr(nested_exc, "repository_id", None) + if repository_id == "eustlb/higgs-audio-v2-tokenizer": + repair_checkpoint = repository_id + break + asset_label = ( + "audio tokenizer" + if repair_checkpoint != checkpoint + else "TTS model" + ) + if repair_checkpoint in _FORCED_REDOWNLOAD_ATTEMPTED: + # Already re-fetched this repo once this process and it is + # still unparseable. Re-downloading again would be the same + # gigabytes for the same result, once per generate request. + raise RuntimeError( + f"The {asset_label} files for {repair_checkpoint} are damaged and a " + "re-download did not fix them. Open Settings → Models, " + "delete the VoiceStudio TTS model, and install it again." + f"{_manual_cache_delete_hint(repair_checkpoint)}" + ) from exc + _FORCED_REDOWNLOAD_ATTEMPTED.add(repair_checkpoint) + logger.warning( + "%s files for %s are present but unparseable (%s) — a " + "download that stopped mid-file, or a file altered on disk " + "after it arrived. Re-fetching them.", + asset_label, + repair_checkpoint, + exc, + ) + _set_loading("loading_weights", "Model files are damaged — re-downloading…") + if not _repair_model_cache(repair_checkpoint, force=True): + raise RuntimeError( + f"The {asset_label} files for {repair_checkpoint} are damaged — a " + "download that stopped part-way, or a file changed on " + "disk after it arrived — and could not be re-downloaded " + f"automatically.{_repair_failure_detail()} Open Settings " + "→ Models, delete the VoiceStudio TTS model, and install " + f"it again.{_manual_cache_delete_hint(repair_checkpoint)}" + ) from exc + _set_loading("loading_weights", f"Loading TTS weights on {device}…") + try: + return _load() + except Exception as exc2: + if not _is_corrupt_weights_error(exc2): + raise + raise RuntimeError( + f"The {asset_label} files for {repair_checkpoint} are still damaged " + "after being re-downloaded. Open Settings → Models, " + "delete the VoiceStudio TTS model, and install it again." + f"{_manual_cache_delete_hint(repair_checkpoint)}" + ) from exc2 + try: _model = _load() except OSError as e: @@ -1905,78 +2001,117 @@ def _load_model_sync(): # interrupted download leaves the cache missing only some files, # and snapshot_download() resumes/fills exactly those (a complete # cache never reaches this branch, so the fast path is untouched). - if not _is_incomplete_cache_error(e): + if _is_corrupt_weights_error(e): + # Present-but-unparseable wearing an OSError (#1406) — + # transformers wraps a tensor-library failure in one. The + # resume ladder below is the wrong repair (it would trust the + # bad blob), so divert before the missing-shard check drops + # this as unrecognised and 500s. + _model = _recover_corrupt_weights(e) + elif not _is_incomplete_cache_error(e): raise - # Rung 0: broken snapshot links — the blobs are on disk but the - # snapshot entries don't resolve (dangling symlinks / zero-byte - # stand-ins). Delete exactly the broken entries, restore, and - # retry the load ONCE (guarded per repo per process). A cache - # without broken links falls straight through to the resume - # ladder below. - _model = None - if _selfheal_broken_snapshot_links(checkpoint): - _set_loading( - "loading_weights", - "Model cache had broken file links — repaired " - "automatically, retrying…", - ) - try: - _model = _load() - except OSError as e_link: - if not _is_incomplete_cache_error(e_link): - raise - logger.warning( - "Load still failing after snapshot-link repair of %s — " - "falling back to resume repair.", checkpoint, + else: + # Rung 0: broken snapshot links — the blobs are on disk but the + # snapshot entries don't resolve (dangling symlinks / zero-byte + # stand-ins). Delete exactly the broken entries, restore, and + # retry the load ONCE (guarded per repo per process). A cache + # without broken links falls straight through to the resume + # ladder below. + _model = None + if _selfheal_broken_snapshot_links(checkpoint): + _set_loading( + "loading_weights", + "Model cache had broken file links — repaired " + "automatically, retrying…", ) - e = e_link - _model = None - if _model is None: - _set_loading("loading_weights", "Repairing incomplete model cache…") - if not _repair_model_cache(checkpoint): - raise RuntimeError( - f"The TTS model cache for {checkpoint} is incomplete " - "(weights missing — usually an interrupted download)." - f"{_repair_failure_detail()} " - "Open Settings → Models, delete the VoiceStudio TTS model, " - f"and install it again.{_manual_cache_delete_hint(checkpoint)}" - ) from e - _set_loading("loading_weights", f"Loading TTS weights on {device}…") - try: - _model = _load() - except OSError as e2: - # Resume-repair ran but the cache is still unusable. The usual - # cause beyond "repo genuinely lacks weights" is a blob that's - # present with the right size but corrupt — snapshot_download's - # resume trusts it and never re-fetches it (#739). Force a full - # re-download (replaces corrupt blobs) and retry once more before - # falling back to the manual delete-and-reinstall message. - if _is_incomplete_cache_error(e2): - _set_loading("loading_weights", "Re-downloading model files…") - if _repair_model_cache(checkpoint, force=True): - try: - _model = _load() - except OSError as e3: + try: + _model = _load() + except OSError as e_link: + if not _is_incomplete_cache_error(e_link): + raise + logger.warning( + "Load still failing after snapshot-link repair of %s — " + "falling back to resume repair.", checkpoint, + ) + e = e_link + _model = None + if _model is None: + _set_loading("loading_weights", "Repairing incomplete model cache…") + if not _repair_model_cache(checkpoint): + raise RuntimeError( + f"The TTS model cache for {checkpoint} is incomplete " + "(weights missing — usually an interrupted download)." + f"{_repair_failure_detail()} " + "Open Settings → Models, delete the VoiceStudio TTS model, " + f"and install it again.{_manual_cache_delete_hint(checkpoint)}" + ) from e + _set_loading("loading_weights", f"Loading TTS weights on {device}…") + try: + _model = _load() + except OSError as e2: + # Resume-repair ran but the cache is still unusable. The usual + # cause beyond "repo genuinely lacks weights" is a blob that's + # present with the right size but corrupt — snapshot_download's + # resume trusts it and never re-fetches it (#739). Force a full + # re-download (replaces corrupt blobs) and retry once more before + # falling back to the manual delete-and-reinstall message. + if _is_corrupt_weights_error(e2): + # The resume filled the missing files, then exposed a + # present-but-damaged blob. A second resume would trust + # that blob, so switch to the forced corruption repair. + _model = _recover_corrupt_weights(e2) + elif _is_incomplete_cache_error(e2): + _set_loading("loading_weights", "Re-downloading model files…") + if _repair_model_cache(checkpoint, force=True): + try: + _model = _load() + except OSError as e3: + raise RuntimeError( + f"The TTS model cache for {checkpoint} is incomplete " + "and could not be auto-repaired. Open Settings → " + "Models, delete the VoiceStudio TTS model, and install " + f"it again.{_manual_cache_delete_hint(checkpoint)}" + ) from e3 + else: raise RuntimeError( - f"The TTS model cache for {checkpoint} is incomplete " - "and could not be auto-repaired. Open Settings → " - "Models, delete the VoiceStudio TTS model, and install " - f"it again.{_manual_cache_delete_hint(checkpoint)}" - ) from e3 + f"The TTS model cache for {checkpoint} is incomplete and " + f"could not be auto-repaired.{_repair_failure_detail()} " + "Open Settings → Models, delete the VoiceStudio TTS model, " + f"and install it again.{_manual_cache_delete_hint(checkpoint)}" + ) from e2 else: raise RuntimeError( f"The TTS model cache for {checkpoint} is incomplete and " - f"could not be auto-repaired.{_repair_failure_detail()} " - "Open Settings → Models, delete the VoiceStudio TTS model, " - f"and install it again.{_manual_cache_delete_hint(checkpoint)}" + "could not be auto-repaired. Open Settings → Models, delete " + "the VoiceStudio TTS model, and install it again." + f"{_manual_cache_delete_hint(checkpoint)}" ) from e2 - else: - raise RuntimeError( - f"The TTS model cache for {checkpoint} is incomplete and " - "could not be auto-repaired. Open Settings → Models, delete " - "the VoiceStudio TTS model, and install it again." - f"{_manual_cache_delete_hint(checkpoint)}" - ) from e2 + except Exception as e_corrupt: + # safetensors raises SafetensorError from a Rust extension and + # torch raises UnpicklingError — neither is an OSError, so the + # ladder above never saw them and the load 500'd with a raw + # "Error while deserializing header: header too large" (#1406). + # Anything that is not this class re-raises untouched, so no + # unrelated failure is swallowed by the broad clause. + if not _is_corrupt_weights_error(e_corrupt): + raise + _model = _recover_corrupt_weights(e_corrupt) + + if preload_asr: + # Keep ASR outside `from_pretrained`: if its separate HF cache is + # corrupt, it must never be mistaken for the TTS checkpoint and + # trigger a second multi-GB TTS load/re-download (CodeRabbit). + try: + _model.load_asr_model() + except Exception as asr_exc: + if not _is_corrupt_model_file_error(asr_exc): + raise + raise RuntimeError( + "The transcription model's files are damaged. Open " + "Settings → Models, delete the transcription (ASR) model, " + "and install it again; or set OMNIVOICE_PRELOAD_TTS_ASR=0 " + "to stop preloading it alongside TTS." + ) from asr_exc try: # plan-02 (#65): gate on Triton availability (+ user setting), not diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 0462745e..0666abcc 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -32,7 +32,6 @@ WORKDIR /app # Enable unbuffered logs and optimizations ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 -ENV UV_SYSTEM_PYTHON=1 ENV HF_HOME=/app/omnivoice_data/huggingface # Allow bare imports (from core.config, from services.*, etc.) when # uvicorn is started as `backend.main:app` from WORKDIR /app. @@ -54,9 +53,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # PEP 668: the ROCm base (Ubuntu 24.04) marks its system Python -# EXTERNALLY-MANAGED, which would refuse `pip install` / `uv pip install -# --system`. Inside a single-purpose container image installing into the -# base env is exactly what we want. No-ops on the conda-based CUDA image. +# EXTERNALLY-MANAGED, which would refuse installing into the selected base +# interpreter. Inside a single-purpose container image that is exactly what +# we want. No-ops on the conda-based CUDA image. ENV PIP_BREAK_SYSTEM_PACKAGES=1 ENV UV_BREAK_SYSTEM_PACKAGES=1 @@ -71,6 +70,11 @@ COPY deploy/torch-constraints.txt ./deploy/torch-constraints.txt # Install the project (non-editable — no need for -e in containers). # Uses `uv` for exponentially faster resolution than plain pip. # +# Target the exact interpreter selected by the base image. The ROCm image has +# both /opt/venv/bin/python3 (ROCm torch) and /usr/bin/python (a CUDA-default +# environment); `--system` used the latter while the build guard used the +# former, so a green image launched a CPU-only backend on AMD (#1274). +# # NOTE: `uv pip install` (without --upgrade) keeps already-installed packages # that satisfy the requirements, so the base image's GPU-built torch/torchaudio # (2.8.0, satisfying our `torch>=2.4`) survive this step instead of being @@ -83,7 +87,8 @@ COPY deploy/torch-constraints.txt ./deploy/torch-constraints.txt # stays put — an ABI mismatch at import (#1357). The pins carry no local # segment, so they match the base image's +cu128 / +rocm6.4 builds rather than # replacing them. -RUN uv pip install --system --no-cache --constraint deploy/torch-constraints.txt . +RUN uv pip install --python "$(command -v python3)" --no-cache \ + --constraint deploy/torch-constraints.txt . # Guard (fails the build, not the user at runtime): assert the dependency # install did NOT replace the base image's GPU torch. A future dep bump that @@ -121,5 +126,6 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=5 \ # Mount points for persistent data (sqlite db, user voices, huggingface cache) VOLUME ["/app/omnivoice_data"] -# Bind to 0.0.0.0 for external access -ENTRYPOINT ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "3900"] +# Bind to 0.0.0.0 for external access. `python3 -m` keeps runtime imports on +# the same interpreter whose torch flavor the build guard validated (#1274). +ENTRYPOINT ["python3", "-m", "uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "3900"] diff --git a/docs/install/docker.md b/docs/install/docker.md index adf64d6e..ec2714c1 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -109,16 +109,26 @@ the CUDA tags exactly. Verify the container sees the GPU: ```bash -docker exec omnivoice python3 -c \ - "import torch; print(torch.cuda.is_available(), torch.cuda.get_device_name(0))" +docker exec python3 -c \ + "import torch; ok = torch.cuda.is_available(); print(ok, torch.cuda.get_device_name(0) if ok else 'unavailable')" ``` +Use `omnivoice` for the `docker run` examples above. Docker Compose names the +ROCm container `omnivoice-studio-rocm` (CPU: `omnivoice-studio`, NVIDIA: +`omnivoice-studio-gpu`); `docker compose ps` shows the exact active name. + (ROCm-built PyTorch reports through `torch.cuda.*` — `True` plus your card's name means torch can see the GPU.) That check alone isn't proof the app is using it: **Settings → System** shows the device VoiceStudio actually resolved. If it reads `cpu` while the command above prints `True`, the backend log line starting `Falling back to CPU:` names the architecture mismatch it hit. +The image installs and launches VoiceStudio through that same `python3` +interpreter. To verify this invariant on an older or custom image, compare +`docker exec python3 -c "import sys, torch; print(sys.executable, +torch.version.hip)"` with `docker exec sh -c 'tr "\\0" " " + python3 -c "import importlib.metadata; print(importlib.metadata.version('omnivoice'))"`, or hit the `/health` endpoint — it returns `{"status": "ok", "device": ..., "version": "0.3.x"}`. Use the container name listed by `docker compose ps` (or `omnivoice` for the `docker run` examples). - **"Loopback origin required" errors (and a blank version):** the desktop build restricts the `/system/*` and `/api/settings/*` routes to a loopback origin, but Docker's NAT makes every request look non-loopback, so the gate diff --git a/docs/install/troubleshooting.md b/docs/install/troubleshooting.md index a0eab928..7518abca 100644 --- a/docs/install/troubleshooting.md +++ b/docs/install/troubleshooting.md @@ -247,6 +247,23 @@ faster than app releases, so when video-URL imports start failing, press **Update** there — the new version survives app updates, and **Restore tested version** reverts to the build the app shipped with. +### YouTube asks you to sign in or confirm you are not a bot + +First update yt-dlp under **Settings → Audio tools**. If YouTube still requires +your signed-in session, export its cookies in Netscape `cookies.txt` format, +then choose that file beside the URL field before importing. VoiceStudio uses +the export for that import only and makes two best-effort attempts to delete +its temporary copy. + +Cookie exports are login credentials. VoiceStudio never reads a browser's +cookie database automatically, never saves the export in your project, and +never uploads it anywhere except to your own VoiceStudio backend. Use an export +limited to YouTube where your browser extension supports domain filtering. +For a backend on another machine, the picker is enabled only over HTTPS; plain +HTTP is accepted solely on the desktop app's loopback connection. +Remote backends must also use `OMNIVOICE_API_KEY` as the bearer key and remain +restricted to a private tailnet; see [API authentication](../api-auth.md). + ## 8. Docker LAN access — media preview 404 **Symptom:** VoiceStudio loads on `http://:3900` but the audio preview diff --git a/frontend/src-tauri/src/tools.rs b/frontend/src-tauri/src/tools.rs index 974afaf6..4ebab9ef 100644 --- a/frontend/src-tauri/src/tools.rs +++ b/frontend/src-tauri/src/tools.rs @@ -3,7 +3,7 @@ use std::fs; use std::io; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::process::{Command, Output, Stdio}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -410,7 +410,7 @@ pub fn resolve_uv( log::info!("Using bundled uv at {}", p.display()); return Ok(p); } - if no_window(Command::new("uv").arg("--version")).output().is_ok() { + if uv_is_usable(Path::new("uv")) { log::info!("Using system uv from PATH"); return Ok(PathBuf::from("uv")); } @@ -427,11 +427,11 @@ pub fn resolve_uv( /// Windows: `powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/{version}/install.ps1 | iex"` /// /// The installer handles platform detection, checksums, and extraction -/// automatically. We control the install directory via `UV_INSTALL_DIR`. -/// Idempotent: if the binary is already present, returns its path immediately. +/// automatically. `UV_UNMANAGED_INSTALL` keeps this app-private tool out of +/// the user's PATH and shell profiles on every platform. fn install_uv_standalone(dest: &Path, _region: &str) -> io::Result { let uv_bin = dest.join(if cfg!(windows) { "uv.exe" } else { "uv" }); - if uv_bin.is_file() { + if uv_is_usable(&uv_bin) { return Ok(uv_bin); } fs::create_dir_all(dest)?; @@ -439,28 +439,24 @@ fn install_uv_standalone(dest: &Path, _region: &str) -> io::Result { #[cfg(unix)] { - let status = Command::new("sh") - .args([ + let output = configure_uv_installer( + Command::new("sh").args([ "-c", &format!( - "curl -LsSf https://astral.sh/uv/{}/install.sh | sh -s -- --no-modify-path", + "curl -LsSf https://astral.sh/uv/{}/install.sh | sh", UV_VERSION ), - ]) - .env("UV_INSTALL_DIR", dest) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .status() - .map_err(|e| io::Error::new( + ]), + dest, + ) + .output() + .map_err(|e| { + io::Error::new( io::ErrorKind::Other, format!("uv installer launch failed (is curl installed?): {}", e), - ))?; - if !status.success() { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("uv installer exited with code {:?}", status.code()), - )); - } + ) + })?; + return finish_uv_install(dest, &uv_bin, output); } #[cfg(windows)] @@ -472,39 +468,328 @@ fn install_uv_standalone(dest: &Path, _region: &str) -> io::Result { // Windows: `CREATE_NO_WINDOW` so the uv installer's PowerShell doesn't // flash a console window during first-run bootstrap. stdout/stderr are // piped, so nothing is lost. - let status = no_window( - Command::new("powershell") - .args(["-ExecutionPolicy", "ByPass", "-c", &script]) - .env("UV_INSTALL_DIR", dest) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()), - ) - .status() - .map_err(|e| io::Error::new( - io::ErrorKind::Other, - format!("uv PowerShell installer failed: {}", e), - ))?; - if !status.success() { - return Err(io::Error::new( + let mut command = Command::new("powershell"); + command.args([ + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "ByPass", + "-c", + &script, + ]); + configure_uv_installer(&mut command, dest); + let output = no_window(&mut command).output().map_err(|e| { + io::Error::new( io::ErrorKind::Other, - format!("uv installer exited with code {:?}", status.code()), - )); - } + format!("uv PowerShell installer failed: {}", e), + ) + })?; + return finish_uv_install(dest, &uv_bin, output); } - if uv_bin.is_file() { - log::info!("uv installed successfully at {}", uv_bin.display()); - Ok(uv_bin) - } else { - let alt = dest.join("bin").join(if cfg!(windows) { "uv.exe" } else { "uv" }); - if alt.is_file() { - fs::rename(&alt, &uv_bin)?; - log::info!("uv moved from bin/ to {}", uv_bin.display()); - return Ok(uv_bin); + #[allow(unreachable_code)] + Err(io::Error::new( + io::ErrorKind::Unsupported, + "unsupported uv install platform", + )) +} + +fn configure_uv_installer<'a>(command: &'a mut Command, dest: &Path) -> &'a mut Command { + // The official unmanaged mode is designed for app-private/CI installs: it + // selects the destination and disables PATH, profile, and self-update + // mutations. Explicitly remove the legacy variable so a parent shell + // cannot leave the installer in two conflicting modes. + command + .env_remove("UV_INSTALL_DIR") + .env("UV_UNMANAGED_INSTALL", dest) + .env("UV_NO_MODIFY_PATH", "1") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) +} + +fn uv_is_usable(path: &Path) -> bool { + no_window( + Command::new(path) + .arg("--version") + .stdout(Stdio::piped()) + .stderr(Stdio::null()), + ) + .output() + .map(|output| output.status.success() && uv_version_matches(&output.stdout)) + .unwrap_or(false) +} + +fn uv_version_matches(output: &[u8]) -> bool { + let Ok(text) = std::str::from_utf8(output) else { + return false; + }; + let mut fields = text.split_whitespace(); + fields.next() == Some("uv") && fields.next() == Some(UV_VERSION) +} + +fn finish_uv_install(dest: &Path, uv_bin: &Path, output: Output) -> io::Result { + finish_uv_install_with_probe(dest, uv_bin, output, uv_is_usable) +} + +fn finish_uv_install_with_probe( + dest: &Path, + uv_bin: &Path, + output: Output, + is_usable: F, +) -> io::Result +where + F: Fn(&Path) -> bool, +{ + let alt = dest.join("bin").join(if cfg!(windows) { "uv.exe" } else { "uv" }); + if !is_usable(uv_bin) && is_usable(&alt) { + fs::rename(&alt, uv_bin).or_else(|_| fs::copy(&alt, uv_bin).map(|_| ()))?; + } + + // Some installer failures happen after extraction (for example while + // editing a Windows shell profile). The installed executable is the real + // postcondition: accepting a verified binary makes first run self-heal in + // this process instead of requiring a restart. Never accept a partial or + // corrupt file merely because it exists. + if is_usable(uv_bin) { + if output.status.success() { + log::info!("uv installed successfully at {}", uv_bin.display()); + } else { + log::warn!( + "uv installer exited with {:?}, but the installed binary passed validation at {}", + output.status.code(), + uv_bin.display() + ); } - Err(io::Error::new( - io::ErrorKind::NotFound, - format!("uv binary not found at {} after installer completed", uv_bin.display()), - )) + return Ok(uv_bin.to_path_buf()); + } + + let detail = installer_output_detail(&output); + Err(io::Error::new( + io::ErrorKind::Other, + if output.status.success() { + format!( + "uv installer completed but no usable binary was found at {}{}", + uv_bin.display(), + detail + ) + } else { + format!("uv installer exited with code {:?}{}", output.status.code(), detail) + }, + )) +} + +fn installer_output_detail(output: &Output) -> String { + let bytes = if output.stderr.is_empty() { + &output.stdout + } else { + &output.stderr + }; + let text = String::from_utf8_lossy(bytes); + let mut text = text.trim().to_string(); + if text.is_empty() { + return String::new(); + } + for key in ["USERPROFILE", "HOME"] { + if let Some(home) = std::env::var_os(key).and_then(|value| value.into_string().ok()) { + text = redact_home_prefix(&text, &home); + } + } + let start = text + .char_indices() + .rev() + .nth(1999) + .map(|(index, _)| index) + .unwrap_or(0); + format!(": {}", &text[start..]) +} + +fn redact_home_prefix(text: &str, home: &str) -> String { + if home.len() < 3 { + return text.to_string(); + } + let mut redacted = text.replace(home, "~"); + let forward = home.replace('\\', "/"); + let backward = home.replace('/', "\\"); + if forward != home { + redacted = redacted.replace(&forward, "~"); + } + if backward != home { + redacted = redacted.replace(&backward, "~"); + } + redacted +} + +#[cfg(test)] +mod uv_tests { + use super::*; + use std::ffi::OsStr; + + #[test] + fn installer_uses_app_private_unmanaged_mode() { + let mut command = Command::new("installer"); + configure_uv_installer(&mut command, Path::new("private-tools")); + let envs: std::collections::HashMap<_, _> = command.get_envs().collect(); + + assert_eq!(envs.get(OsStr::new("UV_INSTALL_DIR")), Some(&None)); + assert_eq!( + envs.get(OsStr::new("UV_UNMANAGED_INSTALL")).and_then(|value| *value), + Some(OsStr::new("private-tools")) + ); + assert_eq!( + envs.get(OsStr::new("UV_NO_MODIFY_PATH")).and_then(|value| *value), + Some(OsStr::new("1")) + ); + } + + #[test] + fn uv_version_probe_requires_the_pinned_version() { + assert!(uv_version_matches( + format!("uv {} (build-id)\n", UV_VERSION).as_bytes() + )); + assert!(!uv_version_matches(b"uv 0.10.0 (older)\n")); + assert!(!uv_version_matches(b"not-uv 0.11.7\n")); + assert!(!uv_version_matches(b"uv\n")); + assert!(!uv_version_matches(&[0xff, 0xfe])); + } + + #[test] + fn installer_error_includes_captured_stderr() { + let output = Output { + status: failure_status(), + stdout: Vec::new(), + stderr: b"profile update denied".to_vec(), + }; + assert_eq!(installer_output_detail(&output), ": profile update denied"); + } + + #[test] + fn installer_error_redacts_unix_and_windows_home_paths() { + assert_eq!( + redact_home_prefix( + "installed into /Users/alice/.local/bin", + "/Users/alice" + ), + "installed into ~/.local/bin" + ); + assert_eq!( + redact_home_prefix( + r"installed into C:\Users\alice\.local\bin", + r"C:\Users\alice" + ), + r"installed into ~\.local\bin" + ); + assert_eq!( + redact_home_prefix( + "installed into C:/Users/alice/.local/bin", + r"C:\Users\alice" + ), + "installed into ~/.local/bin" + ); + } + + #[test] + fn installer_exit_one_is_accepted_when_downloaded_uv_is_usable() { + let dest = Path::new("private-tools"); + let uv_bin = dest.join(if cfg!(windows) { "uv.exe" } else { "uv" }); + let output = Output { + status: failure_status(), + stdout: Vec::new(), + stderr: b"later installer step failed".to_vec(), + }; + + let result = finish_uv_install_with_probe(dest, &uv_bin, output, |candidate| { + candidate == uv_bin + }); + + assert_eq!(result.unwrap(), uv_bin); + } + + #[test] + fn successful_installer_without_usable_uv_is_rejected() { + let dest = Path::new("private-tools"); + let uv_bin = dest.join(if cfg!(windows) { "uv.exe" } else { "uv" }); + let output = Output { + status: success_status(), + stdout: Vec::new(), + stderr: Vec::new(), + }; + + let error = finish_uv_install_with_probe(dest, &uv_bin, output, |_| false) + .expect_err("installer success is insufficient without a usable binary"); + + assert!(error.to_string().contains("no usable binary was found")); + } + + #[test] + fn failed_installer_with_unusable_uv_reports_captured_error() { + let dest = Path::new("private-tools"); + let uv_bin = dest.join(if cfg!(windows) { "uv.exe" } else { "uv" }); + let output = Output { + status: failure_status(), + stdout: Vec::new(), + stderr: b"downloaded executable was corrupt".to_vec(), + }; + + let error = finish_uv_install_with_probe(dest, &uv_bin, output, |_| false) + .expect_err("an unusable download must not be accepted"); + + assert!(error.to_string().contains("downloaded executable was corrupt")); + } + + #[test] + fn usable_legacy_bin_location_is_relocated() { + let unique = format!( + "voicestudio-uv-test-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let dest = std::env::temp_dir().join(unique); + let uv_bin = dest.join(if cfg!(windows) { "uv.exe" } else { "uv" }); + let legacy = dest + .join("bin") + .join(if cfg!(windows) { "uv.exe" } else { "uv" }); + fs::create_dir_all(legacy.parent().unwrap()).unwrap(); + fs::write(&legacy, b"verified test executable").unwrap(); + let output = Output { + status: success_status(), + stdout: Vec::new(), + stderr: Vec::new(), + }; + + let result = finish_uv_install_with_probe(&dest, &uv_bin, output, |candidate| { + candidate.is_file() + }); + + assert_eq!(result.unwrap(), uv_bin); + assert!(uv_bin.is_file()); + assert!(!legacy.exists()); + fs::remove_dir_all(dest).unwrap(); + } + + #[cfg(unix)] + fn success_status() -> std::process::ExitStatus { + use std::os::unix::process::ExitStatusExt; + std::process::ExitStatus::from_raw(0) + } + + #[cfg(windows)] + fn success_status() -> std::process::ExitStatus { + use std::os::windows::process::ExitStatusExt; + std::process::ExitStatus::from_raw(0) + } + + #[cfg(unix)] + fn failure_status() -> std::process::ExitStatus { + use std::os::unix::process::ExitStatusExt; + std::process::ExitStatus::from_raw(1 << 8) + } + + #[cfg(windows)] + fn failure_status() -> std::process::ExitStatus { + use std::os::windows::process::ExitStatusExt; + std::process::ExitStatus::from_raw(1) } } diff --git a/frontend/src/api/dub.ts b/frontend/src/api/dub.ts index fb7034c9..53aafd95 100644 --- a/frontend/src/api/dub.ts +++ b/frontend/src/api/dub.ts @@ -19,6 +19,26 @@ export interface IngestUrlOptions { fetchSubs?: boolean; /** Limit caption fetch to specific lang codes; defaults to all available. */ subLangs?: string[]; + /** Explicit cookies.txt export used only for this import. */ + cookieFile?: File; +} + +export const DUB_COOKIE_TRANSPORT_ERROR = 'DUB_COOKIE_TRANSPORT'; +export const DUB_COOKIE_SIZE_ERROR = 'DUB_COOKIE_TOO_LARGE'; +export const MAX_COOKIE_EXPORT_BYTES = 1024 * 1024; + +function cookieSelectionError(code: string): Error & { code: string } { + return Object.assign(new Error(code), { code }); +} + +export function _cookieTransportAllowed(apiBase: string): boolean { + const endpoint = new URL(apiBase, window.location.href); + return ( + endpoint.protocol === 'https:' || + endpoint.hostname === 'localhost' || + endpoint.hostname === '127.0.0.1' || + endpoint.hostname === '[::1]' + ); } export async function dubIngestUrl( @@ -26,7 +46,14 @@ export async function dubIngestUrl( jobId: string, opts: IngestUrlOptions = {}, ): Promise { - const { signal, fetchSubs, subLangs } = opts; + const { signal, fetchSubs, subLangs, cookieFile } = opts; + if (cookieFile && !_cookieTransportAllowed(API)) { + throw cookieSelectionError(DUB_COOKIE_TRANSPORT_ERROR); + } + if (cookieFile && cookieFile.size > MAX_COOKIE_EXPORT_BYTES) { + throw cookieSelectionError(DUB_COOKIE_SIZE_ERROR); + } + const cookieText = cookieFile ? await cookieFile.text() : undefined; return apiPost( '/dub/ingest-url', { @@ -34,6 +61,7 @@ export async function dubIngestUrl( job_id: jobId, fetch_subs: fetchSubs || undefined, sub_langs: subLangs && subLangs.length ? subLangs : undefined, + cookie_file: cookieText, }, { signal }, ); diff --git a/frontend/src/components/dub/IdleSkeleton.jsx b/frontend/src/components/dub/IdleSkeleton.jsx index 22e65412..02a96bd7 100644 --- a/frontend/src/components/dub/IdleSkeleton.jsx +++ b/frontend/src/components/dub/IdleSkeleton.jsx @@ -19,6 +19,7 @@ import { Download, } from 'lucide-react'; import { Button, Badge } from '../../ui'; +import { useEffect, useRef } from 'react'; import WaveformTimeline from '../WaveformTimeline'; import DubbingDemo from '../DubbingDemo'; import DubFailureNotice from './DubFailureNotice'; @@ -62,6 +63,8 @@ export default function IdleSkeleton({ onIngestUrl, fetchYtSubs, setFetchYtSubs, + youtubeCookieFile, + setYoutubeCookieFile, dubLangCode, setDubLangCode, setDubLang, @@ -70,6 +73,12 @@ export default function IdleSkeleton({ dubInstruct, setDubInstruct, }) { + const youtubeCookieInputRef = useRef(null); + useEffect(() => { + if (!youtubeCookieFile && youtubeCookieInputRef.current) { + youtubeCookieInputRef.current.value = ''; + } + }, [youtubeCookieFile]); return (
{/* Header bar */} @@ -376,6 +385,34 @@ export default function IdleSkeleton({ /> {t('dub.pull_captions')} +
{ + e.preventDefault(); + e.stopPropagation(); + }} + > + {t('dub.youtube_auth')} + e.stopPropagation()} + onChange={(e) => setYoutubeCookieFile(e.target.files?.[0] || null)} + /> + {youtubeCookieFile && ( + + )} +
{/* One decision up front: the target language. Everything else diff --git a/frontend/src/hooks/useDubWorkflow.js b/frontend/src/hooks/useDubWorkflow.js index 1c487d41..030873ec 100644 --- a/frontend/src/hooks/useDubWorkflow.js +++ b/frontend/src/hooks/useDubWorkflow.js @@ -11,6 +11,8 @@ import { tasksCancel, transcribeStreamUrl, dubImportSrt, + DUB_COOKIE_TRANSPORT_ERROR, + DUB_COOKIE_SIZE_ERROR, } from '../api/dub'; import { dialectMatchesLang } from '../api/dialects'; import { segmentGenInputs, applySpeakerCloneDefaults } from '../utils/segments'; @@ -553,6 +555,7 @@ export default function useDubWorkflow({ signal: ctrl.signal, fetchSubs: !!opts.fetchSubs, subLangs: opts.subLangs, + cookieFile: opts.cookieFile, }); setDubJobId(data.job_id); setDubTaskId(data.task_id); @@ -592,10 +595,17 @@ export default function useDubWorkflow({ toastAsrModelMissing(asrMissingPayload(err)); useAppStore.getState().errorPill(t('asr_missing.message')); } else { - setDubError(err.message); + const cookieErrorKey = + err?.code === DUB_COOKIE_TRANSPORT_ERROR + ? 'dub.cookie_transport_error' + : err?.code === DUB_COOKIE_SIZE_ERROR + ? 'dub.cookie_size_error' + : null; + const message = cookieErrorKey ? t(cookieErrorKey) : err.message; + setDubError(message); setDubStep('idle'); - toastErrorWithReport(t('dub_workflow.ingest_failed', { message: err.message }), err); - useAppStore.getState().errorPill(err.message); + toastErrorWithReport(t('dub_workflow.ingest_failed', { message }), err); + useAppStore.getState().errorPill(message); } setTranscribeStart(null); } finally { diff --git a/frontend/src/i18n/locales/ar.json b/frontend/src/i18n/locales/ar.json index eacefbbf..94e3a8b9 100644 --- a/frontend/src/i18n/locales/ar.json +++ b/frontend/src/i18n/locales/ar.json @@ -751,6 +751,11 @@ "paste_url": "... أو الصق عنوان URL للفيديو/YouTube", "ingest": "استيعاب", "pull_captions": "سحب التسميات التوضيحية على YouTube + الترجمات التلقائية", + "youtube_auth": "تسجيل الدخول إلى YouTube (اختياري)", + "youtube_cookie_file": "اختيار ملف cookies.txt مُصدَّر", + "remove_cookie_file": "إزالة ملف تعريف الارتباط", + "cookie_transport_error": "يتطلب تصدير ملفات تعريف الارتباط HTTPS أو تطبيق سطح المكتب المحلي.", + "cookie_size_error": "يجب ألا يتجاوز تصدير ملفات تعريف الارتباط 1 ميغابايت.", "save": "حفظ", "reset": "إعادة تعيين", "save_project": "حفظ المشروع", diff --git a/frontend/src/i18n/locales/de.json b/frontend/src/i18n/locales/de.json index 0a51a785..da9dc976 100644 --- a/frontend/src/i18n/locales/de.json +++ b/frontend/src/i18n/locales/de.json @@ -751,6 +751,11 @@ "paste_url": "…oder fügen Sie die YouTube-/Video-URL ein", "ingest": "Verschlucken", "pull_captions": "Rufen Sie YouTube-Untertitel und automatische Übersetzungen ab", + "youtube_auth": "YouTube-Anmeldung (optional)", + "youtube_cookie_file": "cookies.txt-Export auswählen", + "remove_cookie_file": "Cookie-Datei entfernen", + "cookie_transport_error": "Cookie-Exporte erfordern HTTPS oder die lokale Desktop-App.", + "cookie_size_error": "Der Cookie-Export darf höchstens 1 MB groß sein.", "save": "Speichern", "reset": "Zurücksetzen", "save_project": "Projekt speichern", diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index cacc85e0..2ecdf470 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -1031,6 +1031,11 @@ "paste_url": "…or paste YouTube / video URL", "ingest": "Ingest", "pull_captions": "Pull YouTube captions + auto-translations", + "youtube_auth": "YouTube sign-in (optional)", + "youtube_cookie_file": "Choose a cookies.txt export", + "remove_cookie_file": "Remove cookie file", + "cookie_transport_error": "Cookie exports require HTTPS or the local desktop app.", + "cookie_size_error": "The cookie export must be 1 MB or smaller.", "save": "Save", "reset": "Reset", "save_project": "Save project", diff --git a/frontend/src/i18n/locales/es.json b/frontend/src/i18n/locales/es.json index 80fe742f..f9dff902 100644 --- a/frontend/src/i18n/locales/es.json +++ b/frontend/src/i18n/locales/es.json @@ -751,6 +751,11 @@ "paste_url": "…o pegue la URL de YouTube/vídeo", "ingest": "Ingerir", "pull_captions": "Extraiga subtítulos de YouTube + traducciones automáticas", + "youtube_auth": "Inicio de sesión en YouTube (opcional)", + "youtube_cookie_file": "Elegir una exportación cookies.txt", + "remove_cookie_file": "Quitar archivo de cookies", + "cookie_transport_error": "Las cookies exportadas requieren HTTPS o la aplicación de escritorio local.", + "cookie_size_error": "El archivo de cookies debe tener 1 MB o menos.", "save": "Guardar", "reset": "Reiniciar", "save_project": "Guardar proyecto", diff --git a/frontend/src/i18n/locales/fr.json b/frontend/src/i18n/locales/fr.json index 90a72c4d..9c51f63f 100644 --- a/frontend/src/i18n/locales/fr.json +++ b/frontend/src/i18n/locales/fr.json @@ -751,6 +751,11 @@ "paste_url": "…ou collez l’URL YouTube/vidéo", "ingest": "Ingérer", "pull_captions": "Extrayez les sous-titres YouTube + les traductions automatiques", + "youtube_auth": "Connexion YouTube (facultative)", + "youtube_cookie_file": "Choisir un export cookies.txt", + "remove_cookie_file": "Supprimer le fichier de cookies", + "cookie_transport_error": "L’export de cookies nécessite HTTPS ou l’application de bureau locale.", + "cookie_size_error": "L’export de cookies doit faire 1 Mo maximum.", "save": "Enregistrer", "reset": "Réinitialiser", "save_project": "Enregistrer le projet", diff --git a/frontend/src/i18n/locales/hi.json b/frontend/src/i18n/locales/hi.json index 743251fc..b278ba5d 100644 --- a/frontend/src/i18n/locales/hi.json +++ b/frontend/src/i18n/locales/hi.json @@ -751,6 +751,11 @@ "paste_url": "...या यूट्यूब/वीडियो यूआरएल पेस्ट करें", "ingest": "निगलना", "pull_captions": "YouTube कैप्शन + ऑटो-अनुवाद खींचें", + "youtube_auth": "YouTube साइन-इन (वैकल्पिक)", + "youtube_cookie_file": "cookies.txt निर्यात चुनें", + "remove_cookie_file": "कुकी फ़ाइल हटाएँ", + "cookie_transport_error": "कुकी निर्यात के लिए HTTPS या स्थानीय डेस्कटॉप ऐप आवश्यक है।", + "cookie_size_error": "कुकी निर्यात 1 MB या उससे छोटा होना चाहिए।", "save": "सहेजें", "reset": "रीसेट करें", "save_project": "प्रोजेक्ट सहेजें", diff --git a/frontend/src/i18n/locales/id.json b/frontend/src/i18n/locales/id.json index 5364daf1..cf1ebfe4 100644 --- a/frontend/src/i18n/locales/id.json +++ b/frontend/src/i18n/locales/id.json @@ -751,6 +751,11 @@ "paste_url": "…atau tempel URL YouTube/video", "ingest": "Menelan", "pull_captions": "Tarik teks YouTube + terjemahan otomatis", + "youtube_auth": "Masuk YouTube (opsional)", + "youtube_cookie_file": "Pilih ekspor cookies.txt", + "remove_cookie_file": "Hapus berkas kuki", + "cookie_transport_error": "Ekspor kuki memerlukan HTTPS atau aplikasi desktop lokal.", + "cookie_size_error": "Ekspor kuki harus berukuran 1 MB atau kurang.", "save": "Simpan", "reset": "Setel ulang", "save_project": "Simpan proyek", diff --git a/frontend/src/i18n/locales/it.json b/frontend/src/i18n/locales/it.json index c1ad72fd..8a95d3ce 100644 --- a/frontend/src/i18n/locales/it.json +++ b/frontend/src/i18n/locales/it.json @@ -751,6 +751,11 @@ "paste_url": "...o incolla l'URL di YouTube/video", "ingest": "Ingerire", "pull_captions": "Estrai sottotitoli YouTube + traduzioni automatiche", + "youtube_auth": "Accesso a YouTube (facoltativo)", + "youtube_cookie_file": "Scegli un'esportazione cookies.txt", + "remove_cookie_file": "Rimuovi file dei cookie", + "cookie_transport_error": "L’esportazione dei cookie richiede HTTPS o l’app desktop locale.", + "cookie_size_error": "L’esportazione dei cookie deve essere al massimo di 1 MB.", "save": "Salva", "reset": "Ripristina", "save_project": "Salva progetto", diff --git a/frontend/src/i18n/locales/ja.json b/frontend/src/i18n/locales/ja.json index 8425db15..be832f11 100644 --- a/frontend/src/i18n/locales/ja.json +++ b/frontend/src/i18n/locales/ja.json @@ -751,6 +751,11 @@ "paste_url": "…または YouTube / ビデオの URL を貼り付けます", "ingest": "摂取する", "pull_captions": "YouTube のキャプションと自動翻訳を取得します", + "youtube_auth": "YouTube ログイン（任意）", + "youtube_cookie_file": "cookies.txt のエクスポートを選択", + "remove_cookie_file": "Cookie ファイルを削除", + "cookie_transport_error": "Cookie のエクスポートには HTTPS またはローカルのデスクトップアプリが必要です。", + "cookie_size_error": "Cookie のエクスポートは 1 MB 以下にしてください。", "save": "保存", "reset": "リセット", "save_project": "プロジェクトの保存", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index c54e8a2f..51addff6 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -751,6 +751,11 @@ "paste_url": "...또는 YouTube/동영상 URL을 붙여넣으세요.", "ingest": "섭취", "pull_captions": "YouTube 캡션 및 자동 번역 가져오기", + "youtube_auth": "YouTube 로그인(선택 사항)", + "youtube_cookie_file": "cookies.txt 내보내기 선택", + "remove_cookie_file": "쿠키 파일 제거", + "cookie_transport_error": "쿠키 내보내기에는 HTTPS 또는 로컬 데스크톱 앱이 필요합니다.", + "cookie_size_error": "쿠키 내보내기 파일은 1MB 이하여야 합니다.", "save": "저장", "reset": "재설정", "save_project": "프로젝트 저장", diff --git a/frontend/src/i18n/locales/nl.json b/frontend/src/i18n/locales/nl.json index 62989efe..363907a4 100644 --- a/frontend/src/i18n/locales/nl.json +++ b/frontend/src/i18n/locales/nl.json @@ -751,6 +751,11 @@ "paste_url": "…of plak de YouTube-/video-URL", "ingest": "Innemen", "pull_captions": "Haal YouTube-ondertitels en automatische vertalingen op", + "youtube_auth": "YouTube-aanmelding (optioneel)", + "youtube_cookie_file": "Een cookies.txt-export kiezen", + "remove_cookie_file": "Cookiebestand verwijderen", + "cookie_transport_error": "Cookie-exports vereisen HTTPS of de lokale desktop-app.", + "cookie_size_error": "De cookie-export mag maximaal 1 MB zijn.", "save": "Opslaan", "reset": "Opnieuw instellen", "save_project": "Project opslaan", diff --git a/frontend/src/i18n/locales/pl.json b/frontend/src/i18n/locales/pl.json index fdc44b9d..b01fc2ff 100644 --- a/frontend/src/i18n/locales/pl.json +++ b/frontend/src/i18n/locales/pl.json @@ -751,6 +751,11 @@ "paste_url": "…lub wklej adres URL YouTube/wideo", "ingest": "Połknąć", "pull_captions": "Pobieraj napisy z YouTube + automatyczne tłumaczenia", + "youtube_auth": "Logowanie do YouTube (opcjonalne)", + "youtube_cookie_file": "Wybierz eksport cookies.txt", + "remove_cookie_file": "Usuń plik cookie", + "cookie_transport_error": "Eksport plików cookie wymaga HTTPS lub lokalnej aplikacji komputerowej.", + "cookie_size_error": "Eksport plików cookie może mieć najwyżej 1 MB.", "save": "Zapisz", "reset": "Zresetuj", "save_project": "Zapisz projekt", diff --git a/frontend/src/i18n/locales/pt.json b/frontend/src/i18n/locales/pt.json index 2e0749d8..d534c34c 100644 --- a/frontend/src/i18n/locales/pt.json +++ b/frontend/src/i18n/locales/pt.json @@ -751,6 +751,11 @@ "paste_url": "…ou cole o URL do YouTube/vídeo", "ingest": "Ingerir", "pull_captions": "Obtenha legendas + traduções automáticas do YouTube", + "youtube_auth": "Login no YouTube (opcional)", + "youtube_cookie_file": "Escolher uma exportação cookies.txt", + "remove_cookie_file": "Remover arquivo de cookies", + "cookie_transport_error": "A exportação de cookies requer HTTPS ou o aplicativo local.", + "cookie_size_error": "A exportação de cookies deve ter no máximo 1 MB.", "save": "Salvar", "reset": "Redefinir", "save_project": "Salvar projeto", diff --git a/frontend/src/i18n/locales/ru.json b/frontend/src/i18n/locales/ru.json index c37b91a4..500b1c1f 100644 --- a/frontend/src/i18n/locales/ru.json +++ b/frontend/src/i18n/locales/ru.json @@ -751,6 +751,11 @@ "paste_url": "…или вставьте URL-адрес YouTube/видео", "ingest": "Заглотить", "pull_captions": "Получение титров YouTube + автопереводы", + "youtube_auth": "Вход в YouTube (необязательно)", + "youtube_cookie_file": "Выбрать экспорт cookies.txt", + "remove_cookie_file": "Удалить файл cookie", + "cookie_transport_error": "Для экспорта cookie требуется HTTPS или локальное приложение.", + "cookie_size_error": "Размер экспорта cookie не должен превышать 1 МБ.", "save": "Сохранять", "reset": "Перезагрузить", "save_project": "Сохранить проект", diff --git a/frontend/src/i18n/locales/sv.json b/frontend/src/i18n/locales/sv.json index ef442ffd..0f3e0075 100644 --- a/frontend/src/i18n/locales/sv.json +++ b/frontend/src/i18n/locales/sv.json @@ -751,6 +751,11 @@ "paste_url": "…eller klistra in YouTube/videons URL", "ingest": "Inta", "pull_captions": "Dra YouTube-textning + automatiska översättningar", + "youtube_auth": "YouTube-inloggning (valfritt)", + "youtube_cookie_file": "Välj en cookies.txt-export", + "remove_cookie_file": "Ta bort cookie-filen", + "cookie_transport_error": "Cookie-exporter kräver HTTPS eller den lokala skrivbordsappen.", + "cookie_size_error": "Cookie-exporten får vara högst 1 MB.", "save": "Spara", "reset": "Återställ", "save_project": "Spara projekt", diff --git a/frontend/src/i18n/locales/th.json b/frontend/src/i18n/locales/th.json index 1b60a2c8..e6cae4f4 100644 --- a/frontend/src/i18n/locales/th.json +++ b/frontend/src/i18n/locales/th.json @@ -751,6 +751,11 @@ "paste_url": "…หรือวาง URL ของ YouTube / วิดีโอ", "ingest": "นำเข้า", "pull_captions": "ดึงคำบรรยาย YouTube + การแปลอัตโนมัติ", + "youtube_auth": "ลงชื่อเข้าใช้ YouTube (ไม่บังคับ)", + "youtube_cookie_file": "เลือกไฟล์ส่งออก cookies.txt", + "remove_cookie_file": "ลบไฟล์คุกกี้", + "cookie_transport_error": "การส่งออกคุกกี้ต้องใช้ HTTPS หรือแอปเดสก์ท็อปในเครื่อง", + "cookie_size_error": "ไฟล์ส่งออกคุกกี้ต้องมีขนาดไม่เกิน 1 MB", "save": "บันทึก", "reset": "รีเซ็ต", "save_project": "บันทึกโครงการ", diff --git a/frontend/src/i18n/locales/tr.json b/frontend/src/i18n/locales/tr.json index ed0cb13a..a9ad5c28 100644 --- a/frontend/src/i18n/locales/tr.json +++ b/frontend/src/i18n/locales/tr.json @@ -751,6 +751,11 @@ "paste_url": "…veya YouTube / video URL'sini yapıştırın", "ingest": "Al", "pull_captions": "YouTube altyazılarını + otomatik çevirileri çekin", + "youtube_auth": "YouTube oturumu (isteğe bağlı)", + "youtube_cookie_file": "Bir cookies.txt dışa aktarımı seçin", + "remove_cookie_file": "Çerez dosyasını kaldır", + "cookie_transport_error": "Çerez dışa aktarımları HTTPS veya yerel masaüstü uygulamasını gerektirir.", + "cookie_size_error": "Çerez dışa aktarımı en fazla 1 MB olmalıdır.", "save": "Kaydet", "reset": "Sıfırla", "save_project": "Projeyi kaydet", diff --git a/frontend/src/i18n/locales/uk.json b/frontend/src/i18n/locales/uk.json index e597d5d2..4c6cdb1d 100644 --- a/frontend/src/i18n/locales/uk.json +++ b/frontend/src/i18n/locales/uk.json @@ -751,6 +751,11 @@ "paste_url": "…або вставте URL-адресу YouTube/відео", "ingest": "Проковтнути", "pull_captions": "Витягніть субтитри YouTube + автоматичний переклад", + "youtube_auth": "Вхід у YouTube (необов’язково)", + "youtube_cookie_file": "Вибрати експорт cookies.txt", + "remove_cookie_file": "Видалити файл cookie", + "cookie_transport_error": "Для експорту cookie потрібен HTTPS або локальний застосунок.", + "cookie_size_error": "Розмір експорту cookie не повинен перевищувати 1 МБ.", "save": "зберегти", "reset": "Скинути", "save_project": "Зберегти проект", diff --git a/frontend/src/i18n/locales/vi.json b/frontend/src/i18n/locales/vi.json index 83c35ec4..3f96f9bf 100644 --- a/frontend/src/i18n/locales/vi.json +++ b/frontend/src/i18n/locales/vi.json @@ -751,6 +751,11 @@ "paste_url": "…hoặc dán URL YouTube/video", "ingest": "Nhập", "pull_captions": "Kéo phụ đề YouTube + bản dịch tự động", + "youtube_auth": "Đăng nhập YouTube (tùy chọn)", + "youtube_cookie_file": "Chọn bản xuất cookies.txt", + "remove_cookie_file": "Xóa tệp cookie", + "cookie_transport_error": "Tệp cookie xuất yêu cầu HTTPS hoặc ứng dụng máy tính cục bộ.", + "cookie_size_error": "Tệp cookie xuất phải có dung lượng không quá 1 MB.", "save": "Lưu", "reset": "Đặt lại", "save_project": "Lưu dự án", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 65613efc..ce813b87 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -710,6 +710,11 @@ "paste_url": "…或粘贴 YouTube / 视频链接", "ingest": "导入", "pull_captions": "拉取 YouTube 字幕 + 自动翻译", + "youtube_auth": "YouTube 登录（可选）", + "youtube_cookie_file": "选择导出的 cookies.txt", + "remove_cookie_file": "移除 Cookie 文件", + "cookie_transport_error": "Cookie 导出文件只能通过 HTTPS 或本地桌面应用发送。", + "cookie_size_error": "Cookie 导出文件必须小于或等于 1 MB。", "save": "保存", "reset": "重置", "save_project": "保存项目", diff --git a/frontend/src/i18n/locales/zh-TW.json b/frontend/src/i18n/locales/zh-TW.json index cd7a5523..edff5cd5 100644 --- a/frontend/src/i18n/locales/zh-TW.json +++ b/frontend/src/i18n/locales/zh-TW.json @@ -751,6 +751,11 @@ "paste_url": "…或貼上 YouTube/影片 URL", "ingest": "攝取", "pull_captions": "擷取 YouTube 字幕 + 自動翻譯", + "youtube_auth": "YouTube 登入（選用）", + "youtube_cookie_file": "選擇匯出的 cookies.txt", + "remove_cookie_file": "移除 Cookie 檔案", + "cookie_transport_error": "Cookie 匯出檔只能透過 HTTPS 或本機桌面應用程式傳送。", + "cookie_size_error": "Cookie 匯出檔必須小於或等於 1 MB。", "save": "儲存", "reset": "重置", "save_project": "保存項目", diff --git a/frontend/src/pages/DubTab.jsx b/frontend/src/pages/DubTab.jsx index 42afa813..7555747a 100644 --- a/frontend/src/pages/DubTab.jsx +++ b/frontend/src/pages/DubTab.jsx @@ -415,13 +415,20 @@ export default function DubTab(props) { // component instead of the global store to avoid polluting cross-project // prefs with what's really a per-ingest choice. const [fetchYtSubs, setFetchYtSubs] = useState(false); + const [youtubeCookieFile, setYoutubeCookieFile] = useState(null); + const resetDubAndCredentials = useCallback(() => { + setYoutubeCookieFile(null); + resetDub?.(); + }, [resetDub]); const onIngestUrl = () => { if (!ingestUrl.trim() || !handleDubIngestUrl) return; handleDubIngestUrl(ingestUrl.trim(), { fetchSubs: fetchYtSubs, subLangs: undefined, + cookieFile: youtubeCookieFile || undefined, }); setIngestUrl(''); + setYoutubeCookieFile(null); }; // Track-switcher visibility is keyed to the persisted tracks ONLY — not the // language dropdown. Restored projects can carry finished tracks while @@ -558,6 +565,8 @@ export default function DubTab(props) { onIngestUrl={onIngestUrl} fetchYtSubs={fetchYtSubs} setFetchYtSubs={setFetchYtSubs} + youtubeCookieFile={youtubeCookieFile} + setYoutubeCookieFile={setYoutubeCookieFile} dubLangCode={dubLangCode} setDubLangCode={switchDubLangCode} setDubLang={setDubLang} @@ -578,7 +587,7 @@ export default function DubTab(props) { dubSegments={dubSegments} activeProjectName={activeProjectName} saveProject={saveProject} - resetDub={resetDub} + resetDub={resetDubAndCredentials} dubStep={dubStep} handleDubStop={handleDubStop} dubProgress={dubProgress} diff --git a/frontend/src/test/DubIdleSkeleton.test.jsx b/frontend/src/test/DubIdleSkeleton.test.jsx index d2263fbd..7b009f7c 100644 --- a/frontend/src/test/DubIdleSkeleton.test.jsx +++ b/frontend/src/test/DubIdleSkeleton.test.jsx @@ -1,10 +1,16 @@ import React from 'react'; import { describe, it, expect, vi } from 'vitest'; -import { render, screen } from '@testing-library/react'; +import { fireEvent, render, screen } from '@testing-library/react'; import { I18nextProvider } from 'react-i18next'; import i18n from '../i18n'; import IdleSkeleton from '../components/dub/IdleSkeleton'; +import { + _cookieTransportAllowed, + dubIngestUrl, + DUB_COOKIE_SIZE_ERROR, + MAX_COOKIE_EXPORT_BYTES, +} from '../api/dub'; // Regression guard for the Dub "transcribe-idle-desync" bug: on the // URL-ingest (and restored-job) path there is no local `dubVideoFile`, so the @@ -53,6 +59,8 @@ function baseProps(overrides = {}) { onIngestUrl: noop, fetchYtSubs: false, setFetchYtSubs: noop, + youtubeCookieFile: null, + setYoutubeCookieFile: noop, dubLangCode: 'en', setDubLangCode: noop, setDubLang: noop, @@ -73,11 +81,46 @@ function renderIdle(overrides) { } describe('IdleSkeleton — pipeline-stage vs idle dropzone', () => { + it('never sends cookie credentials over remote plaintext HTTP', () => { + expect(_cookieTransportAllowed('http://127.0.0.1:3900')).toBe(true); + expect(_cookieTransportAllowed('https://studio.example.test')).toBe(true); + expect(_cookieTransportAllowed('http://studio.example.test')).toBe(false); + }); + it('rejects an oversized cookie export before reading it', async () => { + const cookieFile = { + size: MAX_COOKIE_EXPORT_BYTES + 1, + text: vi.fn(), + }; + await expect( + dubIngestUrl('https://youtube.com/watch?v=abc', 'job', { cookieFile }), + ).rejects.toMatchObject({ + code: DUB_COOKIE_SIZE_ERROR, + }); + expect(cookieFile.text).not.toHaveBeenCalled(); + }); it('shows the idle dropzone only when the pipeline is truly idle (no job)', () => { const { container } = renderIdle({ dubStep: 'idle', dubJobId: null }); expect(container.querySelector('.dub-idle-drop')).not.toBeNull(); expect(screen.getByText(DROP_HINT)).toBeInTheDocument(); expect(screen.getByPlaceholderText(URL_PLACEHOLDER)).toBeInTheDocument(); + expect(screen.getByLabelText('Choose a cookies.txt export')).toBeInTheDocument(); + }); + + it('clears the native cookie picker when the selection is removed', () => { + const selected = new File(['# Netscape HTTP Cookie File\n'], 'cookies.txt', { + type: 'text/plain', + }); + const { rerender } = renderIdle({ youtubeCookieFile: selected }); + const input = screen.getByLabelText('Choose a cookies.txt export'); + fireEvent.change(input, { target: { files: [selected] } }); + expect(input.files).toHaveLength(1); + + rerender( + + + , + ); + expect(input.value).toBe(''); }); it('does NOT show the idle dropzone while transcribing a URL-ingested job', () => { diff --git a/frontend/src/test/DubTrackTabsRestore.test.jsx b/frontend/src/test/DubTrackTabsRestore.test.jsx index 73107957..84bd15cf 100644 --- a/frontend/src/test/DubTrackTabsRestore.test.jsx +++ b/frontend/src/test/DubTrackTabsRestore.test.jsx @@ -1,6 +1,6 @@ import React from 'react'; import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { render } from '@testing-library/react'; +import { act, fireEvent, render, screen } from '@testing-library/react'; import { useAppStore } from '../store'; // Regression guard for the "completed dub tracks' tabs hidden until the @@ -23,11 +23,29 @@ vi.mock('../components/dub/DubLeftColumn', () => ({ return
; }, })); -vi.mock('../components/dub/DubHeader', () => ({ default: () => null })); +vi.mock('../components/dub/DubHeader', () => ({ + default: ({ resetDub }) => ( + + ), +})); vi.mock('../components/dub/DubRightColumn', () => ({ default: () => null })); vi.mock('../components/dub/DubFooter', () => ({ default: () => null })); vi.mock('../components/dub/DubPipelineStepper', () => ({ default: () => null })); -vi.mock('../components/dub/IdleSkeleton', () => ({ default: () => null })); +vi.mock('../components/dub/IdleSkeleton', () => ({ + default: ({ youtubeCookieFile, setYoutubeCookieFile }) => ( +
+ {youtubeCookieFile?.name || 'none'} + +
+ ), +})); vi.mock('../components/ExportModal', () => ({ default: () => null })); vi.mock('../hooks/useTimelineOnsets', () => ({ default: () => ({ onsets: [] }) })); vi.mock('../api/dub', () => ({ @@ -146,4 +164,20 @@ describe('DubTab — completed tracks always show their tabs (restore P0)', () = expect(left.hasDubbedTrack).toBe(false); expect(left.previewMode).toBe('original'); }); + + it('clears a selected cookie export when a completed dub is reset', () => { + const resetDub = vi.fn(() => + useAppStore.setState({ dubJobId: null, dubStep: 'idle', dubTracks: [] }), + ); + useAppStore.setState({ dubJobId: null, dubStep: 'idle', dubTracks: [] }); + render(); + + fireEvent.click(screen.getByTestId('select-cookie')); + expect(screen.getByTestId('cookie-name')).toHaveTextContent('cookies.txt'); + act(() => useAppStore.setState({ dubJobId: 'job1', dubStep: 'done' })); + fireEvent.click(screen.getByTestId('reset-dub')); + + expect(resetDub).toHaveBeenCalledOnce(); + expect(screen.getByTestId('cookie-name')).toHaveTextContent('none'); + }); }); diff --git a/omnivoice/models/omnivoice.py b/omnivoice/models/omnivoice.py index b93913c2..08210139 100644 --- a/omnivoice/models/omnivoice.py +++ b/omnivoice/models/omnivoice.py @@ -75,6 +75,16 @@ from omnivoice.utils.voice_design import ( logger = logging.getLogger(__name__) +_AUDIO_TOKENIZER_FALLBACK_REPO = "eustlb/higgs-audio-v2-tokenizer" + + +class OmniVoiceModelAssetError(RuntimeError): + """A fixed nested model repository failed while OmniVoice was loading.""" + + def __init__(self, repository_id: str): + super().__init__(f"Failed to load OmniVoice model asset: {repository_id}") + self.repository_id = repository_id + # --------------------------------------------------------------------------- # Dataclasses @@ -345,18 +355,25 @@ class OmniVoice(PreTrainedModel): if not os.path.isdir(audio_tokenizer_path): # Fallback to the HuggingFace Hub path of transformers' # HiggsAudioV2Tokenizer if the local subdirectory doesn't exist. - audio_tokenizer_path = "eustlb/higgs-audio-v2-tokenizer" + audio_tokenizer_path = _AUDIO_TOKENIZER_FALLBACK_REPO # higgs-audio-v2-tokenizer does not support MPS (output channels > 65536) tokenizer_device = ( "cpu" if str(model.device).startswith("mps") else model.device ) - model.audio_tokenizer = _audio_tokenizer_cls().from_pretrained( - audio_tokenizer_path, device_map=tokenizer_device - ) - model.feature_extractor = AutoFeatureExtractor.from_pretrained( - audio_tokenizer_path - ) + try: + model.audio_tokenizer = _audio_tokenizer_cls().from_pretrained( + audio_tokenizer_path, device_map=tokenizer_device + ) + model.feature_extractor = AutoFeatureExtractor.from_pretrained( + audio_tokenizer_path + ) + except Exception as exc: + if audio_tokenizer_path != _AUDIO_TOKENIZER_FALLBACK_REPO: + raise + raise OmniVoiceModelAssetError( + _AUDIO_TOKENIZER_FALLBACK_REPO + ) from exc model.sampling_rate = model.feature_extractor.sampling_rate diff --git a/tests/test_corrupt_weights_recovery_1406.py b/tests/test_corrupt_weights_recovery_1406.py new file mode 100644 index 00000000..86c98852 --- /dev/null +++ b/tests/test_corrupt_weights_recovery_1406.py @@ -0,0 +1,355 @@ +"""A weight file that is present but unparseable is repairable (#1406). + +Two failure shapes come out of an interrupted or mangled model download, and +only one of them was handled: + +* the shard is **missing** — transformers says "does not appear to have a file + named …", and a whole recovery ladder repairs it; and +* the shard is **present with wrong bytes** — a download that stopped + mid-file, an antivirus that truncated it, a proxy that saved an HTML error + page under its name. transformers opens it happily and safetensors then + fails parsing its header-length prefix. + +The second reached the user as a raw 500 — "Error while deserializing header: +header too large" — on every generation, from voice design and gallery +previews alike, with no repair attempted. It could not reach the ladder for +two independent reasons: the wording is not the missing-shard wording, and +``SafetensorError`` is a Rust-extension exception, not an ``OSError``. + +It also needs the *opposite* repair. The ladder resumes a download, and a +resume trusts a blob that is already the expected size — so it would never +re-fetch the one file that is actually wrong. +""" +from __future__ import annotations + +import pytest + + +@pytest.fixture(autouse=True) +def failure(): + """Resolved at run time: other suites reset `sys.modules` for app modules, + and a module-level binding here could assert against a stale phrase table.""" + import core.failure as _failure + + return _failure + + +# ── classification ───────────────────────────────────────────────────────── + +REPORTED = "Error while deserializing header: header too large" + +CORRUPT_WORDINGS = [ + REPORTED, + "SafetensorError: Error while deserializing header: HeaderTooLarge", + "safetensors_rust.SafetensorError: MetadataIncompleteBuffer", + "InvalidHeaderDeserialization", + "UnpicklingError: invalid load key, '<'.", + "RuntimeError: unexpected end of file while loading model.safetensors", + "It looks like the config file at 'models/snapshots/rev/config.json' " + "is not a valid JSON file.", +] + + +@pytest.mark.parametrize("text", CORRUPT_WORDINGS) +def test_corrupt_wordings_are_recognised(failure, text): + assert failure.is_corrupt_weights_message(text) + + +@pytest.mark.parametrize("text", CORRUPT_WORDINGS) +def test_corrupt_wordings_classify_as_a_damaged_cache(failure, text): + """Same taxonomy class as the missing-shard half: same cause, same remedy, + same docs deeplink. Before the fix these classified as "" and shipped with + no hint and no docs link.""" + assert failure.classify(text) == "MODEL_CACHE_CORRUPT" + + +def test_the_two_halves_stay_distinct(failure): + """They are one class to the user and two repairs to the code — a resume + for the missing half, a forced re-download for the damaged half. If these + ever start matching each other's wording, the wrong repair runs.""" + missing = "repo does not appear to have a file named model.safetensors" + assert failure.is_incomplete_cache_message(missing) + assert not failure.is_corrupt_weights_message(missing) + assert failure.is_corrupt_weights_message(REPORTED) + assert not failure.is_incomplete_cache_message(REPORTED) + + +@pytest.mark.parametrize( + "text", + [ + "connection reset by peer", + "CUDA out of memory", + "No such file or directory", + "", + # Generic enough that zipfile, tarfile, gzip and a JSON parser all say + # it — on its own it must NOT trigger a multi-GB re-download. + "BadZipFile: unexpected end of file", + ], +) +def test_unrelated_failures_are_not_swallowed(failure, text): + """The load's new clause is `except Exception`, so a false positive here + would divert an unrelated failure into a multi-GB re-download.""" + assert not failure.is_corrupt_weights_message(text) + + +# ── the load path ────────────────────────────────────────────────────────── + +class _SafetensorError(Exception): + """Stands in for safetensors_rust.SafetensorError — the point being that + it is NOT an OSError, which is why the ladder never saw the real one.""" + + +@pytest.fixture +def mm(monkeypatch): + import services.model_manager as mm + + monkeypatch.setattr(mm, "_set_loading", lambda *a, **kw: None) + monkeypatch.setattr(mm, "_manual_cache_delete_hint", lambda *a, **kw: "") + monkeypatch.setattr(mm, "_repair_failure_detail", lambda *a, **kw: "") + # Per-process guards must not leak between cases. + monkeypatch.setattr(mm, "_FORCED_REDOWNLOAD_ATTEMPTED", set(), raising=False) + return mm + + +def _drive_load(mm, monkeypatch, raise_first, repair_ok=True): + """Run `_load_model_sync` with a checkpoint load that fails once.""" + calls = {"load": 0, "repair": []} + + def _fake_from_pretrained(*a, **kw): + calls["load"] += 1 + if calls["load"] == 1: + raise raise_first + return object() + + class _FakeModelClass: + from_pretrained = staticmethod(_fake_from_pretrained) + + def _fake_repair(checkpoint, force=False): + calls["repair"].append(force) + return repair_ok + + monkeypatch.setattr(mm, "_lazy_omnivoice", lambda: _FakeModelClass) + monkeypatch.setattr(mm, "_lazy_torch", lambda: __import__("types").SimpleNamespace(float16="f16")) + monkeypatch.setattr(mm, "get_best_device", lambda: "cpu") + monkeypatch.setattr(mm, "resolve_omnivoice_checkpoint", lambda: "org/model") + monkeypatch.setattr(mm, "should_preload_tts_asr", lambda: False) + monkeypatch.setattr(mm, "_repair_model_cache", _fake_repair) + monkeypatch.setattr(mm, "_selfheal_broken_snapshot_links", lambda *a, **kw: False) + return calls + + +def test_a_corrupt_shard_is_re_downloaded_and_the_load_retried(mm, monkeypatch): + """The reported bug. Before the fix this propagated as a raw 500.""" + calls = _drive_load(mm, monkeypatch, _SafetensorError(REPORTED)) + mm._load_model_sync() + assert calls["load"] == 2, "the load was not retried after the repair" + assert calls["repair"] == [True], ( + "the repair must be FORCED — a resume trusts the corrupt blob, which " + "is already the size it expects, and would never re-fetch it" + ) + + +def test_the_same_shape_wrapped_in_an_oserror_is_also_repaired(mm, monkeypatch): + """transformers wraps tensor-library failures in OSError, where the + missing-shard check would drop it as unrecognised and re-raise.""" + calls = _drive_load(mm, monkeypatch, OSError(f"Unable to load weights: {REPORTED}")) + mm._load_model_sync() + assert calls["load"] == 2 + assert calls["repair"] == [True] + + +def test_corrupt_fallback_tokenizer_repairs_its_own_repository(mm, monkeypatch): + """A nested tokenizer failure must not re-download the TTS checkpoint.""" + from omnivoice.models.omnivoice import OmniVoiceModelAssetError + + corrupt = _SafetensorError(REPORTED) + nested = OmniVoiceModelAssetError("eustlb/higgs-audio-v2-tokenizer") + nested.__cause__ = corrupt + calls = _drive_load(mm, monkeypatch, nested) + + repaired = [] + + def _repair(repository_id, force=False): + repaired.append((repository_id, force)) + return True + + monkeypatch.setattr(mm, "_repair_model_cache", _repair) + mm._load_model_sync() + + assert calls["load"] == 2 + assert repaired == [("eustlb/higgs-audio-v2-tokenizer", True)] + + +def test_unrecognized_nested_repository_cannot_redirect_repair(mm, monkeypatch): + from omnivoice.models.omnivoice import OmniVoiceModelAssetError + + nested = OmniVoiceModelAssetError("attacker/unreviewed") + nested.__cause__ = _SafetensorError(REPORTED) + calls = _drive_load(mm, monkeypatch, nested) + repaired = [] + monkeypatch.setattr( + mm, + "_repair_model_cache", + lambda repository_id, force=False: repaired.append( + (repository_id, force) + ) or True, + ) + + mm._load_model_sync() + + assert calls["load"] == 2 + assert repaired == [("org/model", True)] + + +def test_fallback_tokenizer_failure_identifies_its_repository(monkeypatch, tmp_path): + from types import SimpleNamespace + + from omnivoice.models import omnivoice as model_module + + model = SimpleNamespace(device="cpu") + monkeypatch.setattr( + model_module.PreTrainedModel, + "from_pretrained", + classmethod(lambda cls, *args, **kwargs: model), + ) + monkeypatch.setattr( + model_module.AutoTokenizer, + "from_pretrained", + lambda *args, **kwargs: object(), + ) + monkeypatch.setattr( + model_module, + "_resolve_snapshot_dir", + lambda _checkpoint: str(tmp_path), + ) + + corrupt = _SafetensorError(REPORTED) + + class BrokenTokenizer: + @classmethod + def from_pretrained(cls, *args, **kwargs): + raise corrupt + + monkeypatch.setattr(model_module, "_audio_tokenizer_cls", lambda: BrokenTokenizer) + + with pytest.raises(model_module.OmniVoiceModelAssetError) as exc_info: + model_module.OmniVoice.from_pretrained("org/model") + + assert exc_info.value.repository_id == "eustlb/higgs-audio-v2-tokenizer" + assert exc_info.value.__cause__ is corrupt + + +def test_resume_that_exposes_corruption_switches_to_forced_repair(mm, monkeypatch): + """A missing shard can mask a corrupt one until resume fills the gap.""" + calls = _drive_load( + mm, + monkeypatch, + OSError("repo does not appear to have a file named model.safetensors"), + ) + + def _load_sequence(*a, **kw): + calls["load"] += 1 + if calls["load"] == 1: + raise OSError("repo does not appear to have a file named model.safetensors") + if calls["load"] == 2: + raise OSError(f"Unable to load weights: {REPORTED}") + return object() + + monkeypatch.setattr(mm, "_lazy_omnivoice", lambda: type( + "C", (), {"from_pretrained": staticmethod(_load_sequence)} + )) + + mm._load_model_sync() + + assert calls["load"] == 3 + assert calls["repair"] == [False, True] + + +def test_the_cause_is_matched_through_the_exception_chain(mm, monkeypatch): + """transformers re-raises with the tensor error as __cause__; matching only + the outermost message would miss every wrapped case.""" + inner = _SafetensorError(REPORTED) + outer = RuntimeError("could not load the checkpoint") + outer.__cause__ = inner + calls = _drive_load(mm, monkeypatch, outer) + mm._load_model_sync() + assert calls["load"] == 2 + + +def test_an_unrepairable_shard_says_what_to_do(mm, monkeypatch): + calls = _drive_load(mm, monkeypatch, _SafetensorError(REPORTED), repair_ok=False) + with pytest.raises(RuntimeError, match="damaged"): + mm._load_model_sync() + assert calls["load"] == 1, "no point retrying a load whose repair failed" + + +def test_an_unrelated_exception_still_propagates(mm, monkeypatch): + """The new clause is broad; this is what stops it becoming a catch-all.""" + _drive_load(mm, monkeypatch, ValueError("something else entirely")) + with pytest.raises(ValueError, match="something else entirely"): + mm._load_model_sync() + + +def test_a_second_failure_does_not_re_download_again(mm, monkeypatch): + """One bad shard must not turn into a full re-download per generate + request. After one forced re-fetch that did not help, say so and stop + (CodeRabbit).""" + calls = _drive_load(mm, monkeypatch, _SafetensorError(REPORTED)) + + # First attempt: repair runs, but the reloaded weights are still bad. + def _always_bad(*a, **kw): + calls["load"] += 1 + raise _SafetensorError(REPORTED) + + monkeypatch.setattr(mm, "_lazy_omnivoice", lambda: type( + "C", (), {"from_pretrained": staticmethod(_always_bad)} + )) + with pytest.raises(RuntimeError, match="still damaged"): + mm._load_model_sync() + assert calls["repair"] == [True] + + # Second attempt: no further download, straight to the manual remedy. + with pytest.raises(RuntimeError, match="did not fix them"): + mm._load_model_sync() + assert calls["repair"] == [True], "the model was re-downloaded a second time" + + +def test_a_damaged_asr_shard_does_not_re_download_the_tts_model(mm, monkeypatch): + """With OMNIVOICE_PRELOAD_TTS_ASR on, `_load()` also pulls the Whisper + checkpoint — a different repo. Blaming (and re-downloading) the TTS model + for its damage is gigabytes that fix nothing (CodeRabbit).""" + calls = _drive_load(mm, monkeypatch, _SafetensorError(REPORTED)) + monkeypatch.setattr(mm, "should_preload_tts_asr", lambda: True) + + loaded = object() + + class _Model: + llm = loaded + + def load_asr_model(self): + raise _SafetensorError(REPORTED) + + def _load_tts_once(*a, **kw): + calls["load"] += 1 + assert kw.get("load_asr") is False + return _Model() + + monkeypatch.setattr(mm, "_lazy_omnivoice", lambda: type( + "C", (), {"from_pretrained": staticmethod(_load_tts_once)} + )) + with pytest.raises(RuntimeError, match="transcription model"): + mm._load_model_sync() + assert calls["load"] == 1, "ASR diagnosis loaded the multi-GB TTS model twice" + assert calls["repair"] == [], "the TTS checkpoint was re-downloaded for an ASR fault" + + +def test_a_corrupt_config_is_force_repaired_and_retried(mm, monkeypatch): + """#1437: a truncated config.json is the same corrupt-cache class.""" + error = OSError( + "It looks like the config file at 'models/snapshots/rev/config.json' " + "is not a valid JSON file." + ) + calls = _drive_load(mm, monkeypatch, error) + mm._load_model_sync() + assert calls["load"] == 2 + assert calls["repair"] == [True] diff --git a/tests/test_hf_cache_repair.py b/tests/test_hf_cache_repair.py index 2951373d..8f3c0149 100644 --- a/tests/test_hf_cache_repair.py +++ b/tests/test_hf_cache_repair.py @@ -499,8 +499,13 @@ def test_classify_missing_weights_signature(): evt = failure.build_failure(_SIGNATURE, stage="model-load", include_diagnostic=False) assert evt["docs_topic"] == "MODEL_CACHE_CORRUPT" - assert "broken file links" in evt["hint"] - assert "repairs this automatically" in evt["hint"] + # The hint covers both halves of the class since #1406 — a file that is + # missing and one that arrived damaged — so it no longer names only the + # broken-link cause. What must survive is that it promises the automatic + # repair and names the manual fallback. + assert "missing or damaged" in evt["hint"] + assert "repairs it automatically" in evt["hint"] + assert "models----" in evt["hint"] def test_classify_repair_messages(): @@ -536,7 +541,7 @@ def test_classify_local_directory_missing_weights_signature(): evt = failure.build_failure(_SIGNATURE_LOCAL_DIR, stage="model-load", include_diagnostic=False) assert evt["docs_topic"] == "MODEL_CACHE_CORRUPT" - assert "repairs this automatically" in evt["hint"] + assert "repairs it automatically" in evt["hint"] def test_self_heal_recognises_both_wordings(): diff --git a/tests/test_model_manager_preload.py b/tests/test_model_manager_preload.py index 6b6f6c75..105d4426 100644 --- a/tests/test_model_manager_preload.py +++ b/tests/test_model_manager_preload.py @@ -61,12 +61,19 @@ def test_load_model_skips_pytorch_whisper_by_default(model_manager, monkeypatch) def test_load_model_can_preload_pytorch_whisper_when_requested(model_manager, monkeypatch): calls = [] + asr_loads = [] + + class DummyModel: + llm = object() + + def load_asr_model(self): + asr_loads.append(True) class DummyOmniVoice: @staticmethod def from_pretrained(*args, **kwargs): calls.append((args, kwargs)) - return SimpleNamespace(llm=object()) + return DummyModel() monkeypatch.setenv("OMNIVOICE_PRELOAD_TTS_ASR", "1") monkeypatch.setattr(model_manager, "_lazy_torch", lambda: SimpleNamespace(float16="float16")) @@ -75,7 +82,8 @@ def test_load_model_can_preload_pytorch_whisper_when_requested(model_manager, mo model_manager._load_model_sync() - assert calls[0][1]["load_asr"] is True + assert calls[0][1]["load_asr"] is False + assert asr_loads == [True] def test_resolve_checkpoint_honors_test_sentinel(model_manager, monkeypatch): diff --git a/tests/test_torch_constraints_are_applied.py b/tests/test_torch_constraints_are_applied.py index 21dd3a77..82f9cdfb 100644 --- a/tests/test_torch_constraints_are_applied.py +++ b/tests/test_torch_constraints_are_applied.py @@ -6,7 +6,7 @@ The coupling lives in `[tool.uv] constraint-dependencies`. That setting is part of the **project** API — `uv sync`, `uv lock`, `uv run`. `uv pip install` is the pip-compatible interface and ignores it. Both install -paths that use `uv pip install --system` (the Colab notebook and the Docker +paths that use `uv pip install` (the Colab notebook and the Docker image) therefore resolved the trio on its bare lower bounds, free to upgrade torch while leaving a torchvision built against an older ABI in place: @@ -99,13 +99,33 @@ def test_the_pins_carry_no_local_version(file_constraints): def test_the_dockerfile_passes_the_constraint(): text = _DOCKERFILE.read_text(encoding="utf-8") - install = [ln for ln in text.splitlines() if "uv pip install" in ln and "--system" in ln] - assert install, "no `uv pip install --system` line found in the Dockerfile" - for line in install: - assert "--constraint" in line and "torch-constraints.txt" in line, ( - f"Docker installs without the torch constraint, so the trio can " - f"drift again:\n {line.strip()}" - ) + install_start = text.index("RUN uv pip install") + install_end = text.index("\n\n", install_start) + install = text[install_start:install_end] + assert "--constraint" in install and "torch-constraints.txt" in install, ( + "Docker installs without the torch constraint, so the trio can " + f"drift again:\n{install}" + ) + + +def test_docker_install_and_runtime_use_the_guarded_python(): + """#1274: ROCm's `python3` had HIP torch, while `--system` installed and + bare `uvicorn` launched through `/usr/bin/python` with CUDA torch.""" + text = _DOCKERFILE.read_text(encoding="utf-8") + install_start = text.index("RUN uv pip install") + install_end = text.index("\n\n", install_start) + install = text[install_start:install_end] + assert '--python "$(command -v python3)"' in install + assert "--system" not in install + assert 'ENTRYPOINT ["python3", "-m", "uvicorn"' in text + + +def test_docker_docs_do_not_assume_the_run_name_for_compose(): + docs = (_ROOT / "docs" / "install" / "docker.md").read_text(encoding="utf-8") + assert "docker exec python3" in docs + assert "torch.cuda.get_device_name(0) if ok else 'unavailable'" in docs + for compose_name in ("omnivoice-studio", "omnivoice-studio-gpu", "omnivoice-studio-rocm"): + assert compose_name in docs def test_the_dockerfile_copies_the_constraints_file(): diff --git a/tests/test_youtube_cookie_import.py b/tests/test_youtube_cookie_import.py new file mode 100644 index 00000000..40037ee2 --- /dev/null +++ b/tests/test_youtube_cookie_import.py @@ -0,0 +1,255 @@ +"""Explicit, ephemeral YouTube authentication for URL ingest (#1429/#1432).""" +import asyncio +import importlib +import os +import stat +import sys + +import pytest +from fastapi import HTTPException + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "backend")) + +COOKIE_TEXT = "# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tSID\tsecret\n" + + +@pytest.fixture +def dub_core(): + """Import the application router only when a test needs it.""" + return importlib.import_module("api.routers.dub_core") + + +@pytest.fixture +def dub_pipeline(): + """Import the application pipeline only when a test needs it.""" + return importlib.import_module("services.dub_pipeline") + + +def test_cookie_export_requires_deliberate_netscape_file_and_is_private(dub_core): + with pytest.raises(HTTPException) as exc: + dub_core._stage_cookie_export('{"cookies": []}') + assert exc.value.status_code == 400 + + path = dub_core._stage_cookie_export(COOKIE_TEXT) + try: + with open(path, encoding="utf-8") as cookie_file: + assert cookie_file.read() == COOKIE_TEXT + if os.name != "nt": + assert stat.S_IMODE(os.stat(path).st_mode) == 0o600 + finally: + os.unlink(path) + + +def test_cookie_export_accepts_a_bom_and_rejects_empty_or_oversized_files(dub_core): + path = dub_core._stage_cookie_export("\ufeff" + COOKIE_TEXT) + try: + assert os.path.exists(path) + finally: + os.unlink(path) + + for contents in ("", "# Netscape HTTP Cookie File\n" + "x" * (1024 * 1024)): + with pytest.raises(HTTPException) as exc: + dub_core._stage_cookie_export(contents) + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize( + ("scheme", "host", "origin", "allowed"), + [ + ("http", "127.0.0.1", "http://tauri.localhost", True), + ("http", "::1", "http://localhost:3901", True), + ("https", "192.0.2.20", "https://studio.example", True), + ("http", "192.0.2.20", "http://localhost", False), + ("http", "127.0.0.1", "http://studio.example", False), + ("http", "127.0.0.1", None, False), + ], +) +def test_cookie_credentials_only_cross_https_or_local_ui( + dub_core, scheme, host, origin, allowed +): + assert dub_core._cookie_transport_allowed(scheme, host, origin) is allowed + + +def test_cookie_export_is_forwarded_to_ytdlp(dub_pipeline, tmp_path, monkeypatch): + import yt_dlp + + captured = {} + + class FakeYDL: + def __init__(self, opts): + captured.update(opts) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def extract_info(self, _url, download=True): + raise RuntimeError("stop after capturing options") + + monkeypatch.setattr(yt_dlp, "YoutubeDL", FakeYDL) + cookie_path = str(tmp_path / "cookies.txt") + with pytest.raises(RuntimeError): + dub_pipeline.yt_download_sync( + "https://youtube.com/watch?v=abc", + str(tmp_path), + cookie_file=cookie_path, + ) + + assert captured["cookiefile"] == cookie_path + + +def test_pipeline_deletes_cookie_export_after_download_failure( + dub_pipeline, tmp_path, monkeypatch +): + cookie_path = tmp_path / "session.cookies.txt" + cookie_path.write_text(COOKIE_TEXT, encoding="utf-8") + + def fail_download(*_args, **_kwargs): + raise RuntimeError("download failed") + + monkeypatch.setattr(dub_pipeline, "yt_download_sync", fail_download) + async def collect_events(): + events = [] + async for event in dub_pipeline.ingest_pipeline( + "cookie-cleanup", + str(tmp_path / "job"), + { + "kind": "url", + "url": "https://youtube.com/watch?v=abc", + "cookie_file": str(cookie_path), + }, + ): + events.append(event) + return events + + events = asyncio.run(collect_events()) + + assert any('"type": "error"' in event for event in events) + assert "secret" not in "".join(events) + assert not cookie_path.exists() + + +def test_cookie_cleanup_is_idempotent(dub_pipeline, tmp_path): + cookie_path = tmp_path / "session.cookies.txt" + cookie_path.write_text(COOKIE_TEXT, encoding="utf-8") + dub_pipeline._delete_cookie_export(str(cookie_path)) + dub_pipeline._delete_cookie_export(str(cookie_path)) + assert not cookie_path.exists() + + +def test_pipeline_cancellation_deletes_cookie_before_download(dub_pipeline, tmp_path): + cookie_path = tmp_path / "cancel.cookies.txt" + cookie_path.write_text(COOKIE_TEXT, encoding="utf-8") + + async def start_then_cancel(): + pipeline = dub_pipeline.ingest_pipeline( + "cookie-cancel", + str(tmp_path / "job-cancel"), + { + "kind": "url", + "url": "https://youtube.com/watch?v=abc", + "cookie_file": str(cookie_path), + }, + ) + await anext(pipeline) + await pipeline.aclose() + + asyncio.run(start_then_cancel()) + assert not cookie_path.exists() + + +def test_enqueue_failure_deletes_staged_cookie(dub_core, tmp_path, monkeypatch): + from schemas.requests import DubIngestUrlRequest + from starlette.requests import Request + + cookie_path = tmp_path / "queued.cookies.txt" + monkeypatch.setattr(dub_core, "_stage_cookie_export", lambda _text: str(cookie_path)) + cookie_path.write_text(COOKIE_TEXT, encoding="utf-8") + monkeypatch.setattr(dub_core, "_safe_job_dir", lambda _job_id: str(tmp_path / "job")) + + async def fail_add(*_args, **_kwargs): + raise RuntimeError("queue closed") + + monkeypatch.setattr(dub_core.task_manager, "add_task", fail_add) + request = Request( + {"type": "http", "scheme": "http", "server": ("127.0.0.1", 80), + "client": ("127.0.0.1", 1234), "path": "/dub/ingest-url", + "headers": [(b"origin", b"http://tauri.localhost")]} + ) + with pytest.raises(RuntimeError, match="queue closed"): + asyncio.run( + dub_core.dub_ingest_url( + DubIngestUrlRequest( + url="https://youtube.com/watch?v=abc", cookie_file=COOKIE_TEXT + ), + request, + ) + ) + assert not cookie_path.exists() + + +def test_job_directory_failure_happens_before_cookie_staging( + dub_core, tmp_path, monkeypatch +): + from schemas.requests import DubIngestUrlRequest + from starlette.requests import Request + + staged = False + + def stage_cookie(_text): + nonlocal staged + staged = True + return str(tmp_path / "should-not-exist.cookies.txt") + + monkeypatch.setattr(dub_core, "_stage_cookie_export", stage_cookie) + monkeypatch.setattr( + dub_core, "_safe_job_dir", lambda _job_id: str(tmp_path / "job") + ) + + def fail_makedirs(*_args, **_kwargs): + raise OSError("disk full") + + monkeypatch.setattr(dub_core.os, "makedirs", fail_makedirs) + request = Request( + { + "type": "http", + "scheme": "http", + "server": ("127.0.0.1", 80), + "client": ("127.0.0.1", 1234), + "path": "/dub/ingest-url", + "headers": [(b"origin", b"http://tauri.localhost")], + } + ) + + with pytest.raises(OSError, match="disk full"): + asyncio.run( + dub_core.dub_ingest_url( + DubIngestUrlRequest( + url="https://youtube.com/watch?v=abc", cookie_file=COOKIE_TEXT + ), + request, + ) + ) + assert staged is False + + +def test_failed_cookie_unlink_can_be_retried(dub_pipeline, tmp_path, monkeypatch): + cookie_path = tmp_path / "retry.cookies.txt" + cookie_path.write_text(COOKIE_TEXT, encoding="utf-8") + real_unlink = os.unlink + attempts = 0 + + def flaky_unlink(path): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise PermissionError("temporarily busy") + real_unlink(path) + + monkeypatch.setattr(dub_pipeline.os, "unlink", flaky_unlink) + assert dub_pipeline._delete_cookie_export(str(cookie_path)) is False + assert cookie_path.exists() + assert dub_pipeline._delete_cookie_export(str(cookie_path)) is True + assert not cookie_path.exists()