From ae85dd36e4f405ac016d7e0e0ed12423798cec68 Mon Sep 17 00:00:00 2001 From: nidhi-singh02 Date: Tue, 8 Sep 2026 13:17:49 +0530 Subject: [PATCH] fix(colab): install ASR model before transcription and dubbing --- CHANGELOG.md | 2 + notebooks/OmniVoice_Studio_Colab.ipynb | 42 +++++++++++++-- tests/test_colab_asr_setup.py | 74 ++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 tests/test_colab_asr_setup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 21bd003e..7e1367d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ the frozen-backend fallback mirror it for their toolchains. **Highlights** +- Colab transcription and dubbing now include an explicit ASR model setup step (#1922) — thanks @nidhi-singh02! + - Validate current-user Windows installers under a standard account on hosted runners (#1883) - The desktop app builds and opens from a fresh clone again (#1818) — thanks @flutterkage2k! diff --git a/notebooks/OmniVoice_Studio_Colab.ipynb b/notebooks/OmniVoice_Studio_Colab.ipynb index 27d01f49..6b7c5b4c 100644 --- a/notebooks/OmniVoice_Studio_Colab.ipynb +++ b/notebooks/OmniVoice_Studio_Colab.ipynb @@ -642,13 +642,43 @@ "play(path, f\"\\nVoice of profile {NARRATOR_ID} (gen_time={h.get('X-Gen-Time')}s):\")\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 12a. Install the transcription model (required for cells 13 and 18)\n", + "\n", + "Running the next cell explicitly downloads **Systran/faster-whisper-large-v3** from Hugging Face (roughly 3 GB) for the notebook's default transcription backend. Skip it if you only want the TTS examples. Rerunning checks Hugging Face for the snapshot and reuses cached files, downloading only missing files.\n", + "\n", + "The notebook and the backend launched in cell 5 share the same Hugging Face cache. If you change cache settings, restart the backend with those settings before continuing. If you select a different ASR engine/model in the app, install that model through Model Catalogue instead; this cell prepares the default large-v3 workflow.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ── 12a. Install the transcription model ────────────────────────────────────\n", + "from huggingface_hub import snapshot_download\n", + "\n", + "ASR_MODEL_READY = False\n", + "ASR_REPO_ID = \"Systran/faster-whisper-large-v3\"\n", + "print(f\"Preparing {ASR_REPO_ID} (roughly 3 GB if not cached)...\")\n", + "# Reuses cached files and downloads any missing files from an interrupted run.\n", + "# Do not pass local_dir: the backend looks in the shared Hugging Face cache.\n", + "asr_path = snapshot_download(repo_id=ASR_REPO_ID)\n", + "ASR_MODEL_READY = True\n", + "print(\"ASR model cached at:\", asr_path)\n" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 13. Transcription (speech-to-text)\n", "\n", - "The round trip: the WAV that TTS produced in cell 9 goes back through `POST /transcribe`, and the recognized text should match the original sentence. Expected runtime: the **first** transcription downloads an ASR model (roughly 1-3 GB, a few minutes); afterwards it's seconds.\n" + "The round trip: the WAV that TTS produced in cell 9 goes back through `POST /transcribe`, and the recognized text should match the original sentence. Run **cell 12a** first to install the ASR model. The backend intentionally rejects transcription when its model is missing; it does not automatically download it. The first transcription loads the installed model into memory.\n" ] }, { @@ -658,13 +688,16 @@ "outputs": [], "source": [ "# ── 13. Transcription: TTS -> ASR round trip ────────────────────────────────\n", + "if not globals().get(\"ASR_MODEL_READY\", False):\n", + " raise SystemExit(\"Run cell 12a (ASR model download) before transcription or dubbing.\")\n", + "\n", "try:\n", " ensure_wav\n", "except NameError:\n", " raise SystemExit(\"Run cell 8 (feature-tour helpers) first.\")\n", "\n", "wav = ensure_wav(\"tts_en.wav\", EN_TEXT)\n", - "print(\"Transcribing... (first run downloads an ASR model — a few minutes)\")\n", + "print(\"Transcribing... (first run loads the installed ASR model)\")\n", "with open(wav, \"rb\") as f:\n", " r = requests.post(f\"{BASE}/transcribe\",\n", " files={\"audio\": (\"tts_en.wav\", f, \"audio/wav\")},\n", @@ -915,7 +948,7 @@ "\n", "The flagship pipeline, kept honest and miniature: a 6-second synthetic clip (color frame + the cell-9 English narration) is dubbed into Spanish — upload → prep (audio extract + Demucs vocal separation) → transcribe → translate → voice-cloned TTS → mux — all through the same job API the app uses.\n", "\n", - "**Run this cell only if you have 5-15 minutes**: the first run downloads the Demucs separation model and (if cell 13 didn't run) an ASR model. Translation here uses the free Google web endpoint via `deep-translator` (installed in-cell); for a fully offline dub the backend also supports `provider=\"nllb\"` (a ~2.5 GB one-time model download).\n" + "**Run this cell only if you have 5-15 minutes**: run **cell 12a** first to install the ASR model (cell 13 itself is optional). The first dubbing run also downloads the Demucs separation model. Translation here uses the free Google web endpoint via `deep-translator` (installed in-cell); for a fully offline dub the backend also supports `provider=\"nllb\"` (a ~2.5 GB one-time model download).\n" ] }, { @@ -925,6 +958,9 @@ "outputs": [], "source": [ "# ── 18. Video dubbing (mini): English clip -> Spanish dub ───────────────────\n", + "if not globals().get(\"ASR_MODEL_READY\", False):\n", + " raise SystemExit(\"Run cell 12a (ASR model download) before transcription or dubbing.\")\n", + "\n", "import subprocess\n", "import sys\n", "import time\n", diff --git a/tests/test_colab_asr_setup.py b/tests/test_colab_asr_setup.py new file mode 100644 index 00000000..3ffd0478 --- /dev/null +++ b/tests/test_colab_asr_setup.py @@ -0,0 +1,74 @@ +"""Execute notebook prerequisite ordering with mocked downloads (#1922).""" +import json +from pathlib import Path +import sys +import types +import unittest +from unittest.mock import patch + + +NOTEBOOK = Path(__file__).resolve().parents[1] / "notebooks/OmniVoice_Studio_Colab.ipynb" + + +class ColabASRSetupTests(unittest.TestCase): + def setup_source(self): + cells = json.loads(NOTEBOOK.read_text())["cells"] + for cell in cells: + source = "".join(cell["source"]) + if cell["cell_type"] == "code" and "12a. Install the transcription model" in source: + return source + self.fail("Notebook needs an explicit ASR setup cell before transcription") + + def test_setup_downloads_exact_repo_and_can_be_rerun(self): + calls = [] + def download(repo_id): + self.assertEqual(repo_id, "Systran/faster-whisper-large-v3") + calls.append(repo_id) + return "/fake/hub/snapshot" + + module = types.ModuleType("huggingface_hub") + module.snapshot_download = download + scope = {} + with patch.dict(sys.modules, {"huggingface_hub": module}): + exec(self.setup_source(), scope) + self.assertTrue(scope["ASR_MODEL_READY"]) + self.assertEqual(len(calls), 1) + calls.clear() + exec(self.setup_source(), scope) + self.assertEqual(len(calls), 1) + + def test_failed_download_clears_previous_ready_state(self): + module = types.ModuleType("huggingface_hub") + + def download(*args, **kwargs): + raise OSError("download interrupted") + + module.snapshot_download = download + scope = {"ASR_MODEL_READY": True} + with patch.dict(sys.modules, {"huggingface_hub": module}): + with self.assertRaises(OSError): + exec(self.setup_source(), scope) + self.assertFalse(scope["ASR_MODEL_READY"]) + + def test_transcription_and_dubbing_gate_before_any_work(self): + cells = json.loads(NOTEBOOK.read_text())["cells"] + for number in (13, 18): + source = next("".join(c["source"]) for c in cells + if c["cell_type"] == "code" + and f" {number}. " in "".join(c["source"]).splitlines()[0]) + with self.subTest(cell=number): + with self.assertRaisesRegex(SystemExit, "12a"): + exec(source, {}) + + def test_setup_precedes_transcription_in_run_all(self): + code = ["".join(c["source"]) for c in json.loads(NOTEBOOK.read_text())["cells"] + if c["cell_type"] == "code"] + setup = next(i for i, s in enumerate(code) + if "12a. Install the transcription model" in s) + transcribe = next(i for i, s in enumerate(code) + if " 13. " in s.splitlines()[0]) + self.assertLess(setup, transcribe) + + +if __name__ == "__main__": + unittest.main()