Merge remote-tracking branch 'origin/main' into codex/pr1562
# Conflicts: # CHANGELOG.md
This commit is contained in:
debpalash
@@ -25,4 +25,6 @@ regexes = [
|
||||
'''^hf_QWERTYUIOPasdfghjklZXCVBNM0123456789xyzAB$''',
|
||||
# NLLB generation length argument, not the value of a credential.
|
||||
'''^max_length=400$''',
|
||||
# cryptography's Ed25519 private-key type name, not key material.
|
||||
'''^Ed25519PrivateKey$''',
|
||||
]
|
||||
|
||||
@@ -18,20 +18,27 @@ the frozen-backend fallback mirror it for their toolchains.
|
||||
- The backend binds its port immediately and reports startup progress live — `/health` answers 503-with-step and a new `/startup/progress` endpoint lists every step while PyTorch, API routes, and database migrations load in the background, so "starting at step X" is never mistakable for "dead"; the desktop splash narrates each step (#1550)
|
||||
|
||||
### Added
|
||||
- Voices you've cloned stay "warm" across restarts — encoded references now persist to disk (~10 KB each), so the first generation of a session skips the re-encode and any transcription pass; `OMNIVOICE_PROMPT_DISK_CACHE=0` opts out (#1565)
|
||||
- Optional FlashInfer acceleration for the default engine on CUDA (`OMNIVOICE_FLASHINFER=1`, ~2.2x measured) — needs the optional `flashinfer-python` package; missing package or kernel failure logs why and falls back to the standard path (#1565)
|
||||
- The bug reporter notices when you're on an outdated build and offers the latest release before filing — with a "File anyway" escape hatch — and stamps a `Build status` line into every report so up-to-date reports are tellable from stale ones (#1547)
|
||||
- Settings → Performance & Device gains a compute-device override (Auto / CUDA / ROCm / XPU / MPS / CPU, or `OMNIVOICE_DEVICE`) — pin the device when auto-detect picks wrong; only devices your machine actually has are offered (#1557)
|
||||
|
||||
### Docs
|
||||
- The READMEs now lead with download buttons and a three-step first-clone walkthrough, and a new benchmarks page anchors measured per-engine/per-device numbers on the in-repo harness (#1555)
|
||||
- Every engine now has its own guide — 21 new pages under docs/engines plus an index covering all 16 TTS and 11 ASR engines, linked from both READMEs (#1556)
|
||||
- The OmniVoice guide now covers combining style attributes with a reference clip (consistent instruct stabilizes cloning; the reference wins conflicts), inline pronunciation control (pinyin / CMU phonemes), and corrects the claim that the default engine can't do voice design — it can, from attributes (#1565)
|
||||
|
||||
### Fixed
|
||||
- A remote browser hitting an API-key-configured server's admin 403 now gets the API-key login form instead of endless console 403s, while desktop and PIN-only/no-key servers keep the plain loopback error so guests are never offered a login no key can satisfy (#1568) — thanks @paoloantinori!
|
||||
- The crash-isolated ASR sidecar and its download preflight now agree on which model to load — setting the shared faster-whisper model variable applies to both variants instead of the sidecar quietly using a different one (#1556)
|
||||
- "Ready" now requires the deep health probe (a working database-backed route), not just the identity probe — a backend whose install broke underneath can no longer be announced up while every real request fails (#1548)
|
||||
- Supervisor restarts after repeat crashes now back off (immediate, then 5s, then 15s) instead of respawning back-to-back, so a tight crash loop can't burn the whole restart budget in seconds (#1548)
|
||||
|
||||
- Renaming, deleting, or revoking consent on a voice (and starring/clearing history, recording exports) now live-updates every open tab again — the sync routes' WebSocket events were silently dropped, which could look like "all my voices are gone" (#1561) — thanks @paoloantinori!
|
||||
|
||||
### CI
|
||||
- Weekly full-history secret scans no longer mistake the Ed25519 private-key type name for committed key material (#1591)
|
||||
|
||||
## [0.5.0] — 2026-08-13
|
||||
|
||||
**Highlights**
|
||||
|
||||
@@ -1,558 +1,385 @@
|
||||
<div align="center">
|
||||
<img src="docs/logo.png" alt="VoiceStudio Logo" width="120" height="120" />
|
||||
<img src="docs/logo.png" alt="VoiceStudio logo" width="120" height="120" />
|
||||
<h1>VoiceStudio</h1>
|
||||
<p><sub><em>previously OmniVoice-Studio</em></sub></p>
|
||||
<h3>Make voices. Tell stories. Keep the files. ♡</h3>
|
||||
<p>Clone, design, dub, dictate, and build audiobooks in one open-source desktop studio.<br/><b>Local-first by default.</b> No subscription or usage meter. Optional online services stay opt-in.</p>
|
||||
<p><sub>Previously OmniVoice-Studio</sub></p>
|
||||
<h3>Local voice cloning, dubbing, dictation, and long-form audio.</h3>
|
||||
<p>16 TTS engines · 11 ASR engines · 646-language catalogue · macOS, Windows, and Linux</p>
|
||||
<p><strong>Local-first.</strong> No account, API key, subscription, or usage meter for the core workflow.</p>
|
||||
|
||||
<p>
|
||||
<a href="#quickstart">Quickstart</a> ·
|
||||
<a href="#install">Install</a> ·
|
||||
<a href="#features">Features</a> ·
|
||||
<a href="#why-voicestudio">Why VoiceStudio</a> ·
|
||||
<a href="#tts-engines">Engines</a> ·
|
||||
<a href="#openai-api">API</a> ·
|
||||
<a href="#sponsor--donate">Donate</a> ·
|
||||
<a href="#contributing">Contributing</a> ·
|
||||
<a href="https://voicestudio.sh">Website</a> ·
|
||||
<a href="https://voicestudio.sh/docs">Docs</a> ·
|
||||
<a href="https://status.voicestudio.sh">Status</a> ·
|
||||
<a href="https://discord.gg/bzQavDfVV9">Discord</a> ·
|
||||
<a href="https://x.com/idebpalash">X</a> ·
|
||||
<a href="#comparison">Compare</a> ·
|
||||
<a href="#requirements">Requirements</a> ·
|
||||
<a href="#engines">Engines</a> ·
|
||||
<a href="#architecture">Architecture</a> ·
|
||||
<a href="#api">API</a> ·
|
||||
<a href="#documentation">Docs</a> ·
|
||||
<a href="README_CN.md"><strong>简体中文</strong></a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/stargazers"><img src="https://img.shields.io/github/stars/debpalash/VoiceStudio?style=flat-square&color=f59e0b" alt="Stars" /></a>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/stargazers"><img src="https://img.shields.io/github/stars/debpalash/VoiceStudio?style=flat-square&color=f59e0b" alt="GitHub stars" /></a>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/releases"><img src="https://img.shields.io/github/downloads/debpalash/VoiceStudio/total?style=flat-square&color=8b5cf6&label=downloads" alt="Total downloads" /></a>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/releases/latest"><img src="https://img.shields.io/github/v/release/debpalash/VoiceStudio?style=flat-square&color=10b981" alt="Release" /></a>
|
||||
<a href="LICENSE"><img src="https://img.shields.io/badge/license-AGPL--3.0-blue?style=flat-square" alt="License" /></a>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/issues"><img src="https://img.shields.io/github/issues/debpalash/VoiceStudio?style=flat-square&color=ef4444" alt="Issues" /></a>
|
||||
<a href="https://discord.gg/bzQavDfVV9"><img src="https://img.shields.io/badge/Discord-Join_Community-5865F2?style=flat-square&logo=discord&logoColor=white" alt="Discord" /></a>
|
||||
<a href="https://x.com/idebpalash"><img src="https://img.shields.io/badge/X-Follow_for_updates-000000?style=flat-square&logo=x&logoColor=white" alt="Follow on X" /></a>
|
||||
<a href="https://ko-fi.com/debpalash"><img src="https://img.shields.io/badge/Ko--fi-Support_Us-FF5E5B?style=flat-square&logo=ko-fi&logoColor=white" alt="Ko-fi" /></a>
|
||||
<a href="https://paypal.me/palashCoder"><img src="https://img.shields.io/badge/PayPal-Donate-00457C?style=flat-square&logo=paypal&logoColor=white" alt="PayPal" /></a>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/releases/latest"><img src="https://img.shields.io/github/v/release/debpalash/VoiceStudio?style=flat-square&color=10b981" alt="Latest release" /></a>
|
||||
<a href="LICENSE"><img src="https://img.shields.io/badge/license-AGPL--3.0-blue?style=flat-square" alt="AGPL-3.0 license" /></a>
|
||||
<a href="https://discord.gg/bzQavDfVV9"><img src="https://img.shields.io/badge/Discord-Community-5865F2?style=flat-square&logo=discord&logoColor=white" alt="Discord community" /></a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/releases/latest"><img src="https://img.shields.io/badge/⬇_Download-macOS_·_Windows_·_Linux-10b981?style=for-the-badge" alt="Download the latest release" /></a>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<a href="https://trendshift.io/repositories/28176?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-28176" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/28176/daily?language=Python" alt="debpalash%2FVoiceStudio | Trendshift" width="250" height="55"/></a>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/releases/latest"><img src="https://img.shields.io/badge/Download-macOS_·_Windows_·_Linux-10b981?style=for-the-badge" alt="Download VoiceStudio" /></a>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<br/>
|
||||
|
||||
<div align="center">
|
||||
<img src="docs/media/0.5.0/quick-switch.gif" alt="VoiceStudio — switching TTS engines from the status bar" width="100%"/>
|
||||
<img src="docs/media/0.5.0/quick-switch.gif" alt="Switching TTS engines from the VoiceStudio status bar" width="100%" />
|
||||
</div>
|
||||
|
||||
> **Your voice is personal. Your studio should feel personal too.** VoiceStudio keeps its core workflow on your hardware: clone, design, dub, dictate, and publish in 646 languages without a subscription or usage meter. Network-backed engines and services are optional, visible choices—not hidden requirements.
|
||||
|
||||
> [!WARNING]
|
||||
> **Active beta.** Things may break between releases — for the newest fixes, run from source. Bug reports and PRs are very welcome: [open an issue](https://github.com/debpalash/VoiceStudio/issues) or [join Discord](https://discord.gg/bzQavDfVV9).
|
||||
> **Active beta.** Use the [latest release](https://github.com/debpalash/VoiceStudio/releases/latest) for stable work or `main` for current fixes. Report problems through [GitHub Issues](https://github.com/debpalash/VoiceStudio/issues).
|
||||
|
||||
<a id="quickstart"></a>
|
||||
## At a glance
|
||||
|
||||
## ⚡ Quickstart
|
||||
| | VoiceStudio |
|
||||
|---|---|
|
||||
| **Workflows** | Voice cloning and design, video dubbing, dictation, stories, audiobooks, batch generation |
|
||||
| **Language catalogue** | 646 TTS languages; actual coverage and quality depend on the selected engine |
|
||||
| **Engines** | 16 TTS · 11 ASR · switch in Model Catalogue or with <kbd>Ctrl</kbd>/<kbd>Cmd</kbd>+<kbd>E</kbd> |
|
||||
| **Platforms** | macOS 13.3+ on Apple Silicon · Windows 10/11 x64 · Linux x86_64 with glibc 2.39+ |
|
||||
| **Compute** | CUDA · Apple Silicon MPS/MLX · ROCm on Linux · CPU · optional remote workers |
|
||||
| **Interfaces** | Desktop app · local REST/SSE/WebSocket API · OpenAI-compatible audio API · MCP Server |
|
||||
| **Storage** | Voices, projects, settings, and outputs stay on the machine by default |
|
||||
| **License** | AGPL-3.0; optional engines keep their own model licenses |
|
||||
|
||||
<div align="center">
|
||||
<a href="https://github.com/debpalash/VoiceStudio/releases/latest"><img src="https://img.shields.io/badge/macOS-DMG_(Apple_Silicon)-000?style=for-the-badge&logo=apple&logoColor=white" alt="Download macOS DMG" /></a>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/releases/latest"><img src="https://img.shields.io/badge/Windows-MSI_(x64)-0078D4?style=for-the-badge&logo=windows&logoColor=white" alt="Download Windows MSI" /></a>
|
||||
<a href="https://github.com/debpalash/VoiceStudio/releases/latest"><img src="https://img.shields.io/badge/Linux-AppImage_(x64)-FCC624?style=for-the-badge&logo=linux&logoColor=black" alt="Download Linux AppImage" /></a>
|
||||
<br/>
|
||||
<sub>Each button opens the latest-release page — download the installer for your OS from the assets list.</sub><br/>
|
||||
<sub><b>macOS:</b> first launch needs a one-time approval — right-click → <b>Open</b> (or System Settings → Privacy & Security → <b>"Open Anyway"</b> on macOS 15). No Terminal needed. <a href="docs/install/macos.md#gatekeeper-quarantine">Why?</a> · <b>Intel Macs:</b> local backend unsupported (<a href="https://github.com/debpalash/VoiceStudio/issues/889">#889</a>) — <a href="docs/install/macos.md">details</a>.</sub>
|
||||
</div>
|
||||
<a id="install"></a>
|
||||
|
||||
**Install guide:** [🍎 macOS](docs/install/macos.md) · [🪟 Windows](docs/install/windows.md) · [🐧 Linux](docs/install/linux.md) · [🐳 Docker](docs/install/docker.md)
|
||||
## Install
|
||||
|
||||
**Your first cloned voice, in three steps:**
|
||||
| Platform | Package | Guide |
|
||||
|---|---|---|
|
||||
| macOS 13.3+ | DMG, Apple Silicon | [Install on macOS](docs/install/macos.md) |
|
||||
| Windows 10/11 | MSI, x64 | [Install on Windows](docs/install/windows.md) |
|
||||
| Linux | AppImage, x86_64 with glibc 2.39+ | [Install on Linux](docs/install/linux.md) |
|
||||
| Docker | CUDA, ROCm, or CPU | [Run with Docker](docs/install/docker.md) |
|
||||
|
||||
1. **Install & launch.** The first launch sets up its own Python runtime and downloads model weights — the splash screen narrates every step (one-time, a few minutes; instant after that).
|
||||
2. **Open Voice Cloning** from the Launchpad and drop in a **3-second clip** of any voice.
|
||||
3. **Type a line, hit Generate.** The audio is yours — created and stored on your machine, in any of 646 languages.
|
||||
Download packages from the [latest release](https://github.com/debpalash/VoiceStudio/releases/latest). First launch creates a managed Python environment and downloads the default model. Later launches reuse both.
|
||||
|
||||
<details>
|
||||
<summary><b>🧰 Troubleshooting · slow generation · HF tokens · restricted networks</b></summary>
|
||||
> [!NOTE]
|
||||
> On macOS, first launch needs a one-time right-click → **Open** approval. Intel Macs cannot run the local Python backend; use a [remote backend](docs/install/macos.md) instead.
|
||||
|
||||
<br/>
|
||||
### First voice
|
||||
|
||||
- **Something broke?** Run the self-check — **Settings → About → "Run self-check"** (or `uv run python backend/main.py --diagnose --deep`) — then the [top 10 install errors](docs/install/troubleshooting.md). **"Save diagnostic bundle"** packages scrubbed logs for a bug report.
|
||||
- **Feels slow?** [docs/performance.md](docs/performance.md) — where the time goes and how to tune it. Measured numbers per engine/device: [docs/benchmarks.md](docs/benchmarks.md).
|
||||
- **Want breaths, laughter, emotion?** [docs/expressive-speech.md](docs/expressive-speech.md) — what each engine can do today.
|
||||
- **HF tokens · diarization · download speed / mirrors:** [tokens](docs/setup/huggingface-token.md) · [diarization](docs/features/diarization.md) · [downloads](docs/downloading-models.md).
|
||||
- **Coming from [Real-Time-Voice-Cloning](https://github.com/CorentinJ/Real-Time-Voice-Cloning)?** [Migration guide](docs/migration/real-time-voice-cloning.md).
|
||||
1. Launch VoiceStudio and open **Voice Cloning**.
|
||||
2. Add a clean voice sample. Three seconds works; 5–15 seconds usually gives a better prompt.
|
||||
3. Enter text, choose a language, then select **Generate**.
|
||||
|
||||
</details>
|
||||
### Run from source
|
||||
|
||||
---
|
||||
Install the [development prerequisites](.github/CONTRIBUTING.md#development-setup), then:
|
||||
|
||||
<a id="whats-new"></a>
|
||||
```bash
|
||||
git clone https://github.com/debpalash/VoiceStudio.git
|
||||
cd VoiceStudio
|
||||
bun install
|
||||
bun run desktop
|
||||
```
|
||||
|
||||
## 🆕 What's new in 0.5.0
|
||||
Use `bun run dev` for the browser UI. See [Contributing](.github/CONTRIBUTING.md) for services, tests, and platform packages.
|
||||
|
||||
The rename release — full notes: [v0.5.0 release](https://github.com/debpalash/VoiceStudio/releases/tag/v0.5.0) · [CHANGELOG](CHANGELOG.md).
|
||||
### If setup fails
|
||||
|
||||
- 🏷️ **A new name** — VoiceStudio (previously OmniVoice-Studio): one waveform-and-spark identity across app, docs, and installers. Your data folder, settings, and Docker image paths stay put.
|
||||
- 📚 **Model Catalogue** — engines and models in one workspace: every TTS, ASR, and LLM engine with its device routing and install state; pick defaults, install or remove weights.
|
||||
- ⚡ **Engine quick-switch** — change TTS/ASR/LLM engines from the status bar or anywhere with <kbd>Ctrl</kbd>/<kbd>Cmd</kbd>+<kbd>E</kbd> — ready-only choices, memory status, environment-pin protection.
|
||||
- 🖧 **Remote GPU workers** — lend another machine's GPU with a join code and a QR scan; a **Compute** control picks where jobs run, and several people can share one GPU box over revocable, certificate-pinned connections.
|
||||
- 🔐 **Hardened server mode** — admin actions require an API key, exchanged for short-lived scoped sessions that never sit in browser storage or WebSocket URLs.
|
||||
- 💾 **Gallery voices → local profiles** — save any gallery voice as a profile of your own and use it in every picker.
|
||||
- 🎤 **Dictation on Wayland** — the portal shortcut actually fires now, and the recording pill is back on every desktop.
|
||||
|
||||
<div align="center">
|
||||
<img src="docs/screenshot-launchpad.png" alt="VoiceStudio — Launchpad" width="640"/>
|
||||
<br/><sub>The Launchpad — every workspace one click away; quick-switch engines anywhere with <kbd>Ctrl</kbd>/<kbd>Cmd</kbd>+<kbd>E</kbd> (shown above)</sub>
|
||||
</div>
|
||||
|
||||
<br/>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="50%"><img src="docs/media/0.5.0/catalogue.png" alt="Model Catalogue — engines pane" width="100%"/></td>
|
||||
<td width="50%"><img src="docs/media/0.5.0/gallery-save.png" alt="Saving a gallery voice as a profile" width="100%"/></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center"><sub><b>Model Catalogue</b> — every engine, its routing and install state</sub></td>
|
||||
<td align="center"><sub><b>Gallery → profile</b> — keep a gallery voice as your own</sub></td>
|
||||
</tr>
|
||||
</table>
|
||||
- Run **Settings → About → Run self-check** or `uv run python backend/main.py --diagnose --deep`.
|
||||
- Check [install troubleshooting](docs/install/troubleshooting.md).
|
||||
- Save a scrubbed diagnostic bundle from the app when opening an issue.
|
||||
- For slow generation, compare [measured benchmarks](docs/benchmarks.md) and [performance settings](docs/performance.md).
|
||||
|
||||
<a id="features"></a>
|
||||
|
||||
## ✨ Features
|
||||
## Features
|
||||
|
||||
Three flagships, five more headliners, and a dozen under the fold.
|
||||
| Area | Included |
|
||||
|---|---|
|
||||
| **Voice Cloning** | Zero-shot synthesis from a short reference clip |
|
||||
| **Voice Design** | Create a voice from age, accent, pitch, style, and delivery instructions |
|
||||
| **Video Dubbing** | Transcribe, translate, preserve speakers, synthesize, and export video |
|
||||
| **Stories and audiobooks** | Multi-voice scripts · EPUB/PDF import · chapter rendering · `.m4b` export |
|
||||
| **Dictation Widget** | System-wide shortcut, live transcription, optional local-LLM cleanup |
|
||||
| **Vocal Isolation** | Demucs speech/background separation |
|
||||
| **Speaker Diarization** | Pyannote and WhisperX speaker assignment |
|
||||
| **Batch Queue** | Queue large sets of audio and video jobs with per-job progress |
|
||||
| **Model Catalogue** | Install, remove, select, and route TTS, ASR, and LLM models |
|
||||
| **Remote Model Downloads** | Install models on enrolled remote workers with live progress |
|
||||
| **GPU Auto-Detect** | CUDA, MPS, ROCm, and CPU routing with per-engine checks |
|
||||
| **AI Watermark** | AudioSeal embedding and detection |
|
||||
| **MCP Server** | Synthesis and transcription tools for MCP clients |
|
||||
| **Diagnostics** | Self-checks, error journal, logs, and scrubbed support bundles |
|
||||
| **Local-first** | Core creation stays local; network-backed features are explicit opt-ins |
|
||||
| **Extensible** | Registry-based TTS, ASR, and plugin interfaces |
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="33%"><img src="docs/features/clone.png" alt="Voice Cloning" width="100%"/></td>
|
||||
<td width="33%"><img src="docs/features/design.png" alt="Voice Design" width="100%"/></td>
|
||||
<td width="33%"><img src="docs/features/dub.png" alt="Video Dubbing" width="100%"/></td>
|
||||
<td width="50%"><img src="docs/media/0.5.0/catalogue.png" alt="VoiceStudio Model Catalogue" width="100%" /></td>
|
||||
<td width="50%"><img src="docs/media/0.5.0/gallery-save.png" alt="Saving a gallery voice as a local profile" width="100%" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">🎙️ <b>Voice Cloning</b><br/><sub>3-sec clip → any voice · 646 languages · zero-shot</sub></td>
|
||||
<td align="center">🎨 <b>Voice Design</b><br/><sub>Describe it — gender, age, accent, emotion</sub></td>
|
||||
<td align="center">🎬 <b>Video Dubbing</b><br/><sub>Transcribe → translate → re-voice → MP4</sub></td>
|
||||
<td align="center"><sub>Model Catalogue: engine, device, and install state</sub></td>
|
||||
<td align="center"><sub>Gallery: save a shared voice as a local profile</sub></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" width="20%">📖<br/><b>Audiobook</b><br/><sub>EPUB/PDF → .m4b, multi-voice cast</sub></td>
|
||||
<td align="center" width="20%">🎭<br/><b>Stories</b><br/><sub>Multi-voice script editor</sub></td>
|
||||
<td align="center" width="20%">⌨️<br/><b>Dictation Widget</b><br/><sub><kbd>⌘⇧Space</kbd> in any app</sub></td>
|
||||
<td align="center" width="20%">🔐<br/><b>Local-first</b><br/><sub>Core creation stays on your machine</sub></td>
|
||||
<td align="center" width="20%">🤖<br/><b>MCP Server</b><br/><sub>Use from Claude, Cursor, …</sub></td>
|
||||
</tr>
|
||||
</table>
|
||||
<a id="comparison"></a>
|
||||
|
||||
<details>
|
||||
<summary><b>…and 12 more</b> — catalogue, remote GPUs, isolation, diarization, batch, watermarking, and friends</summary>
|
||||
## Comparison
|
||||
|
||||
<br/>
|
||||
VoiceStudio trades managed cloud compute for local control. This is the practical difference:
|
||||
|
||||
- 📚 **Model Catalogue** — one workspace for every TTS/ASR/LLM engine and model: defaults, device routing, install or remove weights — and quick-switch engines from anywhere with <kbd>Ctrl</kbd>/<kbd>Cmd</kbd>+<kbd>E</kbd>.
|
||||
- 🖧 **Remote GPU workers** — send jobs to GPUs on your other machines: join code + QR enrolment, Remote Model Downloads with per-worker live progress, chapter-by-chapter audiobook rendering with local fallback. Off by default; see [docs/remote-workers.md](docs/remote-workers.md).
|
||||
- 🔊 **Vocal Isolation** — Demucs-powered: splits speech from music and keeps the background bed.
|
||||
- 👥 **Speaker Diarization** — Pyannote + WhisperX auto-identify who said what.
|
||||
- 📦 **Batch Queue** — drop 50 videos, walk away; per-job progress bars.
|
||||
- 🛡️ **AI Watermark** — AudioSeal (Meta): invisible, survives compression.
|
||||
- 🔬 **Diagnostics** — self-check suite, error journal, scrubbed diagnostic bundles.
|
||||
- ⚡ **GPU Auto-Detect & Routing** — CUDA · MPS · ROCm (Linux, opt-in) · CPU; ≤8 GB VRAM auto-offloads; per-engine GPU preflight, no silent CPU fallback.
|
||||
- 🧩 **Extensible** — subclass `TTSBackend`, add any engine in ~50 lines.
|
||||
- 🎒 **Portable personas** — export voices as `.ovsvoice` bundles: identity + watermark.
|
||||
- ♾️ **Unlimited TTS** — sentence-chunked generation, no length cap, streaming via WebSocket.
|
||||
- 🧠 **Dictation + LLM** — local-LLM cleanup of transcripts, optional echo cancellation.
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<a id="why-voicestudio"></a>
|
||||
|
||||
## ⚖️ Why VoiceStudio
|
||||
|
||||
Cloud voice tools are convenient, but they put your workflow behind an account, a meter, and somebody else's infrastructure. VoiceStudio gives you a capable studio that runs on your hardware, with optional integrations when you choose them.
|
||||
|
||||
| | **ElevenLabs** | **VoiceStudio** |
|
||||
| | **VoiceStudio** | **Typical hosted voice service** |
|
||||
|---|---|---|
|
||||
| **Pricing** | Subscription and usage limits | Free & open-source (AGPL-3.0) · [Commercial license](#license) for proprietary use |
|
||||
| **Voice Cloning** | ✅ 3s clip | ✅ 3s clip, zero-shot |
|
||||
| **Voice Design** | ✅ Gender, age | ✅ Gender, age, accent, pitch, style, dialect |
|
||||
| **Audiobook / Stories** | ❌ | ✅ Full audiobook editor + multi-voice stories (EPUB/PDF import, .m4b export) |
|
||||
| **Languages** | Plan/model dependent | **646** |
|
||||
| **Video Dubbing** | ✅ Cloud-only | ✅ Fully local |
|
||||
| **Data Privacy** | Audio is processed remotely | Core workflow runs locally; online services are explicit opt-ins |
|
||||
| **API Keys** | Account required | Not needed for the local workflow |
|
||||
| **GPU Support** | N/A (cloud) | CUDA · Apple Silicon · ROCm (Linux) · CPU — plus your other machines' GPUs as [remote workers](docs/remote-workers.md) |
|
||||
| **Desktop App** | ❌ | ✅ macOS · Windows · Linux |
|
||||
| **TTS Engines** | 1 | **16** — [full matrix](#tts-engines) |
|
||||
| **ASR Engines** | 1 | **11** — [full lineup](#asr-engines) |
|
||||
| **MCP Server** | ❌ | ✅ Use from Claude, Cursor, any MCP client |
|
||||
| **Self-check** | ❌ | ✅ Diagnostics suite, error journal, scrubbed debug bundles |
|
||||
| **Customizable** | ❌ Closed | ✅ Fork it, extend it, ship it |
|
||||
| **Best fit** | Private, offline, self-hosted, or high-volume work | Fast setup without local model management |
|
||||
| **Data path** | Local by default; remote features are opt-in | Audio and text are processed by the provider |
|
||||
| **Cost model** | Free software; you supply the hardware | Subscription, credits, or metered API use |
|
||||
| **Setup** | Install the app and model weights | Create an account and use the web app or API |
|
||||
| **Performance** | Depends on your engine and hardware | Provider manages compute and scaling |
|
||||
| **Offline use** | Yes, after required models are installed | Usually requires a network connection |
|
||||
| **Customization** | Source, engines, models, API, and routing are open | Limited to provider options |
|
||||
| **Maintenance** | You manage updates, disk, and compute | Provider manages infrastructure |
|
||||
|
||||
Professional-grade voice AI, minus the subscription and the cloud. Convinced? [Come build with us.](https://discord.gg/bzQavDfVV9)
|
||||
<a id="requirements"></a>
|
||||
|
||||
---
|
||||
## Requirements
|
||||
|
||||
## 🖥️ System Requirements
|
||||
Requirements vary by engine. These values cover the default local workflow.
|
||||
|
||||
| | **Minimum** | **Recommended** |
|
||||
|---|---|---|
|
||||
| **OS** | Windows 10, macOS 13.3+ (Apple Silicon), Ubuntu 24.04+ (glibc 2.39+) | Any modern 64-bit OS |
|
||||
| **OS** | Windows 10 x64 · macOS 13.3 Apple Silicon · Linux x86_64 with glibc 2.39+ | Current supported OS release |
|
||||
| **RAM** | 8 GB | 16 GB+ |
|
||||
| **VRAM (GPU)** | 4 GB (auto-offloads TTS to CPU) | 8 GB+ (NVIDIA RTX 3060+) |
|
||||
| **Disk** | 10 GB free (models + cache) | 20 GB+ SSD |
|
||||
| **Python** | 3.10+ (managed by `uv`) | 3.11–3.12 |
|
||||
| **GPU** | Optional — CPU works | NVIDIA CUDA · Apple Silicon MPS · AMD ROCm (Linux only) |
|
||||
| **Disk** | 10 GB free | 20 GB+ SSD |
|
||||
| **GPU** | Optional; CPU mode is supported | NVIDIA CUDA or Apple Silicon |
|
||||
| **VRAM** | 4 GB when using a GPU | 8 GB+; large optional engines need more |
|
||||
| **Python from source** | 3.11+ | 3.11–3.12 |
|
||||
|
||||
> [!NOTE]
|
||||
> **A GPU is optional** — the whole pipeline runs on CPU (just slower), and on ≤8 GB VRAM, TTS auto-offloads to CPU. Caveats: **AMD ROCm** is Linux-only + opt-in ([Linux](docs/install/linux.md#amd-gpu-rocm)) — Windows AMD/Ryzen AI is CPU-only ([Windows](docs/install/windows.md#gpu-support)); **macOS Intel** can't run the local backend, so point it at a remote one ([#889](https://github.com/debpalash/VoiceStudio/issues/889) · [macOS](docs/install/macos.md)).
|
||||
ROCm is Linux-only and opt-in. Windows AMD/Ryzen AI uses CPU. Systems with limited VRAM offload work to CPU when required. See [performance](docs/performance.md), [benchmarks](docs/benchmarks.md), and [engine disk usage](docs/engines/disk-usage.md).
|
||||
|
||||
<a id="engines"></a>
|
||||
|
||||
## Engines
|
||||
|
||||
Engine support is capability-specific. Check cloning, language, platform, memory, and license before choosing one. Full setup guides: [docs/engines](docs/engines/README.md).
|
||||
|
||||
<a id="tts-engines"></a>
|
||||
|
||||
### 🗣️ TTS Engines
|
||||
|
||||
**16 engines, one picker.** VoiceStudio (default, 600+ languages) is always available; seven more are opt-in and auto-detected (CosyVoice 3, GPT-SoVITS, VoxCPM2, MOSS-TTS-Nano, KittenTTS, MLX-Audio, Sherpa-ONNX), plus eight lazy-installed opt-ins (IndexTTS 2.5, OmniVoice GGUF, OmniVoice subprocess, PocketTTS, Supertonic 3, MOSS-TTS-v1.5, dots.tts, Confucius4-TTS). Switch in **Model Catalogue → Engines** — or from anywhere with <kbd>Ctrl</kbd>/<kbd>Cmd</kbd>+<kbd>E</kbd>; the choice applies everywhere synthesis happens. **Every engine has its own guide: [docs/engines](docs/engines/README.md).**
|
||||
|
||||
<details>
|
||||
<summary><b>📊 The full matrix</b> — 16 engines × platform × clone/instruct × license</summary>
|
||||
|
||||
<br/>
|
||||
### Text to speech
|
||||
|
||||
| Engine | Languages | Clone | Instruct | Linux | macOS ARM | Windows | License |
|
||||
|--------|:---------:|:-----:|:--------:|:-----:|:---------:|:-------:|:-------:|
|
||||
| **VoiceStudio** (default, powered by k2-fsa/OmniVoice) | 600+ | ✅ | ✅ | ✅ CUDA/CPU | ✅ MPS | ✅ CUDA/CPU | Built-in |
|
||||
| **CosyVoice 3** | 9 + 18 dialects | ✅ | ✅ | ✅ CUDA/CPU | ✅ CPU | ✅ CUDA/CPU | Apache-2.0 |
|
||||
| **GPT-SoVITS** | 5 | ✅ | — | ✅ CUDA/CPU | — | ✅ CUDA/CPU | MIT |
|
||||
| **VoxCPM2** | 30 | ✅ | ✅ | ✅ CUDA/CPU | ✅ MPS | ✅ CUDA/CPU | Apache-2.0 |
|
||||
| **MOSS-TTS-Nano** | 20 | ✅ | — | ✅ CUDA/CPU | ✅ CPU | ✅ CUDA/CPU | Apache-2.0 |
|
||||
| **KittenTTS** | English | — | — | ✅ CPU | ✅ CPU | ✅ CPU | MIT |
|
||||
| **MLX-Audio** (Kokoro, Qwen3-TTS, CSM, Dia, …) | Multi | Varies | Varies | ❌ | ✅ Native | ❌ | Varies |
|
||||
| **Sherpa-ONNX** | 20+ | — | — | ✅ CUDA/CPU | ✅ CPU | ✅ CUDA/CPU | Apache-2.0 |
|
||||
| **IndexTTS 2.5** ⚡ | ZH · EN · JA · ES · AR | ✅ | — | ✅ CUDA/CPU | ✅ CPU | ✅ CUDA/CPU | Bilibili model license¹ |
|
||||
| **OmniVoice GGUF** ⚡ | 600+ | ✅ | ✅ | ✅ CUDA/CPU | ✅ MPS/CPU | ✅ CUDA/CPU | Built-in |
|
||||
| **OmniVoice (subprocess)** ⚡² | 600+ | ✅ | ✅ | ✅ CUDA/CPU | ✅ MPS | ✅ CUDA/CPU | Built-in |
|
||||
| **PocketTTS** ⚡ (Kyutai) | EN · FR · DE · PT · IT · ES | ✅ | — | ✅ CPU | ✅ CPU | ✅ CPU | CC-BY-4.0 (gated)³ |
|
||||
| **Supertonic 3** ⚡ | 31 | — | — | ✅ CPU | ✅ CPU | ✅ CPU | OpenRAIL-M |
|
||||
| **MOSS-TTS-v1.5** ⚡ (8B) | 31 | ✅ | — | ✅ CUDA/CPU | ✅ CPU | ✅ CUDA/CPU | Apache-2.0 |
|
||||
| **dots.tts** ⚡ (2B) | 24 | ✅ | — | ✅ CUDA/CPU | ✅ CPU | ❌ | Apache-2.0 |
|
||||
| **Confucius4-TTS** ⚡ | 14 | ✅ | — | ✅ CUDA/CPU | ✅ CPU | ✅ CUDA/CPU | Apache-2.0 |
|
||||
|---|:---:|:---:|:---:|:---:|:---:|:---:|---|
|
||||
| **VoiceStudio** (default, powered by k2-fsa/OmniVoice) | 600+ | Yes | Yes | CUDA/CPU | MPS | CUDA/CPU | [AGPL-3.0](LICENSE) app · [Apache-2.0](LICENSE-NOTICE.md) model |
|
||||
| **CosyVoice 3** | 9 + 18 dialects | Yes | Yes | CUDA/CPU | CPU | CUDA/CPU | Apache-2.0 |
|
||||
| **GPT-SoVITS** | 5 | Yes | — | CUDA/CPU | — | CUDA/CPU | MIT |
|
||||
| **VoxCPM2** | 30 | Yes | Yes | CUDA/CPU | MPS | CUDA/CPU | Apache-2.0 |
|
||||
| **MOSS-TTS-Nano** | 20 | Yes | — | CUDA/CPU | CPU | CUDA/CPU | Apache-2.0 |
|
||||
| **KittenTTS** | English | — | — | CPU | CPU | CPU | MIT |
|
||||
| **MLX-Audio** | Model-dependent | Varies | Varies | — | MLX | — | Varies |
|
||||
| **Sherpa-ONNX** | 20+ | — | — | CUDA/CPU | CPU | CUDA/CPU | Apache-2.0 |
|
||||
| **IndexTTS 2.5** ⚡ | ZH · EN · JA · ES · AR | Yes | — | CUDA/CPU | CPU | CUDA/CPU | Bilibili model license¹ |
|
||||
| **OmniVoice GGUF** ⚡ | 600+ | Yes | Yes | CUDA/CPU | MPS/CPU | CUDA/CPU | [AGPL-3.0](LICENSE) app · [Apache-2.0](LICENSE-NOTICE.md) model |
|
||||
| **OmniVoice (subprocess)** ⚡ | 600+ | Yes | Yes | CUDA/CPU | MPS | CUDA/CPU | [AGPL-3.0](LICENSE) app · [Apache-2.0](LICENSE-NOTICE.md) model |
|
||||
| **PocketTTS** ⚡ | EN · FR · DE · PT · IT · ES | Yes | — | CPU | CPU | CPU | CC-BY-4.0, gated² |
|
||||
| **Supertonic 3** ⚡ | 31 | — | — | CPU | CPU | CPU | OpenRAIL-M |
|
||||
| **MOSS-TTS-v1.5** ⚡ | 31 | Yes | — | CUDA/CPU | CPU | CUDA/CPU | Apache-2.0 |
|
||||
| **dots.tts** ⚡ | 24 | Yes | — | CUDA/CPU | CPU | — | Apache-2.0 |
|
||||
| **Confucius4-TTS** ⚡ | 14 | Yes | — | CUDA/CPU | CPU | CUDA/CPU | Apache-2.0 |
|
||||
|
||||
¹ IndexTTS 2.5 requires a separate written Bilibili license above 100 million
|
||||
monthly active users or RMB 1 billion in annual revenue. Review its
|
||||
[model license](https://huggingface.co/IndexTeam/IndexTTS-2.5/blob/main/LICENSE)
|
||||
before enabling the optional sidecar.
|
||||
⚡ Installed or registered on demand.
|
||||
|
||||
² **OmniVoice (subprocess)** is the same resident model as the default engine, run
|
||||
in a crash-isolated child process: a wedged generation can be hard-killed and its
|
||||
VRAM reclaimed. Opt-in for unattended synthesis and VRAM-tight MPS hosts —
|
||||
[docs/engines/omnivoice-subprocess.md](docs/engines/omnivoice-subprocess.md).
|
||||
¹ IndexTTS 2.5 requires a separate written Bilibili license above 100 million monthly active users or RMB 1 billion annual revenue. Review the [model license](https://huggingface.co/IndexTeam/IndexTTS-2.5/blob/main/LICENSE).
|
||||
|
||||
³ **PocketTTS** (Kyutai) is a fast, low-latency CPU engine with zero-shot cloning;
|
||||
its gated model access and CC-BY-4.0 conditions are shown for review in-app before
|
||||
first use.
|
||||
² PocketTTS shows its gated-access and CC-BY-4.0 terms before first use.
|
||||
|
||||
GPT-SoVITS connects to `http://127.0.0.1:9880` by default. To use a server on
|
||||
another machine, set `OMNIVOICE_GPTSOVITS_URL` to its credential-free
|
||||
`http://` or `https://` origin and add that machine's CIDR to
|
||||
`OMNIVOICE_TRUSTED_NETWORKS`; redirects and untrusted destinations are rejected.
|
||||
|
||||
> **CUDA** = GPU-accelerated · **MPS** = Apple Silicon Metal · **CPU** = runs everywhere, slower for large models · KittenTTS, MOSS-TTS-Nano, and PocketTTS run realtime on CPU · MLX-Audio is Apple Silicon only · ⚡ = lazy-registered (installed on first use)
|
||||
>
|
||||
> **Clone** matters beyond single-clip generation: Video Dubbing (and any Batch job with a pinned voice) needs reference-audio cloning to preserve speaker identity, so picking a Clone-less engine (KittenTTS, Sherpa-ONNX, Supertonic 3) as the active engine fails those jobs up front with an actionable message instead of silently falling back to VoiceStudio.
|
||||
>
|
||||
> **MOSS-TTS-v1.5** (8B, ~16 GB), **dots.tts** (2B, ~9 GB), and **Confucius4-TTS** are heavyweight opt-ins that run in their own isolated venv from a local clone. None claims Apple-Silicon MPS (CPU on Macs); dots.tts has no Windows path; Confucius4 wants CUDA (CPU works, ~17× realtime). Details: [MOSS-TTS-v1.5](docs/engines/moss-tts-v15.md) · [dots.tts](docs/engines/dots-tts.md) · [Confucius4-TTS](docs/engines/confucius4-tts.md).
|
||||
|
||||
</details>
|
||||
Clone-less engines cannot preserve a reference speaker in dubbing or pinned-voice batch jobs. VoiceStudio rejects those jobs instead of silently changing engines. Heavy engines have separate memory and platform limits; check their engine guide first.
|
||||
|
||||
<a id="asr-engines"></a>
|
||||
|
||||
### 🎧 ASR Engines
|
||||
### Speech to text
|
||||
|
||||
**11 engines** — they power dictation, video dubbing, and subtitles. **WhisperX** is the cross-platform default (~100 languages, word-level timing); the rest are opt-in and auto-detected. Switch in **Model Catalogue → Engines**. Ten run fully on-device; the eleventh (OpenAI-compatible) is an optional remote client for Qwen3-ASR or any compatible server. **Per-engine guides: [docs/engines](docs/engines/README.md).**
|
||||
| Engine | ID | Languages | Best fit |
|
||||
|---|---|:---:|---|
|
||||
| **WhisperX** (default) | `whisperx` | ~100 | Dubbing, subtitles, word-level timing |
|
||||
| **Faster-Whisper** | `faster-whisper` | ~100 | General cross-platform transcription |
|
||||
| **Faster-Whisper (isolated)** | `faster-whisper-isolated` | ~100 | Crash-isolated batch transcription |
|
||||
| **MLX Whisper** | `mlx-whisper` | ~100 | Apple Silicon |
|
||||
| **PyTorch Whisper** | `pytorch-whisper` | ~100 | CUDA, MPS, and CPU fallback |
|
||||
| **Parakeet TDT** | `nemo-parakeet` | English + 25 EU | Fast CPU/CUDA transcription |
|
||||
| **Parakeet TDT v3 (MLX)** | `parakeet-mlx` | 25 EU | Apple Silicon dictation and word timestamps |
|
||||
| **Moonshine** | `moonshine` | English | Low-power, low-latency ONNX |
|
||||
| **FunASR** | `funasr` | 50+ | VAD and inline diarization |
|
||||
| **sherpa-onnx** (live dictation) | `sherpa-onnx-asr` | Model-dependent | Streaming CPU dictation |
|
||||
| **OpenAI-compatible** ⚠️ remote | `openai-compat-asr` | Server-dependent | Qwen3-ASR or another compatible endpoint; audio leaves the machine |
|
||||
|
||||
<details>
|
||||
<summary><b>📊 The full lineup</b> — 11 engines, what each is best at, and compute-type notes</summary>
|
||||
WhisperX and Faster-Whisper retry with `int8` when efficient `float16` is unavailable. Pin `ASR_COMPUTE_TYPE=int8` or `float32` only if automatic selection still fails.
|
||||
|
||||
<br/>
|
||||
<a id="architecture"></a>
|
||||
|
||||
| Engine | `OMNIVOICE_ASR_BACKEND` | Languages | Best for |
|
||||
|--------|-------------------------|:---------:|----------|
|
||||
| **WhisperX** (default) | `whisperx` | ~100 | Dubbing & subtitles — word-level timing via wav2vec2 forced alignment |
|
||||
| **Faster-Whisper** | `faster-whisper` | ~100 | Fast transcription on Linux / macOS / Windows (CTranslate2) |
|
||||
| **Faster-Whisper (isolated)** | `faster-whisper-isolated` | ~100 | Same as Faster-Whisper but crash-isolated in a subprocess — an ASR crash won't take down the app |
|
||||
| **MLX Whisper** | `mlx-whisper` | ~100 | Native Apple Silicon speed (Apple MLX / Metal) |
|
||||
| **PyTorch Whisper** | `pytorch-whisper` | ~100 | CUDA / CPU fallback via 🤗 Transformers (no cuDNN 8 needed) |
|
||||
| **Parakeet TDT** | `nemo-parakeet` | English + 25 EU | SOTA accuracy at ~10× realtime even on CPU, auto language detection (NVIDIA NeMo, CUDA/CPU) |
|
||||
| **Parakeet TDT v3 (MLX)** | `parakeet-mlx` | 25 EU | The Parakeet tier for Apple Silicon — word timestamps, ~2 GB unified memory, dictation-grade speed via MLX. Dictation prefers it automatically for its 25 European languages; other languages keep multilingual Whisper. |
|
||||
| **Moonshine** | `moonshine` | English | Edge / low-latency, ONNX |
|
||||
| **FunASR** | `funasr` | 50+ | All-in-one multilingual — built-in VAD + inline speaker diarization (SenseVoice) |
|
||||
| **sherpa-onnx** (live dictation) | `sherpa-onnx-asr` | 25 EU + 90+ | Live, faster-than-real-time dictation — small streaming/offline ONNX models, CPU, identical on macOS / Windows / Linux. Picked per-model in **Settings → Voice**. |
|
||||
| **OpenAI-compatible** ⚠️ remote | `openai-compat-asr` | Server-dependent | A path to **Qwen3-ASR** today (self-hosted server), any OpenAI-compatible transcription endpoint, or OpenAI's own API — configure + test in **Model Catalogue → Engines** (ASR tab). Audio leaves your machine to whatever server you point it at; see [docs/engines/openai-compatible-asr.md](docs/engines/openai-compatible-asr.md). |
|
||||
## Architecture
|
||||
|
||||
> If Dubbing needs an ASR model that is not installed yet, it offers the recommended download in place, shows its progress, and retries transcription on the same job when the model is ready.
|
||||
>
|
||||
> **GPU without efficient float16?** On older NVIDIA GPUs (Maxwell/Pascal, GTX 16xx) or after a CTranslate2/cuDNN mismatch, the CTranslate2 ASR engines (WhisperX, Faster-Whisper) can't run `float16` and VoiceStudio automatically retries on `int8` — no config needed. If transcription still fails, pin the compute type with `ASR_COMPUTE_TYPE=int8` (or `float32` for CPU) and restart the backend.
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ Architecture
|
||||
|
||||
A **Tauri v2** desktop shell (Rust) wraps a **React** UI and a bundled **Python/FastAPI** backend that runs as a local sidecar on `localhost:3900`. Every layer runs on your machine by default; the only network paths are the ones you opt into (remote GPU workers, a remote backend, or an OpenAI-compatible ASR endpoint).
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────────┐
|
||||
│ Tauri v2 shell — Rust │
|
||||
│ window state · global dictation hotkey · system tray · │
|
||||
│ signed auto-updater (stable/preview) · single-instance · │
|
||||
│ first-run bootstrap (installs uv + Python venv) · blank guard │
|
||||
├────────────────────────────────────────────────────────────────────┤
|
||||
│ Frontend — React + Vite │
|
||||
│ Studio · Dub · Stories · Audiobook · Gallery · Catalogue · │
|
||||
│ Dictation · Batch · Diagnostics — Zustand store · WS bus │
|
||||
│ ▲ IPC / HTTP + WS │
|
||||
├──────────────────────────┼─────────────────────────────────────────┤
|
||||
│ Backend — FastAPI sidecar @ localhost:3900 │
|
||||
│ 100+ REST endpoints · SSE + WebSocket streaming · │
|
||||
│ SQLite + Alembic (omnivoice_data/) · OpenAI-compatible API │
|
||||
├───────────┬───────────┬───────────┬───────────┬────────────────────┤
|
||||
│ TTS ×16 │ ASR ×11 │ Demucs │ Pyannote │ AudioSeal │
|
||||
│ clone / │ WhisperX │ vocal │ speaker │ watermark │
|
||||
│ design │ +10 more │ isolation│ diariz. │ embed / detect │
|
||||
├───────────┴───────────┴───────────┴───────────┴────────────────────┤
|
||||
│ Engine routing — per-engine GPU preflight, no silent CPU fallback │
|
||||
│ Hardware: CUDA · MPS · ROCm (Linux) · CPU (auto-detected) │
|
||||
│ + optional remote GPU workers on your other machines │
|
||||
└────────────────────────────────────────────────────────────────────┘
|
||||
```text
|
||||
Tauri v2 desktop shell (Rust)
|
||||
│ IPC
|
||||
React + Vite UI
|
||||
│ HTTP · SSE · WebSocket on localhost:3900
|
||||
FastAPI backend
|
||||
├── TTS / ASR engine registries
|
||||
├── dubbing / audio / long-form pipelines
|
||||
├── OpenAI-compatible API and MCP server
|
||||
└── SQLite + Alembic → omnivoice_data/
|
||||
```
|
||||
|
||||
<a id="openai-api"></a>
|
||||
| Layer | Path | Responsibility |
|
||||
|---|---|---|
|
||||
| Desktop shell | `frontend/src-tauri/` | Window lifecycle, tray, shortcuts, updater, sidecar bootstrap |
|
||||
| Frontend | `frontend/src/` | React UI, Zustand state, API and event clients, i18n |
|
||||
| API | `backend/api/` | REST routes, schemas, auth boundaries, streaming |
|
||||
| Core services | `backend/services/` | Generation, dubbing, audio processing, persistence |
|
||||
| Engines | `backend/engines/` | Isolated and optional engine adapters |
|
||||
| Worker system | `backend/worker/` | Authenticated remote compute and job transport |
|
||||
| Data | `omnivoice_data/` | Projects, voices, settings, logs, and SQLite state |
|
||||
| Delivery | `scripts/`, `deploy/`, `.github/workflows/` | Development, packaging, containers, releases, CI |
|
||||
|
||||
## 🔌 OpenAI-compatible API
|
||||
### Network boundary
|
||||
|
||||
<div align="center">
|
||||
- The desktop talks to a loopback-only backend on `localhost:3900`.
|
||||
- Loopback API calls need no server key. Remote access requires a share PIN or API key.
|
||||
- Remote workers and OpenAI-compatible ASR are opt-in. The UI identifies when audio leaves the machine.
|
||||
- Analytics is off until consent. If enabled, it sends allowlisted, content-free usage metadata—not text, audio, file names, or projects.
|
||||
|
||||
**Drop-in replacement for OpenAI / ElevenLabs audio.** One line — no key, no code changes:
|
||||
<a id="api"></a>
|
||||
|
||||
## OpenAI-compatible API
|
||||
|
||||
Point an OpenAI-compatible audio client at the local backend:
|
||||
|
||||
```diff
|
||||
- base_url="https://api.openai.com/v1"
|
||||
+ base_url="http://localhost:3900/v1"
|
||||
```
|
||||
|
||||
</div>
|
||||
|
||||
Your existing scripts, agents, and OpenAI/ElevenLabs SDK calls now run **locally** on whatever engine you have active. What the cloud can't do: `voice` takes **your own cloned-voice profile IDs**, and `model` can pin a **specific engine** per request.
|
||||
|
||||
| Endpoint | What it does |
|
||||
| Endpoint | Purpose |
|
||||
|---|---|
|
||||
| `POST /v1/audio/speech` | TTS — text in; `mp3` / `opus` / `aac` / `flac` / `wav` / `pcm` out. `model`: `tts-1`/`tts-1-hd` (active engine) or a specific one (`voxcpm2`, `cosyvoice`, …). `voice`: a cloned profile ID, `default`, or an OpenAI name (`alloy`, …). `speed` supported. |
|
||||
| `POST /v1/audio/transcriptions` | STT — audio file in; `json` / `text` / `verbose_json` / `srt` / `vtt` out (`verbose_json` adds word-level timings). `whisper-1` maps to your active ASR engine. |
|
||||
| `GET /v1/audio/voices` | VoiceStudio extension — lists every voice profile and engine, so clients can discover your clones. |
|
||||
|
||||
**Speak with your own cloned voice:**
|
||||
| `POST /v1/audio/speech` | TTS to `mp3`, `opus`, `aac`, `flac`, `wav`, or `pcm`; select a profile with `voice` and an engine with `model` |
|
||||
| `POST /v1/audio/transcriptions` | STT to `json`, `text`, `verbose_json`, `srt`, or `vtt` |
|
||||
| `GET /v1/audio/voices` | List local voice profiles and engines |
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
client = OpenAI(base_url="http://localhost:3900/v1", api_key="none") # any string — nothing checks it
|
||||
|
||||
# Find your cloned voices: GET /v1/audio/voices lists profile IDs
|
||||
client = OpenAI(base_url="http://localhost:3900/v1", api_key="local")
|
||||
|
||||
with client.audio.speech.with_streaming_response.create(
|
||||
model="tts-1", voice="<profile-id>", input="Made on my own hardware.") as r:
|
||||
r.stream_to_file("speech.wav")
|
||||
|
||||
# STT
|
||||
print(client.audio.transcriptions.create(model="whisper-1", file=open("clip.wav", "rb")).text)
|
||||
model="tts-1",
|
||||
voice="<profile-id>",
|
||||
input="Made on my own hardware.",
|
||||
response_format="wav",
|
||||
) as response:
|
||||
response.stream_to_file("speech.wav")
|
||||
```
|
||||
|
||||
Want the whole surface (100+ endpoints)? The full REST API reference is embedded in the app — **Settings → OpenAPI Reference** (Scalar-powered), or the `{}` button in the footer.
|
||||
The full API reference is in **Settings → OpenAPI Reference**. For LAN, Tailscale, or proxy access, read [API authentication](docs/api-auth.md) before exposing the backend.
|
||||
|
||||
Calling the backend from **another machine** (LAN, Tailscale, behind a proxy)? It's loopback-only and unauthenticated by default; to reach it remotely you set a share PIN or an API key, and admin actions require the key — exchanged for short-lived scoped sessions. [docs/api-auth.md](docs/api-auth.md) covers the exact headers, query params, `401`/`403`/`429` meanings, and the `OMNIVOICE_TRUSTED_NETWORKS` exemption.
|
||||
### Agent skills
|
||||
|
||||
### 📓 Run on Google Colab
|
||||
Install the VoiceStudio skills for Claude Code, Codex, Cursor, and other [skills.sh](https://skills.sh)-compatible agents:
|
||||
|
||||
[](https://colab.research.google.com/github/debpalash/VoiceStudio/blob/main/notebooks/OmniVoice_Studio_Colab.ipynb)
|
||||
|
||||
No local GPU? The [official notebook](notebooks/OmniVoice_Studio_Colab.ipynb) boots the full app — web UI included — on a free Colab T4, then walks the whole feature surface as a guided tour with inline playback. No tunnels, no API keys.
|
||||
|
||||
### 🤝 Agent Skills
|
||||
|
||||
Teach your coding agent to speak and listen through your local VoiceStudio — one command, works with **Claude Code, Codex, Cursor, Grok, Kimi, opencode**, and any [skills.sh](https://skills.sh)-compatible agent:
|
||||
|
||||
```sh
|
||||
```bash
|
||||
npx skills add debpalash/omnivoice-studio
|
||||
```
|
||||
|
||||
Ships two skills: **`omnivoice`** — generate speech (including your cloned voices) and transcribe audio from any agent, free and fully offline — and **`oss-maintainer`** — the maintainer methodology this project is run with.
|
||||
- `omnivoice`: synthesize speech and transcribe audio through local VoiceStudio.
|
||||
- `oss-maintainer`: the repository's open-source maintenance workflow.
|
||||
|
||||
---
|
||||
### Google Colab
|
||||
|
||||
<a id="roadmap"></a>
|
||||
[](https://colab.research.google.com/github/debpalash/VoiceStudio/blob/main/notebooks/OmniVoice_Studio_Colab.ipynb)
|
||||
|
||||
## 🗺️ Roadmap
|
||||
The [notebook](notebooks/OmniVoice_Studio_Colab.ipynb) runs the app and web UI on a Colab GPU. Colab is remote compute, so uploaded audio and project data do not remain local to your machine.
|
||||
|
||||
What's up next (lip-sync v2, hosted demo, plugin marketplace, real-time voice changer) and the full history of everything shipped so far live in **[docs/ROADMAP.md](docs/ROADMAP.md)**.
|
||||
<a id="documentation"></a>
|
||||
|
||||
---
|
||||
## Documentation
|
||||
|
||||
<a id="sponsor--donate"></a>
|
||||
| Need | Read |
|
||||
|---|---|
|
||||
| Install | [macOS](docs/install/macos.md) · [Windows](docs/install/windows.md) · [Linux](docs/install/linux.md) · [Docker](docs/install/docker.md) |
|
||||
| Fix setup | [Troubleshooting](docs/install/troubleshooting.md) · [model downloads](docs/downloading-models.md) · [Hugging Face token](docs/setup/huggingface-token.md) |
|
||||
| Choose an engine | [Engine guides](docs/engines/README.md) · [benchmarks](docs/benchmarks.md) · [expressive speech](docs/expressive-speech.md) |
|
||||
| Tune hardware | [Performance](docs/performance.md) · [remote workers](docs/remote-workers.md) |
|
||||
| Build integrations | [API auth](docs/api-auth.md) · [MCP](docs/mcp.md) · [examples](examples/README.md) |
|
||||
| Build VoiceStudio | [Contributing](.github/CONTRIBUTING.md) · [engine acceptance](docs/engine-acceptance.md) |
|
||||
| Track changes | [Changelog](CHANGELOG.md) · [roadmap](docs/ROADMAP.md) · [latest release](https://github.com/debpalash/VoiceStudio/releases/latest) |
|
||||
| Remove everything | [Uninstall guide](docs/install/uninstall.md) |
|
||||
|
||||
## 💜 Sponsor / Donate
|
||||
## FAQ
|
||||
|
||||
One developer, real AI-agent bills. If VoiceStudio is useful to you, chipping in keeps development full-time — every dollar goes straight to the bills.
|
||||
<details>
|
||||
<summary><strong>Does it work on Apple Silicon and Intel Macs?</strong></summary>
|
||||
|
||||
Apple Silicon is supported with MPS and MLX options. Intel Macs cannot run the local backend because current PyTorch wheels are unavailable; they can connect to a remote backend. See [macOS installation](docs/install/macos.md).
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>How much VRAM do I need?</strong></summary>
|
||||
|
||||
A GPU is optional. Use 4 GB VRAM as the minimum for accelerated work and 8 GB+ for the default multi-stage workflow. Large optional engines can require 12–16 GB or more. Check the [benchmarks](docs/benchmarks.md) and engine guide.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Why does a longer reference clip not always improve the clone?</strong></summary>
|
||||
|
||||
Cloning is zero-shot: the clip is a prompt, not training data. Use 5–15 seconds of one speaker, close to the microphone, without music, noise, or reverb. Match the tone and pace you want in the output. For training, see [data preparation](docs/data_preparation.md) and [training](docs/training.md).
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Can I use generated audio commercially?</strong></summary>
|
||||
|
||||
Yes under VoiceStudio's AGPL-3.0 terms. Optional engines and model weights may use different licenses; review the selected engine's license before commercial use.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Does VoiceStudio collect data?</strong></summary>
|
||||
|
||||
Not unless you opt in. Analytics is off by default and skipping consent keeps it off. When enabled, the app sends allowlisted, content-free usage metadata. Text, audio, file names, voices, and projects are excluded. Change this at **Settings → Privacy**.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>How do I remove VoiceStudio and its data?</strong></summary>
|
||||
|
||||
Use `scripts/uninstall.sh` on macOS/Linux or `scripts\uninstall.ps1` on Windows. Both show a dry run before deletion. See the [uninstall guide](docs/install/uninstall.md) for every path.
|
||||
</details>
|
||||
|
||||
## Community and contributing
|
||||
|
||||
- [GitHub Issues](https://github.com/debpalash/VoiceStudio/issues) for reproducible bugs and feature requests.
|
||||
- [Discord](https://discord.gg/bzQavDfVV9) for setup help and project discussion.
|
||||
- [Good first issues](https://github.com/debpalash/VoiceStudio/labels/good%20first%20issue) for a scoped starting point.
|
||||
- [Contributing guide](.github/CONTRIBUTING.md) for setup, tests, and pull requests.
|
||||
|
||||
## Support development
|
||||
|
||||
VoiceStudio is free and has no paid tier. Donations fund development and infrastructure.
|
||||
|
||||
[Ko-fi](https://ko-fi.com/debpalash) · [PayPal](https://paypal.me/palashCoder) · [Sponsorship details](SPONSORS.md)
|
||||
|
||||
## License
|
||||
|
||||
VoiceStudio is licensed under [AGPL-3.0](LICENSE). You may run it, modify it, use it internally, and sell generated audio. If you modify VoiceStudio and provide that modified version as a network service, AGPL requires you to offer the corresponding source under the same license. A commercial license is available for proprietary embedding; contact **VoiceStudio@palash.dev**. See [LICENSE-NOTICE.md](LICENSE-NOTICE.md) for the plain-language scope.
|
||||
|
||||
Optional engines and downloaded models retain their own licenses. The bundled `omnivoice/` model remains Apache-2.0 upstream.
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
VoiceStudio builds on [OmniVoice](https://github.com/k2-fsa/OmniVoice), [WhisperX](https://github.com/m-bain/whisperX), [Demucs](https://github.com/facebookresearch/demucs), [Pyannote](https://github.com/pyannote/pyannote-audio), [CTranslate2](https://github.com/OpenNMT/CTranslate2), [AudioSeal](https://github.com/facebookresearch/audioseal), [Tauri](https://tauri.app), [Supertonic](https://huggingface.co/Supertone/supertonic-3), [Sherpa-ONNX](https://github.com/k2-fsa/sherpa-onnx), [GPT-SoVITS](https://github.com/RVC-Boss/GPT-SoVITS), and [PocketTTS](https://kyutai.org).
|
||||
|
||||
<div align="center">
|
||||
|
||||
<img src="https://img.shields.io/badge/raised_%2410_of_%24200-5%25-EAB308?style=for-the-badge" alt="This month's agent-bill fund: $10 / $200" />
|
||||
|
||||
<br/><br/>
|
||||
|
||||
<a href="https://ko-fi.com/debpalash"><img src="https://img.shields.io/badge/Ko--fi-Support_❤️-FF5E5B?style=for-the-badge&logo=ko-fi&logoColor=white" alt="Ko-fi" /></a>
|
||||
|
||||
<a href="https://paypal.me/palashCoder"><img src="https://img.shields.io/badge/PayPal-Donate-00457C?style=for-the-badge&logo=paypal&logoColor=white" alt="PayPal" /></a>
|
||||
|
||||
</div>
|
||||
|
||||
<a id="sponsors"></a>
|
||||
|
||||
### 🌟 Sponsors
|
||||
|
||||
VoiceStudio is **free** and **AGPL-3.0** — no paid tier, no SaaS revenue. Sponsors keep development going, and in return get a logo slot here, in the app, and (for top tiers) on the project website. It's a thank-you, never a paywall. **[See tiers & become a sponsor →](SPONSORS.md)**
|
||||
|
||||
<div align="center">
|
||||
|
||||
<!-- SPONSORS:START — logo slots are filled here as sponsors come aboard; see SPONSORS.md -->
|
||||
|
||||
**Your logo here** — [become a sponsor](SPONSORS.md)
|
||||
|
||||
<!-- SPONSORS:END -->
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## 💬 Community
|
||||
|
||||
<div align="center">
|
||||
<a href="https://discord.gg/bzQavDfVV9"><img src="https://img.shields.io/badge/💬_Discord-Join_Community-5865F2?style=for-the-badge&logo=discord&logoColor=white" alt="Join Discord" /></a>
|
||||
<a href="https://x.com/idebpalash"><img src="https://img.shields.io/badge/𝕏_Follow-for_updates-000000?style=for-the-badge&logo=x&logoColor=white" alt="Follow on X" /></a>
|
||||
<br/>
|
||||
<sub>Release news, setup help, GPU troubleshooting, feature votes, and showing off your dubs. We respond to setup questions within hours, not days.</sub>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
<a id="contributing"></a>
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
Yes please — bug fixes, new TTS engine adapters, UI improvements, docs, translations. All of it. Start with the **[Contributing Guide](.github/CONTRIBUTING.md)** (setup, code style, PR workflow), browse [good first issues](https://github.com/debpalash/VoiceStudio/labels/good%20first%20issue), or ask in [Discord](https://discord.gg/bzQavDfVV9).
|
||||
|
||||
---
|
||||
|
||||
## ❓ FAQ
|
||||
|
||||
<details>
|
||||
<summary><b>Does it work on Apple Silicon (M1/M2/M3/M4)?</b></summary>
|
||||
<br/>
|
||||
Yes. MPS acceleration is auto-detected. MLX-optimized Whisper models are available for faster transcription on Apple hardware. <b>Intel Macs are not supported</b>: the app UI installs, but the local Python backend cannot run because PyTorch no longer ships Intel-Mac wheels (<a href="https://github.com/debpalash/VoiceStudio/issues/889">#889</a>) — an Intel Mac can only be used with a remote backend.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>How much VRAM do I need?</b></summary>
|
||||
<br/>
|
||||
<b>4 GB minimum.</b> With ≤8 GB, the TTS model is automatically offloaded to CPU during transcription. With 8+ GB, everything runs on GPU simultaneously. No GPU at all? CPU mode works — just slower (~3× for TTS). You can also lend a GPU from another machine you own via <a href="docs/remote-workers.md">remote workers</a>.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>What languages are supported?</b></summary>
|
||||
<br/>
|
||||
646 languages for TTS via the VoiceStudio model. Transcription (WhisperX) supports 99 languages. Translation coverage depends on the target language pair.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Why doesn't a longer reference clip sound more like me?</b></summary>
|
||||
<br/>
|
||||
Because VoiceStudio's cloning is <b>zero-shot</b>: your clip is a <i>prompt</i> the model conditions on — it is never trained on, and past a short window extra audio is simply unused (the dubbing pipeline targets ~8 s and hard-caps at 15 s). <b>What moves clone quality is the clip, not its length</b>: record 5–15 seconds of continuous natural speech, close to the mic, in a quiet room with no reverb or music, one speaker, delivered in the tone and pace you want — the clone copies your delivery, not just your timbre. Want trained-on-your-voice fidelity? That's offline fine-tuning, not an in-app button: <a href="docs/data_preparation.md">docs/data_preparation.md</a> + <a href="docs/training.md">docs/training.md</a>.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Can I use this commercially?</b></summary>
|
||||
<br/>
|
||||
<b>Yes — commercial use is free</b> under the <a href="https://www.gnu.org/licenses/agpl-3.0.html">AGPL-3.0</a>: run it, sell the audio you make, dub client videos, deploy it across your team. One obligation: if you <b>modify</b> VoiceStudio and offer the modified version to others over a network, you must share that modified source under the same terms. Embedding it in a closed-source product instead? A commercial license is available — see <a href="#license">License</a>.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Can I add my own TTS engine?</b></summary>
|
||||
<br/>
|
||||
Yes. Subclass <code>TTSBackend</code> in <code>backend/services/tts_backend.py</code> and add it to the <code>_REGISTRY</code> dictionary — ~50 lines. The sixteen built-in engines all work this way; see <a href="#tts-engines">TTS Engines</a> and <a href="docs/engine-acceptance.md">docs/engine-acceptance.md</a>.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Does VoiceStudio collect any data about me?</b></summary>
|
||||
<br/>
|
||||
<b>Not unless you explicitly say yes.</b> On first run the app <i>asks</i> — one screen, two equal-weight buttons, no pre-ticked box — and until you answer yes, VoiceStudio sends nothing: no analytics, no telemetry, no accounts, no phone-home. Skipping the question means no. Your text, audio, voices, and projects never leave your machine either way.
|
||||
|
||||
If you do opt in (also togglable anytime under <b>Settings → Privacy → "Help improve VoiceStudio"</b>), what's sent is anonymous, content-free usage stats: generations (engine, language, generation time, character <i>count</i>, error <i>type</i>), plus app lifecycle — an install ping, updates (version-to-version), crashes (error class and a <i>bucketed</i> uptime, never logs), error <i>types</i> (capped, deduplicated), and a single uninstall ping if you remove it. Never your text, audio, file names, or anything identifying — enforced in code by a property allowlist (<code>backend/core/analytics.py</code>), not just a promise. Every build — installer, Docker, or built from source — asks the same first-run question and stays off unless you say yes. Your own numbers live in <b>Settings → Usage</b>, computed locally, sent nowhere.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>How do I uninstall it / remove all its data?</b></summary>
|
||||
<br/>
|
||||
VoiceStudio is fully local — uninstalling is just deleting the app plus the folders it wrote (model cache, Python env, your voices/projects, config). Run <code>scripts/uninstall.sh</code> (macOS/Linux) or <code>scripts\uninstall.ps1</code> (Windows) — it prints every folder with its size as a dry-run first, then deletes on <code>--yes</code>. The full per-platform path list and app-removal steps are in <a href="docs/install/uninstall.md"><b>docs/install/uninstall.md</b></a>.
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<a id="license"></a>
|
||||
|
||||
## 📜 License
|
||||
|
||||
VoiceStudio is free and open-source software under the [**GNU Affero General Public License v3.0 (AGPL-3.0)**](https://www.gnu.org/licenses/agpl-3.0.html).
|
||||
|
||||
**Free for any use — including commercial and internal business use.** Run it, sell the audio you produce with it, dub your own or clients' videos, roll it out across your team — all free, no license needed. As a **network copyleft** license, AGPL adds one obligation: if you **modify** VoiceStudio and offer that modified version to others over a network, you must make the complete corresponding source of your modified version available to them under the same AGPL-3.0 terms.
|
||||
|
||||
A **commercial license** is available for organizations that want to embed VoiceStudio in a **closed-source or proprietary** product or service without the AGPL-3.0 copyleft obligations. **Pricing tiers coming soon.** Inquiries: **VoiceStudio@palash.dev**.
|
||||
|
||||
The bundled `omnivoice/` TTS model by Han Zhu remains Apache-2.0 upstream. See [`LICENSE`](LICENSE) for the full, binding terms, and [`LICENSE-NOTICE.md`](LICENSE-NOTICE.md) for the plain-language summary and scope.
|
||||
|
||||
---
|
||||
|
||||
## 🙏 Acknowledgments
|
||||
|
||||
VoiceStudio stands on exceptional open-source work: [OmniVoice (k2-fsa)](https://github.com/k2-fsa/OmniVoice) — the core zero-shot TTS model · [WhisperX](https://github.com/m-bain/whisperX) · [Demucs](https://github.com/facebookresearch/demucs) · [Pyannote](https://github.com/pyannote/pyannote-audio) · [CTranslate2](https://github.com/OpenNMT/CTranslate2) · [AudioSeal](https://github.com/facebookresearch/audioseal) · [Tauri](https://tauri.app) · [Supertonic](https://huggingface.co/Supertone/supertonic-3) · [Sherpa-ONNX](https://github.com/k2-fsa/sherpa-onnx) · [GPT-SoVITS](https://github.com/RVC-Boss/GPT-SoVITS) · [Kyutai PocketTTS](https://kyutai.org) — thank you.
|
||||
|
||||
<a id="more-from-the-maker"></a>
|
||||
|
||||
### 🧰 More local open-source from the maker
|
||||
|
||||
[**Opal** 💠](https://github.com/debpalash/Opal) — play everything: the media player for the AI era · [**memxt** 🧠](https://github.com/debpalash/memxt) — local long-term memory for coding agents. Same rule: **your data stays on your machine.** All of it lives at [palash.dev](https://palash.dev).
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
|
||||
<br/>
|
||||
|
||||
If you read this far, you're our kind of person.<br/>
|
||||
**[⭐ Star this repo](https://github.com/debpalash/VoiceStudio)** so others can find it too.<br/>
|
||||
**[💬 Join the Discord](https://discord.gg/bzQavDfVV9)** to share what you build.<br/>
|
||||
**[❤️ Support development](https://ko-fi.com/debpalash)** — fund the AI agent bills that keep VoiceStudio shipping.
|
||||
|
||||
<br/>
|
||||
|
||||
<a href="https://star-history.com/#debpalash/VoiceStudio&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=debpalash/VoiceStudio&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=debpalash/VoiceStudio&type=Date" />
|
||||
<img alt="Star History" src="https://api.star-history.com/svg?repos=debpalash/VoiceStudio&type=Date&theme=dark" width="600" />
|
||||
</picture>
|
||||
</a>
|
||||
<strong><a href="https://github.com/debpalash/VoiceStudio/releases/latest">Download VoiceStudio</a></strong> ·
|
||||
<a href="https://github.com/debpalash/VoiceStudio">Star the project</a> ·
|
||||
<a href="https://discord.gg/bzQavDfVV9">Join Discord</a>
|
||||
</div>
|
||||
|
||||
@@ -157,6 +157,31 @@ def require_loopback(request: Request) -> None:
|
||||
raise HTTPException(status_code=403, detail="loopback origin required")
|
||||
|
||||
|
||||
def _admin_gate_403() -> None:
|
||||
"""Raise the admin-gate 403 with a detail that states what would ACTUALLY
|
||||
satisfy the gate. The bundled UI routes any 403 whose detail mentions
|
||||
"admin api key" to the API-key login form (frontend ``client.ts``; the
|
||||
literal contract is locked by ``tests/test_auth_gate_detail_lockstep.py``),
|
||||
so the wording must not name a key where presenting one cannot help.
|
||||
|
||||
The detail names the key only when the gate would accept one: server mode
|
||||
WITH an API key configured. Every other rejection — desktop mode (the
|
||||
credential checks in the callers only run under server mode) and a
|
||||
server-mode deployment with only a share PIN or nothing configured — keeps
|
||||
the plain loopback detail, because only loopback can use admin there.
|
||||
Naming the key in those cases would trap a LAN-share guest in a login
|
||||
form that can never succeed (#1213, #1525; PR #1569 review).
|
||||
"""
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=(
|
||||
"loopback origin or admin API key required"
|
||||
if _server_mode() and remote_api_key()
|
||||
else "loopback origin required"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def require_admin(request: Request) -> None:
|
||||
"""Gate RCE/filesystem-capable admin routers.
|
||||
|
||||
@@ -180,7 +205,7 @@ def require_admin(request: Request) -> None:
|
||||
return
|
||||
if _request_presents_admin_credential(request):
|
||||
return
|
||||
raise HTTPException(status_code=403, detail="loopback origin or admin API key required")
|
||||
_admin_gate_403()
|
||||
|
||||
|
||||
def require_admin_action(request: Request) -> None:
|
||||
@@ -198,7 +223,7 @@ def require_admin_action(request: Request) -> None:
|
||||
side_effectful_get=True,
|
||||
):
|
||||
return
|
||||
raise HTTPException(status_code=403, detail="loopback origin or admin API key required")
|
||||
_admin_gate_403()
|
||||
|
||||
|
||||
def require_desktop(request: Request) -> None:
|
||||
|
||||
@@ -57,6 +57,99 @@ def _force_compile_requested() -> bool:
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
# ── FlashInfer opt-in (upstream k2-fsa port) ────────────────────────────────
|
||||
# Explicit power-user opt-in, CUDA-only: OMNIVOICE_FLASHINFER=1 patches the
|
||||
# OmniVoice model with flashinfer packed attention (~2x per upstream's
|
||||
# benchmarks); =graph additionally captures CUDA graphs (best at batch=1).
|
||||
# Off by default — `flashinfer` is not a shipped dependency, and an
|
||||
# optimization must never be a point of failure. Session-sticky failure
|
||||
# latch mirrors torch.compile's (#278).
|
||||
_FLASHINFER_ENV = "OMNIVOICE_FLASHINFER"
|
||||
_flashinfer_runtime_failure: Optional[str] = None
|
||||
|
||||
|
||||
def flashinfer_mode() -> str:
|
||||
"""The user's ``OMNIVOICE_FLASHINFER`` request: 'off' | 'on' | 'graph'.
|
||||
|
||||
Unknown values normalize to 'off' with a log line naming the env var, so
|
||||
a typo degrades to the default path instead of half-applying.
|
||||
"""
|
||||
value = os.environ.get(_FLASHINFER_ENV, "").strip().lower()
|
||||
if value in {"", "0", "false", "no", "off"}:
|
||||
return "off"
|
||||
if value in {"1", "true", "yes", "on"}:
|
||||
return "on"
|
||||
if value == "graph":
|
||||
return "graph"
|
||||
logger.warning(
|
||||
"%s=%r not recognized (valid: 0, 1, graph) — FlashInfer stays off.",
|
||||
_FLASHINFER_ENV, value,
|
||||
)
|
||||
return "off"
|
||||
|
||||
|
||||
def should_flashinfer(device: str) -> str:
|
||||
"""Resolve the FlashInfer request against this host: 'off' | 'on' | 'graph'.
|
||||
|
||||
Requires all of: the ``OMNIVOICE_FLASHINFER`` opt-in, device == "cuda"
|
||||
(flashinfer is CUDA-only), the ``flashinfer`` package importable, and no
|
||||
earlier runtime failure this session. Every refusal is logged with the
|
||||
reason and the knob's name — the user asked for it, so silence would read
|
||||
as "the setting doesn't work".
|
||||
"""
|
||||
mode = flashinfer_mode()
|
||||
if mode == "off":
|
||||
return "off"
|
||||
if device != "cuda":
|
||||
logger.warning(
|
||||
"%s requested but the compute device is %r — FlashInfer is "
|
||||
"CUDA-only, continuing without it.", _FLASHINFER_ENV, device,
|
||||
)
|
||||
return "off"
|
||||
if importlib.util.find_spec("flashinfer") is None:
|
||||
logger.warning(
|
||||
"%s requested but the `flashinfer` package is not installed — "
|
||||
"continuing without it. Install with: uv pip install "
|
||||
"flashinfer-python flashinfer-jit-cache "
|
||||
"--extra-index-url https://flashinfer.ai/whl/cu128/ "
|
||||
"(pick the index matching your CUDA build).", _FLASHINFER_ENV,
|
||||
)
|
||||
return "off"
|
||||
if _flashinfer_runtime_failure is not None:
|
||||
logger.info(
|
||||
"FlashInfer skipped: failed earlier this session (%s) — using the "
|
||||
"standard path.", _flashinfer_runtime_failure,
|
||||
)
|
||||
return "off"
|
||||
return mode
|
||||
|
||||
|
||||
def mark_flashinfer_runtime_failure(reason: str) -> None:
|
||||
"""Latch a FlashInfer apply/runtime failure for the rest of the process,
|
||||
same contract as ``mark_compile_runtime_failure``."""
|
||||
global _flashinfer_runtime_failure
|
||||
try:
|
||||
# Import/kernel errors embed absolute paths (wheels under the user's
|
||||
# home) — redact before latching, since the reason is logged here and
|
||||
# re-logged on every later skip.
|
||||
from core.failure import sanitize
|
||||
|
||||
reason = sanitize(reason)
|
||||
except Exception:
|
||||
# Fail closed: if the redactor itself breaks, latching the raw text
|
||||
# would defeat the redaction. Keep only the exception class (the part
|
||||
# before ':' in our "Type: message" reasons) and drop the message.
|
||||
reason = (
|
||||
f"{(reason or '').split(':', 1)[0][:80]} "
|
||||
"(details redacted: sanitizer unavailable)"
|
||||
).strip()
|
||||
_flashinfer_runtime_failure = reason or "unknown FlashInfer runtime failure"
|
||||
logger.warning(
|
||||
"FlashInfer disabled for this session after a runtime failure: %s",
|
||||
_flashinfer_runtime_failure,
|
||||
)
|
||||
|
||||
|
||||
def _cuda_arch_supported_for_compile() -> "tuple[bool, str]":
|
||||
"""Check the GPU's architecture against this torch build's arch list.
|
||||
|
||||
|
||||
@@ -1376,6 +1376,122 @@ def _install_compile_fallback(_model) -> None:
|
||||
_model.generate = _generate_with_compile_fallback
|
||||
|
||||
|
||||
# ── FlashInfer runtime fallback (upstream k2-fsa port) ──────────────────────
|
||||
|
||||
|
||||
def _is_flashinfer_runtime_failure(exc: BaseException) -> bool:
|
||||
"""True when an exception originates in the FlashInfer fast path (the
|
||||
flashinfer package, our omnivoice_flashinfer patch module, or CUDA-graph
|
||||
capture/replay) rather than in the model or the request itself. Same
|
||||
chain/traceback walk as ``_is_compile_runtime_failure``."""
|
||||
import traceback as _tb
|
||||
|
||||
tb_markers = ("/flashinfer/", "omnivoice_flashinfer")
|
||||
msg_markers = ("flashinfer", "cuda graph", "cudagraph")
|
||||
seen: set[int] = set()
|
||||
cur: BaseException | None = exc
|
||||
while cur is not None and id(cur) not in seen:
|
||||
seen.add(id(cur))
|
||||
mod = type(cur).__module__ or ""
|
||||
if mod.startswith("flashinfer"):
|
||||
return True
|
||||
msg = str(cur).lower()
|
||||
if any(marker in msg for marker in msg_markers):
|
||||
return True
|
||||
try:
|
||||
for frame in _tb.extract_tb(cur.__traceback__):
|
||||
filename = (frame.filename or "").replace("\\", "/")
|
||||
if any(marker in filename for marker in tb_markers):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
if cur.__cause__ is not None:
|
||||
cur = cur.__cause__
|
||||
elif not cur.__suppress_context__:
|
||||
cur = cur.__context__
|
||||
else:
|
||||
cur = None
|
||||
return False
|
||||
|
||||
|
||||
def _unapply_flashinfer(_model) -> None:
|
||||
"""Restore the standard execution path on a FlashInfer-patched model.
|
||||
|
||||
``apply_flashinfer`` works entirely through *instance-level* state —
|
||||
MethodType-bound ``forward``/``_generate_iterative`` overrides and
|
||||
``_fi_*`` attributes — so deleting those attributes restores the class
|
||||
implementations exactly. The attention implementation is restored to the
|
||||
one captured before apply (``_fi_orig_attn_impl`` — could be
|
||||
flash_attention_2, not just sdpa), and use_cache is re-enabled."""
|
||||
llm = getattr(_model, "llm", None)
|
||||
orig_attn = getattr(_model, "_fi_orig_attn_impl", None) or "sdpa"
|
||||
if llm is not None:
|
||||
for module in llm.modules():
|
||||
if "forward" in vars(module):
|
||||
del module.forward
|
||||
for attr in ("_fi_w_qkv", "_fi_qkv_split", "_fi_rope_theta", "_fi_w_gate_up"):
|
||||
if attr in vars(module):
|
||||
delattr(module, attr)
|
||||
try:
|
||||
llm.set_attn_implementation(orig_attn)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"failed to restore %s attention after FlashInfer", orig_attn
|
||||
)
|
||||
llm.config.use_cache = True
|
||||
for attr in (
|
||||
"_fi_orig_attn_impl",
|
||||
"_generate_iterative",
|
||||
"_fi_runner",
|
||||
"_fi_graph_cache",
|
||||
"_fi_enable_cuda_graph",
|
||||
"_fi_graph_buckets",
|
||||
"_fi_overhead_budget",
|
||||
):
|
||||
if attr in vars(_model):
|
||||
delattr(_model, attr)
|
||||
|
||||
|
||||
def _install_flashinfer_fallback(_model) -> None:
|
||||
"""Wrap ``model.generate`` so a FlashInfer failure at inference time falls
|
||||
back to the standard path instead of failing the generation — the same
|
||||
contract as ``_install_compile_fallback`` (#278): an optimization must
|
||||
never turn a working generation into an error."""
|
||||
orig_generate = _model.generate
|
||||
|
||||
def _generate_with_flashinfer_fallback(*args, **kwargs):
|
||||
try:
|
||||
return orig_generate(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
if not _is_flashinfer_runtime_failure(exc):
|
||||
raise
|
||||
logger.warning(
|
||||
"FlashInfer runtime failure during generation (%s: %s) — "
|
||||
"restoring the standard path and disabling FlashInfer for "
|
||||
"this session. Generation is being retried without it.",
|
||||
type(exc).__name__, exc,
|
||||
)
|
||||
from services import engine_env
|
||||
engine_env.mark_flashinfer_runtime_failure(
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
# Unapply BEFORE exposing the eager path: while the teardown
|
||||
# mutates modules, _model.generate still routes through the
|
||||
# thread-affinity wrapper, so a concurrent render queues behind
|
||||
# this call instead of racing the half-restored model (Greptile,
|
||||
# #1565 round 2). Only a fully restored model is published.
|
||||
_unapply_flashinfer(_model)
|
||||
_model.generate = orig_generate
|
||||
try:
|
||||
return orig_generate(*args, **kwargs)
|
||||
except Exception as plain_exc:
|
||||
# `from None`: a genuine standard-path failure must not be
|
||||
# chained to — and misread as — the FlashInfer error.
|
||||
raise plain_exc from None
|
||||
|
||||
_model.generate = _generate_with_flashinfer_fallback
|
||||
|
||||
|
||||
# ── #315: thread affinity for cudagraph-compiled models ─────────────────────
|
||||
# `torch.compile(mode="reduce-overhead")` captures CUDA graphs, and captured
|
||||
# graph state is **thread-local** (torch/_inductor/cudagraph_trees keys its
|
||||
@@ -2117,6 +2233,57 @@ def _load_model_sync():
|
||||
"to stop preloading it alongside TTS."
|
||||
) from asr_exc
|
||||
|
||||
# FlashInfer opt-in (upstream k2-fsa port): packed CFG attention +
|
||||
# fused kernels, ~2x on upstream's benchmarks. Applied INSTEAD of
|
||||
# torch.compile — both rewrite the llm's execution and they do not
|
||||
# compose. Best-effort: any apply failure latches the session off and
|
||||
# the standard path continues untouched.
|
||||
flashinfer_applied = False
|
||||
try:
|
||||
from services.engine_env import (
|
||||
mark_flashinfer_runtime_failure,
|
||||
should_flashinfer,
|
||||
)
|
||||
|
||||
fi_mode = should_flashinfer(device)
|
||||
if fi_mode != "off":
|
||||
_set_loading("compiling", "Applying FlashInfer kernels…")
|
||||
try:
|
||||
from omnivoice.models.omnivoice_flashinfer import apply_flashinfer
|
||||
|
||||
# Captured BEFORE apply so unapply (either the failure
|
||||
# branch below or the generate-time fallback) restores
|
||||
# the true prior implementation.
|
||||
_model._fi_orig_attn_impl = getattr(
|
||||
_model.llm.config, "_attn_implementation", "sdpa"
|
||||
)
|
||||
apply_flashinfer(_model, enable_cuda_graph=(fi_mode == "graph"))
|
||||
except Exception as fi_exc: # noqa: BLE001 — perf opt, never fatal
|
||||
mark_flashinfer_runtime_failure(
|
||||
f"{type(fi_exc).__name__}: {fi_exc}"
|
||||
)
|
||||
# apply_flashinfer mutates the model as it goes — a
|
||||
# failure partway leaves half-patched modules that would
|
||||
# crash the next render (Greptile, #1565). Restore fully.
|
||||
_unapply_flashinfer(_model)
|
||||
else:
|
||||
flashinfer_applied = True
|
||||
_install_flashinfer_fallback(_model)
|
||||
# BOTH modes pin inference to one thread. Graph mode for
|
||||
# the #315 reason (captured CUDA-graph state is
|
||||
# thread-local); eager mode because the FlashInfer
|
||||
# attention wrapper and packed position ids are planned
|
||||
# per generation in module state — two _gpu_pool workers
|
||||
# interleaving plan() and run() would corrupt each
|
||||
# other's layout (CodeRabbit/Greptile, #1565).
|
||||
_install_compile_thread_affinity(_model)
|
||||
logger.info(
|
||||
"FlashInfer applied (mode=%s) — torch.compile skipped "
|
||||
"for this load.", fi_mode,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("FlashInfer opt-in check failed; continuing without")
|
||||
|
||||
try:
|
||||
# plan-02 (#65): gate on Triton availability (+ user setting), not
|
||||
# just device==cuda. Triton has no Windows wheel, so the old
|
||||
@@ -2124,7 +2291,7 @@ def _load_model_sync():
|
||||
# falls back to eager there.
|
||||
from services.engine_env import should_torch_compile
|
||||
|
||||
if should_torch_compile(device):
|
||||
if not flashinfer_applied and should_torch_compile(device):
|
||||
_set_loading("compiling", "Compiling model (torch.compile)…")
|
||||
try:
|
||||
_model.llm = torch.compile(_model.llm, mode=_TORCH_COMPILE_MODE)
|
||||
|
||||
@@ -412,6 +412,95 @@ _PROMPT_CACHE_MAX = 8
|
||||
_prompt_cache: "OrderedDict[tuple, object]" = OrderedDict()
|
||||
_prompt_cache_lock = threading.Lock()
|
||||
|
||||
# Disk layer under the in-memory LRU (upstream k2-fsa VoiceClonePrompt.save/
|
||||
# load format). The in-memory cache dies with the process, so the first
|
||||
# generation of every session re-encodes each voice (~0.4 s + an ASR pass when
|
||||
# ref_text is missing). Encoded prompts are tiny (a (8, T) int token tensor +
|
||||
# transcript), so we persist them and reload across restarts. Keyed by the
|
||||
# same tuple as the memory cache — the ref file's mtime is inside the key, so
|
||||
# an edited reference never matches a stale file; stale files age out via the
|
||||
# mtime prune. Best-effort like the memory cache: any failure means "no disk
|
||||
# hit / no disk write", never a failed generation. OMNIVOICE_PROMPT_DISK_CACHE=0
|
||||
# disables the layer entirely.
|
||||
_PROMPT_DISK_CACHE_MAX = 32
|
||||
|
||||
|
||||
def _prompt_disk_dir():
|
||||
"""Return the prompt-cache directory (created on first use), or None when
|
||||
the layer is disabled or the directory can't be created."""
|
||||
if os.environ.get("OMNIVOICE_PROMPT_DISK_CACHE", "1") == "0":
|
||||
return None
|
||||
try:
|
||||
from core.config import DATA_DIR
|
||||
|
||||
path = os.path.join(str(DATA_DIR), "prompt_cache")
|
||||
os.makedirs(path, exist_ok=True)
|
||||
return path
|
||||
except Exception as e: # noqa: BLE001 — cache layer must never break synthesis
|
||||
logger.debug("prompt disk cache unavailable: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _prompt_disk_path(cache_dir: str, key: tuple) -> str:
|
||||
import hashlib
|
||||
|
||||
digest = hashlib.sha256(repr(key).encode("utf-8")).hexdigest()[:32]
|
||||
return os.path.join(cache_dir, f"{digest}.pt")
|
||||
|
||||
|
||||
def _prompt_disk_load(key: tuple):
|
||||
"""Load a persisted prompt for ``key``, or None. Never raises."""
|
||||
cache_dir = _prompt_disk_dir()
|
||||
if cache_dir is None:
|
||||
return None
|
||||
path = _prompt_disk_path(cache_dir, key)
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
from omnivoice.models.omnivoice import VoiceClonePrompt
|
||||
|
||||
prompt = VoiceClonePrompt.load(path)
|
||||
# Freshen so the LRU prune (by mtime) keeps actively used voices.
|
||||
os.utime(path, None)
|
||||
return prompt
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("failed to load cached voice prompt %s: %s", path, e)
|
||||
try:
|
||||
os.remove(path) # corrupt/incompatible file — don't retry it forever
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _prompt_disk_save(key: tuple, prompt) -> None:
|
||||
"""Persist ``prompt`` under ``key`` and prune old entries. Never raises."""
|
||||
cache_dir = _prompt_disk_dir()
|
||||
if cache_dir is None:
|
||||
return
|
||||
path = _prompt_disk_path(cache_dir, key)
|
||||
try:
|
||||
# Unique per write: two GPU-pool threads missing the same key must not
|
||||
# interleave writes into one tmp file (os.replace stays atomic).
|
||||
import uuid
|
||||
|
||||
tmp = f"{path}.tmp.{os.getpid()}.{uuid.uuid4().hex[:8]}"
|
||||
prompt.save(tmp)
|
||||
os.replace(tmp, path)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("failed to persist voice prompt to %s: %s", path, e)
|
||||
return
|
||||
try:
|
||||
entries = [
|
||||
os.path.join(cache_dir, f)
|
||||
for f in os.listdir(cache_dir)
|
||||
if f.endswith(".pt")
|
||||
]
|
||||
entries.sort(key=lambda p: os.path.getmtime(p), reverse=True)
|
||||
for old in entries[_PROMPT_DISK_CACHE_MAX:]:
|
||||
os.remove(old)
|
||||
except OSError as e:
|
||||
logger.debug("prompt disk cache prune skipped: %s", e)
|
||||
|
||||
|
||||
def _clone_prompt_key(ref_audio: str, ref_text, preprocess_prompt: bool = True):
|
||||
try:
|
||||
@@ -450,15 +539,24 @@ def _get_clone_prompt(
|
||||
if hit is not None:
|
||||
_prompt_cache.move_to_end(key)
|
||||
return hit
|
||||
try:
|
||||
# Encode outside the lock (slow). Mirrors exactly what generate() would
|
||||
# do inline for this ref (omnivoice.py:964-978), so output is identical.
|
||||
prompt = model.create_voice_clone_prompt(
|
||||
ref_audio, ref_text=ref_text, preprocess_prompt=preprocess_prompt
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 — fall back, never break synthesis
|
||||
logger.warning("voice-clone prompt precompute failed; using inline ref: %s", e)
|
||||
return None
|
||||
# Memory miss → disk (survives restarts). A disk hit skips the encode AND
|
||||
# the ASR transcription pass a ref_text-less reference would trigger.
|
||||
prompt = _prompt_disk_load(key)
|
||||
if prompt is None:
|
||||
try:
|
||||
# Encode outside the lock (slow). Mirrors exactly what generate()
|
||||
# would do inline for this ref (omnivoice.py:964-978), so output is
|
||||
# identical.
|
||||
prompt = model.create_voice_clone_prompt(
|
||||
ref_audio, ref_text=ref_text, preprocess_prompt=preprocess_prompt
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 — fall back, never break synthesis
|
||||
logger.warning(
|
||||
"voice-clone prompt precompute failed; using inline ref: %s", e
|
||||
)
|
||||
return None
|
||||
if store:
|
||||
_prompt_disk_save(key, prompt)
|
||||
if not store:
|
||||
return prompt
|
||||
with _prompt_cache_lock:
|
||||
|
||||
+40
-15
@@ -460,30 +460,55 @@ class TaskExecutor:
|
||||
|
||||
@staticmethod
|
||||
def _synthesize(backend, text: str, params: dict):
|
||||
"""Call the engine through the same serial GPU gate local jobs use.
|
||||
"""Render through the same seeded pipeline as local ``/generate``.
|
||||
|
||||
Held against the idle sweep for the duration: a long generation touches
|
||||
the instance cache once, at the start, so on elapsed time alone it is
|
||||
indistinguishable from a model nobody wants any more.
|
||||
|
||||
Do not reduce this to ``backend.generate()``. The control plane sends
|
||||
a complete render contract (pinned gallery seed, synthetic reference,
|
||||
quality controls, chunking, effects); calling the adapter directly
|
||||
silently turns a selected gallery voice into a fresh random take.
|
||||
"""
|
||||
from services import tts_backend # noqa: PLC0415
|
||||
from api.routers.generation import _run_backend_inference, _run_inference # noqa: PLC0415
|
||||
|
||||
kwargs = {
|
||||
key: params[key]
|
||||
for key in (
|
||||
"ref_audio",
|
||||
"ref_text",
|
||||
"instruct",
|
||||
"language",
|
||||
"duration",
|
||||
"description",
|
||||
"speed",
|
||||
)
|
||||
if params.get(key) is not None
|
||||
}
|
||||
language = params.get("language")
|
||||
ref_audio = params.get("ref_audio")
|
||||
ref_text = params.get("ref_text")
|
||||
instruct = params.get("instruct")
|
||||
duration = params.get("duration")
|
||||
num_step = params.get("num_step", 16)
|
||||
guidance_scale = params.get("guidance_scale", 2.0)
|
||||
speed = params.get("speed", 1.0)
|
||||
denoise = params.get("denoise", True)
|
||||
postprocess_output = params.get("postprocess_output", True)
|
||||
used_seed = params.get("seed")
|
||||
effect_preset = params.get("effect_preset", "broadcast")
|
||||
max_chunk_chars = params.get("max_chunk_chars")
|
||||
crossfade_ms = params.get("crossfade_ms")
|
||||
try:
|
||||
with tts_backend.engine_in_use(backend):
|
||||
return backend.generate(text, **kwargs)
|
||||
if isinstance(backend, tts_backend.OmniVoiceBackend):
|
||||
# The OSS default engine has an extended native surface;
|
||||
# preserving it is required for a gallery preview and a
|
||||
# GPU-worker take to share the same voice identity.
|
||||
return _run_inference(
|
||||
backend._model, text, language, ref_audio, ref_text,
|
||||
instruct, duration, num_step, guidance_scale, speed,
|
||||
params.get("t_shift"), denoise, postprocess_output,
|
||||
params.get("layer_penalty_factor"),
|
||||
params.get("position_temperature"),
|
||||
params.get("class_temperature"), used_seed,
|
||||
effect_preset, max_chunk_chars, crossfade_ms,
|
||||
)
|
||||
return _run_backend_inference(
|
||||
backend, text, language, ref_audio, ref_text, instruct,
|
||||
duration, num_step, guidance_scale, speed, denoise,
|
||||
postprocess_output, used_seed, effect_preset,
|
||||
max_chunk_chars, crossfade_ms,
|
||||
)
|
||||
except Exception as exc:
|
||||
from worker import errors as worker_errors # noqa: PLC0415
|
||||
|
||||
|
||||
@@ -57,6 +57,10 @@ REQUIRED_FEATURES = frozenset({
|
||||
"task_progress_v1",
|
||||
"task_inputs_v1",
|
||||
"remote_model_download_v1",
|
||||
# A generic backend.generate() call accepts the same wire shape but drops
|
||||
# profile conditioning controls. Require the canonical worker render path
|
||||
# so an older peer cannot successfully return a different voice.
|
||||
"remote_tts_render_v1",
|
||||
})
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -342,7 +342,7 @@ arbitrary path merely because it ends in `/ws/events` or `/ws/transcribe`.
|
||||
| Code | Meaning | What to do |
|
||||
|---|---|---|
|
||||
| **401** | Consumption auth failed — `{"detail": "PIN required"}` or `{"detail": "API key required"}`. | Supply the PIN / key (header, cookie, or query param above). A WebSocket surfaces this as close code **1008**. |
|
||||
| **403** | Authorization failed: loopback/native access was required, cookie Origin/CSRF validation failed, a server-mode mutation lacked an admin credential, or a native path capability was invalid/expired. | A PIN cannot grant admin or filesystem access. Re-authenticate the UI; scripts should use the API-key header; run native operations from the desktop app. |
|
||||
| **403** | Authorization failed: loopback/native access was required, cookie Origin/CSRF validation failed, a server-mode mutation lacked an admin credential, or a native path capability was invalid/expired. | A PIN cannot grant admin or filesystem access. Re-authenticate the UI; scripts should use the API-key header; run native operations from the desktop app. The admin gate names the key only when one can satisfy it: server mode with `OMNIVOICE_API_KEY` configured answers `{"detail": "loopback origin or admin API key required"}` (the bundled UI routes it to the API-key login form); PIN-only/no-key server mode and the desktop build answer `{"detail": "loopback origin required"}` (only loopback can satisfy the gate). |
|
||||
| **429** | A failed administrator-session exchange exceeded its per-client limit, the GPU pool is saturated, or a model download is rate-limited. Ships with `Retry-After`; workload throttles also carry `X-VoiceStudio-Retryable: true`. | Back off for `Retry-After` seconds. For authentication, verify the master before retrying; a correct master is never locked out. |
|
||||
|
||||
---
|
||||
|
||||
@@ -47,13 +47,39 @@ The env var overrides the persisted UI choice.
|
||||
co-loaded for the cloning path.
|
||||
- Output is 24 kHz mono; the shared mastering chain (highpass + compressor)
|
||||
is tuned for this rate and applied automatically.
|
||||
- Cloning takes a short reference clip (`ref_audio`); an optional transcript
|
||||
of the clip improves conditioning.
|
||||
- Cloning takes a short reference clip (`ref_audio`); 3–10 seconds is the
|
||||
sweet spot. A transcript of the clip improves conditioning — if the profile
|
||||
has none, VoiceStudio transcribes the clip automatically on first use and
|
||||
saves the result to the profile.
|
||||
- Encoded voice references persist on disk (`prompt_cache/` in the app data
|
||||
dir), so the first generation with a known voice after a restart skips the
|
||||
re-encode and any transcription pass. Set `OMNIVOICE_PROMPT_DISK_CACHE=0`
|
||||
to keep the cache in memory only.
|
||||
- Style attributes (`instruct`) and a reference clip can be **combined**:
|
||||
when they agree, the instruct stabilizes cloning for the attributes it
|
||||
names (upstream documents dialect cloning as the canonical case — dialect
|
||||
reference + matching dialect instruct). When they conflict, the reference
|
||||
audio wins.
|
||||
- Inline pronunciation control: Chinese via pinyin with tone numbers
|
||||
(`打ZHE2出售`), English via bracketed CMU phonemes (`[B EY1 S]`). Non-verbal
|
||||
tags like `[laughter]` are covered in
|
||||
[expressive-speech.md](../expressive-speech.md).
|
||||
- Voice design works from attributes (gender, age, pitch, whisper, English
|
||||
accents, Chinese dialects) via the Design tab — no reference audio needed.
|
||||
- Optional FlashInfer acceleration on CUDA: set `OMNIVOICE_FLASHINFER=1`
|
||||
(or `=graph` for CUDA-graph capture, best for one render at a time) after
|
||||
installing the `flashinfer-python` package — see
|
||||
[performance.md](../performance.md). Off by default; if the package is
|
||||
missing or a kernel fails, the app logs why and continues on the standard
|
||||
path.
|
||||
|
||||
## Known limits
|
||||
|
||||
- No voice design from a text description — use [VoxCPM2](voxcpm2.md) for
|
||||
that.
|
||||
- Voice design understands only the fixed attribute vocabulary — free-form
|
||||
design *prose* is mapped onto those attributes, and wording outside them
|
||||
is ignored. Design is trained on English and Chinese and can be unstable
|
||||
in low-resource languages; for description-driven design in other cases
|
||||
try [VoxCPM2](voxcpm2.md).
|
||||
- Below the 6 GB VRAM floor, expect very slow renders or budget timeouts;
|
||||
prefer [OmniVoice GGUF](omnivoice-gguf.md) or a CPU engine such as
|
||||
[PocketTTS](pockettts.md).
|
||||
|
||||
@@ -81,6 +81,8 @@ None of them are required — the defaults are chosen for the common case.
|
||||
| Variable | Default | What it does |
|
||||
|---|---|---|
|
||||
| `OMNIVOICE_DEVICE` | `auto` | Pin the compute device (`cuda` / `rocm` / `xpu` / `mps` / `cpu`) instead of auto-detect. Same control lives in **Settings → Performance & Device** (the env var wins over the UI pick). Honored only for devices the host actually has — a family that isn't detected is noted and ignored, never obeyed blindly. Applies at the next backend start. |
|
||||
| `OMNIVOICE_FLASHINFER` | `0` | CUDA-only accelerated decoding for the default engine via [FlashInfer](https://github.com/flashinfer-ai/flashinfer) kernels (packed CFG attention, fused RMSNorm/RoPE/GEMM) — ~2x on upstream's benchmarks. `1` enables it; `graph` also captures CUDA graphs (best when you render one thing at a time). Requires installing the optional `flashinfer-python` package into the backend environment first (`uv pip install flashinfer-python flashinfer-jit-cache --extra-index-url https://flashinfer.ai/whl/cu128/`, matching your CUDA build). Replaces `torch.compile` for that session, pins inference to a single GPU thread (the FlashInfer attention plan is per-generation state), and keeps fused copies of the attention/MLP weights resident (~roughly half the LLM's weight size extra VRAM) — leave it off on tight-VRAM cards. If the package is missing or a FlashInfer/CUDA-graph kernel fails at runtime, the app logs the reason and falls back to the standard path; failures outside those kernels (e.g. a genuine out-of-memory) surface normally. |
|
||||
| `OMNIVOICE_PROMPT_DISK_CACHE` | `1` | Persist encoded voice-clone references (`prompt_cache/` in the app data dir, ~10 KB per voice, 32 newest kept) so the first generation with a known voice after a restart skips the reference re-encode and any auto-transcription. Set `0` to keep the cache in memory only. |
|
||||
| `OMNIVOICE_IDLE_TIMEOUT_S` | `900` | Seconds of idle before the TTS model unloads to free memory. Raise it (e.g. `3600`) if you generate in bursts and dislike the ~8 s reload; lower it on tight-memory machines. |
|
||||
| `OMNIVOICE_SIDECAR_IDLE_TIMEOUT_S` | `300` | Same idea for sidecar engines (IndexTTS 2.5 etc.). |
|
||||
| `OMNIVOICE_LLM_CONCURRENCY` | `6` | Parallel LLM translation calls during a dub. Raise for a fast API endpoint, lower if your provider rate-limits. |
|
||||
|
||||
+13
-2
@@ -153,6 +153,15 @@ fallback is reported once. ASR, diarization and translation also remain local. D
|
||||
runs here, deliberately and permanently, because there latency *is* the
|
||||
feature. The remaining operations are being ported one at a time.
|
||||
|
||||
### Voice identity parity
|
||||
|
||||
For TTS, the worker receives the complete local rendering contract: the voice
|
||||
profile's reference audio and transcript, its pinned seed, model quality
|
||||
controls, text chunking/crossfade settings, and output effect preset. The
|
||||
worker runs the same native or generic rendering pipeline as local
|
||||
`/generate`; selecting a gallery voice therefore does not turn it into a new
|
||||
random voice merely because it was rendered on another GPU.
|
||||
|
||||
The picker knows this. It resolves against the surface you are on, so a chosen
|
||||
worker reads **Local** on a tab whose work has no remote path yet and names the
|
||||
reason, instead of showing a green dot next to a GPU that receives nothing. The
|
||||
@@ -210,10 +219,12 @@ what is genuinely still in flight.
|
||||
**Version or feature mismatch.** The protocol keeps a two-release compatibility
|
||||
window, but release numbers alone do not prove that a worker understands every
|
||||
additive command. Registration therefore also declares named features for task
|
||||
inputs, progress leases, and remote model downloads. A worker outside the
|
||||
inputs, progress leases, remote model downloads, and the voice-identity render
|
||||
pipeline. A worker outside the
|
||||
version window, or one missing a required feature, is refused with
|
||||
`UPGRADE_REQUIRED` and an update instruction before any task runs. It can never
|
||||
silently render without reference audio or leave a download stuck at 0%.
|
||||
silently render without reference audio, substitute a different voice, or leave
|
||||
a download stuck at 0%.
|
||||
|
||||
Every remote failure includes a concrete next step. Capacity, missing models,
|
||||
expired leases or sessions, authentication, rejected inputs, and result upload
|
||||
|
||||
@@ -182,15 +182,16 @@ describe('apiFetch 401 routing', () => {
|
||||
dispatch.mockRestore();
|
||||
});
|
||||
|
||||
const stub401 = (detail: string) =>
|
||||
const stubStatus = (status: number, statusText: string, detail: string) =>
|
||||
vi.fn(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 401,
|
||||
statusText: 'Unauthorized',
|
||||
status,
|
||||
statusText,
|
||||
text: async () => JSON.stringify({ detail }),
|
||||
}),
|
||||
) as any;
|
||||
const stub401 = (detail: string) => stubStatus(401, 'Unauthorized', detail);
|
||||
|
||||
const authEvent = () =>
|
||||
dispatch.mock.calls.map((c) => c[0]).find((e) => (e as Event).type === 'ov:auth-required');
|
||||
@@ -227,6 +228,70 @@ describe('apiFetch 401 routing', () => {
|
||||
expect(authEvent()).toBeTruthy();
|
||||
expect((authEvent() as any).detail.mode).toBe('pin');
|
||||
});
|
||||
|
||||
const stub403 = (detail: string) => stubStatus(403, 'Forbidden', detail);
|
||||
|
||||
it('dispatches ov:auth-required {mode:"apikey"} on an admin-gate 403 (#1525)', async () => {
|
||||
globalThis.fetch = stub403('loopback origin or admin API key required');
|
||||
const { apiFetch } = await import('./client');
|
||||
try {
|
||||
await apiFetch('/system/info');
|
||||
} catch {
|
||||
/* ApiError expected */
|
||||
}
|
||||
expect(authEvent()).toBeTruthy();
|
||||
expect((authEvent() as any).detail.mode).toBe('apikey');
|
||||
});
|
||||
|
||||
it('does not dispatch ov:auth-required on other 403s (CSRF / desktop-only)', async () => {
|
||||
globalThis.fetch = stub403('browser origin rejected');
|
||||
const { apiFetch } = await import('./client');
|
||||
try {
|
||||
await apiFetch('/system/info');
|
||||
} catch {
|
||||
/* ApiError expected */
|
||||
}
|
||||
expect(authEvent()).toBeFalsy();
|
||||
});
|
||||
|
||||
it('a stale 403 neither clears a new session nor reopens the auth gate (PR #1569 race)', async () => {
|
||||
// The request goes out with an old credential; while it is in flight the
|
||||
// user completes another key exchange. A late 403 may only invalidate the
|
||||
// credentials the failed request actually carried — wiping the fresh
|
||||
// session or reopening the gate would undo the successful login.
|
||||
sessionStorage.setItem(
|
||||
ADMIN_SESSION_STORAGE_KEY,
|
||||
JSON.stringify({
|
||||
token: `ovs_admin_session_${'O'.repeat(43)}`,
|
||||
expiresAt: Date.now() / 1000 + 3600,
|
||||
apiBase: API,
|
||||
}),
|
||||
);
|
||||
globalThis.fetch = vi.fn(() => {
|
||||
sessionStorage.setItem(
|
||||
ADMIN_SESSION_STORAGE_KEY,
|
||||
JSON.stringify({
|
||||
token: `ovs_admin_session_${'N'.repeat(43)}`,
|
||||
expiresAt: Date.now() / 1000 + 3600,
|
||||
apiBase: API,
|
||||
}),
|
||||
);
|
||||
return Promise.resolve({
|
||||
ok: false,
|
||||
status: 403,
|
||||
statusText: 'Forbidden',
|
||||
text: async () => JSON.stringify({ detail: 'loopback origin or admin API key required' }),
|
||||
});
|
||||
}) as any;
|
||||
const { apiFetch } = await import('./client');
|
||||
try {
|
||||
await apiFetch('/system/info');
|
||||
} catch {
|
||||
/* ApiError expected */
|
||||
}
|
||||
expect(authEvent()).toBeFalsy();
|
||||
expect(sessionStorage.getItem(ADMIN_SESSION_STORAGE_KEY)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('apiFetch 404 from a non-VoiceStudio server (#1385)', () => {
|
||||
|
||||
@@ -528,15 +528,37 @@ export async function apiFetch(path: string, opts: ApiFetchOptions = {}): Promis
|
||||
// "API key required" (BearerKeyMiddleware, OMNIVOICE_API_KEY) vs anything
|
||||
// else, i.e. "PIN required" (NetworkAccessMiddleware). Both are 401; the
|
||||
// detail is the only discriminator (only two 401 sites exist backend-side).
|
||||
if (backendTarget && res.status === 401 && typeof window !== 'undefined') {
|
||||
// The router-level admin gates answer 403 "loopback origin or admin API
|
||||
// key required" (require_admin/require_admin_action) — same situation, the
|
||||
// client just isn't admin-authenticated — so it routes to the API-key form
|
||||
// too. Other 403s (CSRF "browser origin rejected", loopback-only routes)
|
||||
// are NOT credential gaps; presenting a key won't help, so they stay plain
|
||||
// errors.
|
||||
const adminGate403 =
|
||||
res.status === 403 &&
|
||||
typeof detail === 'string' &&
|
||||
detail.toLowerCase().includes('admin api key');
|
||||
if (backendTarget && (res.status === 401 || adminGate403) && typeof window !== 'undefined') {
|
||||
// readError's declared `string` return isn't guaranteed at runtime —
|
||||
// `j.detail` can be a structured object/array on a future 401. Match only
|
||||
// real strings (avoids both a `.toLowerCase()` crash and `String()` itself
|
||||
// throwing on a malformed object); anything else falls back to PIN.
|
||||
// (No adminGate403 arm here: "admin api key" ⊇ "api key", so the sniff
|
||||
// below already yields 'apikey' for every admin-gate 403.)
|
||||
const mode =
|
||||
typeof detail === 'string' && detail.toLowerCase().includes('api key') ? 'apikey' : 'pin';
|
||||
if (mode === 'apikey') clearAdminSession();
|
||||
window.dispatchEvent(new CustomEvent('ov:auth-required', { detail: { mode } }));
|
||||
// A failed response may only invalidate the credentials it actually
|
||||
// carried (`session` is captured at send time). Clearing blindly let
|
||||
// a stale 403 that landed after a key exchange wipe the fresh
|
||||
// session, reloading a successful login straight back into the gate.
|
||||
const currentSession = getAdminSession(API);
|
||||
const staleAdminResponse = mode === 'apikey' && currentSession?.token !== session?.token;
|
||||
if (mode === 'apikey' && !staleAdminResponse && session) {
|
||||
clearAdminSession();
|
||||
}
|
||||
if (!staleAdminResponse) {
|
||||
window.dispatchEvent(new CustomEvent('ov:auth-required', { detail: { mode } }));
|
||||
}
|
||||
}
|
||||
// Structured details (e.g. the typed asr_model_missing 409) carry a
|
||||
// human-readable `message` — use it for the Error message instead of
|
||||
|
||||
@@ -10,7 +10,7 @@ import { useModelStatus } from '../api/hooks';
|
||||
import useRealtimeEvents from './useRealtimeEvents';
|
||||
import { mergeDescribedAttrs } from '../utils/voiceInstruct';
|
||||
import { sanitizeOmniUi } from '../utils/omniUiSchema';
|
||||
import { retryInitialLoad } from '../utils/initialLoadRetry';
|
||||
import { loadLatest, retryInitialLoad } from '../utils/initialLoadRetry';
|
||||
import { queueJsonWrite } from '../utils/coalescedJsonStorage';
|
||||
|
||||
/**
|
||||
@@ -176,16 +176,15 @@ export default function useAppData() {
|
||||
const loadersRef = useRef({ profiles: 0, history: 0, dub: 0, projects: 0, exports: 0 });
|
||||
const makeLoader =
|
||||
(key, fetch, set, label) =>
|
||||
async ({ rethrow } = {}) => {
|
||||
const gen = ++loadersRef.current[key];
|
||||
try {
|
||||
const data = await fetch();
|
||||
if (gen === loadersRef.current[key]) set(data);
|
||||
} catch (e) {
|
||||
console.warn(`Failed to load ${label}:`, e);
|
||||
if (rethrow) throw e;
|
||||
}
|
||||
};
|
||||
({ rethrow } = {}) =>
|
||||
loadLatest({
|
||||
generations: loadersRef.current,
|
||||
key,
|
||||
fetch,
|
||||
apply: set,
|
||||
label,
|
||||
rethrow,
|
||||
});
|
||||
const loadProfiles = makeLoader('profiles', listProfiles, setProfiles, 'voice profiles');
|
||||
const loadHistory = makeLoader('history', listHistory, setHistory, 'generation history');
|
||||
const loadDubHistory = makeLoader('dub', listDubHistory, setDubHistory, 'dub history');
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { retryInitialLoad } from '../utils/initialLoadRetry';
|
||||
import { loadLatest, retryInitialLoad } from '../utils/initialLoadRetry';
|
||||
|
||||
// #1158 class: the initial data loads (profiles/history/…) ran exactly once
|
||||
// after the backend became reachable. A transient failure on that single call
|
||||
@@ -61,7 +61,44 @@ describe('retryInitialLoad (#1158 class)', () => {
|
||||
]) {
|
||||
expect(initialBlock).toContain(`${loader}({ rethrow: true })`);
|
||||
}
|
||||
// and the loader factory must actually rethrow when asked
|
||||
expect(src).toContain('if (rethrow) throw e;');
|
||||
expect(src).toContain('rethrow,');
|
||||
});
|
||||
});
|
||||
|
||||
describe('loadLatest', () => {
|
||||
it('does not let a stale initial response overwrite a WebSocket reload', async () => {
|
||||
let resolveInitial;
|
||||
let resolveReload;
|
||||
const initial = new Promise((resolve) => {
|
||||
resolveInitial = resolve;
|
||||
});
|
||||
const reload = new Promise((resolve) => {
|
||||
resolveReload = resolve;
|
||||
});
|
||||
const generations = {};
|
||||
const apply = vi.fn();
|
||||
|
||||
const initialLoad = loadLatest({
|
||||
generations,
|
||||
key: 'profiles',
|
||||
fetch: () => initial,
|
||||
apply,
|
||||
label: 'profiles',
|
||||
});
|
||||
const websocketReload = loadLatest({
|
||||
generations,
|
||||
key: 'profiles',
|
||||
fetch: () => reload,
|
||||
apply,
|
||||
label: 'profiles',
|
||||
});
|
||||
|
||||
resolveReload(['new']);
|
||||
await websocketReload;
|
||||
resolveInitial(['stale']);
|
||||
await initialLoad;
|
||||
|
||||
expect(apply).toHaveBeenCalledOnce();
|
||||
expect(apply).toHaveBeenCalledWith(['new']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -27,3 +27,16 @@ export async function retryInitialLoad(loader, opts = {}) {
|
||||
delay = Math.min((delay || baseDelayMs) * 2, maxDelayMs);
|
||||
}
|
||||
}
|
||||
|
||||
/** Apply only the newest invocation's response for a named list. */
|
||||
export async function loadLatest({ generations, key, fetch, apply, label, rethrow = false }) {
|
||||
const generation = (generations[key] ?? 0) + 1;
|
||||
generations[key] = generation;
|
||||
try {
|
||||
const data = await fetch();
|
||||
if (generation === generations[key]) apply(data);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to load ${label}:`, error);
|
||||
if (rethrow) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,12 +91,60 @@ class OmniVoiceModelAssetError(RuntimeError):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_VOICE_CLONE_PROMPT_FORMAT_VERSION = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoiceClonePrompt:
|
||||
ref_audio_tokens: torch.Tensor # (C, T)
|
||||
ref_text: str
|
||||
ref_rms: float
|
||||
|
||||
def save(self, path: str) -> None:
|
||||
"""Save this prompt to ``path`` for reuse in a later session.
|
||||
|
||||
The file stores a plain dict with the audio tokens moved to CPU, so
|
||||
it can be loaded with ``torch.load(weights_only=True)`` (the default
|
||||
since torch 2.6) and is portable across devices.
|
||||
|
||||
Args:
|
||||
path: Destination file path (e.g. ``"my_voice.pt"``).
|
||||
"""
|
||||
torch.save(
|
||||
{
|
||||
"format_version": _VOICE_CLONE_PROMPT_FORMAT_VERSION,
|
||||
"ref_audio_tokens": self.ref_audio_tokens.detach().cpu(),
|
||||
"ref_text": self.ref_text,
|
||||
"ref_rms": float(self.ref_rms),
|
||||
},
|
||||
path,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str, map_location: str = "cpu") -> "VoiceClonePrompt":
|
||||
"""Load a prompt saved with :meth:`save`.
|
||||
|
||||
The returned prompt can be passed directly to
|
||||
:meth:`OmniVoice.generate`; the audio tokens are moved to the model
|
||||
device automatically during generation, so no manual ``.to(device)``
|
||||
is needed.
|
||||
|
||||
Args:
|
||||
path: File path previously written by :meth:`save`.
|
||||
map_location: Device to load the audio tokens onto.
|
||||
Returns:
|
||||
The restored :class:`VoiceClonePrompt`.
|
||||
"""
|
||||
data = torch.load(path, map_location=map_location, weights_only=True)
|
||||
version = data.get("format_version")
|
||||
if version != _VOICE_CLONE_PROMPT_FORMAT_VERSION:
|
||||
raise ValueError(f"Unsupported VoiceClonePrompt format version: {version}")
|
||||
return cls(
|
||||
ref_audio_tokens=data["ref_audio_tokens"],
|
||||
ref_text=data["ref_text"],
|
||||
ref_rms=data["ref_rms"],
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OmniVoiceGenerationConfig:
|
||||
|
||||
@@ -0,0 +1,667 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FlashInfer-accelerated iterative decoding for OmniVoice.
|
||||
|
||||
Approach (mirrors CosyVoice/runtime/triton_trtllm/token2wav_dit_flashinfer.py):
|
||||
|
||||
- Sequence packing: the baseline pads the uncond (CFG) sequence to the cond
|
||||
length and runs batch=2 with a (2,1,S,S) bool mask. Here cond+uncond are
|
||||
packed into ONE row of length c_len+u_len with per-document positions and
|
||||
flashinfer ragged attention (qo_indptr = document boundaries) — no pad
|
||||
compute, no S^2 mask materialization.
|
||||
- Attention: registered as a custom HF attention implementation
|
||||
("omnivoice_fi") via AttentionInterface; reads the wrapper planned
|
||||
once per generation from a module-level context. HF mask construction is
|
||||
bypassed by passing attention_mask={"full_attention": None}.
|
||||
- KV cache: disabled (llm.config.use_cache=False). Iterative bidirectional
|
||||
decoding recomputes the full sequence every step, so the DynamicCache the
|
||||
baseline builds each forward is pure overhead.
|
||||
- Optional CUDA graphs: one graph per packed shape; all 32 denoising steps
|
||||
replay the same graph (input_ids/audio_mask/position_ids are copied into
|
||||
static buffers). Each shape owns a private flashinfer wrapper, since a
|
||||
plan bakes its launch metadata into the captured graph.
|
||||
|
||||
Usage:
|
||||
from omnivoice_flashinfer import apply_flashinfer
|
||||
apply_flashinfer(model, enable_cuda_graph=True)
|
||||
|
||||
Ported from upstream k2-fsa/OmniVoice master with one behavioural change:
|
||||
the unmasking schedule uses ``num_step + 1`` timesteps to match this repo's
|
||||
``_generate_iterative``. VoiceStudio enables it via ``OMNIVOICE_FLASHINFER``
|
||||
(see services/model_manager.py); ``flashinfer`` is an optional dependency and
|
||||
this module must only be imported after that opt-in.
|
||||
"""
|
||||
|
||||
import math
|
||||
import time
|
||||
from types import MethodType
|
||||
from typing import List
|
||||
|
||||
import flashinfer
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers.modeling_utils import AttentionInterface
|
||||
|
||||
from omnivoice.models.omnivoice import (
|
||||
GenerationTask,
|
||||
OmniVoiceGenerationConfig,
|
||||
_get_time_steps,
|
||||
_gumbel_sample,
|
||||
)
|
||||
|
||||
_WORKSPACE_SIZE = 128 * 1024 * 1024
|
||||
# Context read by the registered attention function. "wrapper" must be planned
|
||||
# for the current packed layout before any llm forward.
|
||||
_CTX = {"wrapper": None}
|
||||
|
||||
|
||||
def _flashinfer_attention(
|
||||
module, query, key, value, attention_mask, scaling=None, dropout=0.0, **kwargs
|
||||
):
|
||||
"""query (1, Hq, S, D), key/value (1, Hkv, S, D) — packed documents."""
|
||||
_b, hq, s, d = query.shape
|
||||
hkv = key.shape[1]
|
||||
q = query.transpose(1, 2).reshape(s, hq, d)
|
||||
k = key.transpose(1, 2).reshape(s, hkv, d)
|
||||
v = value.transpose(1, 2).reshape(s, hkv, d)
|
||||
out = _CTX["wrapper"].run(q, k, v) # (S, Hq, D)
|
||||
return out.view(1, s, hq, d), None
|
||||
|
||||
|
||||
AttentionInterface.register("omnivoice_fi", _flashinfer_attention)
|
||||
|
||||
|
||||
def _fi_rmsnorm_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Single-kernel replacement for Qwen3RMSNorm.forward (a 7-kernel
|
||||
fp32-upcast chain in eager mode). flashinfer.norm.rmsnorm computes in
|
||||
fp32 internally and matches to fp16 rounding."""
|
||||
shape = hidden_states.shape
|
||||
out = flashinfer.norm.rmsnorm(
|
||||
hidden_states.reshape(-1, shape[-1]).contiguous(),
|
||||
self.weight,
|
||||
eps=self.variance_epsilon,
|
||||
)
|
||||
return out.view(shape)
|
||||
|
||||
|
||||
def _patch_rmsnorm(llm):
|
||||
from transformers.models.qwen3.modeling_qwen3 import Qwen3RMSNorm
|
||||
|
||||
n = 0
|
||||
for module in llm.modules():
|
||||
if isinstance(module, Qwen3RMSNorm):
|
||||
module.forward = MethodType(_fi_rmsnorm_forward, module)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def _fi_attention_module_forward(
|
||||
self,
|
||||
hidden_states,
|
||||
position_embeddings=None,
|
||||
attention_mask=None,
|
||||
past_key_values=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""NHD-layout replacement for Qwen3Attention.forward (packed batch=1).
|
||||
|
||||
The stock forward works in (B, H, S, D): the rotate-half RoPE costs a cat
|
||||
plus four elementwise passes, and handing (B,H,S,D) to the ragged wrapper
|
||||
costs three transpose copies. Keeping everything in (S, H, D) removes all
|
||||
of that; RoPE is one fused in-place kernel driven by packed position ids
|
||||
(read from _CTX, set per generation / baked per graph)."""
|
||||
s = hidden_states.shape[1]
|
||||
x = hidden_states[0] # (S, hidden)
|
||||
if getattr(self, "_fi_w_qkv", None) is not None:
|
||||
qkv = F.linear(x, self._fi_w_qkv)
|
||||
q, k, v = qkv.split(self._fi_qkv_split, dim=-1)
|
||||
# split views are strided; reshape materializes contiguous copies
|
||||
# (q/k would be copied inside the fused rmsnorm anyway)
|
||||
q = self.q_norm(q.reshape(s, -1, self.head_dim))
|
||||
k = self.k_norm(k.reshape(s, -1, self.head_dim))
|
||||
v = v.reshape(s, -1, self.head_dim)
|
||||
else:
|
||||
q = self.q_norm(self.q_proj(x).view(s, -1, self.head_dim))
|
||||
k = self.k_norm(self.k_proj(x).view(s, -1, self.head_dim))
|
||||
v = self.v_proj(x).view(s, -1, self.head_dim)
|
||||
flashinfer.rope.apply_rope_pos_ids_inplace(
|
||||
q, k, _CTX["pos_ids"], rope_theta=self._fi_rope_theta, interleave=False
|
||||
)
|
||||
slots = _CTX.get("doc_slots")
|
||||
if slots is not None:
|
||||
# bucketed-graph mode: a flashinfer plan bakes document boundaries
|
||||
# into the graph, so attention runs per fixed-length document slot as
|
||||
# SDPA with an O(slot) key-padding mask whose contents are rewritten
|
||||
# per generation. (A dense (S,S) block-diag mask scales quadratically
|
||||
# and the enable_gqa+mask combo drops SDPA to the math backend, so
|
||||
# k/v are pre-expanded to full heads instead.)
|
||||
ng = self.num_key_value_groups
|
||||
k = k.repeat_interleave(ng, dim=1) # (S, Hq, D)
|
||||
v = v.repeat_interleave(ng, dim=1)
|
||||
out = torch.empty_like(q)
|
||||
for start, slot_len, m in slots:
|
||||
od = F.scaled_dot_product_attention(
|
||||
q[start : start + slot_len].transpose(0, 1).unsqueeze(0),
|
||||
k[start : start + slot_len].transpose(0, 1).unsqueeze(0),
|
||||
v[start : start + slot_len].transpose(0, 1).unsqueeze(0),
|
||||
attn_mask=m,
|
||||
)
|
||||
out[start : start + slot_len] = od.squeeze(0).transpose(0, 1)
|
||||
else:
|
||||
out = _CTX["wrapper"].run(q, k, v) # (S, Hq, D)
|
||||
return self.o_proj(out.reshape(s, -1)).unsqueeze(0), None
|
||||
|
||||
|
||||
def _patch_attention_forward(llm, fuse_qkv=True):
|
||||
theta = llm.config.rope_parameters["rope_theta"]
|
||||
for layer in llm.layers:
|
||||
attn = layer.self_attn
|
||||
attn._fi_rope_theta = theta
|
||||
if fuse_qkv:
|
||||
attn._fi_w_qkv = torch.cat(
|
||||
[attn.q_proj.weight, attn.k_proj.weight, attn.v_proj.weight], dim=0
|
||||
)
|
||||
attn._fi_qkv_split = [
|
||||
attn.q_proj.weight.shape[0],
|
||||
attn.k_proj.weight.shape[0],
|
||||
attn.v_proj.weight.shape[0],
|
||||
]
|
||||
attn.forward = MethodType(_fi_attention_module_forward, attn)
|
||||
|
||||
|
||||
def _fi_mlp_forward(self, x):
|
||||
"""Qwen3MLP with fused gate+up GEMM and flashinfer silu_and_mul
|
||||
(2 GEMMs + silu + mul -> 1 GEMM + 1 fused kernel)."""
|
||||
y = F.linear(x[0], self._fi_w_gate_up) # (S, 2*inter)
|
||||
y = flashinfer.activation.silu_and_mul(y)
|
||||
return self.down_proj(y).unsqueeze(0)
|
||||
|
||||
|
||||
def _patch_mlp(llm):
|
||||
for layer in llm.layers:
|
||||
mlp = layer.mlp
|
||||
mlp._fi_w_gate_up = torch.cat([mlp.gate_proj.weight, mlp.up_proj.weight], dim=0)
|
||||
mlp.forward = MethodType(_fi_mlp_forward, mlp)
|
||||
|
||||
|
||||
class PackedAttnRunner:
|
||||
def __init__(
|
||||
self,
|
||||
num_qo_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
device,
|
||||
workspace_size=_WORKSPACE_SIZE,
|
||||
):
|
||||
self.num_qo_heads = num_qo_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_dim = head_dim
|
||||
self.device = device
|
||||
self._workspace = torch.zeros(workspace_size, dtype=torch.uint8, device=device)
|
||||
self.wrapper = flashinfer.BatchPrefillWithRaggedKVCacheWrapper(
|
||||
self._workspace, "NHD"
|
||||
)
|
||||
self._planned_key = None
|
||||
|
||||
def plan(self, doc_lens: List[int], dtype: torch.dtype):
|
||||
key = (tuple(doc_lens), dtype)
|
||||
if key == self._planned_key:
|
||||
return
|
||||
indptr = torch.zeros(len(doc_lens) + 1, dtype=torch.int32, device=self.device)
|
||||
indptr[1:] = torch.cumsum(
|
||||
torch.tensor(doc_lens, dtype=torch.int32, device=self.device), dim=0
|
||||
)
|
||||
self.wrapper.plan(
|
||||
indptr,
|
||||
indptr,
|
||||
self.num_qo_heads,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
causal=False,
|
||||
sm_scale=self.head_dim**-0.5,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
)
|
||||
self._planned_key = key
|
||||
|
||||
|
||||
def _generate_iterative_packed(
|
||||
self, task: GenerationTask, gen_config: OmniVoiceGenerationConfig
|
||||
) -> List[torch.Tensor]:
|
||||
"""Packed-sequence rewrite of OmniVoice._generate_iterative.
|
||||
|
||||
Documents are packed as [cond_0, uncond_0, cond_1, uncond_1, ...] into a
|
||||
single batch row; the scoring/unmasking math is identical to the original.
|
||||
"""
|
||||
B = task.batch_size
|
||||
inputs_list = [
|
||||
self._prepare_inference_inputs(
|
||||
task.texts[i],
|
||||
task.target_lens[i],
|
||||
task.ref_texts[i],
|
||||
task.ref_audio_tokens[i],
|
||||
task.langs[i],
|
||||
task.instructs[i],
|
||||
gen_config.denoise,
|
||||
)
|
||||
for i in range(B)
|
||||
]
|
||||
|
||||
c_lens = [inp["input_ids"].size(2) for inp in inputs_list]
|
||||
u_lens = list(task.target_lens)
|
||||
doc_lens = []
|
||||
for c, u in zip(c_lens, u_lens):
|
||||
doc_lens.extend([c, u])
|
||||
|
||||
use_graph = getattr(self, "_fi_enable_cuda_graph", False)
|
||||
buckets = getattr(self, "_fi_graph_buckets", None) # durations in seconds
|
||||
|
||||
# Choose the packed layout. Bucketed-graph mode places each item in fixed
|
||||
# slots [C_budget | U_budget] so one graph per (batch, duration bucket)
|
||||
# serves any sample that fits; otherwise pack tightly.
|
||||
bucket_U = None
|
||||
if use_graph and buckets is not None:
|
||||
frame_rate = self.audio_tokenizer.config.frame_rate
|
||||
t_max = max(u_lens)
|
||||
overhead_max = max(c - u for c, u in zip(c_lens, u_lens))
|
||||
bucket_U = next(
|
||||
(int(d * frame_rate) for d in sorted(buckets) if d * frame_rate >= t_max),
|
||||
None,
|
||||
)
|
||||
if bucket_U is None or overhead_max > self._fi_overhead_budget:
|
||||
bucket_U = None
|
||||
use_graph = False # too long for the buckets: eager fallback
|
||||
|
||||
if bucket_U is not None:
|
||||
U_b = bucket_U
|
||||
C_b = U_b + self._fi_overhead_budget
|
||||
offsets = []
|
||||
for i in range(B):
|
||||
offsets.extend([i * (C_b + U_b), i * (C_b + U_b) + C_b])
|
||||
total_len = B * (C_b + U_b)
|
||||
else:
|
||||
offsets = [0]
|
||||
for l in doc_lens[:-1]:
|
||||
offsets.append(offsets[-1] + l)
|
||||
total_len = sum(doc_lens)
|
||||
|
||||
C = self.config.num_audio_codebook
|
||||
packed_ids = torch.full(
|
||||
(1, C, total_len),
|
||||
self.config.audio_mask_id,
|
||||
dtype=torch.long,
|
||||
device=self.device,
|
||||
)
|
||||
packed_audio_mask = torch.zeros(
|
||||
(1, total_len), dtype=torch.bool, device=self.device
|
||||
)
|
||||
position_ids = torch.zeros((1, total_len), dtype=torch.long, device=self.device)
|
||||
|
||||
for i, inp in enumerate(inputs_list):
|
||||
c_off, u_off = offsets[2 * i], offsets[2 * i + 1]
|
||||
c_len, u_len = c_lens[i], u_lens[i]
|
||||
packed_ids[0, :, c_off : c_off + c_len] = inp["input_ids"][0]
|
||||
packed_audio_mask[0, c_off : c_off + c_len] = inp["audio_mask"][0]
|
||||
position_ids[0, c_off : c_off + c_len] = torch.arange(c_len, device=self.device)
|
||||
# uncond doc = target region only
|
||||
packed_ids[0, :, u_off : u_off + u_len] = inp["input_ids"][0, :, -u_len:]
|
||||
packed_audio_mask[0, u_off : u_off + u_len] = inp["audio_mask"][0, -u_len:]
|
||||
position_ids[0, u_off : u_off + u_len] = torch.arange(u_len, device=self.device)
|
||||
|
||||
# num_step + 1 mirrors our _generate_iterative's schedule (a local
|
||||
# divergence from upstream k2-fsa): packed decoding must unmask on exactly
|
||||
# the same schedule as the eager path or outputs differ between the two.
|
||||
timesteps = _get_time_steps(
|
||||
t_start=0.0,
|
||||
t_end=1.0,
|
||||
num_step=gen_config.num_step + 1,
|
||||
t_shift=gen_config.t_shift,
|
||||
).tolist()
|
||||
schedules = []
|
||||
for t_len in task.target_lens:
|
||||
total_mask = t_len * C
|
||||
rem = total_mask
|
||||
sched = []
|
||||
for step in range(gen_config.num_step):
|
||||
num = (
|
||||
rem
|
||||
if step == gen_config.num_step - 1
|
||||
else min(
|
||||
math.ceil(total_mask * (timesteps[step + 1] - timesteps[step])), rem
|
||||
)
|
||||
)
|
||||
sched.append(int(num))
|
||||
rem -= int(num)
|
||||
schedules.append(sched)
|
||||
|
||||
layer_ids = torch.arange(C, device=self.device).view(1, -1, 1)
|
||||
|
||||
# gather indices of the logits-consuming positions, laid out as
|
||||
# [all cond-target blocks | all uncond blocks] so the guidance/scoring
|
||||
# math can run over every item in one batched pass. flat_spans[i] gives
|
||||
# the item's (start, len) within each half; in bucket mode items sit at a
|
||||
# fixed stride U_b with junk rows (pointing at position 0) in between.
|
||||
cond_ranges, uncond_ranges = [], []
|
||||
flat_spans = []
|
||||
for i in range(B):
|
||||
c_off, u_off = offsets[2 * i], offsets[2 * i + 1]
|
||||
c_len, t_len = c_lens[i], task.target_lens[i]
|
||||
if bucket_U is not None:
|
||||
flat_spans.append((U_b * i, t_len))
|
||||
cond_rows = torch.zeros(U_b, dtype=torch.long, device=self.device)
|
||||
cond_rows[:t_len] = torch.arange(
|
||||
c_off + c_len - t_len, c_off + c_len, device=self.device
|
||||
)
|
||||
uncond_rows = torch.zeros(U_b, dtype=torch.long, device=self.device)
|
||||
uncond_rows[:t_len] = torch.arange(u_off, u_off + t_len, device=self.device)
|
||||
cond_ranges.append(cond_rows)
|
||||
uncond_ranges.append(uncond_rows)
|
||||
else:
|
||||
prev = 0 if i == 0 else flat_spans[-1][0] + flat_spans[-1][1]
|
||||
flat_spans.append((prev, t_len))
|
||||
cond_ranges.append(
|
||||
torch.arange(c_off + c_len - t_len, c_off + c_len, device=self.device)
|
||||
)
|
||||
uncond_ranges.append(torch.arange(u_off, u_off + t_len, device=self.device))
|
||||
T_flat = (U_b * B) if bucket_U is not None else sum(task.target_lens)
|
||||
tgt_index = torch.cat(cond_ranges + uncond_ranges)
|
||||
|
||||
# flat per-position token state aligned with the cond half of the gathered
|
||||
# layout. Junk positions (bucket-mode slot padding) are initialized to -1
|
||||
# so the global "already unmasked" fill gives them -inf scores and topk
|
||||
# never selects them.
|
||||
tokens_flat = torch.full((C, T_flat), -1, dtype=torch.long, device=self.device)
|
||||
for st, t_len in flat_spans:
|
||||
tokens_flat[:, st : st + t_len] = self.config.audio_mask_id
|
||||
|
||||
if use_graph and bucket_U is not None:
|
||||
graph_entry = _get_or_capture_bucket_graph(self, B, U_b, C_b)
|
||||
# refresh the per-generation static contents (shape-invariant, data-variant)
|
||||
graph_entry["audio_mask"].copy_(packed_audio_mask)
|
||||
graph_entry["position_ids"].copy_(position_ids)
|
||||
graph_entry["pos_ids_i32"].copy_(position_ids[0].to(torch.int32))
|
||||
graph_entry["tgt_index"].copy_(tgt_index)
|
||||
for d_idx, m in enumerate(graph_entry["doc_masks"]):
|
||||
length = c_lens[d_idx // 2] if d_idx % 2 == 0 else u_lens[d_idx // 2]
|
||||
m[..., :length] = True
|
||||
m[..., length:] = False
|
||||
elif use_graph:
|
||||
graph_entry = _get_or_capture_graph(self, tuple(doc_lens), tgt_index)
|
||||
graph_entry["audio_mask"].copy_(packed_audio_mask)
|
||||
graph_entry["position_ids"].copy_(position_ids)
|
||||
else:
|
||||
self._fi_runner.plan(doc_lens, torch.float16)
|
||||
_CTX["wrapper"] = self._fi_runner.wrapper
|
||||
_CTX["pos_ids"] = position_ids[0].to(torch.int32)
|
||||
_CTX["doc_slots"] = None
|
||||
|
||||
# optional llm timing hook (set by the benchmark; graph replays bypass
|
||||
# model.forward, so wrapping forward would miss them)
|
||||
stats = getattr(self, "_fi_llm_stats", None)
|
||||
|
||||
for step in range(gen_config.num_step):
|
||||
if stats is not None:
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.perf_counter()
|
||||
if use_graph:
|
||||
graph_entry["input_ids"].copy_(packed_ids)
|
||||
graph_entry["graph"].replay()
|
||||
batch_logits = graph_entry["logits"].to(torch.float32)
|
||||
else:
|
||||
batch_logits = _forward_logits(
|
||||
self, packed_ids, packed_audio_mask, position_ids, tgt_index
|
||||
).to(torch.float32)
|
||||
if stats is not None:
|
||||
torch.cuda.synchronize()
|
||||
stats["seconds"] += time.perf_counter() - t0
|
||||
stats["calls"] += 1
|
||||
|
||||
# batched scoring over every item at once: the guidance/log_softmax/
|
||||
# argmax/gumbel chain (the GPU-heavy part) runs on the whole
|
||||
# [cond | uncond] halves; only topk + scatter stay per item.
|
||||
c_logits_all = batch_logits[:, :, :T_flat, :]
|
||||
u_logits_all = batch_logits[:, :, T_flat:, :]
|
||||
pred_all, scores_all = self._predict_tokens_with_scoring(
|
||||
c_logits_all, u_logits_all, gen_config
|
||||
)
|
||||
scores_all = scores_all - (layer_ids * gen_config.layer_penalty_factor)
|
||||
if gen_config.position_temperature > 0.0:
|
||||
scores_all = _gumbel_sample(scores_all, gen_config.position_temperature)
|
||||
# -inf for already-unmasked positions AND bucket-slot junk (-1)
|
||||
scores_all.masked_fill_(
|
||||
(tokens_flat != self.config.audio_mask_id).unsqueeze(0), -float("inf")
|
||||
)
|
||||
pred_all, scores_all = pred_all[0], scores_all[0] # (C, T_flat)
|
||||
|
||||
for i in range(B):
|
||||
k = schedules[i][step]
|
||||
if k <= 0:
|
||||
continue
|
||||
c_off, u_off = offsets[2 * i], offsets[2 * i + 1]
|
||||
c_len, t_len = c_lens[i], task.target_lens[i]
|
||||
st, _ = flat_spans[i]
|
||||
|
||||
_, topk_idx = torch.topk(scores_all[:, st : st + t_len].reshape(-1), k)
|
||||
flat_tokens = tokens_flat[:, st : st + t_len].reshape(-1)
|
||||
flat_tokens[topk_idx] = pred_all[:, st : st + t_len].reshape(-1)[topk_idx]
|
||||
new_tokens = flat_tokens.view(C, t_len)
|
||||
tokens_flat[:, st : st + t_len] = new_tokens
|
||||
|
||||
packed_ids[0, :, c_off + c_len - t_len : c_off + c_len] = new_tokens
|
||||
packed_ids[0, :, u_off : u_off + t_len] = new_tokens
|
||||
|
||||
return [tokens_flat[:, st : st + t_len] for (st, t_len) in flat_spans]
|
||||
|
||||
|
||||
def _forward_logits(model, input_ids, audio_mask, position_ids, tgt_index):
|
||||
"""LLM forward + audio head over target positions only.
|
||||
|
||||
The scoring step consumes logits at the cond-target and uncond ranges
|
||||
(2*sum(t_len) of the packed positions); running the 1024->8200 audio_heads
|
||||
GEMM and the fp32 upcast on the full packed length is wasted work.
|
||||
Returns logits of shape (1, C, 2*sum(t_len), V) laid out as
|
||||
[all cond-target blocks | all uncond blocks] — matching tgt_index
|
||||
(torch.cat(cond_ranges + uncond_ranges)) and the caller's split at T_flat.
|
||||
"""
|
||||
inputs_embeds = model._prepare_embed_inputs(input_ids, audio_mask)
|
||||
hidden = model.llm(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask={"full_attention": None},
|
||||
return_dict=True,
|
||||
position_ids=position_ids,
|
||||
)[0]
|
||||
tgt_hidden = hidden[0, tgt_index] # (2T, hidden)
|
||||
logits_flat = model.audio_heads(tgt_hidden)
|
||||
n = tgt_hidden.shape[0]
|
||||
return logits_flat.view(
|
||||
1, n, model.config.num_audio_codebook, model.config.audio_vocab_size
|
||||
).permute(0, 2, 1, 3)
|
||||
|
||||
|
||||
def _get_or_capture_graph(model, doc_lens_key, tgt_index):
|
||||
cache = model._fi_graph_cache
|
||||
entry = cache.get(doc_lens_key)
|
||||
if entry is not None:
|
||||
return entry
|
||||
|
||||
device = model.device
|
||||
total_len = sum(doc_lens_key)
|
||||
C = model.config.num_audio_codebook
|
||||
llm_cfg = model.config.llm_config
|
||||
runner = PackedAttnRunner(
|
||||
llm_cfg.num_attention_heads,
|
||||
llm_cfg.num_key_value_heads,
|
||||
llm_cfg.head_dim,
|
||||
device,
|
||||
workspace_size=64 * 1024 * 1024,
|
||||
)
|
||||
runner.plan(list(doc_lens_key), torch.float16)
|
||||
_CTX["wrapper"] = runner.wrapper
|
||||
|
||||
# positions are fully determined by doc_lens (the cache key), so both the
|
||||
# long buffer (model-level rotary) and the int32 copy (fused rope) can be
|
||||
# baked with their final values
|
||||
positions = torch.cat([torch.arange(l, device=device) for l in doc_lens_key])
|
||||
static = {
|
||||
"input_ids": torch.full(
|
||||
(1, C, total_len),
|
||||
model.config.audio_mask_id,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
),
|
||||
"audio_mask": torch.zeros((1, total_len), dtype=torch.bool, device=device),
|
||||
"position_ids": positions.unsqueeze(0).contiguous(),
|
||||
}
|
||||
pos_ids_i32 = positions.to(torch.int32)
|
||||
_CTX["pos_ids"] = pos_ids_i32
|
||||
_CTX["doc_slots"] = None
|
||||
|
||||
side_stream = torch.cuda.Stream()
|
||||
side_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side_stream):
|
||||
for _ in range(2):
|
||||
_forward_logits(
|
||||
model,
|
||||
static["input_ids"],
|
||||
static["audio_mask"],
|
||||
static["position_ids"],
|
||||
tgt_index,
|
||||
)
|
||||
torch.cuda.current_stream().wait_stream(side_stream)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
# tgt_index depends only on doc_lens (the cache key), so it is safe
|
||||
# to bake into the graph
|
||||
logits = _forward_logits(
|
||||
model,
|
||||
static["input_ids"],
|
||||
static["audio_mask"],
|
||||
static["position_ids"],
|
||||
tgt_index,
|
||||
)
|
||||
|
||||
# tgt_index is baked into the captured gather by pointer — the entry must
|
||||
# keep it alive or the allocator will reuse its memory for later samples
|
||||
entry = {
|
||||
"graph": graph,
|
||||
"logits": logits,
|
||||
"runner": runner,
|
||||
"tgt_index": tgt_index,
|
||||
"pos_ids_i32": pos_ids_i32,
|
||||
**static,
|
||||
}
|
||||
cache[doc_lens_key] = entry
|
||||
return entry
|
||||
|
||||
|
||||
def _get_or_capture_bucket_graph(model, B, U_b, C_b):
|
||||
"""One graph per (batch, duration-bucket): items sit in fixed
|
||||
[C_budget | U_budget] slots; attention runs as SDPA over a runtime-updated
|
||||
block-diagonal mask, so any sample that fits the slots replays exactly."""
|
||||
key = ("bucket", B, U_b)
|
||||
cache = model._fi_graph_cache
|
||||
entry = cache.get(key)
|
||||
if entry is not None:
|
||||
return entry
|
||||
|
||||
device = model.device
|
||||
total_len = B * (C_b + U_b)
|
||||
C = model.config.num_audio_codebook
|
||||
|
||||
static = {
|
||||
"input_ids": torch.full(
|
||||
(1, C, total_len),
|
||||
model.config.audio_mask_id,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
),
|
||||
"audio_mask": torch.zeros((1, total_len), dtype=torch.bool, device=device),
|
||||
"position_ids": torch.zeros((1, total_len), dtype=torch.long, device=device),
|
||||
"pos_ids_i32": torch.zeros(total_len, dtype=torch.int32, device=device),
|
||||
"tgt_index": torch.zeros(2 * B * U_b, dtype=torch.long, device=device),
|
||||
}
|
||||
# per-document key-padding masks (contents updated per generation);
|
||||
# init all-True so warmup/capture has no fully-masked softmax rows
|
||||
doc_masks, doc_slots = [], []
|
||||
for i in range(B):
|
||||
for slot_start, slot_len in (
|
||||
(i * (C_b + U_b), C_b),
|
||||
(i * (C_b + U_b) + C_b, U_b),
|
||||
):
|
||||
m = torch.ones(1, 1, 1, slot_len, dtype=torch.bool, device=device)
|
||||
doc_masks.append(m)
|
||||
doc_slots.append((slot_start, slot_len, m))
|
||||
|
||||
_CTX["wrapper"] = None
|
||||
_CTX["pos_ids"] = static["pos_ids_i32"]
|
||||
_CTX["doc_slots"] = doc_slots
|
||||
|
||||
side_stream = torch.cuda.Stream()
|
||||
side_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side_stream):
|
||||
for _ in range(2):
|
||||
_forward_logits(
|
||||
model,
|
||||
static["input_ids"],
|
||||
static["audio_mask"],
|
||||
static["position_ids"],
|
||||
static["tgt_index"],
|
||||
)
|
||||
torch.cuda.current_stream().wait_stream(side_stream)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
logits = _forward_logits(
|
||||
model,
|
||||
static["input_ids"],
|
||||
static["audio_mask"],
|
||||
static["position_ids"],
|
||||
static["tgt_index"],
|
||||
)
|
||||
|
||||
entry = {
|
||||
"graph": graph,
|
||||
"logits": logits,
|
||||
"doc_masks": doc_masks,
|
||||
"doc_slots": doc_slots,
|
||||
**static,
|
||||
}
|
||||
cache[key] = entry
|
||||
return entry
|
||||
|
||||
|
||||
def apply_flashinfer(
|
||||
model,
|
||||
enable_cuda_graph: bool = False,
|
||||
fuse_rmsnorm: bool = True,
|
||||
fuse_attention: bool = True,
|
||||
cuda_graph_buckets=None,
|
||||
overhead_budget: int = 512,
|
||||
):
|
||||
"""Patch an OmniVoice instance to use flashinfer packed attention."""
|
||||
model.llm.set_attn_implementation("omnivoice_fi")
|
||||
if fuse_rmsnorm:
|
||||
_patch_rmsnorm(model.llm)
|
||||
if fuse_attention:
|
||||
_patch_attention_forward(model.llm)
|
||||
_patch_mlp(model.llm)
|
||||
# Bidirectional iterative decoding recomputes everything each step; the
|
||||
# DynamicCache the baseline allocates+fills per forward is pure overhead.
|
||||
model.llm.config.use_cache = False
|
||||
|
||||
llm_cfg = model.config.llm_config
|
||||
model._fi_runner = PackedAttnRunner(
|
||||
llm_cfg.num_attention_heads,
|
||||
llm_cfg.num_key_value_heads,
|
||||
llm_cfg.head_dim,
|
||||
model.device,
|
||||
)
|
||||
model._fi_graph_cache = {}
|
||||
model._fi_enable_cuda_graph = enable_cuda_graph or cuda_graph_buckets is not None
|
||||
model._fi_graph_buckets = cuda_graph_buckets
|
||||
model._fi_overhead_budget = overhead_budget
|
||||
model._generate_iterative = MethodType(_generate_iterative_packed, model)
|
||||
return model
|
||||
@@ -627,7 +627,7 @@ def test_engine_health_is_admin_gated(fresh_app):
|
||||
client = _client(fresh_app, host="10.0.0.5")
|
||||
r = client.get("/engines/omnivoice/health")
|
||||
assert r.status_code == 403
|
||||
assert r.json()["detail"] == "loopback origin or admin API key required"
|
||||
assert r.json()["detail"] == "loopback origin required"
|
||||
|
||||
|
||||
def test_server_mode_engine_mutations_require_api_key(fresh_app, monkeypatch):
|
||||
@@ -802,7 +802,7 @@ def test_selftest_unknown_id_is_404(fresh_app):
|
||||
def test_selftest_is_admin_gated(fresh_app):
|
||||
r = _client(fresh_app, host="10.0.0.9").post("/engines/omnivoice/selftest")
|
||||
assert r.status_code == 403
|
||||
assert r.json()["detail"] == "loopback origin or admin API key required"
|
||||
assert r.json()["detail"] == "loopback origin required"
|
||||
|
||||
|
||||
def test_selftest_captures_synth_exception_without_500(fresh_app):
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Cross-layer contract lock for the admin-gate 403 detail string.
|
||||
|
||||
The backend's ``require_admin``/``require_admin_action`` answer 403 with a
|
||||
mode-distinct ``detail`` (``_admin_gate_403`` in backend/api/dependencies.py):
|
||||
"loopback origin or admin API key required" in server mode, plain
|
||||
"loopback origin required" on the desktop build. The SPA's ``apiFetch`` routes
|
||||
a 403 to the API-key login gate exactly when the detail contains the substring
|
||||
"admin api key" (frontend/src/api/client.ts) — i.e. when presenting the key
|
||||
could actually satisfy the gate. The per-mode behaviour is pinned by
|
||||
tests/test_loopback_server_mode.py; this file pins the LITERAL contract across
|
||||
layers: a backend reword keeps backend tests green while the frontend matcher
|
||||
silently stops firing, and a LAN user is back to raw 403 spam instead of the
|
||||
login form.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEPS = ROOT / "backend" / "api" / "dependencies.py"
|
||||
CLIENT = ROOT / "frontend" / "src" / "api" / "client.ts"
|
||||
|
||||
|
||||
def _frontend_sniff() -> str:
|
||||
"""The substring apiFetch matches on a 403 to admit it to the auth gate."""
|
||||
text = CLIENT.read_text(encoding="utf-8")
|
||||
# adminGate403 = ... detail.toLowerCase().includes('<sniff>')
|
||||
m = re.search(r"adminGate403 =.*?includes\('([^']+)'\)", text, re.DOTALL)
|
||||
assert m, "adminGate403 matcher not found in frontend/src/api/client.ts"
|
||||
return m.group(1)
|
||||
|
||||
|
||||
def _key_named_details() -> set[str]:
|
||||
"""Every quoted string in dependencies.py that names the admin API key."""
|
||||
return set(re.findall(r'"([^"]*admin API key[^"]*)"', DEPS.read_text(encoding="utf-8")))
|
||||
|
||||
|
||||
def test_key_named_details_match_frontend_sniff():
|
||||
"""Every backend literal naming the admin key must contain the SPA matcher."""
|
||||
details = _key_named_details()
|
||||
assert details, (
|
||||
"no 'admin API key' detail literal left in dependencies.py — moved or "
|
||||
"reworded? Update frontend/src/api/client.ts in the same change."
|
||||
)
|
||||
sniff = _frontend_sniff()
|
||||
for detail in details:
|
||||
# Case-insensitive substring, mirroring apiFetch's toLowerCase match.
|
||||
assert sniff in detail.lower(), (
|
||||
f"backend detail {detail!r} no longer contains the frontend matcher "
|
||||
f"{sniff!r} — the SPA would stop routing it to the API-key gate. "
|
||||
"Update frontend/src/api/client.ts in the same change."
|
||||
)
|
||||
|
||||
|
||||
def test_frontend_sniff_rejects_details_a_key_cannot_fix():
|
||||
"""The sniff must not swallow 403s an API key cannot satisfy.
|
||||
|
||||
The desktop admin-gate arm (loopback-only regardless of credentials), the
|
||||
legacy require_loopback desktop 403, the CSRF rejection, and the
|
||||
desktop-only filesystem gate: routing any of these to the login form would
|
||||
trap the user in a form that can never succeed.
|
||||
"""
|
||||
sniff = _frontend_sniff()
|
||||
unfixable = (
|
||||
"loopback origin required", # desktop admin arm + require_loopback
|
||||
"browser origin rejected", # BearerKeyMiddleware CSRF (main.py)
|
||||
"desktop origin required", # require_desktop — loopback-only forever
|
||||
"native filesystem access requires loopback origin", # require_native
|
||||
)
|
||||
for detail in unfixable:
|
||||
assert sniff not in detail.lower(), (
|
||||
f"frontend matcher {sniff!r} now also matches {detail!r}, which "
|
||||
"an API key cannot satisfy — the login gate would loop."
|
||||
)
|
||||
@@ -0,0 +1,214 @@
|
||||
"""The FlashInfer opt-in (OMNIVOICE_FLASHINFER, upstream k2-fsa port).
|
||||
|
||||
An optimization must never be a point of failure (#278 contract, same as
|
||||
torch.compile): the env knob is CUDA-only, off by default, refuses with a
|
||||
named reason when the host can't honor it, latches off for the session after
|
||||
a runtime failure, and a mid-generation FlashInfer error unapplies the patch
|
||||
and retries the standard path once.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
|
||||
def _ee():
|
||||
import services.engine_env as m
|
||||
return m
|
||||
|
||||
|
||||
def _mm():
|
||||
import services.model_manager as m
|
||||
return m
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_latch(monkeypatch):
|
||||
monkeypatch.setattr(_ee(), "_flashinfer_runtime_failure", None)
|
||||
monkeypatch.delenv("OMNIVOICE_FLASHINFER", raising=False)
|
||||
|
||||
|
||||
# ── the env knob ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value,expected",
|
||||
[
|
||||
("", "off"), ("0", "off"), ("false", "off"), ("off", "off"),
|
||||
("1", "on"), ("true", "on"), ("ON", "on"),
|
||||
("graph", "graph"), ("GRAPH", "graph"),
|
||||
("banana", "off"), # typo → default path, not a crash
|
||||
],
|
||||
)
|
||||
def test_flashinfer_mode_parsing(monkeypatch, value, expected):
|
||||
if value:
|
||||
monkeypatch.setenv("OMNIVOICE_FLASHINFER", value)
|
||||
assert _ee().flashinfer_mode() == expected
|
||||
|
||||
|
||||
def test_should_flashinfer_refuses_non_cuda(monkeypatch):
|
||||
monkeypatch.setenv("OMNIVOICE_FLASHINFER", "1")
|
||||
assert _ee().should_flashinfer("cpu") == "off"
|
||||
assert _ee().should_flashinfer("mps") == "off"
|
||||
|
||||
|
||||
def test_should_flashinfer_refuses_without_the_package(monkeypatch):
|
||||
monkeypatch.setenv("OMNIVOICE_FLASHINFER", "1")
|
||||
ee = _ee()
|
||||
monkeypatch.setattr(ee.importlib.util, "find_spec", lambda name: None)
|
||||
assert ee.should_flashinfer("cuda") == "off"
|
||||
|
||||
|
||||
def test_latched_reason_is_sanitized(monkeypatch):
|
||||
# Wheel import errors embed the user's home path — the latch must store
|
||||
# the redacted form (core.failure.sanitize maps $HOME → "~").
|
||||
import os
|
||||
|
||||
home = os.path.expanduser("~")
|
||||
_ee().mark_flashinfer_runtime_failure(
|
||||
f"ImportError: {home}/.venv/lib/flashinfer/_kernels.so: bad ELF"
|
||||
)
|
||||
latched = _ee()._flashinfer_runtime_failure
|
||||
assert home not in latched
|
||||
assert "ImportError" in latched
|
||||
|
||||
|
||||
def test_sanitizer_failure_never_latches_the_raw_reason(monkeypatch):
|
||||
# Fail closed: a broken redactor must not leak the original message.
|
||||
import core.failure
|
||||
|
||||
def _boom(_):
|
||||
raise RuntimeError("sanitizer exploded (test)")
|
||||
|
||||
monkeypatch.setattr(core.failure, "sanitize", _boom)
|
||||
_ee().mark_flashinfer_runtime_failure(
|
||||
"ImportError: /home/someone/secret-project/creds.so missing"
|
||||
)
|
||||
latched = _ee()._flashinfer_runtime_failure
|
||||
assert "secret-project" not in latched and "/home/" not in latched
|
||||
assert latched.startswith("ImportError")
|
||||
assert "redacted" in latched
|
||||
|
||||
|
||||
def test_runtime_failure_latches_the_session_off(monkeypatch):
|
||||
monkeypatch.setenv("OMNIVOICE_FLASHINFER", "graph")
|
||||
ee = _ee()
|
||||
monkeypatch.setattr(ee.importlib.util, "find_spec", lambda name: object())
|
||||
assert ee.should_flashinfer("cuda") == "graph"
|
||||
ee.mark_flashinfer_runtime_failure("boom")
|
||||
assert ee.should_flashinfer("cuda") == "off"
|
||||
|
||||
|
||||
# ── failure classification ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_classifier_matches_flashinfer_markers():
|
||||
mm = _mm()
|
||||
assert mm._is_flashinfer_runtime_failure(RuntimeError("flashinfer plan failed"))
|
||||
assert mm._is_flashinfer_runtime_failure(RuntimeError("CUDA graph capture aborted"))
|
||||
assert not mm._is_flashinfer_runtime_failure(ValueError("Unsupported instruct items"))
|
||||
assert not mm._is_flashinfer_runtime_failure(RuntimeError("CUDA out of memory"))
|
||||
|
||||
|
||||
def test_classifier_walks_the_cause_chain():
|
||||
mm = _mm()
|
||||
inner = RuntimeError("flashinfer workspace too small")
|
||||
outer = RuntimeError("generation failed")
|
||||
outer.__cause__ = inner
|
||||
assert mm._is_flashinfer_runtime_failure(outer)
|
||||
# `raise ... from None` severs the chain — a genuine error must not be
|
||||
# re-classified via a suppressed FlashInfer context.
|
||||
severed = RuntimeError("generation failed")
|
||||
severed.__context__ = inner
|
||||
severed.__suppress_context__ = True
|
||||
assert not mm._is_flashinfer_runtime_failure(severed)
|
||||
|
||||
|
||||
# ── unapply restores the class implementations ──────────────────────────────
|
||||
|
||||
|
||||
class _MiniModel:
|
||||
class _Llm(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.lin = torch.nn.Linear(2, 2)
|
||||
self.config = type("C", (), {"use_cache": False})()
|
||||
self.attn_impl = None
|
||||
|
||||
def set_attn_implementation(self, name):
|
||||
self.attn_impl = name
|
||||
|
||||
def __init__(self):
|
||||
self.llm = self._Llm()
|
||||
|
||||
def _generate_iterative(self, *a):
|
||||
return "class-impl"
|
||||
|
||||
|
||||
def test_unapply_flashinfer_restores_instance_state():
|
||||
from types import MethodType
|
||||
|
||||
m = _MiniModel()
|
||||
# Simulate apply_flashinfer's instance-level patching.
|
||||
m.llm.lin.forward = MethodType(lambda self, x: "patched", m.llm.lin)
|
||||
m.llm.lin._fi_w_qkv = torch.zeros(1)
|
||||
m._generate_iterative = MethodType(lambda self, *a: "patched", m)
|
||||
m._fi_runner = object()
|
||||
m._fi_graph_cache = {}
|
||||
m._fi_enable_cuda_graph = True
|
||||
|
||||
_mm()._unapply_flashinfer(m)
|
||||
|
||||
assert "forward" not in vars(m.llm.lin), "instance forward override must go"
|
||||
assert not hasattr(m.llm.lin, "_fi_w_qkv")
|
||||
assert m._generate_iterative() == "class-impl"
|
||||
assert not hasattr(m, "_fi_runner")
|
||||
assert m.llm.attn_impl == "sdpa"
|
||||
assert m.llm.config.use_cache is True
|
||||
|
||||
|
||||
def test_unapply_restores_the_captured_attention_impl():
|
||||
# The pre-apply impl may be flash_attention_2, not sdpa — unapply must
|
||||
# put back what was actually there (CodeRabbit/Greptile, #1565).
|
||||
m = _MiniModel()
|
||||
m._fi_orig_attn_impl = "flash_attention_2"
|
||||
_mm()._unapply_flashinfer(m)
|
||||
assert m.llm.attn_impl == "flash_attention_2"
|
||||
assert not hasattr(m, "_fi_orig_attn_impl")
|
||||
|
||||
|
||||
# ── generate-time fallback ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_generate_fallback_unapplies_and_retries_once():
|
||||
mm = _mm()
|
||||
calls = {"n": 0}
|
||||
|
||||
class _Model(_MiniModel):
|
||||
def generate(self, **kw):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise RuntimeError("flashinfer ragged attention failed")
|
||||
return ["ok"]
|
||||
|
||||
m = _Model()
|
||||
m._fi_runner = object()
|
||||
mm._install_flashinfer_fallback(m)
|
||||
assert m.generate() == ["ok"]
|
||||
assert calls["n"] == 2
|
||||
assert not hasattr(m, "_fi_runner"), "fallback must unapply the patch"
|
||||
assert _ee()._flashinfer_runtime_failure is not None
|
||||
|
||||
|
||||
def test_generate_fallback_leaves_real_errors_alone():
|
||||
mm = _mm()
|
||||
|
||||
class _Model(_MiniModel):
|
||||
def generate(self, **kw):
|
||||
raise ValueError("Unsupported instruct items")
|
||||
|
||||
m = _Model()
|
||||
mm._install_flashinfer_fallback(m)
|
||||
with pytest.raises(ValueError):
|
||||
m.generate()
|
||||
@@ -17,6 +17,7 @@ EXPECTED_EXACT_REGEXES = {
|
||||
"^hf_abcdefghijklmnopqrstuvwxyz0123456789ABCDEF$",
|
||||
"^hf_QWERTYUIOPasdfghjklZXCVBNM0123456789xyzAB$",
|
||||
"^max_length=400$",
|
||||
"^Ed25519PrivateKey$",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -253,6 +253,89 @@ def test_side_effectful_get_rejects_remote_api_key_outside_server_mode(monkeypat
|
||||
assert exc.value.status_code == 403
|
||||
|
||||
|
||||
# Mode-distinct admin-gate detail: the 403 message must state what would
|
||||
# ACTUALLY satisfy the gate. The bundled UI routes any 403 whose detail
|
||||
# mentions "admin api key" to the API-key login form (frontend client.ts;
|
||||
# the literal contract is locked by tests/test_auth_gate_detail_lockstep.py).
|
||||
# Server mode accepts the key, so naming it is right. Desktop mode rejects
|
||||
# every non-loopback client regardless of credentials — the checks above only
|
||||
# run under server mode — so it must keep the plain loopback detail: naming
|
||||
# the key there invites a login form that can never succeed (a desktop
|
||||
# LAN-share guest would lose the whole consumption UI to it, #1213).
|
||||
|
||||
|
||||
def test_require_admin_desktop_detail_is_plain_loopback(monkeypatch):
|
||||
"""Desktop build: no presented key can satisfy the gate."""
|
||||
monkeypatch.delenv("OMNIVOICE_SERVER_MODE", raising=False)
|
||||
monkeypatch.setenv("OMNIVOICE_API_KEY", "s3cret") # a valid key can't help here
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
require_admin(
|
||||
_req_full("10.0.0.5", headers={"authorization": "Bearer s3cret"})
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
assert exc.value.detail == "loopback origin required"
|
||||
|
||||
|
||||
def test_require_admin_server_mode_detail_names_the_key(monkeypatch):
|
||||
"""Server mode with an API key configured: the 403 names the key."""
|
||||
monkeypatch.setenv("OMNIVOICE_SERVER_MODE", "1")
|
||||
monkeypatch.setenv("OMNIVOICE_API_KEY", "s3cret")
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
require_admin(_req_full("172.17.0.1")) # credential configured, none presented
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
assert exc.value.detail == "loopback origin or admin API key required"
|
||||
|
||||
|
||||
def test_require_admin_pin_only_server_mode_detail_is_plain_loopback(monkeypatch):
|
||||
"""Server mode with ONLY a share PIN (Greptile P1, PR #1569): the PIN
|
||||
closes read-only bootstrap but no API key exists to present, so naming
|
||||
the key would send the browser to a login form that can never succeed.
|
||||
Only loopback can use admin here — the plain detail says so, and the
|
||||
SPA leaves it a plain error instead of gating the whole UI."""
|
||||
monkeypatch.setenv("OMNIVOICE_SERVER_MODE", "1")
|
||||
monkeypatch.delenv("OMNIVOICE_API_KEY", raising=False)
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
require_admin(_req_full("172.17.0.1", pin="424242")) # PIN ≠ admin credential
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
assert exc.value.detail == "loopback origin required"
|
||||
|
||||
|
||||
def test_require_admin_action_desktop_detail_is_plain_loopback(monkeypatch):
|
||||
"""Desktop build, side-effectful GET: plain loopback detail."""
|
||||
monkeypatch.delenv("OMNIVOICE_SERVER_MODE", raising=False)
|
||||
monkeypatch.setenv("OMNIVOICE_API_KEY", "s3cret")
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
require_admin_action(
|
||||
_req_full(
|
||||
"10.0.0.5",
|
||||
method="GET",
|
||||
headers={"authorization": "Bearer s3cret"},
|
||||
)
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
assert exc.value.detail == "loopback origin required"
|
||||
|
||||
|
||||
def test_require_admin_action_server_mode_detail_names_the_key(monkeypatch):
|
||||
"""Server mode + key configured, side-effectful GET: names the key."""
|
||||
monkeypatch.setenv("OMNIVOICE_SERVER_MODE", "1")
|
||||
monkeypatch.setenv("OMNIVOICE_API_KEY", "s3cret")
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
require_admin_action(_req_full("172.17.0.1", method="GET"))
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
assert exc.value.detail == "loopback origin or admin API key required"
|
||||
|
||||
|
||||
def test_side_effectful_get_rejects_pin_and_trusted_network(monkeypatch):
|
||||
monkeypatch.setenv("OMNIVOICE_SERVER_MODE", "1")
|
||||
monkeypatch.setenv("OMNIVOICE_TRUSTED_NETWORKS", "10.0.0.0/8")
|
||||
|
||||
@@ -50,6 +50,7 @@ _ALLOWED_FILES = {
|
||||
"README_CN.md", # Chinese README (a translation)
|
||||
"docs/data_preparation.md", # multilingual example payloads
|
||||
"docs/voice-design.md", # EN/CJK attribute mapping table
|
||||
"docs/engines/omnivoice.md", # pinyin pronunciation-control example (functional CJK)
|
||||
"docs/superpowers/specs/2026-05-31-voice-gallery-design.md", # Chinese-dialect taxonomy reference table
|
||||
"examples/README.md", # multilingual example payloads
|
||||
# Text-processing (CJK punctuation inside sentence/clause-splitting regexes)
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Voice-clone prompts persist across restarts (upstream VoiceClonePrompt port).
|
||||
|
||||
The in-memory prompt cache (#427/#473) dies with the process, so the first
|
||||
generation of every session re-encoded each voice — and re-ran ASR when the
|
||||
profile had no stored transcript. Upstream k2-fsa added
|
||||
``VoiceClonePrompt.save()/.load()`` for exactly this; we port the format
|
||||
(version-tagged dict, ``torch.load(weights_only=True)``-safe) and put a disk
|
||||
layer under the memory LRU, keyed identically (ref path + mtime + ref_text +
|
||||
preprocess flag). Restart is simulated here by clearing the memory cache: a
|
||||
second lookup must come from disk, not a re-encode.
|
||||
|
||||
The layer is best-effort by contract: disabled (env), unwritable, or corrupt
|
||||
disk state must never fail a generation — worst case is the old re-encode.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
|
||||
def _tb():
|
||||
"""The *live* services.tts_backend (same rationale as
|
||||
test_clone_prompt_wiring._tb: other suites purge services.* modules)."""
|
||||
import services.tts_backend as m
|
||||
return m
|
||||
|
||||
|
||||
def _VoiceClonePrompt():
|
||||
"""Resolved at call time — a module-level binding could go stale when
|
||||
another suite purges omnivoice.* from sys.modules (CodeRabbit, #1565)."""
|
||||
from omnivoice.models.omnivoice import VoiceClonePrompt
|
||||
return VoiceClonePrompt
|
||||
|
||||
|
||||
def _prompt():
|
||||
return _VoiceClonePrompt()(
|
||||
ref_audio_tokens=torch.arange(24, dtype=torch.long).reshape(8, 3),
|
||||
ref_text="Nice to meet you.",
|
||||
ref_rms=0.123,
|
||||
)
|
||||
|
||||
|
||||
class _StubModel:
|
||||
def __init__(self):
|
||||
self.encodes = 0
|
||||
|
||||
def create_voice_clone_prompt(self, ref_audio, ref_text=None, preprocess_prompt=True):
|
||||
self.encodes += 1
|
||||
return _prompt()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated(tmp_path, monkeypatch):
|
||||
"""Point the disk layer at a per-test dir and start with empty caches."""
|
||||
monkeypatch.setattr("core.config.DATA_DIR", tmp_path / "data")
|
||||
monkeypatch.delenv("OMNIVOICE_PROMPT_DISK_CACHE", raising=False)
|
||||
_tb().clear_clone_prompt_cache()
|
||||
yield
|
||||
_tb().clear_clone_prompt_cache()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def ref_wav(tmp_path):
|
||||
p = tmp_path / "ref.wav"
|
||||
p.write_bytes(b"\x00" * 256)
|
||||
return str(p)
|
||||
|
||||
|
||||
def _disk_files(tmp_path):
|
||||
d = tmp_path / "data" / "prompt_cache"
|
||||
return sorted(d.glob("*.pt")) if d.is_dir() else []
|
||||
|
||||
|
||||
# ── the ported save/load format ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_prompt_save_load_roundtrip(tmp_path):
|
||||
p = _prompt()
|
||||
path = str(tmp_path / "voice.pt")
|
||||
p.save(path)
|
||||
loaded = _VoiceClonePrompt().load(path)
|
||||
assert torch.equal(loaded.ref_audio_tokens, p.ref_audio_tokens)
|
||||
assert loaded.ref_text == p.ref_text
|
||||
assert loaded.ref_rms == pytest.approx(p.ref_rms)
|
||||
# The file must stay loadable under torch's safe default (weights_only=True
|
||||
# since 2.6) — a pickled dataclass would not be.
|
||||
raw = torch.load(path, weights_only=True)
|
||||
assert raw["format_version"] == 1
|
||||
|
||||
|
||||
def test_prompt_load_rejects_unknown_format_version(tmp_path):
|
||||
path = str(tmp_path / "future.pt")
|
||||
torch.save({"format_version": 999}, path)
|
||||
with pytest.raises(ValueError, match="format version"):
|
||||
_VoiceClonePrompt().load(path)
|
||||
|
||||
|
||||
def test_saved_tokens_are_cpu_even_from_dataclass_on_another_device(tmp_path):
|
||||
# save() must detach+CPU the tokens so the file is portable. On CUDA hosts
|
||||
# this exercises the real device move; CI (CPU-only) still verifies the
|
||||
# detach and that the persisted payload is CPU-resident.
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
p = _VoiceClonePrompt()(
|
||||
ref_audio_tokens=torch.zeros(8, 3, requires_grad=True).to(device),
|
||||
ref_text="x",
|
||||
ref_rms=0.5,
|
||||
)
|
||||
path = str(tmp_path / "v.pt")
|
||||
p.save(path)
|
||||
loaded = _VoiceClonePrompt().load(path)
|
||||
assert not loaded.ref_audio_tokens.requires_grad
|
||||
assert loaded.ref_audio_tokens.device.type == "cpu"
|
||||
# The device move must happen at SAVE time (portability of the file
|
||||
# itself), not merely at load: the raw payload carries CPU tensors.
|
||||
assert torch.load(path, weights_only=True)["ref_audio_tokens"].device.type == "cpu"
|
||||
|
||||
|
||||
# ── the disk layer under the memory cache ───────────────────────────────────
|
||||
|
||||
|
||||
def test_disk_hit_survives_restart(tmp_path, ref_wav):
|
||||
tb = _tb()
|
||||
model = _StubModel()
|
||||
|
||||
first = tb._get_clone_prompt(model, ref_wav, "hello", True)
|
||||
assert model.encodes == 1
|
||||
assert len(_disk_files(tmp_path)) == 1
|
||||
|
||||
tb.clear_clone_prompt_cache() # "restart": memory gone, disk remains
|
||||
second = tb._get_clone_prompt(model, ref_wav, "hello", True)
|
||||
assert model.encodes == 1, "restart re-encoded despite a persisted prompt"
|
||||
assert torch.equal(second.ref_audio_tokens, first.ref_audio_tokens)
|
||||
assert second.ref_text == first.ref_text
|
||||
|
||||
|
||||
def test_edited_reference_is_not_served_a_stale_prompt(tmp_path, ref_wav):
|
||||
import os
|
||||
|
||||
tb = _tb()
|
||||
model = _StubModel()
|
||||
tb._get_clone_prompt(model, ref_wav, "hello", True)
|
||||
tb.clear_clone_prompt_cache()
|
||||
|
||||
# Same path, new content+mtime → new key → the old file must not match.
|
||||
with open(ref_wav, "wb") as f:
|
||||
f.write(b"\x01" * 512)
|
||||
os.utime(ref_wav, (1, 1))
|
||||
tb._get_clone_prompt(model, ref_wav, "hello", True)
|
||||
assert model.encodes == 2
|
||||
|
||||
|
||||
def test_single_use_refs_never_touch_disk(tmp_path, ref_wav):
|
||||
tb = _tb()
|
||||
tb._get_clone_prompt(_StubModel(), ref_wav, "hello", True, store=False)
|
||||
assert _disk_files(tmp_path) == [], (
|
||||
"store=False (dub per-segment clips) must not spray single-use "
|
||||
"prompts onto disk — same scan-resistance as the memory LRU"
|
||||
)
|
||||
|
||||
|
||||
def test_env_kill_switch_disables_the_layer(tmp_path, ref_wav, monkeypatch):
|
||||
monkeypatch.setenv("OMNIVOICE_PROMPT_DISK_CACHE", "0")
|
||||
tb = _tb()
|
||||
model = _StubModel()
|
||||
tb._get_clone_prompt(model, ref_wav, "hello", True)
|
||||
assert _disk_files(tmp_path) == []
|
||||
tb.clear_clone_prompt_cache()
|
||||
tb._get_clone_prompt(model, ref_wav, "hello", True)
|
||||
assert model.encodes == 2 # no disk → honest re-encode
|
||||
|
||||
|
||||
def test_corrupt_disk_entry_is_dropped_and_reencoded(tmp_path, ref_wav):
|
||||
tb = _tb()
|
||||
model = _StubModel()
|
||||
tb._get_clone_prompt(model, ref_wav, "hello", True)
|
||||
tb.clear_clone_prompt_cache()
|
||||
|
||||
disk = _disk_files(tmp_path)
|
||||
assert len(disk) == 1
|
||||
disk[0].write_bytes(b"not a torch file")
|
||||
|
||||
prompt = tb._get_clone_prompt(model, ref_wav, "hello", True)
|
||||
assert prompt is not None
|
||||
assert model.encodes == 2, "corrupt file must fall back to encoding"
|
||||
# ...and the corrupt file was removed, then replaced by the fresh save.
|
||||
fresh = _disk_files(tmp_path)
|
||||
assert len(fresh) == 1
|
||||
assert torch.load(str(fresh[0]), weights_only=True)["format_version"] == 1
|
||||
|
||||
|
||||
def test_prune_keeps_only_the_newest(tmp_path, monkeypatch):
|
||||
import os
|
||||
import time
|
||||
|
||||
tb = _tb()
|
||||
monkeypatch.setattr(tb, "_PROMPT_DISK_CACHE_MAX", 3)
|
||||
model = _StubModel()
|
||||
refs = []
|
||||
for i in range(5):
|
||||
p = tmp_path / f"ref{i}.wav"
|
||||
p.write_bytes(bytes([i]) * 64)
|
||||
os.utime(p, (i + 1, i + 1))
|
||||
refs.append(str(p))
|
||||
for i, r in enumerate(refs):
|
||||
tb._get_clone_prompt(model, r, f"text {i}", True)
|
||||
# mtime is the prune order; keep saves strictly ordered even on
|
||||
# filesystems with coarse timestamps.
|
||||
files = _disk_files(tmp_path)
|
||||
newest = max(files, key=lambda f: f.stat().st_mtime)
|
||||
os.utime(newest, (1000 + i, 1000 + i))
|
||||
assert len(_disk_files(tmp_path)) == 3
|
||||
|
||||
|
||||
def test_unwritable_cache_dir_never_breaks_prompt_building(ref_wav, monkeypatch):
|
||||
# Simulate an unwritable data dir: the layer must vanish, not raise.
|
||||
monkeypatch.setattr(
|
||||
"core.config.DATA_DIR", "/proc/omnivoice-definitely-not-writable"
|
||||
)
|
||||
tb = _tb()
|
||||
model = _StubModel()
|
||||
assert tb._get_clone_prompt(model, ref_wav, "hello", True) is not None
|
||||
assert model.encodes == 1
|
||||
@@ -21,7 +21,9 @@ tests/test_synthetic_audio_watermark_1169.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import tokenize
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -75,6 +77,46 @@ _PRODUCERS = [
|
||||
]
|
||||
|
||||
|
||||
def _code_only(src: str) -> str:
|
||||
"""``src`` with comments and string literals blanked to spaces.
|
||||
|
||||
ee35d238 made a module a "producer" by *mentioning* ``backend.generate()``
|
||||
in a comment — prose can't synthesize audio. Only real call sites may
|
||||
match ``_SYNTH_CALL``, so blank every COMMENT/STRING token span (spaces,
|
||||
not deletion, to keep the layout the regexes were written against).
|
||||
Unparseable source falls back to the raw text — fail closed, a module we
|
||||
can't tokenize still gets scanned.
|
||||
|
||||
f-strings stay conservative (Greptile P1 on #1564): on Python ≤3.11 the
|
||||
whole f-string — replacement expressions included — is ONE STRING token,
|
||||
so blanking it would let ``f"{backend.generate(t)}"`` evade the guard.
|
||||
f-prefixed strings are therefore kept raw there (a literal f-string
|
||||
*mentioning* a primitive false-positives toward the allowlist — fail
|
||||
closed). On 3.12+ (PEP 701) replacement code arrives as ordinary tokens
|
||||
and only the literal FSTRING_MIDDLE text is blanked.
|
||||
"""
|
||||
fstring_middle = getattr(tokenize, "FSTRING_MIDDLE", None)
|
||||
lines = src.splitlines(keepends=True)
|
||||
try:
|
||||
tokens = list(tokenize.generate_tokens(io.StringIO(src).readline))
|
||||
except (tokenize.TokenError, IndentationError, SyntaxError):
|
||||
return src
|
||||
for tok in tokens:
|
||||
if tok.type == tokenize.STRING:
|
||||
prefix = tok.string.split(tok.string[-1], 1)[0].rstrip("\"'")
|
||||
if "f" in prefix.lower():
|
||||
continue # pre-3.12 f-string: may contain executable code
|
||||
elif tok.type not in (tokenize.COMMENT, fstring_middle):
|
||||
continue
|
||||
(srow, scol), (erow, ecol) = tok.start, tok.end
|
||||
for row in range(srow - 1, erow):
|
||||
line = lines[row]
|
||||
lo = scol if row == srow - 1 else 0
|
||||
hi = ecol if row == erow - 1 else len(line.rstrip("\r\n"))
|
||||
lines[row] = line[:lo] + " " * (hi - lo) + line[hi:]
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def _py_files():
|
||||
for sub in ("api", "services", "worker"):
|
||||
for p in sorted((_BACKEND / sub).rglob("*.py")):
|
||||
@@ -84,9 +126,11 @@ def _py_files():
|
||||
def test_every_synthesis_module_routes_through_mark_synthetic():
|
||||
offenders = []
|
||||
for rel, src in _py_files():
|
||||
if not _SYNTH_CALL.search(src):
|
||||
if not _SYNTH_CALL.search(_code_only(src)):
|
||||
continue
|
||||
if rel in _ALLOWED or "mark_synthetic" in src:
|
||||
# The satisfying reference must be code too — a comment saying
|
||||
# "mark_synthetic" must not certify a module (CodeRabbit, #1564).
|
||||
if rel in _ALLOWED or "mark_synthetic" in _code_only(src):
|
||||
continue
|
||||
offenders.append(rel)
|
||||
assert not offenders, (
|
||||
@@ -100,7 +144,7 @@ def test_every_synthesis_module_routes_through_mark_synthetic():
|
||||
@pytest.mark.parametrize("rel", _PRODUCERS)
|
||||
def test_known_producer_still_marks(rel):
|
||||
src = (_BACKEND / rel).read_text(encoding="utf-8")
|
||||
assert "mark_synthetic" in src, (
|
||||
assert "mark_synthetic" in _code_only(src), (
|
||||
f"{rel} lost its mark_synthetic call — its synthetic audio would ship "
|
||||
"without the Art. 50(2) provenance mark (#1169)."
|
||||
)
|
||||
@@ -127,12 +171,34 @@ def test_allowlist_is_not_stale():
|
||||
p = _BACKEND / rel
|
||||
assert p.is_file(), f"watermark-coverage list names a missing file: {rel}"
|
||||
for rel in _ALLOWED:
|
||||
assert _SYNTH_CALL.search((_BACKEND / rel).read_text(encoding="utf-8")), (
|
||||
assert _SYNTH_CALL.search(_code_only((_BACKEND / rel).read_text(encoding="utf-8"))), (
|
||||
f"{rel} no longer matches a synthesis primitive — remove it from "
|
||||
"tests/test_watermark_route_coverage.py so the guard stays sharp."
|
||||
)
|
||||
|
||||
|
||||
def test_prose_mentions_are_not_producers():
|
||||
"""The ee35d238 regression: a comment (or log string / docstring) naming a
|
||||
synthesis primitive must not make a module a producer — only a call can."""
|
||||
prose = (
|
||||
"# A generic backend.generate() call accepts the same wire shape\n"
|
||||
'MSG = "route through generate_with_cached_ref(model) instead"\n'
|
||||
"def f():\n"
|
||||
' """Docs may mention _run_inference( freely."""\n'
|
||||
" return 1\n"
|
||||
)
|
||||
assert not _SYNTH_CALL.search(_code_only(prose))
|
||||
real = "def f(backend):\n return backend.generate(text='hi')\n"
|
||||
assert _SYNTH_CALL.search(_code_only(real))
|
||||
# Greptile P1: a call inside an f-string replacement field is code and
|
||||
# must still be caught, on every supported Python (≤3.11 tokenizes the
|
||||
# whole f-string as one STRING; 3.12+ splits out the expression tokens).
|
||||
fstring_call = 'def f(backend):\n return f"{backend.generate(text=\'hi\')}"\n'
|
||||
assert _SYNTH_CALL.search(_code_only(fstring_call))
|
||||
# ...and a comment claiming mark_synthetic must not certify a producer.
|
||||
assert "mark_synthetic" not in _code_only("# routes via mark_synthetic\nx = 1\n")
|
||||
|
||||
|
||||
# ── mark_synthetic unit contract (delegation, not new policy) ────────────────
|
||||
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ def test_registration_declares_semantic_features(proto):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_worker_is_visibly_refused_before_running_wrong_audio():
|
||||
"""An old peer can share v1's protobuf shape while missing inputs/progress.
|
||||
"""An old peer can share v1's protobuf shape while missing render parity.
|
||||
|
||||
Registration must fail by name, before authentication or task dispatch,
|
||||
instead of allowing a clone with no reference audio to report SUCCESS.
|
||||
@@ -95,6 +95,11 @@ async def test_old_worker_is_visibly_refused_before_running_wrong_audio():
|
||||
assert REQUIRED_FEATURES
|
||||
|
||||
|
||||
def test_remote_tts_render_parity_is_a_required_worker_feature():
|
||||
"""Do not let an old worker silently bypass the canonical TTS pipeline."""
|
||||
assert "remote_tts_render_v1" in REQUIRED_FEATURES
|
||||
|
||||
|
||||
# ── Control / data plane separation ────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Worker TTS must preserve the same rendering contract as local `/generate`.
|
||||
|
||||
The control plane already sends the profile reference, pinned seed and every
|
||||
quality control to a remote worker. This test protects the other half of that
|
||||
contract: the worker must call the canonical render helpers rather than a bare
|
||||
``backend.generate()`` call that silently discards the controls.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import nullcontext
|
||||
|
||||
|
||||
def _gallery_params():
|
||||
return {
|
||||
"ref_audio": "/worker-inputs/whisper-gallery.wav",
|
||||
"ref_text": "The gallery sample transcript.",
|
||||
"instruct": "female, whispering, warm",
|
||||
"language": "English",
|
||||
"duration": 3.5,
|
||||
"speed": 0.9,
|
||||
"num_step": 32,
|
||||
"guidance_scale": 2.0,
|
||||
"denoise": True,
|
||||
"postprocess_output": True,
|
||||
"t_shift": 0.4,
|
||||
"layer_penalty_factor": 1.1,
|
||||
"position_temperature": 0.7,
|
||||
"class_temperature": 0.8,
|
||||
"seed": 42,
|
||||
"max_chunk_chars": 180,
|
||||
"crossfade_ms": 55,
|
||||
"effect_preset": "broadcast",
|
||||
}
|
||||
|
||||
|
||||
def test_worker_omnivoice_preserves_gallery_identity_contract(monkeypatch):
|
||||
"""A selected Whisper archetype must reach native render unchanged."""
|
||||
from services import tts_backend
|
||||
from worker.executor import TaskExecutor
|
||||
import api.routers.generation as generation
|
||||
|
||||
model = object()
|
||||
backend = tts_backend.OmniVoiceBackend(model=model)
|
||||
captured = {}
|
||||
|
||||
monkeypatch.setattr(tts_backend, "engine_in_use", lambda _backend: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
generation,
|
||||
"_run_inference",
|
||||
lambda *args, **kwargs: captured.update(args=args, kwargs=kwargs) or "audio",
|
||||
)
|
||||
|
||||
assert TaskExecutor._synthesize(backend, "Whispered test line.", _gallery_params()) == "audio"
|
||||
|
||||
args = captured["args"]
|
||||
assert args[0] is model
|
||||
assert args[1] == "Whispered test line."
|
||||
assert args[3] == "/worker-inputs/whisper-gallery.wav"
|
||||
assert args[4] == "The gallery sample transcript."
|
||||
assert args[5] == "female, whispering, warm"
|
||||
assert args[7:11] == (32, 2.0, 0.9, 0.4)
|
||||
assert args[13:17] == (1.1, 0.7, 0.8, 42)
|
||||
assert args[17:20] == ("broadcast", 180, 55)
|
||||
|
||||
|
||||
def test_worker_generic_engine_preserves_seeded_render_controls(monkeypatch):
|
||||
"""Non-native engines use the generic canonical helper with the same knobs."""
|
||||
from services import tts_backend
|
||||
from worker.executor import TaskExecutor
|
||||
import api.routers.generation as generation
|
||||
|
||||
class Backend:
|
||||
applies_own_mastering = False
|
||||
|
||||
backend = Backend()
|
||||
captured = {}
|
||||
monkeypatch.setattr(tts_backend, "engine_in_use", lambda _backend: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
generation,
|
||||
"_run_backend_inference",
|
||||
lambda *args, **kwargs: captured.update(args=args, kwargs=kwargs) or "audio",
|
||||
)
|
||||
|
||||
assert TaskExecutor._synthesize(backend, "Whispered test line.", _gallery_params()) == "audio"
|
||||
|
||||
args = captured["args"]
|
||||
assert args[0] is backend
|
||||
assert args[1] == "Whispered test line."
|
||||
assert args[3] == "/worker-inputs/whisper-gallery.wav"
|
||||
assert args[4] == "The gallery sample transcript."
|
||||
assert args[5] == "female, whispering, warm"
|
||||
assert args[7:10] == (32, 2.0, 0.9)
|
||||
assert args[12:16] == (42, "broadcast", 180, 55)
|
||||
Reference in New Issue
Block a user