feat(electron): add full VoiceStudio desktop app

This commit is contained in:
Palash Debnath
2026-09-14 10:22:23 -07:00
parent eaf8bb9538
commit f832d616f7
824 changed files with 152701 additions and 771 deletions
+28
View File
@@ -172,6 +172,12 @@ jobs:
working-directory: frontend
run: node --experimental-strip-types --no-warnings --test ../tests/frontend/*.test.mjs
# Electron used to be built only after a release started, so renderer,
# preload and packaging regressions could pass the required PR gate.
# Keep this command shared with release.yml through the root script.
- name: Electron typecheck, tests and production contract
run: bun run check:electron
# Production-bundle blank-screen gate. Everything above runs UN-minified
# (dev server + Vitest/jsdom), so a crash that exists ONLY in the minified
# release bundle — a TDZ reorder that throws before React mounts — passes
@@ -186,6 +192,28 @@ jobs:
- name: Production-bundle smoke — no blank screen
working-directory: frontend
run: bun run test:prod-bundle
- name: Electron renderer workflow smokes
shell: bash
run: |
set -euo pipefail
export OMNIVOICE_PORT=3999
export VOICESTUDIO_UI_URL=http://localhost:3912
export PLAYWRIGHT_CHANNEL=chromium
bun run --cwd electron smoke:server > /tmp/voicestudio-electron-smoke.log 2>&1 &
server_pid=$!
trap 'kill "$server_pid" 2>/dev/null || true' EXIT
for _ in {1..60}; do
if curl --fail --silent --show-error "$VOICESTUDIO_UI_URL" >/dev/null; then
break
fi
sleep 0.25
done
curl --fail --silent --show-error "$VOICESTUDIO_UI_URL" >/dev/null || {
cat /tmp/voicestudio-electron-smoke.log
exit 1
}
node electron/tests/playback-smoke.mjs
node electron/tests/dub-smoke.mjs
# ── Cross-platform Tauri shell check ────────────────────────────────────
# Catches platform-specific Rust regressions on PR (cfg(target_os=...)
+131 -2
View File
@@ -142,6 +142,9 @@ jobs:
working-directory: frontend
run: node --experimental-strip-types --no-warnings --test ../tests/frontend/*.test.mjs
- name: Electron typecheck, tests and production contract
run: bun run check:electron
# Decide preview-vs-stable, and for nightly runs whether `main` actually
# moved in the last day. Outputs gate the expensive matrix (`build`) and the
# `preview-notes` job, so a no-commit night costs only this ~30s job.
@@ -311,7 +314,7 @@ jobs:
libwebkit2gtk-4.1-dev \
build-essential curl wget file libxdo-dev libssl-dev \
libayatana-appindicator3-dev librsvg2-dev \
libasound2-dev ffmpeg
libasound2-dev ffmpeg xvfb
# ── Frontend build ─────────────────────────────────────────────────
- name: Cache bun deps
@@ -344,7 +347,7 @@ jobs:
- name: Bundle uv (${{ matrix.rust_target }})
shell: bash
env:
UV_VERSION: "0.11.7"
UV_VERSION: "0.12.13"
TRIPLE: ${{ matrix.rust_target }}
run: |
set -euo pipefail
@@ -663,6 +666,83 @@ jobs:
updaterJsonPreferNsis: false
includeUpdaterJson: true
- name: Build + publish Electron desktop
shell: bash
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
RELEASE_TAG: ${{ (needs.preview-gate.outputs.is_preview == 'true') && 'preview' || github.ref_name }}
IS_PREVIEW: ${{ needs.preview-gate.outputs.is_preview }}
VOICESTUDIO_RUST_TARGET: ${{ matrix.rust_target }}
run: |
set -euo pipefail
case "${{ matrix.rust_target }}" in
aarch64-apple-darwin) ELECTRON_OS=darwin; ELECTRON_ARCH=arm64; FLAGS="--mac --arm64" ;;
x86_64-apple-darwin) ELECTRON_OS=darwin; ELECTRON_ARCH=x64; FLAGS="--mac --x64" ;;
x86_64-pc-windows-msvc) ELECTRON_OS=win32; ELECTRON_ARCH=x64; FLAGS="--win --x64" ;;
x86_64-unknown-linux-gnu) ELECTRON_OS=linux; ELECTRON_ARCH=x64; FLAGS="--linux --x64" ;;
*) echo "Unsupported Electron target: ${{ matrix.rust_target }}"; exit 1 ;;
esac
if [ "$IS_PREVIEW" = "true" ]; then
export VOICESTUDIO_UPDATE_CHANNEL="electron-preview-${ELECTRON_OS}-${ELECTRON_ARCH}"
else
export VOICESTUDIO_UPDATE_CHANNEL="electron-stable-${ELECTRON_OS}-${ELECTRON_ARCH}"
fi
if [ -n "${APPLE_CERTIFICATE:-}" ]; then
export CSC_LINK="$APPLE_CERTIFICATE"
export CSC_KEY_PASSWORD="${APPLE_CERTIFICATE_PASSWORD:-}"
fi
bun install --frozen-lockfile
(
cd electron
bun run build
node tests/packaging-contract.mjs
bun x electron-builder \
--config electron-builder.config.mjs $FLAGS --publish never
node tests/packaging-contract.mjs --artifact
if [ "$RUNNER_OS" = "Linux" ]; then
xvfb-run -a node tests/packaged-smoke.mjs --setup
else
node tests/packaged-smoke.mjs --setup
fi
node tests/update-package-contract.mjs \
--channel "$VOICESTUDIO_UPDATE_CHANNEL" \
--platform "$ELECTRON_OS" \
--arch "$ELECTRON_ARCH"
)
# A rolling preview reuses one release. Remove only this platform /
# architecture's older Electron artifacts before publishing the new
# version; sibling matrix legs own different names and metadata.
if [ "$IS_PREVIEW" = "true" ]; then
gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --json assets \
--jq '.assets[].name' > electron-assets.txt
case "${{ runner.os }}" in
Windows) OS_TOKEN=win ;;
macOS) OS_TOKEN=mac ;;
Linux) OS_TOKEN=linux ;;
esac
while IFS= read -r asset; do
case "$asset" in
VoiceStudio-Electron-*-${OS_TOKEN}-${ELECTRON_ARCH}.*|${VOICESTUDIO_UPDATE_CHANNEL}*.yml)
gh release delete-asset "$RELEASE_TAG" "$asset" --yes --repo "$GITHUB_REPOSITORY"
;;
esac
done < electron-assets.txt
fi
electron_artifact_count=0
while IFS= read -r artifact; do
gh release upload "$RELEASE_TAG" "$artifact" --clobber --repo "$GITHUB_REPOSITORY"
electron_artifact_count=$((electron_artifact_count + 1))
done < <(find electron/release -maxdepth 1 -type f \
\( -name 'VoiceStudio-Electron-*' -o -name "${VOICESTUDIO_UPDATE_CHANNEL}*.yml" \) | sort)
if [ "$electron_artifact_count" -eq 0 ]; then
echo "FAIL — Electron build produced no publishable artifacts"
find electron/release -maxdepth 1 -type f -print || true
exit 1
fi
- name: Build per-user Windows MSI
if: runner.os == 'Windows'
shell: bash
@@ -914,6 +994,10 @@ jobs:
-o -name "*.msi" -o -name "*.msi.sig" \
-o -name "*.AppImage" -o -name "*.AppImage.sig" \
-o -name "*.deb" \) 2>/dev/null | sort)
while IFS= read -r artifact; do
ARTIFACTS+=("$artifact")
done < <(find electron/release -maxdepth 1 -type f \
\( -name 'VoiceStudio-Electron-*' -o -name 'electron-*.yml' \) 2>/dev/null | sort)
if [ ${#ARTIFACTS[@]} -eq 0 ]; then
echo "FAIL — no artifacts found under $BUNDLE_DIR"
@@ -1031,6 +1115,51 @@ jobs:
# This job runs once after the whole matrix as the single final writer:
# it makes the manifest's linux signature agree with the .sig asset that
# actually shipped, and refuses to leave a mismatch behind.
# The matrix validates each Electron package before upload. This final read-only
# check validates the other half of the contract: GitHub must actually serve
# all four manifests and every payload they name. Without it a green release
# can leave the in-app updater with four 404 feeds.
electron-publish-contract:
needs: [build, preview-gate]
if: >-
needs.build.result == 'success' &&
(needs.preview-gate.outputs.is_preview == 'true' || startsWith(github.ref, 'refs/tags/v'))
runs-on: ubuntu-22.04
permissions:
contents: read
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ (needs.preview-gate.outputs.is_preview == 'true') && 'preview' || github.ref_name }}
CHANNEL: ${{ (needs.preview-gate.outputs.is_preview == 'true') && 'preview' || 'stable' }}
STABLE_TAG: ${{ needs.preview-gate.outputs.stable_tag }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Verify published Electron updater assets
shell: bash
run: |
set -euo pipefail
WORK="$(mktemp -d)"
mkdir -p "$WORK/manifests"
gh release view "$TAG" --repo "$GITHUB_REPOSITORY" \
--json tagName,isPrerelease,assets > "$WORK/release.json"
gh release download "$TAG" --repo "$GITHUB_REPOSITORY" \
--pattern "electron-${CHANNEL}-*.yml" --dir "$WORK/manifests"
if [ "$CHANNEL" = "preview" ]; then
VERSION=$(python3 scripts/stamp-preview-version.py \
--package-json frontend/package.json \
--stable-tag "$STABLE_TAG" \
--run-number "${{ github.run_number }}")
else
VERSION=$(python3 -c 'import json; print(json.load(open("frontend/package.json"))["version"])')
fi
python3 scripts/check_electron_release_assets.py \
--release-json "$WORK/release.json" \
--manifest-dir "$WORK/manifests" \
--channel "$CHANNEL" \
--version "$VERSION"
repair-updater-manifest:
needs: [build, preview-gate]
runs-on: ubuntu-latest
+3
View File
@@ -22,6 +22,8 @@ node_modules
.turbo/
bun.lockb
frontend/src-tauri/target/
electron/.tmp-native-target/
native/**/target*/
# ─────────────────────────────────────────────────────────────────────────
# Secrets & env
@@ -55,6 +57,7 @@ memxt.db-wal
!.claude/agents/**
/.cache*
/.tmp/
/.tmp-*
# ─────────────────────────────────────────────────────────────────────────
# Research clones — upstream repos used as reference, not shipped
+2
View File
@@ -10,6 +10,8 @@ the frozen-backend fallback mirror it for their toolchains.
**Highlights**
- Electron now ships as a complete cross-platform VoiceStudio desktop app with local-first cloning, production workspaces, model packs, repair agents, native integrations, updates, parity checks, and the shared backend contracts required by those workflows (#1823)
- The Model Catalogue is one page: what you use now on top, then each family's engines and weights (#2013)
- VoxCPM2 installs in one click into its own environment, with the CUDA build of PyTorch on NVIDIA GPUs (#2021)
- MOSS-TTS-Nano installs in one click into its own environment, pinned to a reviewed upstream commit it works with (#2022)
+6 -1
View File
@@ -420,8 +420,11 @@ def _omnivoice_sampling_kwargs(opts: ExpressiveOptions) -> dict:
today exactly: num_step 32, guidance 2.0, and NO temperature/postprocess
kwargs (the model keeps its own defaults). Emotion is never forwarded —
the VoiceStudio config rejects unknown kwargs."""
from services.performance_profiles import tts_defaults
defaults = tts_defaults()
kw = {
"num_step": opts.num_step if opts.num_step is not None else LONGFORM_NUM_STEP,
"num_step": opts.num_step if opts.num_step is not None else defaults.get("num_step", LONGFORM_NUM_STEP),
"guidance_scale": (
opts.guidance_scale if opts.guidance_scale is not None else LONGFORM_GUIDANCE_SCALE
),
@@ -432,6 +435,8 @@ def _omnivoice_sampling_kwargs(opts: ExpressiveOptions) -> dict:
kw["class_temperature"] = opts.class_temperature
if opts.postprocess_output is not None:
kw["postprocess_output"] = opts.postprocess_output
elif "postprocess_output" in defaults:
kw["postprocess_output"] = defaults["postprocess_output"]
return kw
+463 -146
View File
@@ -9,6 +9,8 @@ the SQLite `jobs` table for history, but the queue itself restarts empty
on backend restart — intentional, since GPU jobs can't be safely resumed.
"""
import os
import json
import shutil
import uuid
import time
import asyncio
@@ -16,20 +18,42 @@ import logging
from typing import Optional, List
from fastapi import APIRouter, File, UploadFile, HTTPException, Form
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from core.config import DATA_DIR
from core import failure
from core.logging_utils import log_safe
from core.file_cleanup import FileCleanupError, unlink_if_present
from services.dub_batching import (
BATCH_WIDTH_ENV,
batch_timeout_s as _batch_timeout_s,
native_batch_width as _native_batch_width,
)
from services import gpu_gateway
from services.segment_bundle import extract_segment_wavs, remove_segment_wavs
from services.tts_backend import active_backend_id, resolve_generation_backend
router = APIRouter()
logger = logging.getLogger("omnivoice.batch")
# Compatibility values emitted by the established Tauri Batch picker. They
# are taxonomy tokens, not arbitrary prose, and are resolved server-side so
# native watch-folder uploads and both desktop clients use the same voice.
_BATCH_PRESET_INSTRUCT = {
"narrator": "male, middle-aged, low pitch, british accent",
"excited_child": "child, high pitch",
"anxious_whisper": "young adult, whisper",
"surprised_woman": "female, young adult, high pitch",
"elderly_story": "male, elderly, very low pitch",
"sichuan": "female, young adult, moderate pitch, \u56db\u5ddd\u8bdd",
}
# ── In-memory queue ─────────────────────────────────────────────────────
_queue: asyncio.Queue = None # Lazily initialised
_worker_task: asyncio.Task = None # Background consumer
_processing_job_ids: set[str] = set()
_jobs: dict = {} # job_id → status dict
@@ -40,11 +64,15 @@ class BatchJobStatus(BaseModel):
langs: List[str]
voice_id: Optional[str] = None
preserve_bg: bool = True
translation_provider: Optional[str] = None
created_at: float
started_at: Optional[float] = None
finished_at: Optional[float] = None
error: Optional[str] = None
progress: Optional[dict] = None
attempts: int = 1
retry_ready: bool = True
setup_required: Optional[dict] = None
def _ensure_queue():
@@ -66,6 +94,7 @@ async def _worker():
job["status"] = "running"
job["started_at"] = time.time()
_processing_job_ids.add(job_id)
logger.info("Batch job %s starting: %s", job_id, job["filename"])
try:
@@ -95,6 +124,9 @@ async def _worker():
job["finished_at"] = time.time()
logger.error("Batch job %s failed: %s", job_id, e, exc_info=True)
finally:
_processing_job_ids.discard(job_id)
if job["status"] == "cancelled":
job["retry_ready"] = True
_queue.task_done()
@@ -104,18 +136,62 @@ def _set_progress(job, stage, percent=0, **extra):
#: Override for the native dub batch width. Set to 1 to disable batching.
BATCH_WIDTH_ENV = "OMNIVOICE_DUB_BATCH_WIDTH"
#: Hard ceiling on the override — a batch this wide is already amortizing
#: almost all of the per-call setup, and beyond it the failure mode is an OOM
#: that costs more than the saving.
_MAX_BATCH_WIDTH = 16
# Bound each allocation while persisting multipart uploads. Video inputs can
# be many gigabytes; `await UploadFile.read()` with no size used to mirror the
# entire file in process memory before writing it back out.
_UPLOAD_CHUNK_BYTES = 1024 * 1024
_REMOTE_BATCH_OPERATION = "batch_segments"
async def _resolve_batch_execution(voice: dict):
"""Resolve Batch's TTS target without loading local weights remotely."""
engine_id = active_backend_id()
decision = gpu_gateway.decide("batch")
if decision.remote:
await gpu_gateway.preflight(
engine_id,
decision,
operation=_REMOTE_BATCH_OPERATION,
)
return engine_id, decision, None
backend = await resolve_generation_backend(
require_cloning=voice["requires_cloning"],
cloning_purpose="this batch job's pinned voice",
)
return engine_id, decision, backend
def _decode_remote_batch(
result: gpu_gateway.RemoteResult,
batch_dir: str,
expected: set[int],
) -> tuple[dict[int, str], int]:
"""Validate and unpack one worker result before accepting remote success."""
import soundfile as sf
target = os.path.join(batch_dir, ".remote", result.task_id)
paths = extract_segment_wavs(result.path or "", target)
try:
if set(paths) != expected:
missing = sorted(expected - set(paths))
extra = sorted(set(paths) - expected)
raise ValueError(
f"segment bundle mismatch (missing={missing}, extra={extra})"
)
rates = {int(sf.info(path).samplerate) for path in paths.values()}
if len(rates) != 1 or next(iter(rates), 0) <= 0:
raise ValueError("segment bundle has inconsistent sample rates")
return paths, rates.pop()
except BaseException:
remove_segment_wavs(paths)
raise
async def _save_upload(upload: UploadFile, destination: str) -> None:
try:
@@ -130,64 +206,63 @@ async def _save_upload(upload: UploadFile, destination: str) -> None:
raise
def _native_batch_width(backend) -> int:
"""How many segments to render in one native batch on THIS host.
def _batch_voice(voice_id: str | None) -> dict:
"""Resolve one queue-wide voice into concrete generation inputs.
A native batch widens the forward pass, so the width cannot be a constant.
The default engine declares ``min_vram_gb = 6.0`` for a SINGLE job; an
unconditional 8-wide batch would OOM the 4-8 GB CUDA cards and the MPS
Macs where the per-segment path succeeds today — turning a throughput
optimization into a regression on exactly the hardware that already
struggles (#1616 is a 4 GB card reporting capacity failures). Default
behaviour must not get riskier on a host, so the width is derived from
measured headroom and falls back to 1 (no batching) when unknown.
CPU hosts get 1: batching there buys no kernel amortization and only
multiplies peak RAM.
Clone profiles contribute their reference; designed profiles contribute
their healed instruction and seed. Legacy ``preset:`` selections become
the same instruction used by Dubbing instead of falling through to the
engine default.
"""
override = os.environ.get(BATCH_WIDTH_ENV, "").strip()
if override:
try:
return max(1, min(_MAX_BATCH_WIDTH, int(override)))
except (TypeError, ValueError):
logger.warning(
"%s=%r is not an integer — deriving the batch width from the host instead.",
BATCH_WIDTH_ENV, override,
)
try:
from core.device_caps import detect_host_caps
caps = detect_host_caps()
except Exception: # noqa: BLE001 — an unprobeable host takes the safe path
return 1
if caps.family == "cpu" or not caps.vram_gb:
return 1
headroom = caps.vram_gb - float(getattr(backend, "min_vram_gb", 0.0) or 0.0)
if headroom < 2.0:
return 1
if headroom < 6.0:
return 2
if headroom < 12.0:
return 4
return 8
resolved = {
"ref_audio": None,
"ref_text": None,
"instruct": "",
"seed": None,
"requires_cloning": False,
}
if not voice_id:
return resolved
if voice_id.startswith("preset:"):
preset_id = voice_id.removeprefix("preset:")
instruct = _BATCH_PRESET_INSTRUCT.get(preset_id)
if instruct is None:
raise ValueError("That built-in voice preset no longer exists")
from omnivoice.utils.voice_design import sanitize_instruct
resolved["instruct"] = sanitize_instruct(instruct)
return resolved
def _batch_timeout_s(texts: list[str], backend) -> float:
"""Execution budget for one native batch.
from core.config import VOICES_DIR
from core.db import db_conn
Not the sum of the per-item budgets: ``generate_timeout_s`` returns a
floor (300s GPU / 600s CPU) plus per-length overage, so summing it across
eight items yields a ~2400s budget — and a wedged batch would hold a
GPU-pool worker for forty minutes before the reset this file depends on
(#730). One floor covers wedge detection for the whole call; only the
length-driven overage is genuinely additive.
"""
from services.model_manager import generate_timeout_s
with db_conn() as conn:
row = conn.execute(
"SELECT * FROM voice_profiles WHERE id=?",
(voice_id,),
).fetchone()
if row is None:
raise ValueError("That saved voice no longer exists")
floor = generate_timeout_s("", engine=backend)
overage = sum(
max(0.0, generate_timeout_s(text, engine=backend) - floor) for text in texts
)
return floor + overage
if row["kind"] == "design":
from omnivoice.utils.voice_design import heal_design_instruct
resolved["instruct"] = heal_design_instruct(row["instruct"], row["vd_states"])
resolved["seed"] = int(row["seed"]) if row["seed"] is not None else None
return resolved
relative = row["locked_audio_path"] if row["is_locked"] else row["ref_audio_path"]
if not relative:
raise ValueError("That saved voice has no reference audio")
ref_audio = os.path.join(VOICES_DIR, relative)
if not os.path.isfile(ref_audio):
raise ValueError("That saved voice's reference audio is missing")
resolved.update({
"ref_audio": ref_audio,
"ref_text": row["ref_text"],
"requires_cloning": True,
})
return resolved
async def _run_batch_pipeline(job_id: str, job: dict):
@@ -280,19 +355,30 @@ async def _run_batch_pipeline(job_id: str, job: dict):
return
# ── Engine resolution (issue #312 class) ────────────────────────────
# Batch used to hardcode VoiceStudio via get_model() regardless of the
# engine selected in Model Catalogue. require_cloning only when a
# specific voice is pinned (job["voice_id"]) — an unpinned job is fine on
# any active engine. Resolved ONCE for the whole job (every language
# Batch used to hardcode VoiceStudio regardless of the engine selected in
# Model Catalogue. Clone profiles require a cloning-capable engine; presets
# and designed voices use instruction mode. Resolve once for the whole job.
# below shares the same active engine); an uncaught ValueError here
# propagates to _worker()'s existing except-Exception handling, which
# already records a structured job failure via core.failure.build_failure.
from services.tts_backend import resolve_generation_backend
backend = await resolve_generation_backend(
require_cloning=bool(job.get("voice_id")),
cloning_purpose="this batch job's pinned voice",
)
sr = backend.sample_rate
voice = _batch_voice(job.get("voice_id"))
engine_id, execution_target, backend = await _resolve_batch_execution(voice)
sr = backend.sample_rate if backend is not None else 0
from services.performance_profiles import tts_defaults
_profile_defaults = tts_defaults(engine_id)
_batch_num_step = _profile_defaults.get("num_step", 16)
_batch_postprocess = _profile_defaults.get("postprocess_output", True)
batch_run = gpu_gateway.JobRun("batch")
async def _prepare_local_batch() -> gpu_gateway.LocalCall:
nonlocal backend, sr
if backend is None:
backend = await resolve_generation_backend(
require_cloning=voice["requires_cloning"],
cloning_purpose="this batch job's pinned voice",
)
sr = backend.sample_rate
return gpu_gateway.LocalCall(fn=lambda: None, what="Batch TTS fallback")
# ── 3. Translate + Generate per language ───────────────────────────
total_langs = len(langs)
@@ -311,40 +397,65 @@ async def _run_batch_pipeline(job_id: str, job: dict):
translated_segments = list(segments) # copy
if target_lang != source_lang:
try:
def _translate_batch(segs, src, tgt):
"""Translate segment texts via Google Translate."""
from deep_translator import GoogleTranslator
TRANSLATE_CODES = {
"en": "en", "es": "es", "fr": "fr", "de": "de",
"it": "it", "pt": "pt", "ru": "ru", "ja": "ja",
"ko": "ko", "zh": "zh-CN", "ar": "ar", "hi": "hi",
"tr": "tr", "pl": "pl", "nl": "nl", "sv": "sv",
}
src_code = TRANSLATE_CODES.get(src, src) or "auto"
tgt_code = TRANSLATE_CODES.get(tgt, tgt)
translator = GoogleTranslator(source=src_code, target=tgt_code)
out = []
for s in segs:
s_copy = dict(s)
text = s.get("text", "").strip()
if text:
try:
s_copy["text"] = translator.translate(text) or text
except Exception as e:
logger.warning("Translate seg failed: %s", e)
out.append(s_copy)
return out
# Use the same provider dispatch as interactive Dubbing. The old
# batch-only implementation hardcoded Google and silently kept the
# source text on failure, which could make an English track labelled
# "es" while also sending text online despite an offline selection.
from api.routers.dub_translate import dub_translate
from schemas.requests import TranslateRequest
translated_segments = await loop.run_in_executor(
_cpu_pool, _translate_batch,
segments, source_lang, target_lang,
from core import prefs
provider = job.get("translation_provider") or prefs.get("translation_backend", "argos")
translation = await dub_translate(TranslateRequest(
segments=[
{
"id": str(segment["id"]),
"text": segment.get("text", ""),
"start": segment.get("start"),
"end": segment.get("end"),
}
for segment in segments
],
source_lang=source_lang,
target_lang=target_lang,
provider=provider,
quality="fast",
))
if isinstance(translation, JSONResponse):
try:
payload = json.loads(translation.body)
detail = payload.get("error") or payload.get("detail")
if payload.get("code") == "argos_pack_missing":
job["setup_required"] = {
"kind": "argos_packs",
"source_lang": source_lang,
"target_langs": [
pair["target_lang"]
for pair in payload.get("pairs", [])
if isinstance(pair, dict) and pair.get("target_lang")
],
}
except Exception: # noqa: BLE001 — retain the stable fallback
detail = None
raise RuntimeError(
detail or f"{provider} could not translate this batch"
)
except ImportError:
logger.warning("deep_translator not installed, skipping translation for %s", target_lang)
except Exception as e:
logger.warning("Translation failed for %s: %s, using original", target_lang, e)
translated_segments = segments
rows = {
str(row.get("id")): row
for row in translation.get("translated", [])
if isinstance(row, dict)
}
failed = [row for row in rows.values() if row.get("error")]
if failed or len(rows) != len(segments):
raise RuntimeError(
f"{provider} translation failed for "
f"{len(failed) or len(segments) - len(rows)} segment(s)"
)
translated_segments = [
{**segment, "text": rows[str(segment["id"])]["text"]}
for segment in segments
]
if job["status"] == "cancelled":
return
@@ -362,6 +473,90 @@ async def _run_batch_pipeline(job_id: str, job: dict):
from services.audio_io import atomic_save_wav
import torch
remote_segments: dict[int, str] = {}
valid_rows = [
(i, segment)
for i, segment in enumerate(translated_segments)
if segment.get("end", 0) - segment.get("start", 0) > 0.05
and segment.get("text", "").strip()
]
if execution_target.remote and valid_rows:
remote_rows = [
{
"index": i,
"text": segment.get("text", "").strip(),
"language": target_lang,
"ref_text": voice["ref_text"],
"instruct": voice["instruct"] or None,
"duration": segment.get("end", 0) - segment.get("start", 0),
"num_step": _batch_num_step,
"postprocess_output": _batch_postprocess,
"guidance_scale": 2.0,
"speed": 1.0,
"effect_preset": "batch",
"seed": (
voice["seed"] + i if voice["seed"] is not None else None
),
# The assembled track receives one watermark below. Marking
# each line here would double-process remote output.
"watermark": False,
}
for i, segment in valid_rows
]
expected = {row["index"] for row in remote_rows}
def _remote_state(state: dict) -> None:
fraction = max(0.0, min(1.0, float(state.get("progress") or 0.0)))
_set_progress(
job,
"generate",
percent=int(((lang_idx + fraction) / total_langs) * 100),
current_lang=target_lang,
current_segment=min(len(remote_rows), round(fraction * len(remote_rows))),
total_segments=len(remote_rows),
execution_target=execution_target.label,
execution_phase=state.get("phase"),
)
route_task = asyncio.create_task(
gpu_gateway.run(
"batch",
local=gpu_gateway.LocalCall(prepare=_prepare_local_batch),
remote=gpu_gateway.RemoteCall(
engine=engine_id,
operation=_REMOTE_BATCH_OPERATION,
params={
"segments": remote_rows,
"ref_audio": [voice["ref_audio"] for _ in remote_rows],
"input_seconds": sum(
float(row.get("duration") or 0.0) for row in remote_rows
),
},
idempotency_key=f"batch:{job_id}:{target_lang}",
decode=lambda result: _decode_remote_batch(
result, batch_dir, expected
),
),
decision=execution_target,
job=batch_run,
on_state=_remote_state,
)
)
while not route_task.done():
await asyncio.wait({route_task}, timeout=0.25)
if job["status"] == "cancelled":
route_task.cancel()
try:
await route_task
except asyncio.CancelledError:
pass
return
routed = route_task.result()
if routed is not None:
remote_segments, sr = routed
# A remote-only empty transcript still needs a valid silent-track rate.
sr = sr or 24_000
total_samples = int(duration * sr)
full_audio = torch.zeros(1, total_samples)
total_segs = len(translated_segments)
@@ -372,26 +567,15 @@ async def _run_batch_pipeline(job_id: str, job: dict):
# the established one-segment behavior below.
from services.tts_backend import TTSBackend
batched_audio: dict[int, torch.Tensor] = {}
has_native_batch = type(backend).generate_batch is not TTSBackend.generate_batch
has_native_batch = (
backend is not None
and type(backend).generate_batch is not TTSBackend.generate_batch
)
if has_native_batch:
from services.text_normalization import normalize_for_tts
batch_ref_audio = None
batch_ref_text = None
if job.get("voice_id"):
from core.db import db_conn
from core.config import VOICES_DIR as _VD
with db_conn() as conn:
row = conn.execute(
"SELECT * FROM voice_profiles WHERE id=?",
(job["voice_id"],),
).fetchone()
if row:
if row["is_locked"] and row["locked_audio_path"]:
batch_ref_audio = os.path.join(_VD, row["locked_audio_path"])
elif row["ref_audio_path"]:
batch_ref_audio = os.path.join(_VD, row["ref_audio_path"])
batch_ref_text = row["ref_text"]
batch_ref_audio = voice["ref_audio"]
batch_ref_text = voice["ref_text"]
batch_width = _native_batch_width(backend)
@@ -428,17 +612,20 @@ async def _run_batch_pipeline(job_id: str, job: dict):
]
def _render_native_batch():
if voice["seed"] is not None:
torch.manual_seed(voice["seed"])
generated = backend.generate_batch(
batch_texts,
language=target_lang,
ref_audio=batch_ref_audio,
ref_text=batch_ref_text,
instruct=voice["instruct"] or None,
duration=batch_durations,
num_step=16,
num_step=_batch_num_step,
guidance_scale=2.0,
speed=1.0,
denoise=True,
postprocess_output=True,
postprocess_output=_batch_postprocess,
)
if len(generated) != len(batch_indices):
raise RuntimeError(
@@ -473,6 +660,7 @@ async def _run_batch_pipeline(job_id: str, job: dict):
for i, seg in enumerate(translated_segments):
if job["status"] == "cancelled":
remove_segment_wavs(remote_segments)
return
_set_progress(
@@ -499,32 +687,18 @@ async def _run_batch_pipeline(job_id: str, job: dict):
from services.text_normalization import normalize_for_tts
text = normalize_for_tts(text, lang)
ref_audio = None
ref_text = None
# Use voice_id if provided
if job.get("voice_id"):
from core.db import db_conn
from core.config import VOICES_DIR as _VD
with db_conn() as conn:
row = conn.execute(
"SELECT * FROM voice_profiles WHERE id=?",
(job["voice_id"],),
).fetchone()
if row:
if row["is_locked"] and row["locked_audio_path"]:
ref_audio = os.path.join(_VD, row["locked_audio_path"])
elif row["ref_audio_path"]:
ref_audio = os.path.join(_VD, row["ref_audio_path"])
ref_text = row.get("ref_text")
try:
if backend is None:
raise RuntimeError("the local TTS fallback was not prepared")
if voice["seed"] is not None:
torch.manual_seed(voice["seed"] + i)
audio_out = backend.generate(
text=text, language=lang,
ref_audio=ref_audio, ref_text=ref_text,
duration=dur, num_step=16,
ref_audio=voice["ref_audio"], ref_text=voice["ref_text"],
instruct=voice["instruct"] or None,
duration=dur, num_step=_batch_num_step,
guidance_scale=2.0, speed=1.0,
denoise=True, postprocess_output=True,
denoise=True, postprocess_output=_batch_postprocess,
)
if not getattr(backend, "applies_own_mastering", False):
audio_out = apply_mastering(audio_out, sample_rate=sr)
@@ -548,15 +722,43 @@ async def _run_batch_pipeline(job_id: str, job: dict):
# Budget is the shared length-scaled one (#1190): a long segment
# on CPU-class hardware no longer dies on the flat 300s.
from services.model_manager import generate_timeout_s
if has_native_batch and i not in batched_audio:
await _prefetch_batch(i)
if i in batched_audio:
audio_tensor = batched_audio.pop(i)
remote_path = remote_segments.pop(i, None)
if remote_path is not None:
import soundfile as sf
try:
audio_array, remote_sr = sf.read(
remote_path,
dtype="float32",
always_2d=True,
)
if int(remote_sr) != sr:
raise ValueError(
f"remote segment sample rate changed from {sr} to {remote_sr}"
)
audio_tensor = torch.from_numpy(audio_array.T).mean(
dim=0,
keepdim=True,
)
finally:
remove_segment_wavs({i: remote_path})
else:
audio_tensor = await run_on_gpu_pool_guarded(
_gen, what="Batch generate",
timeout=generate_timeout_s(seg_text, engine=backend),
)
if backend is None:
await _prepare_local_batch()
# This path means a validated remote bundle lost a row
# after dispatch. Recover only that row; native batches
# were not planned for this language.
has_native_batch = False
if has_native_batch and i not in batched_audio:
await _prefetch_batch(i)
if i in batched_audio:
audio_tensor = batched_audio.pop(i)
else:
audio_tensor = await run_on_gpu_pool_guarded(
_gen,
what="Batch generate",
timeout=generate_timeout_s(seg_text, engine=backend),
)
# Fit to slot
target_samples_seg = int(seg_duration * sr)
@@ -604,6 +806,8 @@ async def _run_batch_pipeline(job_id: str, job: dict):
f"left silent: {e}"
)
remove_segment_wavs(remote_segments)
# ── 3c. Save dubbed audio track ───────────────────────────────
# Invisible provenance mark on the assembled track (#1169), tensor
# stage, before the WAV write / aac mux — batch dubs used to ship
@@ -670,6 +874,7 @@ async def _run_batch_pipeline(job_id: str, job: dict):
outputs[target_lang] = output_path
job["outputs"] = outputs
job.pop("setup_required", None)
_set_progress(job, "done", 100)
@@ -681,6 +886,7 @@ async def enqueue_batch_job(
langs: str = Form("es"), # comma-separated lang codes
voice_id: Optional[str] = Form(None),
preserve_bg: bool = Form(True),
translation_provider: Optional[str] = Form(None),
):
"""Enqueue a video for batch dubbing.
@@ -694,6 +900,14 @@ async def enqueue_batch_job(
if not lang_list:
raise HTTPException(400, "At least one target language is required")
# Validate the snapshot before persisting a potentially large upload.
# Resolve it again in the worker so deleting or editing a queued profile
# cannot silently fall back to the engine's default voice.
try:
await asyncio.to_thread(_batch_voice, voice_id)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
# TTS-only install: no ASR model on disk → typed 409 with a download CTA
# now, instead of accepting the job and having the transcribe stage
# silently auto-download multi-GB whisper weights (or fail) in the worker.
@@ -702,6 +916,19 @@ async def enqueue_batch_job(
if missing is not None:
raise HTTPException(409, {**missing, "message": asr_model_missing_detail(missing)})
# Snapshot the selected translation engine when the user enqueues the job,
# so a later Settings change cannot alter work already waiting in the queue.
from core import prefs
from services import translation_engines
provider = translation_provider or prefs.get("translation_backend", "argos")
if not translation_engines.get_engine(provider):
raise HTTPException(400, "Unknown translation engine")
if not translation_engines.is_installed(provider):
raise HTTPException(409, "Install the selected translation engine before adding this batch")
if not translation_engines.is_ready(provider):
raise HTTPException(409, "Configure the selected translation provider before adding this batch")
# Save the uploaded video
batch_dir = os.path.join(DATA_DIR, "batch")
os.makedirs(batch_dir, exist_ok=True)
@@ -718,7 +945,9 @@ async def enqueue_batch_job(
"langs": lang_list,
"voice_id": voice_id,
"preserve_bg": preserve_bg,
"translation_provider": provider,
"created_at": time.time(),
"attempts": 1,
"started_at": None,
"finished_at": None,
"error": None,
@@ -741,6 +970,8 @@ def list_batch_jobs(status: Optional[str] = None, limit: int = 50):
if status:
if status == "active":
jobs = [j for j in jobs if j["status"] in ("queued", "running")]
elif status == "retryable":
jobs = [j for j in jobs if j["status"] in ("failed", "cancelled")]
else:
jobs = [j for j in jobs if j["status"] == status]
jobs.sort(key=lambda j: j["created_at"], reverse=True)
@@ -764,14 +995,88 @@ def cancel_batch_job(job_id: str):
raise HTTPException(404, "Job not found")
if job["status"] in ("done", "failed", "cancelled"):
return {"already": job["status"]}
was_running = job["status"] == "running" or job_id in _processing_job_ids
job["status"] = "cancelled"
job["retry_ready"] = not was_running
job["finished_at"] = time.time()
return {"cancelled": True}
@router.post("/batch/jobs/{job_id}/retry")
async def retry_batch_job(job_id: str):
"""Retry a terminal job using its original app-owned upload and settings."""
job = _jobs.get(job_id)
if not job:
raise HTTPException(404, "Job not found")
if job["status"] not in ("failed", "cancelled"):
raise HTTPException(409, f"Job is {job['status']}, not retryable")
if job_id in _processing_job_ids or not job.get("retry_ready", True):
raise HTTPException(409, "The cancelled job is still stopping")
if not os.path.isfile(job.get("video_path") or ""):
raise HTTPException(409, "The original batch input is no longer available")
try:
await asyncio.to_thread(_batch_voice, job.get("voice_id"))
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
from services.asr_backend import asr_model_missing_detail, asr_model_missing_error
missing = await asyncio.to_thread(asr_model_missing_error)
if missing is not None:
raise HTTPException(409, {**missing, "message": asr_model_missing_detail(missing)})
from services import translation_engines
provider = job.get("translation_provider") or "argos"
if not translation_engines.is_ready(provider):
raise HTTPException(409, "Configure the selected translation provider before retrying")
if provider == "argos" and job.get("source_lang"):
status = await asyncio.to_thread(
translation_engines.argos_pack_status,
job["source_lang"],
job["langs"],
)
if any(not pair["installed"] for pair in status["pairs"]):
raise HTTPException(409, "Install the required Argos language packs before retrying")
batch_root = os.path.realpath(os.path.join(DATA_DIR, "batch"))
output_dir = os.path.realpath(os.path.join(batch_root, job_id))
if os.path.dirname(output_dir) != batch_root:
raise HTTPException(status_code=400, detail="Invalid batch job path")
try:
if os.path.isdir(output_dir):
await asyncio.to_thread(shutil.rmtree, output_dir)
except OSError as exc:
raise HTTPException(
status_code=500,
detail="Could not reset the batch output files. Close any app using them and retry.",
) from exc
for key in (
"duration",
"segments",
"source_lang",
"outputs",
"warnings",
"setup_required",
"retry_ready",
):
job.pop(key, None)
job.update({
"status": "queued",
"started_at": None,
"finished_at": None,
"error": None,
"progress": None,
"attempts": int(job.get("attempts", 1)) + 1,
})
_ensure_queue()
await _queue.put(job_id)
return {"job_id": job_id, "status": "queued", "queue_position": _queue.qsize()}
@router.delete("/batch/jobs/{job_id}")
def delete_batch_job(job_id: str):
"""Delete a batch job record and its video file."""
"""Delete a batch job record and every app-owned input/output file."""
job = _jobs.get(job_id)
if not job:
raise HTTPException(404, "Job not found")
@@ -783,6 +1088,18 @@ def delete_batch_job(job_id: str):
status_code=500,
detail="Could not delete the batch video file. Close any app using it and retry.",
) from exc
batch_root = os.path.realpath(os.path.join(DATA_DIR, "batch"))
output_dir = os.path.realpath(os.path.join(batch_root, job_id))
if os.path.dirname(output_dir) != batch_root:
raise HTTPException(status_code=400, detail="Invalid batch job path")
try:
if os.path.isdir(output_dir):
shutil.rmtree(output_dir)
except OSError as exc:
raise HTTPException(
status_code=500,
detail="Could not delete the batch output files. Close any app using them and retry.",
) from exc
_jobs.pop(job_id, None)
return {"deleted": True}
+79 -9
View File
@@ -60,7 +60,8 @@ async def transcribe_audio(
language: Optional language hint (not currently used; auto-detected).
model: Whisper model size (legacy; ignored in dual-mode architecture).
mode: 'fast' (default) uses MLX Turbo for speed; 'accurate' uses
WhisperX with forced alignment for word-level timing.
the selected ASR engine with word-level timing. 'reference' uses
the selected ASR engine without word-level timing.
refine: Opt-in local-LLM cleanup of the final text (disfluencies,
self-corrections, punctuation) — same pipeline the live
dictation socket uses. Off by default so MCP/CLI callers don't
@@ -91,7 +92,9 @@ async def transcribe_audio(
tmp.write(content)
tmp.close()
use_accurate = (mode or "").strip().lower() == "accurate"
requested_mode = (mode or "").strip().lower()
use_accurate = requested_mode == "accurate"
use_active_asr = requested_mode in {"accurate", "reference"}
# TTS-only install: no ASR model on disk → typed 409 with a download
# CTA, BEFORE any backend is constructed (the whisper backends
@@ -99,7 +102,8 @@ async def transcribe_audio(
from services.asr_backend import asr_model_missing_detail, asr_model_missing_error
missing = await asyncio.to_thread(
asr_model_missing_error,
purpose="transcribe" if use_accurate else "dictation",
purpose="transcribe" if use_active_asr else "dictation",
require_installed=requested_mode == "reference",
)
if missing is not None:
raise HTTPException(
@@ -108,7 +112,7 @@ async def transcribe_audio(
)
def _run():
if use_accurate:
if use_active_asr:
# Accurate mode: full WhisperX with forced alignment —
# for when the user explicitly wants word-level timing.
# `load_*`, not `get_*`: the selector alone hands back an
@@ -116,8 +120,8 @@ async def transcribe_audio(
# chain is broken, which then 500s at `.transcribe()`. The
# loader degrades to the next healthy engine (#1185).
from services.asr_backend import load_active_asr_backend
backend = load_active_asr_backend()
result = backend.transcribe(tmp.name, word_timestamps=True)
backend = load_active_asr_backend(require_installed=True) if requested_mode == "reference" else load_active_asr_backend()
result = backend.transcribe(tmp.name, word_timestamps=use_accurate)
else:
# Fast mode (default): use the fastest available engine
# (MLX Turbo on Apple Silicon). Skip word_timestamps for
@@ -125,7 +129,8 @@ async def transcribe_audio(
from services.asr_backend import get_capture_asr_backend
backend = get_capture_asr_backend()
result = backend.transcribe(tmp.name, word_timestamps=False)
return result, backend.id
sherpa_model_id = getattr(getattr(backend, "spec", None), "id", None)
return result, backend.id, sherpa_model_id
from services.model_manager import _gpu_pool
from services.asr_backend import (
@@ -135,7 +140,7 @@ async def transcribe_audio(
)
t0 = time.perf_counter()
try:
result, engine_id = await run_transcribe_guarded(
result, engine_id, sherpa_model_id = await run_transcribe_guarded(
_gpu_pool, _run, what="Dictation",
)
except ASRTimeoutError as e:
@@ -151,6 +156,69 @@ async def transcribe_audio(
status_code=409,
detail={**e.payload, "message": asr_model_missing_detail(e.payload)},
)
# Some sherpa-onnx NeMo-TDT builds load successfully but decode an
# entire spoken clip to no tokens. Live dictation already recovers
# from that failure; the shared file endpoint must do the same because
# it also powers uploaded transcription and automatic profile text.
# Retry only through an already-installed fallback, and demote the
# silent model only when the second recognizer actually heard words.
initial_text = str(result.get("text") or "").strip()
if not initial_text and result.get("segments"):
initial_text = " ".join(
str(segment.get("text") or "")
for segment in result["segments"]
if isinstance(segment, dict)
).strip()
recovered_from = None
if not use_active_asr and sherpa_model_id and not initial_text:
fallback_missing = await asyncio.to_thread(
asr_model_missing_error,
purpose="dictation",
skip_sherpa=True,
require_installed=True,
)
if fallback_missing is None:
def _run_fallback():
from services.asr_backend import get_capture_asr_backend
fallback = get_capture_asr_backend(skip_sherpa=True)
return (
fallback.transcribe(tmp.name, word_timestamps=False),
fallback.id,
)
try:
fallback_result, fallback_engine_id = await run_transcribe_guarded(
_gpu_pool,
_run_fallback,
what="Dictation fallback",
)
fallback_text = str(fallback_result.get("text") or "").strip()
if not fallback_text and fallback_result.get("segments"):
fallback_text = " ".join(
str(segment.get("text") or "")
for segment in fallback_result["segments"]
if isinstance(segment, dict)
).strip()
if fallback_text:
from services.sherpa_dictation import demote_model
await asyncio.to_thread(demote_model, sherpa_model_id)
result = fallback_result
engine_id = fallback_engine_id
recovered_from = sherpa_model_id
logger.warning(
"File transcription recovered from silent dictation model %s "
"through installed engine %s",
sherpa_model_id,
fallback_engine_id,
)
except Exception:
logger.exception(
"Installed fallback failed after dictation model %s returned no text",
sherpa_model_id,
)
elapsed = round(time.perf_counter() - t0, 2)
# Normalize result shape
@@ -202,7 +270,7 @@ async def transcribe_audio(
logger.info(
"Capture transcription done: engine=%s, elapsed=%.2fs, duration=%.1fs, mode=%s, refined=%s",
engine_id, elapsed, duration, "accurate" if use_accurate else "fast",
engine_id, elapsed, duration, requested_mode if use_active_asr else "fast",
refined_text is not None,
)
@@ -223,6 +291,8 @@ async def transcribe_audio(
}
if refined_text is not None:
response["refined_text"] = refined_text
if recovered_from is not None:
response["model_silent"] = recovered_from
return response
finally:
try:
+10 -5
View File
@@ -21,7 +21,7 @@ import logging
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from typing import Optional
from typing import Literal, Optional
from api.dependencies import require_local
from api.public_engine_metadata import public_unavailability
@@ -87,13 +87,18 @@ def list_dictation_models():
@router.get("/dictation/readiness", dependencies=[Depends(require_local)])
def dictation_readiness(model_id: str | None = None) -> dict:
"""Check capture's model selection without loading or downloading weights."""
def dictation_readiness(
model_id: str | None = None,
purpose: Literal["dictation", "transcribe"] = "dictation",
) -> dict:
"""Check a selected ASR path without loading or downloading weights."""
from services.asr_backend import asr_model_missing_error
missing = asr_model_missing_error(
purpose="dictation",
sherpa_model_id=model_id or _read_prefs()["model_id"],
purpose=purpose,
sherpa_model_id=(model_id or _read_prefs()["model_id"])
if purpose == "dictation"
else None,
)
return {"ready": missing is None, "missing": missing}
+205 -17
View File
@@ -355,6 +355,138 @@ async def dub_import_srt(job_id: str, file: UploadFile = File(...)):
}
def _select_downloaded_caption_track(
tracks: dict[str, list[dict]], preferred: str | None,
) -> str | None:
"""Choose the closest original-language caption track deterministically."""
available = [key for key, cues in tracks.items() if isinstance(cues, list) and cues]
if not available:
return None
preferred_tag = (preferred or "").strip().lower().replace("_", "-")
preferred_base = preferred_tag.split("-", 1)[0]
def rank(key: str) -> tuple[int, int, int, str]:
tag = key.strip().lower().replace("_", "-")
base = tag.split("-", 1)[0]
if preferred_tag:
language_rank = 0 if tag == preferred_tag else 1 if base == preferred_base else 2
else:
language_rank = 0
return (
language_rank,
0 if tag.endswith("-orig") else 1,
0 if "-" not in tag else 1,
tag,
)
return min(available, key=rank)
def _prepare_downloaded_caption_segments(cues: list[dict], duration: float) -> list[dict]:
"""Normalize downloaded VTT cues into safe, sequential Dub segments."""
def cue_start(cue: dict) -> float:
try:
return float(cue.get("start") or 0.0)
except (TypeError, ValueError):
return 0.0
def remove_repeated_prefix(previous: str, current: str) -> str:
previous_words = previous.split()
current_words = current.split()
folded_previous = [word.casefold() for word in previous_words]
folded_current = [word.casefold() for word in current_words]
for count in range(min(len(previous_words), len(current_words)), 0, -1):
if folded_previous[-count:] == folded_current[:count]:
return " ".join(current_words[count:])
return current
prepared: list[dict] = []
previous_end = 0.0
ordered = sorted((cue for cue in cues if isinstance(cue, dict)), key=cue_start)
for index, cue in enumerate(ordered):
try:
raw_start = max(0.0, float(cue.get("start") or 0.0))
end = float(cue.get("end") or raw_start)
except (TypeError, ValueError):
continue
text = " ".join(str(cue.get("text") or "").split())
if duration > 0:
if raw_start >= duration:
continue
end = min(end, duration)
if prepared and raw_start < previous_end:
text = remove_repeated_prefix(prepared[-1]["text"], text)
if not text:
prepared[-1]["end"] = round(max(previous_end, end), 3)
previous_end = max(previous_end, end)
continue
# Caption hosts commonly emit slightly overlapping cues. Dubbing needs
# a monotonic timeline, so trim the later cue rather than manufacture
# overlapping speech slots.
start = max(raw_start, previous_end)
if not text or end <= start:
continue
prepared.append({
"id": str(index),
"start": round(start, 3),
"end": round(end, 3),
"text": text,
"speaker_id": "Speaker 1",
})
previous_end = end
cleaned = clean_up_segments(prepared)
return [
{
**segment,
"id": index,
"text_original": segment.get("text", ""),
}
for index, segment in enumerate(cleaned)
]
@router.post("/dub/use-downloaded-captions/{job_id}")
def dub_use_downloaded_captions(job_id: str):
"""Seed a prepared Dub job from its downloaded caption track."""
job = _get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
tracks = job.get("youtube_subs")
if not isinstance(tracks, dict):
raise HTTPException(status_code=404, detail="No downloaded captions are available")
caption_lang = _select_downloaded_caption_track(
tracks,
job.get("source_lang_override") or job.get("source_lang"),
)
if caption_lang is None:
raise HTTPException(status_code=404, detail="No downloaded captions are available")
segments = _prepare_downloaded_caption_segments(
tracks[caption_lang],
float(job.get("duration") or 0.0),
)
if not segments:
raise HTTPException(status_code=422, detail="Downloaded captions contain no usable cues")
source_lang = job.get("source_lang_override") or _detected_source_lang(caption_lang)
job["segments"] = segments
job["source_lang"] = source_lang
job["full_transcript"] = " ".join(segment["text"] for segment in segments)
# Caption files contain timing and text, but no trustworthy speaker or
# reference-audio attribution. Never retain stale clone maps from a prior
# transcript on the same job.
job["segment_clones"] = {}
job["speaker_clones"] = {}
job.pop("cast_sources", None)
_save_job(job_id, job)
return {
"segments": segments,
"source_lang": source_lang,
"caption_lang": caption_lang,
"available": sorted(tracks.keys()),
}
@router.post("/dub/cleanup-segments/{job_id}")
def dub_cleanup_segments(job_id: str):
"""Re-run merge/stitch passes on a job's existing segments to drop fragments."""
@@ -375,8 +507,7 @@ def dub_abort(job_id: str):
had_procs = bool(_active_procs.get(job_id))
_kill_job_procs(job_id)
try:
if task_manager.cancel_task(job_id) is False:
raise RuntimeError("task cancellation was declined")
had_task = task_manager.cancel_task(job_id)
except Exception as exc:
logger.warning("Dub task cancellation failed")
raise HTTPException(
@@ -386,7 +517,13 @@ def dub_abort(job_id: str):
job = _dub_jobs.get(job_id)
if job is not None:
job["aborted"] = True
return {"aborted": True, "had_active_procs": had_procs}
# Cancellation is idempotent: a missing active task means it already
# stopped between the renderer aborting its stream and this request.
return {
"aborted": True,
"had_active_procs": had_procs,
"had_active_task": had_task,
}
@router.get("/dub/history")
@@ -615,8 +752,19 @@ async def dub_upload(
os.makedirs(job_dir, exist_ok=True)
video_path = os.path.join(job_dir, f"original{ext}")
with open(video_path, "wb") as f:
f.write(await video.read())
def _stream_upload_to_disk() -> None:
# UploadFile is already a spooled file. Copy it in bounded chunks on a
# worker thread instead of materialising a multi-GB video in RAM and
# blocking every API request while the event loop writes it.
video.file.seek(0)
with open(video_path, "wb") as output:
shutil.copyfileobj(video.file, output, length=1024 * 1024)
try:
await asyncio.to_thread(_stream_upload_to_disk)
finally:
await video.close()
filename = video.filename or f"video{ext}"
task_id = f"prep_{job_id}"
@@ -959,6 +1107,23 @@ async def dub_transcribe_stream(
job = _get_job(job_id)
# The durable job is written before the terminal SSE events below. If
# the renderer, proxy, or backend connection drops in that narrow
# window, reconnecting must replay the completed result instead of
# running a second whole-file ASR pass. This is deliberately gated by
# an explicit completion marker so partial work and imported subtitle
# rows still take their established paths.
if job and job.get("transcription_complete") and isinstance(job.get("segments"), list):
yield _sse_event("final", {
"segments": job["segments"],
"source_lang": job.get("source_lang") or "en",
"full_transcript": job.get("full_transcript") or "",
"speaker_clones": job.get("cast_sources", {}),
"cast_sources": job.get("cast_sources", {}),
})
yield _sse_event("done", {})
return
preflight_error: Optional[str] = None
# Extra machine-readable fields merged into the preflight `error` SSE event
# (e.g. the typed asr_model_missing payload → download-CTA in the UI).
@@ -1453,6 +1618,7 @@ async def dub_transcribe_stream(
from services.model_manager import (
DIARIZATION_ERR_LICENSE,
DIARIZATION_ERR_NO_TOKEN,
DIARIZATION_ERR_MISSING,
)
from core import error_docs_map
@@ -1545,7 +1711,23 @@ async def dub_transcribe_stream(
from services import token_resolver
resolved = token_resolver.resolve()
if err_sentinel == DIARIZATION_ERR_NO_TOKEN or not resolved:
if err_sentinel == DIARIZATION_ERR_MISSING:
from services.diarization_runtime import SORTFORMER, selected_backend
native_selected = selected_backend() == SORTFORMER
detail = (
"Native Sortformer files are missing. Install audiocpp_cli beside "
"the audio.cpp native bundle in Settings > Models > "
"Diarisation, then retry transcription. "
"Using silence gaps for now; rapid speaker turns may be merged."
) if native_selected else (
"Speaker diarization files are missing or incomplete. "
"Install or repair pyannote in Settings > Models > Diarisation, "
"then retry transcription. No models were downloaded during "
"this job. Using silence gaps for now; rapid speaker turns "
"may be merged."
)
error_class = "DIARIZATION_MODEL_MISSING"
elif err_sentinel == DIARIZATION_ERR_NO_TOKEN:
detail = (
"Speaker diarization is disabled because no HuggingFace token "
"was found in any source (Settings → API Keys, the HF_TOKEN "
@@ -1558,12 +1740,12 @@ async def dub_transcribe_stream(
)
error_class = "HF_AUTH_FAILED"
elif err_sentinel == DIARIZATION_ERR_LICENSE:
who = resolved.username or "(whoami suppressed)"
who = resolved.username if resolved else "(not signed in)"
detail = (
f"Speaker diarization model is gated — the "
f"pyannote/speaker-diarization-3.1 license has not been "
f"accepted on HuggingFace by this account "
f"(source={resolved.source}, user={who}). Visit "
f"(user={who}). Visit "
f"huggingface.co/pyannote/speaker-diarization-3.1 AND "
f"huggingface.co/pyannote/segmentation-3.0 while signed "
f"in and click 'Agree and access repository' on both, "
@@ -1575,17 +1757,13 @@ async def dub_transcribe_stream(
else:
# err_sentinel == DIARIZATION_ERR_LOAD (or unexpected None
# with a resolved token — historical safety net).
who = resolved.username or "(whoami suppressed)"
detail = (
f"Speaker diarization model failed to load even though an HF "
f"token was found (source={resolved.source}, user={who}). "
f"Most common causes: the pyannote/speaker-diarization-3.1 "
f"license has not been accepted on HuggingFace, or there is "
f"a pyannote/torch version mismatch. See backend logs for "
f"The installed speaker diarization model failed to load. "
f"See Settings > Logs > Backend for "
f"the underlying error. Falling back to a silence-gap "
f"heuristic; rapid speaker turns may be merged."
)
error_class = "PYANNOTE_LICENSE_REQUIRED"
error_class = "DIARIZATION_LOAD_FAILED"
warning = {
"detail": detail + _hint_suffix(),
"error_class": error_class,
@@ -1606,7 +1784,13 @@ async def dub_transcribe_stream(
# provided (#274). pyannote's apply() accepts num_speakers;
# omit it entirely when None so we don't depend on the kwarg
# existing in every pyannote build.
if num_speakers:
from services.diarization_native import NativeSortformer
if isinstance(diar_pipe, NativeSortformer):
diar = diar_pipe(
asr_audio_target, num_speakers=num_speakers, job_id=job_id,
cancel_check=lambda: bool(job.get("aborted")) or task_manager.is_cancelled(job_id),
)
elif num_speakers:
logger.info("Diarizing with num_speakers=%d (user hint)", num_speakers)
diar = diar_pipe(asr_audio_target, num_speakers=num_speakers)
else:
@@ -1632,7 +1816,7 @@ async def dub_transcribe_stream(
len(asr_phrase_segments), separation,
)
return recovered_segments, None, "phrase_embeddings"
return resplit, None, "pyannote"
return resplit, None, "audiocpp-sortformer" if isinstance(diar_pipe, NativeSortformer) else "pyannote"
except Exception as e:
logger.exception("Diarization failed")
# Inline ASR turns beat the silence-gap heuristic as a crash
@@ -1681,6 +1865,9 @@ async def dub_transcribe_stream(
final_segs, diar_warning, labels_source = done.pop().result()
break
yield _sse_event("ping", {})
if job.get("aborted") or task_manager.is_cancelled(job_id):
yield _sse_event("aborted", {})
return
if diar_warning:
logger.warning("diarization fallback: %s", diar_warning.get("detail"))
payload = {
@@ -1845,6 +2032,7 @@ async def dub_transcribe_stream(
detected_lang
)
job["full_transcript"] = " ".join(s.get("text", "") for s in final_segs)
job["transcription_complete"] = True
_save_job(job_id, job)
# Restore TTS model to GPU now that ASR is done. unload() blocks
+56 -16
View File
@@ -15,7 +15,7 @@ from core.http_headers import content_disposition
from core.logging_utils import log_safe
from core.path_security import UnsafePath, resolve_within
from core.tasks import task_manager
from fastapi import APIRouter, Header, HTTPException, Query, Response
from fastapi import APIRouter, Header, HTTPException, Query, Request, Response
from fastapi.responses import FileResponse, StreamingResponse
from services.ffmpeg_utils import (
bed_mix_filter,
@@ -1050,8 +1050,8 @@ _MEDIA_TYPES = {
}
@router.get("/dub/media/{job_id}")
async def dub_get_media(job_id: str):
@router.api_route("/dub/media/{job_id}", methods=["GET", "HEAD"])
async def dub_get_media(job_id: str, request: Request):
_job_dir_or_400(job_id)
job = _get_job(job_id)
if not job:
@@ -1064,7 +1064,15 @@ async def dub_get_media(job_id: str):
# silent black box. Default to video/mp4 because the ingest pipeline
# remuxes URL downloads to mp4 (dub_pipeline.yt_download_sync).
ext = os.path.splitext(video_path)[1].lower()
return FileResponse(video_path, media_type=_MEDIA_TYPES.get(ext, "video/mp4"))
media_type = _MEDIA_TYPES.get(ext, "video/mp4")
headers = {
"Cache-Control": "private, max-age=31536000, immutable",
"Accept-Ranges": "bytes",
}
if request.method == "HEAD":
headers["Content-Length"] = str(os.path.getsize(video_path))
return Response(media_type=media_type, headers=headers)
return FileResponse(video_path, media_type=media_type, headers=headers)
# One mux at a time per preview file. Without this, two overlapping requests
# (e.g. the <video> element remounting right after a re-dub) both ran ffmpeg
@@ -1081,8 +1089,9 @@ def _preview_lock(path: str) -> asyncio.Lock:
return lock
@router.get("/dub/preview-video/{job_id}")
@router.api_route("/dub/preview-video/{job_id}", methods=["GET", "HEAD"])
async def dub_preview_video(
request: Request,
job_id: str,
lang: str = Query(..., description="Language code of the dubbed track to mux in"),
preserve_bg: bool = Query(True),
@@ -1124,7 +1133,7 @@ async def dub_preview_video(
os.makedirs(exports_dir, exist_ok=True)
bg_suffix = "bg" if (preserve_bg and has_bg) else "nobg"
preview_path = os.path.realpath(
os.path.join(exports_dir, f"preview_{lang}_{bg_suffix}.mp4")
os.path.join(exports_dir, f"preview_v2_{lang}_{bg_suffix}.mp4")
)
if not preview_path.startswith(_base + os.sep):
raise HTTPException(status_code=400, detail="Invalid path")
@@ -1138,6 +1147,18 @@ async def dub_preview_video(
and os.path.getmtime(preview_path) >= track_mtime
)
# Vidstack probes extensionless routes with HEAD before choosing a native
# provider. Confirm that this preview is valid without starting an ffmpeg
# mux; the following GET builds it lazily when needed.
if request.method == "HEAD":
headers = {
"Cache-Control": "private, max-age=31536000, immutable",
"Accept-Ranges": "bytes",
}
if _cache_ok():
headers["Content-Length"] = str(os.path.getsize(preview_path))
return Response(media_type="video/mp4", headers=headers)
async def _mux_preview():
# Mux into a temp file and os.replace() into place so a concurrent
# reader never sees a partially-written preview (#281: video stuck
@@ -1266,7 +1287,7 @@ async def dub_preview_video(
cmd += ["-c:v", "libx264", "-preset", "medium", "-crf", "20", "-pix_fmt", "yuv420p"]
else:
cmd += ["-c:v", "copy"]
cmd += ["-c:a", "aac", "-b:a", "192k"]
cmd += ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"]
# `-shortest` would cut the retimed video at the (slightly different)
# audio length and lose the trailing frame; only use it on the copy path.
if not stretch_entry and retime_decision is None:
@@ -1311,22 +1332,37 @@ async def dub_preview_video(
if not _cache_ok():
await _mux_preview()
# no-store: the URL is stable across re-dubs, so any HTTP-level caching
# in the WebView would keep showing the previous dub after a re-generate
# (#281: "edits don't change the result").
# The renderer includes the segment-fingerprint revision in the URL, so a
# regenerated track gets a fresh cache key. Keep each completed preview:
# switching Original/Dub then reuses local ranges instead of re-reading a
# multi-hundred-megabyte MP4 from the backend.
return FileResponse(
preview_path,
media_type="video/mp4",
headers={"Cache-Control": "no-store"},
headers={"Cache-Control": "private, max-age=31536000, immutable", "Accept-Ranges": "bytes"},
)
def _compute_onsets_sync(src_path: str) -> list[float]:
def _compute_timeline_sync(src_path: str) -> tuple[list[float], list[float]]:
"""Blocking part of onset analysis — runs in a worker thread."""
import numpy as np
import soundfile as sf
from services.onset_align import detect_speech_onsets
audio, sr = sf.read(src_path, dtype="float32")
return detect_speech_onsets(audio, sr)
onsets = detect_speech_onsets(audio, sr)
mono = np.asarray(audio, dtype=np.float32)
if mono.ndim > 1:
mono = mono.mean(axis=1)
mono = mono.reshape(-1)
if mono.size == 0:
return onsets, []
bucket_count = min(2048, int(mono.size))
bucket_width = max(1, (int(mono.size) + bucket_count - 1) // bucket_count)
padded_size = bucket_count * bucket_width
if padded_size != mono.size:
mono = np.pad(mono, (0, padded_size - int(mono.size)))
peaks = np.max(np.abs(mono.reshape(bucket_count, bucket_width)), axis=1)
return onsets, [round(float(value), 5) for value in peaks]
@router.get("/dub/onsets/{job_id}")
@@ -1365,20 +1401,24 @@ async def dub_get_onsets(job_id: str):
):
with open(cache_path, "r", encoding="utf-8") as f:
cached = json.load(f)
if isinstance(cached, dict) and isinstance(cached.get("onsets"), list):
if (
isinstance(cached, dict)
and isinstance(cached.get("onsets"), list)
and isinstance(cached.get("peaks"), list)
):
return cached
except (OSError, ValueError):
pass # unreadable/corrupt cache → recompute below
try:
onsets = await asyncio.to_thread(_compute_onsets_sync, src_path)
onsets, peaks = await asyncio.to_thread(_compute_timeline_sync, src_path)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Onset analysis failed: {str(e)[:200]}",
)
payload = {"onsets": onsets, "source": source}
payload = {"onsets": onsets, "peaks": peaks, "source": source}
try:
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
tmp_path = cache_path + ".tmp"
+323 -53
View File
@@ -5,8 +5,6 @@ import struct
import logging
import time
import asyncio
import shutil
import zipfile
import torch
import torchaudio
from fastapi import APIRouter, HTTPException
@@ -16,7 +14,8 @@ from core.config import DUB_DIR, VOICES_DIR, dub_seg_path
from core.tasks import task_manager
from schemas.requests import DubRequest
from services.model_manager import _gpu_pool, run_on_gpu_pool_guarded
from services.tts_backend import resolve_generation_backend, active_backend_id
from services.tts_backend import TTSBackend, resolve_generation_backend, active_backend_id
from services.dub_batching import batch_timeout_s, native_batch_width
from services import gpu_gateway
from services.audio_dsp import apply_mastering, normalize_audio, apply_effects_chain, get_effect_chain
from services.audio_io import atomic_save_wav, _safe_torchaudio_save
@@ -34,6 +33,7 @@ from services.incremental import segment_fingerprint, fit_fingerprint
from services.fit_planner import UNDERRUN_TOLERANCE, FitParams, plan_fit
from services.watermark import mark_synthetic
from services.speaker_clone import auto_profile_id
from services.segment_bundle import extract_segment_wavs
from api.routers.dub_core import _get_job, _save_job
from omnivoice.utils.voice_design import heal_design_instruct
@@ -49,6 +49,22 @@ logger = logging.getLogger("omnivoice.dub")
MAX_STRETCH_RATIO = 1.8
class _RemoteDubBackend:
"""Sample-rate carrier while Dubbing runs without local TTS weights."""
sample_rate = 24_000
async def _resolve_dub_execution():
"""Resolve routing without loading local weights for a remote dub."""
engine_id = active_backend_id()
decision = gpu_gateway.decide("dub_segments")
if decision.remote:
await gpu_gateway.preflight(engine_id, decision, operation="dub_segments")
return engine_id, decision, _RemoteDubBackend()
return engine_id, decision, await resolve_generation_backend(require_cloning=True)
def _prepare_oom_retry(error: Exception, *, execution_target: str) -> bool:
"""Prepare one *local* low-step retry after a genuine device OOM.
@@ -461,21 +477,12 @@ def _remote_voice(job: dict, profile_id: str | None, seg_id, voice_match: str,
def _decode_remote_dub(result: gpu_gateway.RemoteResult) -> dict[int, str]:
"""Extract the worker bundle into a task-scoped directory, path-safely."""
target = os.path.join(DUB_DIR, ".remote", result.task_id)
os.makedirs(target, exist_ok=True)
paths: dict[int, str] = {}
with zipfile.ZipFile(result.path) as archive:
for member in archive.infolist():
match = re.fullmatch(r"segments/(\d+)\.wav", member.filename)
if not match:
raise ValueError(f"unexpected dub artifact member: {member.filename}")
index = int(match.group(1))
destination = os.path.join(target, f"{index}.wav")
partial = f"{destination}.part"
with archive.open(member) as source, open(partial, "wb") as output:
shutil.copyfileobj(source, output)
os.replace(partial, destination)
paths[index] = destination
return paths
try:
return extract_segment_wavs(result.path or "", target)
except ValueError as exc:
# Preserve the established route-specific error wording consumed by
# diagnostics and regression tests.
raise ValueError(str(exc).replace("segment artifact", "dub artifact")) from exc
router = APIRouter()
@@ -491,16 +498,25 @@ async def dub_generate(job_id: str, req: DubRequest):
)
# ── Engine resolution (issue #312 class) ────────────────────────────────
# Dub used to hardcode VoiceStudio via get_model() regardless of the engine
# selected in Model Catalogue — a SILENT fallback. Every real dub
# segment's ref_audio resolves to either an auto:<speaker>/auto-seg:<id>
# clone cut from the source video or a saved voice-profile row (see
# `_gen` below), so require_cloning=True: an engine that can't clone
# would either mis-clone per segment or fail deep into the job. Checked
# ONCE here, before the streaming task starts, so a doomed job fails fast
# with one clear message instead of N per-segment ones.
# Every rendered segment clones either source speech or a saved profile, so
# local execution still requires a cloning-capable engine. Remote execution
# validates the selected worker here without loading duplicate local weights;
# its local backend is prepared only if gateway fallback actually selects it.
try:
backend = await resolve_generation_backend(require_cloning=True)
engine_id, decision, backend = await _resolve_dub_execution()
except gpu_gateway.ModelNotDownloaded as e:
raise HTTPException(
status_code=409,
detail={
"error": "model_not_downloaded",
"message": str(e),
"engine": e.engine,
"repo_ids": e.repo_ids,
"target": e.target,
"target_label": e.target_label,
"downloadable": e.downloadable,
},
) from e
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
@@ -516,6 +532,14 @@ async def dub_generate(job_id: str, req: DubRequest):
)
raise HTTPException(status_code=503, detail=payload["detail"]) from e
# Resolve the global profile once for this job. Explicit Production
# overrides remain authoritative, while ordinary Dubbing now follows the
# same Fast/Balanced/Quality/Max contract as Clone and long-form work.
from services.performance_profiles import tts_defaults
_profile_defaults = tts_defaults(engine_id)
_job_num_step = req.num_step if req.num_step is not None else _profile_defaults.get("num_step", 16)
_job_postprocess = _profile_defaults.get("postprocess_output", True)
async def _stream(task_id):
total = len(req.segments)
all_segment_wavs = []
@@ -701,11 +725,70 @@ async def dub_generate(job_id: str, req: DubRequest):
_t_start = time.perf_counter()
_t_cache = 0.0
_t_tts = 0.0
_batched_audio: dict[int, torch.Tensor] = {}
_profile_row_cache: dict[str, object | None] = {}
_has_native_batch = (
getattr(type(backend), "generate_batch", TTSBackend.generate_batch)
is not TTSBackend.generate_batch
)
_native_batch_width = native_batch_width(backend) if _has_native_batch else 1
async def _prepare_local_dub():
"""Load local TTS only when gateway fallback actually needs it."""
nonlocal backend, _has_native_batch, _native_batch_width
if isinstance(backend, _RemoteDubBackend):
backend = await resolve_generation_backend(require_cloning=True)
_has_native_batch = (
getattr(type(backend), "generate_batch", TTSBackend.generate_batch)
is not TTSBackend.generate_batch
)
_native_batch_width = (
native_batch_width(backend) if _has_native_batch else 1
)
return gpu_gateway.LocalCall(fn=lambda: {})
def _segment_generation_args(index, segment) -> dict:
"""Resolve the per-row controls shared by serial and native batches."""
current_id = seg_ids[index] if index < len(seg_ids) else f"seg_{index}"
duration = segment.end - segment.start
profile_id = segment.profile_id or None
speed = segment.speed if segment.speed is not None else req.speed
language = segment.target_lang or req.language
instruct = segment.instruct or req.instruct
direction_text = getattr(segment, "direction", None)
if direction_text and direction_text.strip():
try:
from services.director import parse as _parse_direction
direction = _parse_direction(direction_text)
extra = direction.instruct_prompt()
if extra:
instruct = f"{instruct}, {extra}" if instruct else extra
bias = direction.rate_bias()
if (
bias
and abs(bias - 1.0) > 0.01
and strategy == "strict_slot"
):
speed = (speed or 1.0) * bias
except Exception as error:
logger.debug("direction parse skipped for %s: %s", current_id, error)
return {
"seg_id": current_id,
"text": segment.text,
"language": language,
"instruct": instruct,
"duration": duration if strategy == "strict_slot" else None,
"num_step": 8 if req.preview else _job_num_step,
"guidance_scale": req.guidance_scale,
"speed": speed,
"profile_id": profile_id,
"effect_preset": getattr(segment, "effect_preset", None) or "broadcast",
}
# One coarse remote lease for every segment that actually needs fresh
# synthesis. Assembly, fitting and the separately-pooled RVC pass stay
# here; the worker returns a single verified bundle of segment WAVs.
decision = gpu_gateway.decide("dub_segments")
if decision.remote:
remote_rows: list[dict] = []
remote_refs: list[str | None] = []
@@ -740,7 +823,8 @@ async def dub_generate(job_id: str, req: DubRequest):
"ref_text": ref_text, "ref_single_use": ref_single_use,
"instruct": seg_instruct,
"duration": (seg.end - seg.start) if strategy == "strict_slot" else None,
"num_step": 8 if req.preview else req.num_step,
"num_step": 8 if req.preview else _job_num_step,
"postprocess_output": _job_postprocess,
"guidance_scale": req.guidance_scale, "speed": seg_speed,
"effect_preset": seg.effect_preset or "broadcast",
"seed": seed,
@@ -752,13 +836,13 @@ async def dub_generate(job_id: str, req: DubRequest):
if remote_rows:
states: asyncio.Queue = asyncio.Queue()
call = gpu_gateway.RemoteCall(
engine=active_backend_id(), operation="dub_segments",
engine=engine_id, operation="dub_segments",
params={"segments": remote_rows, "ref_audio": remote_refs},
decode=_decode_remote_dub,
)
dub_run = gpu_gateway.JobRun("dub_segments")
run = asyncio.create_task(gpu_gateway.run(
"dub_segments", local=gpu_gateway.LocalCall(fn=lambda: {}),
"dub_segments", local=gpu_gateway.LocalCall(prepare=_prepare_local_dub),
remote=call, decision=decision, job=dub_run,
on_state=states.put_nowait,
))
@@ -777,11 +861,34 @@ async def dub_generate(job_id: str, req: DubRequest):
continue
fraction = float(state.get("progress") or 0.0)
yield f"data: {json.dumps({'type': 'progress', 'current': round(fraction * total, 2), 'total': total, 'text': state.get('stage') or state.get('phase')})}\n\n"
remote_audio = await run
try:
remote_audio = await run
except Exception as error:
from core.public_errors import stream_generation_failure
detail = stream_generation_failure(error)["detail"]
yield f"data: {json.dumps({'type': 'error', 'error': detail})}\n\n"
return
notice = dub_run.notice()
if notice is not None:
yield f"data: {json.dumps({'type': 'routing_notice', 'status': notice[0], 'reason': notice[1]})}\n\n"
if remote_audio and isinstance(backend, _RemoteDubBackend):
first_remote = next(iter(remote_audio.values()))
backend.sample_rate = int(torchaudio.info(first_remote).sample_rate)
elif isinstance(backend, _RemoteDubBackend):
# Fit-only / cache-only reruns synthesize nothing. Keep the cached
# track's native rate when one exists instead of resampling it to
# the carrier's conservative 24 kHz default.
for cached_id in seg_ids:
cached_path = _seg_lang_path(cached_id)
if os.path.exists(cached_path):
try:
backend.sample_rate = int(torchaudio.info(cached_path).sample_rate)
break
except Exception:
continue
for i, seg in enumerate(req.segments):
seg_id = seg_ids[i] if i < len(seg_ids) else f"seg_{i}"
@@ -891,13 +998,14 @@ async def dub_generate(job_id: str, req: DubRequest):
continue
def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id, effect_preset,
*, execution_target="local"):
*, execution_target="local", prepare_only=False, current_seg_id=None):
# Normalize once at the segment's text→engine choke point
# (covers the OOM-retry generate below too, which reuses this
# closure's `text`). Pref-gated, idempotent, never raises.
from services.text_normalization import normalize_for_tts
text = normalize_for_tts(text, lang)
effective_seg_id = seg_id if current_seg_id is None else current_seg_id
ref_audio = None
ref_text = None
used_seed = None
@@ -928,7 +1036,7 @@ async def dub_generate(job_id: str, req: DubRequest):
# CROSS binding (sid != this segment) can only come from an
# explicit request — honour its clip unchanged.
_consistent_alt = None
if voice_match == "consistent" and sid == str(seg_id):
if voice_match == "consistent" and sid == str(effective_seg_id):
_spk_key = _speaker_key_for_segment(job, sid)
if _spk_key:
_consistent_alt = resolve_consistent_ref(
@@ -969,7 +1077,9 @@ async def dub_generate(job_id: str, req: DubRequest):
# editor's Voice dropdown can actually render ("From
# Video → Speaker N"). `seg_id` is closed over from
# the per-segment loop below.
segment_speaker_key = _speaker_key_for_segment(job, seg_id)
segment_speaker_key = _speaker_key_for_segment(
job, effective_seg_id
)
# Legacy jobs may not persist diarized segment rows.
# Preserve their established per-line preference; only
# suppress it when current metadata proves the user
@@ -978,7 +1088,7 @@ async def dub_generate(job_id: str, req: DubRequest):
segment_speaker_key is None or segment_speaker_key == key
)
seg_ref = (
(job.get("segment_clones") or {}).get(str(seg_id))
(job.get("segment_clones") or {}).get(str(effective_seg_id))
if selected_is_segment_speaker
else None
)
@@ -1003,8 +1113,13 @@ async def dub_generate(job_id: str, req: DubRequest):
profile_id = None # prevent the voice_profiles lookup below
if profile_id:
with db_conn() as conn:
row = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone()
if profile_id not in _profile_row_cache:
with db_conn() as conn:
_profile_row_cache[profile_id] = conn.execute(
"SELECT * FROM voice_profiles WHERE id=?",
(profile_id,),
).fetchone()
row = _profile_row_cache[profile_id]
if row:
if row["is_locked"] and row["locked_audio_path"]:
ref_audio = os.path.join(VOICES_DIR, row["locked_audio_path"])
@@ -1024,15 +1139,33 @@ async def dub_generate(job_id: str, req: DubRequest):
_vd = None
instruct_str = heal_design_instruct(row["instruct"], _vd)
if used_seed is not None:
if used_seed is not None and not prepare_only:
torch.manual_seed(used_seed)
# Last gate before the engine: every resolution branch above
# produces a PATH, and none of them can know it still exists.
ref_audio = warn_if_ref_missing(
ref_audio, job_id=job_id, seg_id=seg_id, where="dub render",
ref_audio, job_id=job_id, seg_id=effective_seg_id, where="dub render",
)
if prepare_only:
return {
"text": text,
"language": lang if lang != "Auto" else None,
"ref_audio": ref_audio,
"ref_text": ref_text,
"cache_ref": not ref_single_use,
"instruct": instruct_str if instruct_str else None,
"duration": dur_s,
"num_step": nstep,
"guidance_scale": cfg,
"speed": spd,
"denoise": True,
"postprocess_output": _job_postprocess,
"effect_preset": effect_preset or "broadcast",
"seed": used_seed,
}
try:
audio_out = backend.generate(
text=text, language=lang if lang != "Auto" else None,
@@ -1040,7 +1173,7 @@ async def dub_generate(job_id: str, req: DubRequest):
cache_ref=not ref_single_use,
instruct=instruct_str if instruct_str else None,
duration=dur_s, num_step=nstep, guidance_scale=cfg,
speed=spd, denoise=True, postprocess_output=True,
speed=spd, denoise=True, postprocess_output=_job_postprocess,
)
sr = backend.sample_rate
@@ -1081,7 +1214,7 @@ async def dub_generate(job_id: str, req: DubRequest):
cache_ref=not ref_single_use,
instruct=instruct_str if instruct_str else None,
duration=dur_s, num_step=retry_steps, guidance_scale=cfg,
speed=spd, denoise=True, postprocess_output=True,
speed=spd, denoise=True, postprocess_output=_job_postprocess,
)
sr = backend.sample_rate
@@ -1109,6 +1242,134 @@ async def dub_generate(job_id: str, req: DubRequest):
f"Underlying error: {retry_err}"
) from retry_err
async def _prefetch_native_batch(first_index: int) -> None:
"""Render one bounded batch and retain only its small output window."""
if _native_batch_width < 2 or remote_audio:
return
batch: list[tuple[int, dict]] = []
compatibility = None
for candidate_index in range(first_index, len(req.segments)):
candidate = req.segments[candidate_index]
candidate_id = (
seg_ids[candidate_index]
if candidate_index < len(seg_ids)
else f"seg_{candidate_index}"
)
if (
candidate_index in _batched_audio
or candidate.end - candidate.start <= 0.05
or not candidate.text.strip()
or (
regen_only is not None
and candidate_id not in regen_only
)
):
continue
args = _segment_generation_args(candidate_index, candidate)
try:
prepared = _gen(
args["text"],
args["language"],
args["instruct"],
args["duration"],
args["num_step"],
args["guidance_scale"],
args["speed"],
args["profile_id"],
args["effect_preset"],
prepare_only=True,
current_seg_id=args["seg_id"],
)
except Exception:
if candidate_index == first_index:
raise
break
# Fixed-seed profiles deliberately keep their established
# one-row deterministic RNG contract.
if prepared["seed"] is not None:
if candidate_index == first_index:
return
break
candidate_compatibility = (
prepared["cache_ref"],
bool(prepared["ref_audio"]),
prepared["num_step"],
prepared["guidance_scale"],
prepared["postprocess_output"],
)
if compatibility is None:
compatibility = candidate_compatibility
elif candidate_compatibility != compatibility:
break
batch.append((candidate_index, prepared))
if len(batch) >= _native_batch_width:
break
if len(batch) < 2:
return
def _render_batch() -> list[torch.Tensor]:
prepared_rows = [prepared for _, prepared in batch]
outputs = backend.generate_batch(
[prepared["text"] for prepared in prepared_rows],
language=[prepared["language"] for prepared in prepared_rows],
ref_audio=[prepared["ref_audio"] for prepared in prepared_rows],
ref_text=[prepared["ref_text"] for prepared in prepared_rows],
cache_ref=prepared_rows[0]["cache_ref"],
instruct=[prepared["instruct"] for prepared in prepared_rows],
duration=[prepared["duration"] for prepared in prepared_rows],
num_step=prepared_rows[0]["num_step"],
guidance_scale=prepared_rows[0]["guidance_scale"],
speed=[prepared["speed"] for prepared in prepared_rows],
denoise=True,
postprocess_output=prepared_rows[0]["postprocess_output"],
)
if len(outputs) != len(prepared_rows):
raise RuntimeError(
f"native batch returned {len(outputs)} outputs for "
f"{len(prepared_rows)} segments"
)
rendered = []
for output, prepared in zip(outputs, prepared_rows):
preset = prepared["effect_preset"]
if preset == "raw":
rendered.append(output)
continue
mastered = output
if not getattr(backend, "applies_own_mastering", False):
mastered = apply_mastering(mastered, sample_rate=backend.sample_rate)
effect_chain = get_effect_chain(preset)
if effect_chain:
mastered = apply_effects_chain(
mastered,
sample_rate=backend.sample_rate,
chain=effect_chain,
)
rendered.append(normalize_audio(mastered, target_dBFS=-2.0))
return rendered
try:
outputs = await run_on_gpu_pool_guarded(
_render_batch,
what="Dub generate batch",
timeout=batch_timeout_s(
[prepared["text"] for _, prepared in batch], backend
),
)
except TimeoutError:
raise
except Exception as error:
_prepare_oom_retry(error, execution_target="local")
logger.warning(
"Native dub batch failed for segments %s-%s; falling back: %s",
batch[0][0] + 1,
batch[-1][0] + 1,
error,
)
return
_batched_audio.update(
(index, output) for (index, _), output in zip(batch, outputs)
)
seg_profile = seg.profile_id or None
seg_speed = seg.speed if hasattr(seg, 'speed') and seg.speed is not None else req.speed
seg_lang = seg.target_lang if getattr(seg, 'target_lang', None) else req.language
@@ -1149,8 +1410,8 @@ async def dub_generate(job_id: str, req: DubRequest):
# quality for ~2× speed by dropping flow-matching steps.
# Client sends `preview=true` when the user is iterating;
# before final export the client should re-call without the
# flag to restore num_step=req.num_step quality.
_num_step = 8 if req.preview else req.num_step
# flag to restore the explicit override or shared profile.
_num_step = 8 if req.preview else _job_num_step
_t_tts_0 = time.perf_counter()
seg_effect_preset = getattr(seg, "effect_preset", None) or "broadcast"
@@ -1177,14 +1438,19 @@ async def dub_generate(job_id: str, req: DubRequest):
import torchaudio.functional as AF
audio_tensor = AF.resample(audio_tensor, remote_sr, backend.sample_rate)
else:
audio_tensor = await run_on_gpu_pool_guarded(
lambda: _gen(
seg.text, seg_lang, seg_instruct, _dur_for_tts,
_num_step, req.guidance_scale, seg_speed, seg_profile, seg_effect_preset,
),
what="Dub generate",
timeout=generate_timeout_s(seg.text, engine=backend),
)
if i not in _batched_audio:
await _prefetch_native_batch(i)
if i in _batched_audio:
audio_tensor = _batched_audio.pop(i)
else:
audio_tensor = await run_on_gpu_pool_guarded(
lambda: _gen(
seg.text, seg_lang, seg_instruct, _dur_for_tts,
_num_step, req.guidance_scale, seg_speed, seg_profile, seg_effect_preset,
),
what="Dub generate",
timeout=generate_timeout_s(seg.text, engine=backend),
)
_t_tts += time.perf_counter() - _t_tts_0
# Check abort immediately after GPU work completes
@@ -1194,6 +1460,10 @@ async def dub_generate(job_id: str, req: DubRequest):
target_samples = int(seg_duration * backend.sample_rate)
current_samples = audio_tensor.shape[-1]
# Capture the real spoken duration before strict-slot padding
# or trimming. This is the evidence used by Agent timing and
# keeps sync badges truthful for every timing strategy.
natural_generated_dur = current_samples / backend.sample_rate
if strategy == "strict_slot":
# Legacy: pad short audio + trim long audio so the mix
@@ -1210,7 +1480,7 @@ async def dub_generate(job_id: str, req: DubRequest):
# trim, slip, stretch the video, or split audio/video
# retiming (smart_fit) to accommodate it.
generated_dur = audio_tensor.shape[-1] / backend.sample_rate
generated_dur = natural_generated_dur
sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3)
sync_scores.append(sync_ratio)
+339 -63
View File
@@ -3,10 +3,10 @@ import time
import asyncio
import logging
from typing import Optional
from fastapi import APIRouter
from fastapi import APIRouter, HTTPException
from fastapi.responses import JSONResponse
from schemas.requests import TranslateRequest
from schemas.requests import AgentFitRequest, TranslateRequest
from services.model_manager import _cpu_pool, _gpu_pool
from services.hf_revisions import revision_for
from services.translator import cinematic_available, cinematic_refine_many, _cinematic_budget
@@ -19,10 +19,11 @@ _NLLB_REPO_ID = "facebook/nllb-200-distilled-600M"
def _load_nllb_component(factory):
"""Load a curated NLLB component from its reviewed immutable revision."""
"""Load explicitly installed NLLB weights at their reviewed revision."""
return factory.from_pretrained(
_NLLB_REPO_ID,
revision=revision_for(_NLLB_REPO_ID),
local_files_only=True,
)
TRANSLATE_CODES = {
@@ -39,8 +40,33 @@ FLORES_CODES = {
"hi": "hin_Deva", "tr": "tur_Latn", "pl": "pol_Latn", "nl": "nld_Latn",
"sv": "swe_Latn", "th": "tha_Thai", "vi": "vie_Latn", "id": "ind_Latn",
"uk": "ukr_Cyrl",
"zh-TW": "zho_Hant", "zh-Hant": "zho_Hant", "cmn-Hant": "zho_Hant",
"zh-Hans": "zho_Hans", "yue": "yue_Hant",
"bn": "ben_Beng", "ta": "tam_Taml", "te": "tel_Telu", "ml": "mal_Mlym",
"kn": "kan_Knda", "gu": "guj_Gujr", "mr": "mar_Deva", "ur": "urd_Arab",
"fa": "pes_Arab", "he": "heb_Hebr", "el": "ell_Grek", "cs": "ces_Latn",
"da": "dan_Latn", "fi": "fin_Latn", "nb": "nob_Latn", "nn": "nno_Latn",
"ro": "ron_Latn", "hu": "hun_Latn", "bg": "bul_Cyrl", "sk": "slk_Latn",
"sl": "slv_Latn", "hr": "hrv_Latn", "sr": "srp_Cyrl", "lt": "lit_Latn",
"et": "est_Latn", "sw": "swh_Latn", "af": "afr_Latn", "ms": "zsm_Latn",
}
def _nllb_language(code: str) -> str | None:
"""Resolve aliases or tokenizer-supported FLORES codes without loading weights."""
from transformers.models.nllb.tokenization_nllb import FAIRSEQ_LANGUAGE_CODES
normalized = code.strip().replace("_", "-").lower()
aliases = {key.lower(): value for key, value in FLORES_CODES.items()}
if normalized in aliases:
return aliases[normalized]
exact = [value for value in FAIRSEQ_LANGUAGE_CODES if value.replace("_", "-").lower() == normalized]
if exact:
return exact[0]
# Bare ISO-639-3 codes are safe only when the tokenizer has one script.
matches = [value for value in FAIRSEQ_LANGUAGE_CODES if value.split("_")[0] == normalized]
return matches[0] if len(matches) == 1 else None
# Human-readable language names for LLM prompts. Empirically a tiny / 7B
# local LLM produces Devanagari Hindi reliably when told "translate into
# Hindi" but drifts to German / English / phonetic-Latin when told
@@ -163,9 +189,77 @@ def _looks_like_target(text: str, code: str, threshold: float = 0.5) -> bool:
codepoints alone."""
return _script_ratio(text, code) >= threshold
def _translation_output_error(text: object) -> str | None:
"""Reject provider error pages that arrive with HTTP 200.
Google's mobile endpoint occasionally returns its generic HTML error copy
inside the element deep-translator treats as a successful translation.
Passing that through would replace the user's transcript with the error
page, so treat it like any other transient provider failure and retry.
"""
if not isinstance(text, str) or not text.strip():
return "empty translation"
normalized = " ".join(text.split()).casefold()
error_markers = (
"error 500 (server error)",
"that's an error",
"thats an error",
"there was an error. please try again later",
"no translation was found using the current translator",
)
if "\ufffd" in text or any(marker in normalized for marker in error_markers):
return "translation provider returned invalid output"
return None
_nllb_model = None
_nllb_tokenizer = None
_nllb_device = None
_NLLB_BATCH_SIZE_ENV = "OMNIVOICE_NLLB_BATCH_SIZE"
_NLLB_MAX_BATCH_SIZE = 32
def _nllb_batch_size() -> int:
"""Bound NLLB forward-pass width; explicit overrides remain available."""
configured = os.environ.get(_NLLB_BATCH_SIZE_ENV, "").strip()
if configured:
try:
return max(1, min(_NLLB_MAX_BATCH_SIZE, int(configured)))
except (TypeError, ValueError):
logger.warning("%s=%r is not an integer; using the safe default", _NLLB_BATCH_SIZE_ENV, configured)
# The 600M checkpoint leaves ample room on modern discrete GPUs. Scale the
# forward-pass width there; CPU and unified-memory MPS keep the conservative
# width because their failure recovery moves the whole model.
if _nllb_device == "cuda":
try:
import torch
free_gib = int(torch.cuda.mem_get_info()[0]) / 1024**3
if free_gib >= 16:
return 24
if free_gib >= 8:
return 12
except Exception:
pass
return 8
return 4
def _nllb_hypothesis_budget() -> int:
"""Bound batch × beam hypotheses by currently available device memory."""
if _nllb_device != "cuda":
return 16
try:
import torch
free_gib = int(torch.cuda.mem_get_info()[0]) / 1024**3
if free_gib >= 16:
return 64
if free_gib >= 8:
return 32
except Exception:
pass
return 16
def _dialect_flags(req, applied: bool) -> dict:
@@ -255,10 +349,11 @@ def _resolve_translation_context(req, client, model_name: str, timeout: float,
def _unload_nllb():
"""Release NLLB VRAM so TTS model can reload."""
global _nllb_model, _nllb_tokenizer
global _nllb_device, _nllb_model, _nllb_tokenizer
import gc
_nllb_model = None
_nllb_tokenizer = None
_nllb_device = None
gc.collect()
try:
import torch
@@ -270,10 +365,43 @@ def _unload_nllb():
pass
def _should_unload_nllb() -> bool:
"""Retain a warm local translator only when the accelerator has safe headroom."""
override = os.environ.get("OMNIVOICE_UNLOAD_NLLB")
if override is not None:
return override.strip().lower() not in {"0", "false", "no", "off"}
if _nllb_device != "cuda":
return True
try:
import torch
free_bytes, total_bytes = torch.cuda.mem_get_info()
return total_bytes < 16 * 1024**3 or free_bytes < 8 * 1024**3
except Exception:
return True
@router.post("/dub/translate")
async def dub_translate(req: TranslateRequest):
try:
provider = (req.provider if req.provider else os.environ.get("TRANSLATE_PROVIDER", "google")).lower()
from services import translation_engines
if not translation_engines.get_engine(provider):
return JSONResponse(
status_code=400,
content={"error": "Choose a supported translation engine."},
)
if not translation_engines.is_installed(provider):
return JSONResponse(
status_code=409,
content={"error": "Install the selected translation engine before translating."},
)
if not translation_engines.is_ready(provider):
return JSONResponse(
status_code=409,
content={"error": "Configure the selected translation provider before translating."},
)
lang_code = TRANSLATE_CODES.get(req.target_lang, req.target_lang)
api_key = os.environ.get("TRANSLATE_API_KEY", "")
loop = asyncio.get_running_loop()
@@ -281,8 +409,16 @@ async def dub_translate(req: TranslateRequest):
# Offline NLLB Transformer Translation
if provider == "nllb":
flores_tgt = FLORES_CODES.get(req.target_lang, "eng_Latn")
flores_src = FLORES_CODES.get(src_lang, "eng_Latn")
requested = [src_lang, req.target_lang, *(seg.target_lang for seg in req.segments if seg.target_lang)]
resolved = {code: _nllb_language(code) for code in requested}
unsupported = [code for code, language in resolved.items() if language is None]
if unsupported:
return JSONResponse(status_code=400, content={
"error": "NLLB does not support the requested language.",
"code": "unsupported_translation_language", "languages": unsupported,
})
flores_tgt = resolved[req.target_lang]
flores_src = resolved[src_lang]
def _translate_nllb():
global _nllb_model, _nllb_tokenizer, _nllb_device
@@ -314,44 +450,112 @@ async def dub_translate(req: TranslateRequest):
logger.exception("NLLB model load failed")
return [{"id": seg.id, "text": seg.text, "error": f"Model load error: {str(e)}"} for seg in req.segments]
results = []
for seg in req.segments:
from services.performance_profiles import translation_decode_defaults
# Snapshot once so every segment and device fallback in this
# job uses the same decoding effort even if preferences change.
decode_options = translation_decode_defaults()
def _generate_rows(rows, target_language):
global _nllb_device
_nllb_tokenizer.src_lang = flores_src
inputs = _nllb_tokenizer(
[seg.text for _, seg in rows],
return_tensors="pt",
padding=True,
)
if _nllb_device and _nllb_device != "cpu":
inputs = {key: value.to(_nllb_device) for key, value in inputs.items()}
forced_bos_token_id = _nllb_tokenizer.convert_tokens_to_ids(target_language)
try:
if not seg.text or not seg.text.strip():
results.append({"id": seg.id, "text": seg.text})
continue
tokens = _nllb_model.generate(
**inputs,
forced_bos_token_id=forced_bos_token_id,
max_length=400,
**decode_options,
)
except (RuntimeError, NotImplementedError) as error:
if _nllb_device != "mps":
raise
logger.warning("MPS generate failed, retrying on CPU: %s", error)
_nllb_model.to("cpu")
_nllb_device = "cpu"
inputs = {key: value.to("cpu") for key, value in inputs.items()}
tokens = _nllb_model.generate(
**inputs,
forced_bos_token_id=forced_bos_token_id,
max_length=400,
**decode_options,
)
decoded = _nllb_tokenizer.batch_decode(tokens, skip_special_tokens=True)
if len(decoded) != len(rows):
raise RuntimeError(
f"NLLB returned {len(decoded)} translations for {len(rows)} segments"
)
return decoded
tgt = FLORES_CODES.get(seg.target_lang, flores_tgt) if seg.target_lang else flores_tgt
# A target-language BOS token is shared by a forward pass, so
# group mixed-language rows first. Preserve request order in
# the final response even though groups render independently.
grouped: dict[str, list[tuple[int, object]]] = {}
results_by_index: dict[int, dict] = {}
for index, seg in enumerate(req.segments):
if not seg.text or not seg.text.strip():
results_by_index[index] = {"id": seg.id, "text": seg.text}
continue
target = resolved[seg.target_lang] if seg.target_lang else flores_tgt
grouped.setdefault(target, []).append((index, seg))
_nllb_tokenizer.src_lang = flores_src
inputs = _nllb_tokenizer(seg.text, return_tensors="pt")
if _nllb_device and _nllb_device != "cpu":
inputs = {k: v.to(_nllb_device) for k, v in inputs.items()}
forced_bos_token_id = _nllb_tokenizer.convert_tokens_to_ids(tgt)
# Beam search multiplies decoder memory per row. Keep the
# effective hypothesis count bounded while still widening the
# Fast path aggressively.
beam_count = max(1, int(decode_options.get("num_beams", 1)))
width = min(
_nllb_batch_size(),
max(1, _nllb_hypothesis_budget() // beam_count),
)
for target, rows in grouped.items():
for start in range(0, len(rows), width):
batch = rows[start : start + width]
try:
translated_tokens = _nllb_model.generate(
**inputs, forced_bos_token_id=forced_bos_token_id, max_length=400
translated_texts = _generate_rows(batch, target)
except Exception as batch_error:
if len(batch) == 1:
index, seg = batch[0]
results_by_index[index] = {
"id": seg.id,
"text": seg.text,
"error": str(batch_error),
}
continue
# A single unusually long row must not sink its
# neighbours. Clear a failed device allocation and
# retain the established per-segment degradation.
if torch.cuda.is_available():
torch.cuda.empty_cache()
logger.warning(
"NLLB batch of %d failed; retrying rows individually: %s",
len(batch),
batch_error,
)
except (RuntimeError, NotImplementedError) as e:
if _nllb_device == "mps":
logger.warning("MPS generate failed, retrying on CPU: %s", e)
_nllb_model.to("cpu")
_nllb_device = "cpu"
inputs = {k: v.to("cpu") for k, v in inputs.items()}
translated_tokens = _nllb_model.generate(
**inputs, forced_bos_token_id=forced_bos_token_id, max_length=400
)
else:
raise
translated_text = _nllb_tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]
results.append({"id": seg.id, "text": translated_text})
except Exception as e:
results.append({"id": seg.id, "text": seg.text, "error": str(e)})
return results
for index, seg in batch:
try:
translated_text = _generate_rows([(index, seg)], target)[0]
results_by_index[index] = {"id": seg.id, "text": translated_text}
except Exception as row_error:
results_by_index[index] = {
"id": seg.id,
"text": seg.text,
"error": str(row_error),
}
continue
for (index, seg), translated_text in zip(batch, translated_texts):
results_by_index[index] = {"id": seg.id, "text": translated_text}
return [results_by_index[index] for index in range(len(req.segments))]
translated = await loop.run_in_executor(_gpu_pool, _translate_nllb)
if os.environ.get("OMNIVOICE_UNLOAD_NLLB", "1") == "1":
if _should_unload_nllb():
_unload_nllb()
# Cinematic/Autofit refine + rate-ratio badges must run for NLLB too
# (previously this returned before _maybe_cinematic, so a Cinematic
@@ -594,18 +798,35 @@ async def dub_translate(req: TranslateRequest):
f"switch the Engine dropdown to another provider."
)
return JSONResponse(status_code=400, content={"error": friendly})
target_codes = list(dict.fromkeys(
seg.target_lang if seg.target_lang else req.target_lang
for seg in req.segments
))
try:
pack_status = translation_engines.argos_pack_status(src_lang, target_codes)
except (ImportError, ValueError) as exc:
return JSONResponse(status_code=422, content={"error": str(exc)})
missing_packs = [
pair for pair in pack_status["pairs"] if not pair["installed"]
]
if missing_packs:
pairs = ", ".join(
f'{pair["source_lang"]}{pair["target_lang"]}'
for pair in missing_packs
)
return JSONResponse(
status_code=409,
content={
"error": f"Install the Argos language pack for {pairs} before translating.",
"code": "argos_pack_missing",
"pairs": missing_packs,
},
)
def _translate_argos():
cache_dir = os.environ.get("OMNIVOICE_CACHE_DIR")
if cache_dir:
argos_cache = os.path.join(cache_dir, "argos-translate")
os.makedirs(argos_cache, exist_ok=True)
os.environ.setdefault("ARGOS_PACKAGES_DIR", argos_cache)
os.environ.setdefault("ARGOS_DATA_DIR", argos_cache)
import argostranslate.package
import argostranslate.translate
from_code = src_lang
available_packages = argostranslate.package.get_installed_packages()
from_code = pack_status["source_lang"]
results = []
for seg in req.segments:
@@ -614,19 +835,12 @@ async def dub_translate(req: TranslateRequest):
results.append({"id": seg.id, "text": seg.text})
continue
to_code = seg.target_lang if seg.target_lang else req.target_lang
installed_pkg = next(filter(lambda x: x.from_code == from_code and x.to_code == to_code, available_packages), None)
if installed_pkg is None:
argostranslate.package.update_package_index()
all_packages = argostranslate.package.get_available_packages()
package_to_install = next(filter(lambda x: x.from_code == from_code and x.to_code == to_code, all_packages), None)
if package_to_install:
argostranslate.package.install_from_path(package_to_install.download())
available_packages = argostranslate.package.get_installed_packages()
else:
raise Exception(f"No Argos package available for {from_code} -> {to_code}")
translated_text = argostranslate.translate.translate(seg.text, from_code, to_code)
to_code = translation_engines.argos_lang_code(to_code)
translated_text = (
seg.text
if from_code == to_code
else argostranslate.translate.translate(seg.text, from_code, to_code)
)
results.append({"id": seg.id, "text": translated_text})
except Exception as e:
results.append({"id": seg.id, "text": seg.text, "error": str(e)})
@@ -700,9 +914,10 @@ async def dub_translate(req: TranslateRequest):
for attempt, src in enumerate([src_arg, src_arg, "auto"]):
try:
out = _build_translator(src, seg_lc).translate(seg.text)
if out and out.strip():
output_error = _translation_output_error(out)
if output_error is None:
return {"id": seg.id, "text": out}
last_err = "empty translation"
last_err = output_error
except Exception as e:
last_err = f"{type(e).__name__}: {e}"
logger.warning(
@@ -895,7 +1110,7 @@ async def _apply_fit_pass(rows, req, slots_by_id, source_by_id, quality, loop, d
their current text and get ``rate_error='fit-budget'``. Only rows with a
slot + text + no prior error participate.
"""
strict = (quality == "autofit")
strict = quality in ("autofit", "agent")
items = []
for row in rows:
seg_id = str(row["id"])
@@ -958,7 +1173,7 @@ async def _maybe_cinematic(translated, req, src_lang, loop, *, already_llm=False
# Fast (and anything unrecognised) returns the plain translation unchanged
# (plus the pre-synthesis duration-plan badges — no LLM needed for those).
if quality not in ("cinematic", "autofit"):
if quality not in ("cinematic", "autofit", "agent"):
await _finalize_duration_plan(translated, req, loop)
return base
@@ -1076,3 +1291,64 @@ async def _maybe_cinematic(translated, req, src_lang, loop, *, already_llm=False
"quality_used": quality,
**_dialect_flags(req, applied=bool(dialect_hint)),
}
@router.post("/dub/agent-fit")
async def dub_agent_fit(req: AgentFitRequest):
"""Rewrite rendered lines from real duration evidence.
Synthesis stays in the normal Dubbing pipeline. The client renders each
candidate, measures it, and may request one more bounded correction.
"""
from services import llm_skills
from services.speech_rate import adjust_for_measured_slot_many
readiness = llm_skills.resolve_skill("slot_fitting")
if not readiness.ready:
raise HTTPException(
status_code=409,
detail={
"error": "llm_skill_unavailable",
"skill": "slot_fitting",
"reason": readiness.reason or "unavailable",
},
)
items = [
(
segment.id,
segment.text,
segment.slot_seconds,
segment.measured_seconds,
req.target_lang,
segment.source_text,
segment.context_before,
segment.context_after,
)
for segment in req.segments
]
budget = _cinematic_budget()
try:
call = adjust_for_measured_slot_many(items, executor=_cpu_pool)
rows = await asyncio.wait_for(call, timeout=budget) if budget and budget > 0 else await call
except asyncio.TimeoutError:
rows = {
segment.id: {
"text": segment.text,
"changed": False,
"measured_seconds": round(segment.measured_seconds, 3),
"target_seconds": round(segment.slot_seconds, 3),
"measured_ratio": round(
segment.measured_seconds / max(segment.slot_seconds, 0.001), 3
),
"error": "fit-budget",
}
for segment in req.segments
}
return {
"target_lang": req.target_lang,
"segments": [
{"id": segment.id, **rows[str(segment.id)]}
for segment in req.segments
],
}
+230 -3
View File
@@ -15,6 +15,7 @@ Environment variables (`OMNIVOICE_TTS_BACKEND`, `OMNIVOICE_ASR_BACKEND`,
`OMNIVOICE_LLM_BACKEND`) still win over the UI choice so power-users can pin
a backend without Settings silently undoing it.
"""
import asyncio
import logging
import os
import threading
@@ -23,10 +24,11 @@ from time import perf_counter
from fastapi import APIRouter, Depends, HTTPException
from huggingface_hub import utils as hf_utils
from huggingface_hub.errors import HFValidationError
from pydantic import BaseModel
from pydantic import BaseModel, Field
from api.dependencies import require_admin, require_admin_action, require_desktop
from core import prefs
from core.engine_licenses import LICENSE_GATED_ENGINES
from services import tts_backend, asr_backend, llm_backend, translation_engines
from services.audio_dsp import list_effect_presets
from api.schemas import EffectPresetsResponse
@@ -58,12 +60,49 @@ def _catalogue_active_id(family: str, module) -> str:
def _family_payload(family: str, module):
"""Public inventory plus whether an environment pin owns this family."""
active = _catalogue_active_id(family, module)
model = None
if family == "asr":
model = asr_backend._offline_asr_repo(active)
elif family == "llm" and active != "off":
model = llm_backend.get_active_llm_backend().model_name
elif family == "tts":
if active in {"omnivoice", "omnivoice-subprocess"}:
from services.model_manager import resolve_omnivoice_checkpoint
model = resolve_omnivoice_checkpoint()
elif active == "mlx-audio":
from core import prefs
cls = tts_backend.MLXAudioBackend
key = prefs.resolve("mlx_audio_model_id", env="OMNIVOICE_MLX_AUDIO_MODEL", default=cls.DEFAULT_MODEL_KEY)
model = cls.CURATED_MODELS.get(key, key)
else:
instance = getattr(tts_backend, "_active_instance", None)
if instance is not None and getattr(tts_backend, "_active_instance_id", None) == active:
model = instance.model_identity()
backends = public_backends(module.list_backends())
if family == "tts":
from services import settings_store
for backend in backends:
engine_id = backend.get("id")
if engine_id in LICENSE_GATED_ENGINES:
backend["license_required"] = True
try:
backend["license_accepted"] = settings_store.get_license_accepted(engine_id)
except Exception:
logger.warning(
"Could not read license acceptance for %s",
engine_id,
exc_info=True,
)
backend["license_accepted"] = False
return {
# MPS hides the explicit compatibility row, so legacy configs report
# the visible canonical equivalent as active to picker consumers.
"active": _catalogue_active_id(family, module),
"active": active,
"active_model": model,
"env_override": bool(os.environ.get(f"OMNIVOICE_{family.upper()}_BACKEND")),
"backends": public_backends(module.list_backends()),
"backends": backends,
}
def _is_hf_repo_id(value: str) -> bool:
@@ -126,6 +165,90 @@ def list_effects_presets():
return {"presets": list_effect_presets()}
@router.get("/engines/diarisation")
def diarisation_status():
"""Describe the selected local diarisation runtime without loading weights."""
from services.diarization_runtime import (
PYANNOTE,
SORTFORMER,
selected_backend,
sortformer_status,
)
selected = selected_backend()
native = selected == SORTFORMER
options = []
from api.routers.setup.models import KNOWN_MODELS, cache_is_complete, is_cached
pyannote_repo = "pyannote/speaker-diarization-3.1"
spec = next(model for model in KNOWN_MODELS if model["repo_id"] == pyannote_repo)
pyannote_installed = is_cached(pyannote_repo) and cache_is_complete(spec)
pyannote_reason = None if pyannote_installed else "Install the pyannote model bundle"
options.append({
"id": PYANNOTE,
"label": "pyannote 3.1",
"model": pyannote_repo,
"installed": pyannote_installed,
"reason": pyannote_reason,
})
native_status = sortformer_status()
native_installed = native_status["installed"]
native_model = native_status["model"]
native_reason = native_status["reason"]
options.append({
"id": SORTFORMER,
"label": "Sortformer v1 (audio.cpp)",
"model": native_model,
"model_installed": native_status["model_installed"],
"runtime_installed": native_status["runtime_installed"],
"installed": native_installed,
"reason": native_reason,
})
if native:
from services.diarization_native import is_running
return {"active": SORTFORMER, "label": "Sortformer v1 (audio.cpp)",
"model": native_model, "installed": native_installed, "loaded": False,
"model_installed": native_status["model_installed"],
"runtime_installed": native_status["runtime_installed"],
"busy": is_running(), "reason": native_reason, "options": options}
from services import model_manager
return {"active": PYANNOTE, "label": "pyannote 3.1", "model": pyannote_repo,
"installed": pyannote_installed,
"loaded": model_manager._diar_pipeline is not None, "reason": pyannote_reason,
"options": options}
class DiarisationSelection(BaseModel):
engine_id: str
@router.post("/engines/diarisation/select", dependencies=[Depends(require_admin)])
def select_diarisation_engine(request: DiarisationSelection):
"""Persist an installed diarisation runtime; environment overrides still win."""
from services.diarization_runtime import SORTFORMER, select_backend, selected_backend
status = diarisation_status()
option = next(
(item for item in status["options"] if item["id"] == request.engine_id),
None,
)
if option is None:
raise HTTPException(404, "Unknown diarisation engine")
if not option["installed"]:
raise HTTPException(409, option.get("reason") or "Install this diarisation engine first")
select_backend(request.engine_id)
if request.engine_id == SORTFORMER:
# Native Sortformer is stateless. Release a previously loaded pyannote
# pipeline so Engine Ready cannot hide stale accelerator memory.
from services import model_manager
model_manager.unload_diarization_pipeline()
return {
"active": selected_backend(),
"env_override": bool(os.environ.get("OMNIVOICE_DIARIZATION_BACKEND")),
}
@router.get("/engines/translation")
def list_translation_engines():
"""Translation engines with per-engine pip-package availability.
@@ -136,6 +259,7 @@ def list_translation_engines():
an engine whose Python dependency isn't importable yet.
"""
return {
"active": prefs.get("translation_backend", "argos"),
"engines": [
{**entry, "availability_reason": public_unavailability(entry.get("availability_reason"))}
for entry in translation_engines.list_engines()
@@ -144,6 +268,66 @@ def list_translation_engines():
}
class TranslationSelection(BaseModel):
engine_id: str
class ArgosPackRequest(BaseModel):
source_lang: str | None = None
target_langs: list[str] = Field(min_length=1, max_length=32)
job_id: str | None = None
def _argos_pack_request(request: ArgosPackRequest) -> tuple[str, list[str]]:
source = request.source_lang
if not source and request.job_id:
from api.routers.dub_core import _get_job
job = _get_job(request.job_id)
source = job.get("source_lang") if job else None
if not source:
raise HTTPException(422, "Transcribe the source before installing its language pack")
return source, request.target_langs
@router.post("/engines/translation/argos/packs/status")
def argos_pack_status(request: ArgosPackRequest):
source, targets = _argos_pack_request(request)
try:
return translation_engines.argos_pack_status(source, targets)
except (ImportError, ValueError) as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@router.post(
"/engines/translation/argos/packs/install",
dependencies=[Depends(require_admin)],
)
async def install_argos_packs(request: ArgosPackRequest):
source, targets = _argos_pack_request(request)
try:
return await asyncio.to_thread(
translation_engines.install_argos_packs,
source,
targets,
)
except (ImportError, ValueError) as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@router.post("/engines/translation/select", dependencies=[Depends(require_admin)])
def select_translation_engine(request: TranslationSelection):
entry = translation_engines.get_engine(request.engine_id)
if not entry:
raise HTTPException(404, "Unknown translation engine")
if not translation_engines.is_installed(request.engine_id):
raise HTTPException(409, "Install this translation engine before selecting it")
if not translation_engines.is_ready(request.engine_id):
raise HTTPException(409, "Configure this translation provider before selecting it")
prefs.set_("translation_backend", request.engine_id)
return {"active": request.engine_id}
@router.post(
"/engines/translation/{engine_id}/install",
dependencies=[Depends(require_admin)],
@@ -215,6 +399,32 @@ async def uninstall_translation_engine(engine_id: str):
return {"status": "uninstalled", "engine": engine_id, "package": pkg, "log_tail": out[-800:]}
# ── Checksummed native audio.cpp runtime install ───────────────────────────
@router.get(
"/engines/audiocpp/runtime/install/status",
dependencies=[Depends(require_admin)],
)
def audiocpp_runtime_install_status():
from services import audiocpp_runtime_install
return audiocpp_runtime_install.status()
@router.post(
"/engines/audiocpp/runtime/install",
dependencies=[Depends(require_admin), Depends(require_desktop)],
)
def install_audiocpp_runtime():
from services import audiocpp_runtime_install
try:
return audiocpp_runtime_install.start_install()
except RuntimeError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
# ── One-click sidecar-engine install (IndexTTS-2 & friends) ────────────────
#
# Sidecar engines (dedicated venv + source checkout + weights, isolated from
@@ -693,6 +903,23 @@ def select_engine(req: SelectEngineRequest):
"Hugging Face repo ID like 'owner/name'.",
)
prefs.set_("mlx_audio_model_id", req.model_id)
if req.family == "asr" and req.model_id is not None:
if req.backend_id not in {"faster-whisper", "faster-whisper-isolated"}:
raise HTTPException(400, "This ASR engine does not accept a CTranslate2 model")
from api.routers.setup.models import KNOWN_MODELS, is_cached
model = next((item for item in KNOWN_MODELS if item["repo_id"] == req.model_id), None)
compatible = req.model_id.startswith("Systran/faster-") or req.model_id == (
"deepdml/faster-whisper-large-v3-turbo-ct2"
)
if model is None or str(model.get("role", "")).lower() != "asr" or not compatible:
raise HTTPException(400, "This model is not compatible with Faster-Whisper")
if not is_cached(req.model_id):
raise HTTPException(409, "Install this ASR model before selecting it")
try:
asr_backend.select_faster_whisper_model(req.model_id)
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
prefs.set_(pref_key, req.backend_id)
return {
"family": req.family,
+9 -2
View File
@@ -1366,12 +1366,12 @@ async def generate_speech(
ref_text: Optional[str] = Form(None),
instruct: Optional[str] = Form(None),
duration: Optional[float] = Form(None),
num_step: int = Form(16),
num_step: Optional[int] = Form(None),
guidance_scale: float = Form(2.0),
speed: float = Form(1.0),
t_shift: Optional[float] = Form(None),
denoise: bool = Form(True),
postprocess_output: bool = Form(True),
postprocess_output: Optional[bool] = Form(None),
layer_penalty_factor: Optional[float] = Form(None),
position_temperature: Optional[float] = Form(None),
class_temperature: Optional[float] = Form(None),
@@ -1417,6 +1417,13 @@ async def generate_speech(
)
engine_id = engine or active_backend_id()
from services.performance_profiles import tts_defaults
sampling_defaults = tts_defaults(engine_id)
if num_step is None:
num_step = sampling_defaults.get("num_step", 16)
if postprocess_output is None:
postprocess_output = sampling_defaults.get("postprocess_output", True)
try:
backend_cls = get_backend_class(engine_id)
except ValueError:
+182
View File
@@ -0,0 +1,182 @@
"""Keyless portrait search with bounded, normalized, safe thumbnails."""
import asyncio
import base64
import html
import re
from html.parser import HTMLParser
from urllib.parse import urlsplit
import httpx
from fastapi import APIRouter, HTTPException, Query
from core.profile_images import MAX_IMAGE_BYTES, normalize_portrait
router = APIRouter()
def trusted_thumbnail(url: str) -> bool:
try:
parsed = urlsplit(url)
google = parsed.hostname in {
f"encrypted-tbn{i}.gstatic.com" for i in range(4)
}
openverse = (
parsed.hostname == "api.openverse.org"
and re.fullmatch(r"/v1/images/[0-9a-f-]+/thumb/?", parsed.path) is not None
)
return (
parsed.scheme == "https"
and not parsed.username
and not parsed.password
and parsed.port in (None, 443)
and (google or openverse)
)
except ValueError:
return False
def google_thumbnails(document: str) -> list[tuple[str, str]]:
"""Extract result thumbnails only; never download third-party originals."""
results = []
seen = set()
def add(title, source):
if not isinstance(source, str) or source in seen:
return
if not (trusted_thumbnail(source) or source.startswith("data:image/jpeg;base64,")):
return
seen.add(source)
if len(results) < 20:
results.append((title or "", source))
class Images(HTMLParser):
def handle_starttag(self, tag, attrs):
if tag == "img":
values = dict(attrs)
add(values.get("alt"), values.get("src") or values.get("data-src"))
Images().feed(document)
# Google also assigns thumbnails from script strings after rendering.
decoded = html.unescape(document)
for escaped, literal in ((r"\u003d", "="), (r"\u0026", "&"), (r"\/", "/")):
decoded = decoded.replace(escaped, literal)
for match in re.finditer(r'https://encrypted-tbn[0-3]\.gstatic\.com/[^\s"\'<>\\]+|data:image/jpeg;base64,[A-Za-z0-9+/=]+', decoded):
add("", match.group())
return results
async def openverse_thumbnails(client: httpx.AsyncClient, name: str) -> list[tuple[str, str]]:
"""Public-domain/CC portrait fallback when Google returns its JS-only shell.
Openverse requires no user credential, excludes sensitive results by
default, and can restrict results to licenses that allow modification and
commercial use. We still fetch only its own thumbnail proxy.
"""
response = await client.get(
"https://api.openverse.org/v1/images/",
headers={
"User-Agent": "VoiceStudio/0.5 (+https://github.com/debpalash/VoiceStudio)",
"Accept": "application/json",
},
params={
"q": name,
"page_size": 20,
"mature": "false",
"extension": "jpg,png",
"aspect_ratio": "square",
"license_type": "commercial,modification",
},
)
response.raise_for_status()
if len(response.content) > 4 * 1024 * 1024:
raise ValueError("Search response too large")
payload = response.json()
rows = payload.get("results") if isinstance(payload, dict) else None
if not isinstance(rows, list):
raise ValueError("Invalid search response")
results = []
seen = set()
for row in rows:
if not isinstance(row, dict):
continue
source = row.get("thumbnail")
if not isinstance(source, str) or source in seen or not trusted_thumbnail(source):
continue
seen.add(source)
title = str(row.get("title") or name)
creator = str(row.get("creator") or "").strip()
license_name = str(row.get("license") or "").upper()
credit = " · ".join(value for value in (creator, license_name) if value)
results.append((f"{title}{credit}" if credit else title, source))
return results
@router.get("/profile-images/search")
async def search_profile_images(name: str = Query(min_length=1, max_length=100)):
if not name.strip():
raise HTTPException(422, detail={"code": "image_search_failed"})
async with httpx.AsyncClient(
timeout=15,
follow_redirects=False,
headers={
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 Chrome/140.0.0.0 Safari/537.36",
"Accept-Language": "en-US,en;q=0.9",
},
) as client:
results: list[tuple[str, str]] = []
try:
async with client.stream("GET", "https://www.google.com/search", params={
"q": name.strip(), "udm": "2", "safe": "active", "tbs": "ift:jpg",
}) as response:
response.raise_for_status()
document = bytearray()
async for chunk in response.aiter_bytes():
document.extend(chunk)
if len(document) > 4 * 1024 * 1024:
raise ValueError("Search page too large")
page = document.decode("utf-8", errors="replace")
results = google_thumbnails(page)
except (httpx.HTTPError, ValueError, TypeError):
# Search providers can change their anonymous HTML or reject a
# non-browser request. The fallback below keeps this explicit,
# user-triggered feature useful without requiring credentials.
results = []
if not results:
try:
results = await openverse_thumbnails(client, name.strip())
except (httpx.HTTPError, ValueError, TypeError):
results = []
if not results:
raise HTTPException(502, detail={"code": "image_search_failed"})
async def thumbnail(title, url):
try:
if url.startswith("data:image/jpeg;base64,"):
encoded = url.partition(",")[2]
if len(encoded) > MAX_IMAGE_BYTES * 4 // 3 + 4:
return None
data = base64.b64decode(encoded, validate=True)
else:
if not trusted_thumbnail(url):
return None
async with client.stream("GET", url) as image:
image.raise_for_status()
data = bytearray()
async for chunk in image.aiter_bytes():
data.extend(chunk)
if len(data) > MAX_IMAGE_BYTES:
return None
normalized = await asyncio.to_thread(normalize_portrait, bytes(data))
return {"title": title[:200], "data": base64.b64encode(normalized).decode("ascii")}
except (httpx.HTTPError, HTTPException, ValueError, TypeError):
return None
images = []
for start in range(0, len(results), 5):
batch = await asyncio.gather(*(thumbnail(*result) for result in results[start:start + 5]))
images.extend(image for image in batch if image)
if len(images) >= 5:
break
return {"images": images[:5]}
+109 -7
View File
@@ -1,3 +1,5 @@
import asyncio
import logging
import os
import re
import uuid
@@ -14,8 +16,21 @@ from core import event_bus
from core.personalities import get_personalities
from omnivoice.utils.voice_design import heal_design_instruct, sanitize_instruct
from core.path_security import UnsafePath, resolve_within
from core.profile_images import MAX_IMAGE_BYTES, normalize_portrait
from starlette.datastructures import UploadFile as StarletteUploadFile
router = APIRouter()
logger = logging.getLogger("omnivoice.profiles")
def _profile_record(row):
result = dict(row)
image_path = _voices_path(f"{result['id']}.portrait.jpg")
result["image_url"] = (
f"/profiles/{result['id']}/image?v={os.stat(image_path).st_mtime_ns}"
if image_path and os.path.isfile(image_path) else None
)
return result
class ProfileUpdate(BaseModel):
@@ -35,7 +50,7 @@ def list_personalities():
def list_profiles():
with db_conn() as conn:
rows = conn.execute("SELECT * FROM voice_profiles ORDER BY created_at DESC").fetchall()
return [dict(r) for r in rows]
return [_profile_record(r) for r in rows]
_DESIGN_SEED = 42 # deterministic sample render, same as archetype previews
@@ -51,6 +66,7 @@ async def create_profile(
personality: str = Form(""),
kind: str = Form("clone"),
vd_states: Optional[str] = Form(None),
image: Optional[UploadFile] = File(None),
):
"""Create a voice profile (spec: docs/specs/voice-studio-unification.md §5).
@@ -60,6 +76,9 @@ async def create_profile(
archetype materialization) and stores it as the profile's
reference so the voice identity is stable across runs.
"""
name = name.strip()
if not name:
raise HTTPException(status_code=400, detail="A voice profile needs a name.")
if kind not in ("clone", "design"):
raise HTTPException(status_code=422, detail="kind must be 'clone' or 'design'")
if kind == "clone" and ref_audio is None:
@@ -106,13 +125,38 @@ async def create_profile(
instruct = sanitize_instruct(instruct)
profile_id = str(uuid.uuid4())[:8]
portrait = None
if isinstance(image, StarletteUploadFile):
portrait = normalize_portrait(await image.read(MAX_IMAGE_BYTES + 1))
portrait_path = os.path.join(VOICES_DIR, f"{profile_id}.portrait.jpg")
if kind == "clone":
ext = os.path.splitext(ref_audio.filename or ".wav")[1]
audio_filename = f"{profile_id}{ext}"
audio_path = os.path.join(VOICES_DIR, audio_filename)
# Storage can be removed after startup; recover before persisting uploads.
os.makedirs(VOICES_DIR, exist_ok=True)
with open(audio_path, "wb") as f:
f.write(await ref_audio.read())
# A matching transcript defines the boundary between the reference and
# the requested line. Saving a blank transcript and waiting until the
# first generation made that first take depend on the TTS model's
# internal ASR fallback; short lines could then start with stray words
# from the reference. Resolve it while the profile is being created so
# every synthesis, including the first, uses stable conditioning. This
# remains best-effort and local-only: transcribe_reference considers
# only already-installed ASR/dictation models.
if not ref_text.strip():
try:
from services.asr_backend import transcribe_reference
ref_text = (
await asyncio.to_thread(transcribe_reference, audio_path) or ""
).strip()
except Exception as exc: # noqa: BLE001 — profile save remains usable
logger.warning(
"reference transcription during profile save failed: %s", exc
)
used_seed = seed
else:
# Saving a design profile is a pure persistence operation — it must not
@@ -153,6 +197,10 @@ async def create_profile(
used_seed = seed if seed is not None else _DESIGN_SEED
try:
if portrait:
os.makedirs(VOICES_DIR, exist_ok=True)
with open(portrait_path, "wb") as out:
out.write(portrait)
with db_conn() as conn:
conn.execute(
"INSERT INTO voice_profiles (id, name, ref_audio_path, ref_text, instruct, "
@@ -162,12 +210,14 @@ async def create_profile(
used_seed, personality, kind, vd_states, time.time())
)
except Exception:
if os.path.exists(portrait_path):
os.remove(portrait_path)
# Clean up orphaned audio file if DB insert fails
if os.path.exists(audio_path):
os.remove(audio_path)
raise
event_bus.emit("profiles", {"action": "created", "id": profile_id})
return {"id": profile_id, "name": name, "kind": kind}
return get_profile(profile_id)
@router.get("/profiles/{profile_id}")
def get_profile(profile_id: str):
@@ -181,14 +231,47 @@ def get_profile(profile_id: str):
status_code=404,
detail="That voice profile doesn't exist. It may have been deleted from another tab.",
)
return dict(row)
return _profile_record(row)
@router.get("/profiles/{profile_id}/image")
def get_profile_image(profile_id: str):
get_profile(profile_id)
path = _voices_path(f"{profile_id}.portrait.jpg")
if not path or not os.path.isfile(path):
raise HTTPException(404, "Profile image not found")
return FileResponse(path, media_type="image/jpeg", headers={"Cache-Control": "no-cache"})
@router.put("/profiles/{profile_id}/image")
async def update_profile_image(profile_id: str, image: UploadFile = File(...)):
get_profile(profile_id)
path = _voices_path(f"{profile_id}.portrait.jpg")
if path is None:
raise HTTPException(404, "Profile not found")
portrait = normalize_portrait(await image.read(MAX_IMAGE_BYTES + 1))
os.makedirs(VOICES_DIR, exist_ok=True)
with open(path, "wb") as out:
out.write(portrait)
event_bus.emit("profiles", {"action": "updated", "id": profile_id})
return get_profile(profile_id)
@router.put("/profiles/{profile_id}")
def update_profile(profile_id: str, patch: ProfileUpdate):
"""Partial update — only fields set on the payload are changed."""
with db_conn() as conn:
existing = conn.execute(
"SELECT kind FROM voice_profiles WHERE id = ?", (profile_id,),
).fetchone()
if not existing:
raise HTTPException(
status_code=404,
detail="That voice profile doesn't exist. It may have been deleted from another tab.",
)
fields = []
params = []
edited_instruct = None
for col in ("name", "ref_text", "instruct", "language", "personality"):
val = getattr(patch, col)
if val is None:
@@ -199,12 +282,22 @@ def update_profile(profile_id: str, patch: ProfileUpdate):
# Never let an edit persist a validator-rejecting instruct (prose /
# "[object Object]"); keep only whitelist tags (#550 #571 #594 #596).
val = sanitize_instruct(val)
edited_instruct = val
fields.append(f"{col} = ?")
params.append(val.strip() if col in ("name", "language") else val)
if edited_instruct is not None and existing["kind"] == "design":
# Keep the complete recipe synchronized with the editable instruct.
# Otherwise clients restore a stale vd_states snapshot and a successful
# style edit has no effect on the next generation.
import json
from core.describe_voice import instruct_to_vd_states
fields.append("vd_states = ?")
params.append(json.dumps(instruct_to_vd_states(edited_instruct)))
if not fields:
raise HTTPException(
status_code=400,
detail="PUT /profiles/{id} body contained no editable fields. Include at least one of: name, language, instruct, description.",
detail="PUT /profiles/{id} body contained no editable fields. Include at least one of: name, language, ref_text, instruct, personality.",
)
params.append(profile_id)
with db_conn() as conn:
@@ -221,7 +314,7 @@ def update_profile(profile_id: str, patch: ProfileUpdate):
"SELECT * FROM voice_profiles WHERE id = ?", (profile_id,),
).fetchone()
event_bus.emit("profiles", {"action": "updated", "id": profile_id})
return dict(row)
return _profile_record(row)
@router.get("/profiles/{profile_id}/usage")
@@ -252,8 +345,14 @@ def get_profile_usage(profile_id: str):
state = json.loads(r["state_json"] or "{}")
except Exception:
continue
segs = state.get("segments") or []
n = sum(1 for s in segs if s.get("profile_id") == profile_id)
if not isinstance(state, dict):
continue
# Current desktop snapshots use dubSegments. An explicit empty list
# supersedes legacy segments retained in an older snapshot.
segs = state.get("dubSegments", state.get("segments", []))
if not isinstance(segs, list):
continue
n = sum(1 for s in segs if isinstance(s, dict) and s.get("profile_id") == profile_id)
if n:
project_hits.append({
"project_id": r["id"],
@@ -539,6 +638,9 @@ def delete_profile(profile_id: str):
path = _voices_path(row[col])
if path and os.path.exists(path):
os.remove(path)
portrait_path = _voices_path(f"{profile_id}.portrait.jpg")
if portrait_path and os.path.isfile(portrait_path):
os.remove(portrait_path)
# Prevent FOREIGN KEY constraint failure
conn.execute("UPDATE generation_history SET profile_id = NULL WHERE profile_id=?", (profile_id,))
conn.execute("DELETE FROM voice_profiles WHERE id=?", (profile_id,))
+59 -1
View File
@@ -20,6 +20,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel, Field
from core.logging_utils import log_safe
from core.engine_licenses import LICENSE_GATED_ENGINES
from api.dependencies import require_admin, require_admin_action
logger = logging.getLogger("omnivoice.api.settings")
@@ -96,6 +97,63 @@ def get_hf_token_state(fresh: bool = Query(False)):
_TORCH_COMPILE_KEY = "perf.torch_compile_disabled"
from services.performance_profiles import (
_PERFORMANCE_PROFILE_KEY, _PERFORMANCE_TIERS, _PERFORMANCE_FAMILIES,
activate_performance_tier,
profile_state as _performance_profile_state,
)
class _PerformanceProfileBody(BaseModel):
tier: str = Field(..., description="fast | balanced | quality | max")
family: str | None = Field(None, description="Engine family, or null to set the global tier")
@router.get("/performance-profile")
def get_performance_profile():
"""Return the global speed/quality preference and per-engine overrides."""
return _performance_profile_state()
@router.put("/performance-profile")
def set_performance_profile(body: _PerformanceProfileBody):
"""Persist a performance preference and apply installed Max-capacity picks."""
from core import prefs
tier = body.tier.strip().lower()
if tier not in _PERFORMANCE_TIERS:
raise HTTPException(status_code=400, detail="Unknown performance tier")
family = body.family.strip().lower() if body.family else None
if family is not None and family not in _PERFORMANCE_FAMILIES:
raise HTTPException(status_code=400, detail="Unknown engine family")
state = _performance_profile_state()
applicable = state["applicable_families"]
if (family is not None and family not in applicable) or (family is None and not applicable):
raise HTTPException(status_code=409, detail="The selected engines do not support this performance preset")
from core import job_store
from api.routers.batch import list_batch_jobs
if job_store.list_jobs(status="active", limit=1) or list_batch_jobs(status="active", limit=1):
raise HTTPException(status_code=409, detail="Wait for queued or running jobs to finish before changing performance presets")
try:
if family is None:
# One atomic write clears family overrides together with the global
# choice, so a crash cannot leave half of a global change persisted.
prefs.update_mapping(_PERFORMANCE_PROFILE_KEY, {"global": tier}, replace=True)
else:
prefs.update_mapping(_PERFORMANCE_PROFILE_KEY, {family: tier})
except Exception:
logger.exception("set_performance_profile failed")
raise HTTPException(status_code=500, detail="Failed to persist performance profile")
activations = activate_performance_tier(tier, family)
result = _performance_profile_state()
if activations:
result["runtime_activations"] = activations
if tier == "max":
result["capacity_activations"] = activations
return result
class _TorchCompileBody(BaseModel):
enabled: bool = Field(..., description="True to set TORCH_COMPILE_DISABLE=1 on engine subprocesses")
@@ -653,7 +711,7 @@ def set_llm_skill(skill_id: str, body: _LLMSkillBody):
#: Engines that have an in-tree acceptance dialog. Adding a new engine
#: here means adding a corresponding frontend dialog + a license URLs
#: dict in its constants module. Until that, the API refuses the write.
_LICENSE_ALLOWED_ENGINES: frozenset[str] = frozenset({"supertonic3", "pockettts"})
_LICENSE_ALLOWED_ENGINES = LICENSE_GATED_ENGINES
class _LicenseAcceptBody(BaseModel):
+213 -12
View File
@@ -14,6 +14,7 @@ import logging
import os
import sys
import threading
import time
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
@@ -31,7 +32,7 @@ from utils import download_aggregator
from .models import ( # noqa: F401
KNOWN_MODELS,
invalidate_cache,
snapshot_has_weights,
snapshot_is_complete,
disk_space_error,
_MIN_WEIGHT_BYTES,
_WEIGHT_FLOORS,
@@ -42,6 +43,9 @@ router = APIRouter()
# Cooldown: prevent rapid re-install after a failure. Maps repo_id → last_fail_time.
_install_cooldowns: dict[str, float] = {}
# Last classified failure per repo. The SSE stream carries the same detail live;
# retaining it here keeps recovery useful after navigation or renderer reconnect.
_install_failures: dict[str, dict] = {}
_COOLDOWN_SECS = 60.0
# Evict cooldown entries older than this so the dict can't grow unbounded across
# a long-lived process (MM2-06). Anything past the cooldown window is dead state.
@@ -54,6 +58,14 @@ def _sweep_cooldowns(now: float) -> None:
stale = [k for k, t in _install_cooldowns.items() if (now - t) > _COOLDOWN_TTL_SECS]
for k in stale:
_install_cooldowns.pop(k, None)
_install_failures.pop(k, None)
stale_failures = [
repo_id
for repo_id, failure in _install_failures.items()
if (now - float(failure.get("failed_at") or 0)) > _COOLDOWN_TTL_SECS
]
for repo_id in stale_failures:
_install_failures.pop(repo_id, None)
def clear_install_cooldowns() -> None:
@@ -63,6 +75,7 @@ def clear_install_cooldowns() -> None:
very next action is "retry the failed download on the new mirror", and a
429 there would dead-end the wizard's switch-and-retry flow."""
_install_cooldowns.clear()
_install_failures.clear()
# Repo_ids the user asked to cancel (FDL-11). Checked between retry attempts.
# Note: a single in-flight snapshot_download/Xet fetch is not interruptible
@@ -180,6 +193,28 @@ def _repo_cancelled(repo_id: str) -> bool:
return repo_id in _cancelled
def _create_cache_pointer(blob_path: str, pointer: str) -> None:
"""Keep the canonical blob while exposing it from the snapshot tree.
huggingface_hub's ``new_blob=True`` fallback moves the blob into the
snapshot when Windows symlinks are unavailable. The next model load then
sees a missing blob and downloads the same multi-gigabyte weight again.
NTFS hardlinks preserve both cache paths without doubling disk usage; other
filesystems fall back to Hugging Face's copy/symlink path.
"""
from huggingface_hub.file_download import _create_symlink
if os.name == "nt":
try:
os.link(blob_path, pointer)
return
except FileExistsError:
return
except OSError:
pass
_create_symlink(blob_path, pointer, new_blob=False)
def _segmented_snapshot(repo_id: str, *, endpoint: "str | None", revision: str) -> str:
"""Fetch every file of a repo via the segmented downloader into the HF
cache, mirroring hf_hub_download's blob+snapshot+refs layout so the result
@@ -190,7 +225,7 @@ def _segmented_snapshot(repo_id: str, *, endpoint: "str | None", revision: str)
import asyncio as _asyncio
from huggingface_hub import HfApi, constants as _C
from huggingface_hub.file_download import (
hf_hub_url, get_hf_file_metadata, repo_folder_name, _create_symlink,
hf_hub_url, get_hf_file_metadata, repo_folder_name,
)
from services.segmented_download import segmented_download
from services.token_resolver import resolve as _resolve_token
@@ -229,7 +264,7 @@ def _segmented_snapshot(repo_id: str, *, endpoint: "str | None", revision: str)
cancel_check=lambda: _repo_cancelled(repo_id),
))
if not os.path.lexists(pointer):
_create_symlink(blob_path, pointer, new_blob=True)
_create_cache_pointer(blob_path, pointer)
# refs/main → commit so scan_cache_dir maps the revision correctly.
ref_path = os.path.join(refs_dir, "main")
@@ -279,10 +314,17 @@ def _validate_snapshot_has_weights(repo_id: str, snapshot_path: str) -> None:
retry loop and the UI's re-download path can deal with it, instead of at
first synthesis with an opaque transformers error.
Delegates the weight check to ``models.snapshot_has_weights`` (single source of
the floors); only the install-time error message lives here."""
if snapshot_has_weights(snapshot_path):
Delegates to ``models.snapshot_is_complete`` so configuration-only pipeline
repositories use their declared required files instead of a weight floor."""
model = next((m for m in KNOWN_MODELS if m["repo_id"] == repo_id), {"repo_id": repo_id})
if snapshot_is_complete(model, snapshot_path):
return
if model.get("config_only"):
required = ", ".join(model.get("config_required_files") or ())
raise OSError(f"{repo_id}: download is incomplete; required configuration files: {required}")
if model.get("required_files"):
required = ", ".join(model["required_files"])
raise OSError(f"{repo_id}: required model files are missing or incomplete: {required}")
biggest = 0
try:
for root, _dirs, files in os.walk(snapshot_path, followlinks=True):
@@ -346,6 +388,65 @@ class InstallModelRequest(BaseModel):
target: str | None = None
@router.get("/models/install/status")
def model_install_status():
"""Read local and remote jobs after navigation without starting downloads."""
from services import gpu_gateway # noqa: PLC0415
now = time.time()
_sweep_cooldowns(now)
with _active_installs_lock:
active = tuple(_active_installs)
jobs = []
for repo_id in active:
aggregate = download_aggregator._get(repo_id)
jobs.append(
{
"repo_id": repo_id,
"target": "local",
"state": "downloading",
**(aggregate.snapshot() if aggregate else {}),
}
)
detailed = set()
for repo_id, failure in tuple(_install_failures.items()):
failed_at = float(failure.get("failed_at") or 0)
if repo_id in active or now - failed_at >= _COOLDOWN_SECS:
continue
detailed.add(repo_id)
cooldown_at = _install_cooldowns.get(repo_id)
retry_after = (
max(0, int(_COOLDOWN_SECS - (now - cooldown_at) + 0.999))
if cooldown_at is not None
else 0
)
jobs.append(
{
"repo_id": repo_id,
"target": "local",
"state": "failed",
"retry_after_seconds": retry_after,
**failure,
}
)
# Preserve status for callers/tests that seed the legacy cooldown map alone.
jobs.extend(
{
"repo_id": repo_id,
"target": "local",
"state": "failed",
"retry_after_seconds": max(
0, int(_COOLDOWN_SECS - (now - failed_at) + 0.999)
),
}
for repo_id, failed_at in tuple(_install_cooldowns.items())
if repo_id not in active
and repo_id not in detailed
and now - failed_at < _COOLDOWN_SECS
)
jobs.extend(gpu_gateway.remote_download_jobs())
return {"jobs": jobs}
def _is_retryable_download_error(exc: BaseException) -> bool:
"""Whether a failed download attempt is worth retrying.
@@ -474,6 +575,9 @@ async def install_model(req: InstallModelRequest):
loop = asyncio.get_running_loop()
def _do():
# Failure handling must work even when imports, token resolution or
# revision lookup fail before the heartbeat thread is started.
_resolving = threading.Event()
token = hf_progress.current_repo_id.set(req.repo_id)
target_token = hf_progress.current_target.set("local")
hf_progress.emit({
@@ -500,6 +604,10 @@ async def install_model(req: InstallModelRequest):
"revision": revision_for(req.repo_id),
"max_workers": _download_max_workers(),
}
from services.token_resolver import resolve as resolve_token
resolved_token = resolve_token()
if resolved_token:
dl_kwargs["token"] = resolved_token.token
if allow_patterns:
dl_kwargs["allow_patterns"] = allow_patterns
_tqdm_cls = hf_progress.tracked_tqdm_class()
@@ -513,9 +621,7 @@ async def install_model(req: InstallModelRequest):
# Emit a 'resolving' heartbeat every 2s while snapshot_download
# resolves repo metadata (before any tqdm bars appear).
import threading
import time as _t
_resolving = threading.Event()
def _heartbeat():
_step = 0
@@ -549,8 +655,22 @@ async def install_model(req: InstallModelRequest):
_preflight_kwargs["allow_patterns"] = allow_patterns
if _endpoint:
_preflight_kwargs["endpoint"] = _endpoint
if resolved_token:
_preflight_kwargs["token"] = resolved_token.token
try:
_plan = snapshot_download(**_preflight_kwargs) # nosec B615 -- immutable revision_for pin
_plan = list(snapshot_download(**_preflight_kwargs)) # nosec B615 -- immutable revision_for pin
for dependency in model_spec.get("dependencies") or ():
if req.repo_id in _cancelled:
raise _InstallCancelled()
dependency_plan_kwargs = {
**_preflight_kwargs,
"repo_id": dependency["repo_id"],
"revision": revision_for(dependency["repo_id"]),
}
dependency_plan_kwargs.pop("allow_patterns", None)
if dependency.get("allow_patterns"):
dependency_plan_kwargs["allow_patterns"] = dependency["allow_patterns"]
_plan.extend(snapshot_download(**dependency_plan_kwargs)) # nosec B615 -- immutable revision_for pin
_summary = compute_plan(_plan)
# Disk-space guard (before a single byte flows): the preflight
# gives an exact "to download" size, so reject an install that
@@ -568,6 +688,11 @@ async def install_model(req: InstallModelRequest):
"phase": "install_error",
"error": _disk_err,
})
_install_failures[req.repo_id] = {
"failed_at": time.time(),
"error": _disk_err,
"docs_topic": "DISK_SPACE_LOW",
}
# A disk-full is not a transient network failure — don't set
# a cooldown (freeing space, not waiting, is the fix). The
# outer finally still cleans up the aggregator + context.
@@ -584,6 +709,8 @@ async def install_model(req: InstallModelRequest):
"phase": "install_plan",
**_summary,
})
except _InstallCancelled:
raise
except Exception as _pf_err:
# No preflight (older/gated repo, mirror without dry-run, etc.):
# fall back to today's fill-in-as-files-appear behaviour.
@@ -651,6 +778,27 @@ async def install_model(req: InstallModelRequest):
from huggingface_hub.constants import HF_HUB_CACHE
from services.hf_revisions import remember_revision
remember_revision(req.repo_id, dl_kwargs["revision"], HF_HUB_CACHE)
# A pipeline config is not a runnable installation by itself.
# Download its reviewed dependencies only inside this explicit
# install action, retaining the parent cancellation/retry flow.
for dependency in model_spec.get("dependencies") or ():
if req.repo_id in _cancelled:
raise _InstallCancelled()
dependency_id = dependency["repo_id"]
dependency_kwargs = {
**dl_kwargs,
"repo_id": dependency_id,
"revision": revision_for(dependency_id),
}
dependency_kwargs.pop("allow_patterns", None)
if dependency.get("allow_patterns"):
dependency_kwargs["allow_patterns"] = dependency["allow_patterns"]
dependency_path = snapshot_download(**dependency_kwargs) # nosec B615 -- immutable revision_for pin
if not snapshot_is_complete(dependency, dependency_path):
raise OSError(f"{dependency_id}: required model files are missing or incomplete")
remember_revision(dependency_id, dependency_kwargs["revision"], HF_HUB_CACHE)
if req.repo_id in _cancelled:
raise _InstallCancelled()
break
except Exception as net_err:
# #1224: a truncated body ("peer closed connection without
@@ -714,12 +862,28 @@ async def install_model(req: InstallModelRequest):
"phase": "install_done",
})
_install_cooldowns.pop(req.repo_id, None) # success clears any cooldown (MM2-06)
_install_failures.pop(req.repo_id, None)
invalidate_cache()
# A saved performance pack owns the desired engine/model policy.
# Reconcile after every successful local install so the new model
# becomes usable without a restart or a second manual selection.
try:
from services.performance_profiles import reconcile_active_profile
activated = reconcile_active_profile()
if activated:
logger.info("model install activated performance profile: %s", activated)
except Exception:
# The model is fully installed even if optional preference
# reconciliation fails; readiness refresh and manual selection
# remain available instead of misreporting the download.
logger.exception("performance profile reconciliation failed after model install")
except _InstallCancelled:
_resolving.set()
logger.info("model install cancelled: %s", req.repo_id)
# A cancel is user intent, not a failure — don't set a cooldown.
_install_cooldowns.pop(req.repo_id, None)
_install_failures.pop(req.repo_id, None)
hf_progress.emit({
"repo_id": req.repo_id,
"filename": req.repo_id,
@@ -730,7 +894,8 @@ async def install_model(req: InstallModelRequest):
_resolving.set()
logger.info("model install failed for %s: %s", req.repo_id, e)
import time as _time_fail
_install_cooldowns[req.repo_id] = _time_fail.time()
_failed_at = _time_fail.time()
_install_cooldowns[req.repo_id] = _failed_at
# #874: when the install failed because the configured HF mirror is
# unreachable, name the mirror + the setting instead of leaking the
# raw connectivity error. #959: likewise for the SOCKS-proxy class
@@ -739,15 +904,41 @@ async def install_model(req: InstallModelRequest):
# class so the wizard can react structurally (HF_MIRROR_UNREACHABLE
# raises the inline mirror picker) without string-matching.
from core.failure import append_hint, classify
_error = append_hint(str(e))
_docs_topic = classify(str(e))
# Gated catalogue entries own their recovery topic. Hugging Face
# uses several exception wordings for the same access verdict, so
# the UI must not depend on parsing an English 401/403 message.
_catalogue_topic = str(model_spec.get("failure_topic") or "")
if _catalogue_topic and _docs_topic in {
"",
"HF_AUTH_FAILED",
"PYANNOTE_LICENSE_REQUIRED",
}:
_docs_topic = _catalogue_topic
# Waiting cannot fix an access/token verdict. Let the user accept
# the terms or update the token and retry immediately.
if _docs_topic in {
"HF_AUTH_FAILED",
"PYANNOTE_LICENSE_REQUIRED",
"POCKETTTS_GATED_WEIGHTS",
}:
_install_cooldowns.pop(req.repo_id, None)
_install_failures[req.repo_id] = {
"failed_at": _failed_at,
"error": _error,
"docs_topic": _docs_topic,
}
hf_progress.emit({
"repo_id": req.repo_id,
"filename": req.repo_id,
"downloaded": 0, "total": 0, "pct": 0.0,
"phase": "install_error",
"error": append_hint(str(e)),
"docs_topic": classify(str(e)),
"error": _error,
"docs_topic": _docs_topic,
})
finally:
_resolving.set()
_cancelled.discard(req.repo_id)
download_aggregator.finish(req.repo_id, target=target or "local")
hf_progress.current_repo_id.reset(token)
@@ -762,6 +953,7 @@ async def install_model(req: InstallModelRequest):
# Admission and task publication are one atomic generation boundary:
# cancellation can never observe an admitted install without its task.
_cancelled.discard(req.repo_id)
_install_failures.pop(req.repo_id, None)
try:
task = loop.create_task(asyncio.to_thread(_do))
_install_tasks.add(task)
@@ -811,8 +1003,17 @@ async def cancel_install(req: InstallModelRequest):
in hf_hub 1.7.2, so an already-streaming file finishes; the cancel takes
effect at the next retry boundary. Clears the cooldown so the user can
immediately restart."""
target = (req.target or "local").strip() or "local"
if target != "local":
from services import gpu_gateway # noqa: PLC0415
try:
return await gpu_gateway.cancel_download(req.repo_id, target=target)
except gpu_gateway.GatewayError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
_cancelled.add(req.repo_id)
_install_cooldowns.pop(req.repo_id, None)
_install_failures.pop(req.repo_id, None)
return {"cancelling": req.repo_id}
+122 -24
View File
@@ -15,7 +15,7 @@ import sys
import time
from pathlib import Path
from fastapi import APIRouter
from fastapi import APIRouter, HTTPException, Query
logger = logging.getLogger("omnivoice.setup.models")
router = APIRouter()
@@ -117,7 +117,7 @@ def _target_repo_inventory() -> tuple[str, set[str]] | None:
for capability in live.record.capabilities or []:
if capability.get("downloaded"):
downloaded.update(str(repo) for repo in capability.get("repo_ids") or [])
return live.id, downloaded
return live.worker_id, downloaded
def _current_platform_tags() -> list[str]:
@@ -368,23 +368,44 @@ def _snapshot_dirs(repo_id: str) -> list[str]:
return dirs
def snapshot_is_complete(model: dict, snapshot_path: str) -> bool:
"""Apply the same catalogue requirements during installation and listing."""
config_only = bool(model.get("config_only"))
required = tuple(str(name) for name in (
model.get("config_required_files") if config_only else model.get("required_files")
) or ())
if config_only and not required:
return False
try:
present = all(
os.path.isfile(os.path.join(snapshot_path, name))
and os.path.getsize(os.path.join(snapshot_path, name)) >= (
1 if config_only else _WEIGHT_FLOORS.get(os.path.splitext(name)[1].lower(), 1)
)
for name in required
)
return present and (config_only or snapshot_has_weights(snapshot_path))
except OSError:
return False
def cache_is_complete(model: dict) -> bool:
"""True when this model's on-disk cache is usable (not a truncated download).
Config-only repos (``config_only: true`` in models.yaml e.g. pyannote's
diarisation pipeline, whose real weights live in referenced sub-repos) carry no
weight file of their own, so the weight check would false-positive them as
incomplete (#622 caveat). They're exempt: cache presence alone means complete.
A weight-bearing repo is complete only if at least one of its snapshots has
weights; if no snapshot dir is found on disk we can't prove truncation, so we
don't downgrade (the size-based caller already decided it's cached).
Config-only repos carry no weight file of their own. Their catalogue entry
declares the small files that make the pipeline usable, so a README left by a
gated 403 is not mistaken for a completed install. A weight-bearing repo is
complete only if at least one snapshot has weights; if no snapshot directory
is found, the size-based caller's cached result is preserved.
"""
if model.get("config_only"):
return True
for dependency in model.get("dependencies") or ():
snapshots = _snapshot_dirs(dependency["repo_id"])
if not any(snapshot_is_complete(dependency, path) for path in snapshots):
return False
dirs = _snapshot_dirs(model["repo_id"])
if not dirs:
return True
return any(snapshot_has_weights(d) for d in dirs)
return any(snapshot_is_complete(model, snapshot) for snapshot in dirs)
def _is_cached_on_disk(repo_id: str) -> bool:
@@ -449,6 +470,17 @@ def _scan_cache_on_disk() -> dict[str, dict]:
return out
def _cache_dir_missing(exc: Exception) -> bool:
"""Whether Hugging Face is reporting the normal empty-cache state.
``CacheNotFound`` is expected on a clean installation before the first
download. Treating it like a damaged Windows cache makes every model probe
perform a redundant filesystem fallback and fills the first-run log with
warnings. Unexpected scan failures remain visible and recoverable below.
"""
return type(exc).__name__ == "CacheNotFound"
def is_cached(repo_id: str) -> bool:
"""Best-effort check: does HF have this repo in its cache on disk?"""
try:
@@ -459,6 +491,8 @@ def is_cached(repo_id: str) -> bool:
return True
return False
except Exception as e:
if _cache_dir_missing(e):
return False
# scan_cache_dir can raise on Windows (WinError 448 'untrusted mount
# point'); fall back to a direct disk check so a cached model isn't
# mistaken for missing and re-downloaded in a loop (#117/#118). Logged
@@ -497,6 +531,62 @@ def invalidate_cache() -> None:
# ── Endpoints ──────────────────────────────────────────────────────────────
@router.get("/models/access/status")
def model_access_status(repo_id: str = Query(...)):
"""Check gated Hub access without downloading model files.
This route runs only after an explicit UI action. It never returns the
token or a raw Hub exception; callers need only the per-repository verdict.
"""
model = _catalog.get(repo_id)
if model is None:
raise HTTPException(status_code=404, detail="Unknown model")
if not model.get("gated"):
return {
"repo_id": repo_id,
"token_present": False,
"ready": True,
"repositories": [],
}
from services import token_resolver
resolved = token_resolver.resolve()
repositories = [repo_id]
prerequisite = str(model.get("prerequisite_repo_id") or "").strip()
if prerequisite:
repositories.append(prerequisite)
if not resolved:
return {
"repo_id": repo_id,
"token_present": False,
"ready": False,
"repositories": [
{"repo_id": current, "access": "token_missing"}
for current in repositories
],
}
from huggingface_hub import get_hf_file_metadata, hf_hub_url
results = []
for current in repositories:
try:
url = hf_hub_url(current, filename=".gitattributes")
get_hf_file_metadata(url, token=resolved.token)
access = "granted"
except Exception as exc: # Hub exception types vary across releases.
status = getattr(getattr(exc, "response", None), "status_code", None)
access = "required" if status in {401, 403, 404} else "unavailable"
results.append({"repo_id": current, "access": access})
return {
"repo_id": repo_id,
"token_present": True,
"ready": all(item["access"] == "granted" for item in results),
"repositories": results,
}
@router.get("/models")
def list_models():
"""Catalogue every known model + its on-disk install state.
@@ -532,10 +622,13 @@ def list_models():
"nb_files": entry.nb_files,
}
except Exception as e:
# WinError-448 fallback (#117/#118): use a direct disk scan so installed
# models still show as installed instead of offering a re-download.
logger.warning("scan_cache_dir failed (%s); using disk fallback", e)
cached_by_repo = _scan_cache_on_disk()
if _cache_dir_missing(e):
cached_by_repo = {}
else:
# WinError-448 fallback (#117/#118): use a direct disk scan so installed
# models still show as installed instead of offering a re-download.
logger.warning("scan_cache_dir failed (%s); using disk fallback", e)
cached_by_repo = _scan_cache_on_disk()
out = []
host_tags = set(platform_tags)
@@ -562,6 +655,7 @@ def list_models():
"curated": _model_curated(m, host_tags),
})
response = {
"target": target_key,
"models": out,
"total_installed_bytes": sum(m["size_on_disk_bytes"] for m in out),
"hf_cache_dir": "" if remote_inventory is not None else hf_cache_dir(),
@@ -623,21 +717,21 @@ def recommendations():
rationale = (
"NVIDIA preset: VoiceStudio (required) runs standalone. Optional ASR picks "
"are CUDA-accelerated via CTranslate2 — Whisper large-v3 for dubbing "
"(best word timestamps), Turbo for 5× faster transcription, Parakeet TDT "
"v3 for live dictation. KittenTTS adds CPU-realtime English."
"(best word timestamps) and Turbo for 5× faster transcription. Whisper "
"Tiny provides broad-language local dictation. KittenTTS adds CPU-realtime English."
)
elif has_rocm:
rationale = (
"AMD/ROCm preset: VoiceStudio (required) runs standalone. CTranslate2 has "
"no ROCm backend, so the PyTorch Whisper large-v3 build is the "
"GPU-accelerated ASR route; faster-whisper works on CPU, and Parakeet "
"TDT v3 handles live dictation."
"GPU-accelerated ASR route; faster-whisper works on CPU, and Whisper Tiny "
"provides broad-language local dictation."
)
else:
rationale = (
"CPU preset: VoiceStudio (required) runs standalone. Optional picks favour "
"speed on CPU — Whisper large-v3 (int8) for accuracy, Turbo when speed "
"matters, Parakeet TDT v3 (int8 ONNX) for live dictation, KittenTTS for "
"matters, Whisper Tiny (ONNX) for live dictation, KittenTTS for "
"instant English TTS."
)
@@ -653,9 +747,12 @@ def recommendations():
entry.repo_id for entry in info.repos if entry.size_on_disk > 0
}
except Exception as e:
# WinError-448 fallback (#117/#118): recommend based on the disk scan.
logger.debug("scan_cache_dir failed (%s); using disk fallback", e)
cached_ids = set(_scan_cache_on_disk().keys())
if _cache_dir_missing(e):
cached_ids = set()
else:
# WinError-448 fallback (#117/#118): recommend based on the disk scan.
logger.debug("scan_cache_dir failed (%s); using disk fallback", e)
cached_ids = set(_scan_cache_on_disk().keys())
entries = []
for meta in curated:
@@ -679,6 +776,7 @@ def recommendations():
all_installed = all(e["installed"] for e in entries)
return {
"target": remote_inventory[0] if remote_inventory is not None else "local",
"device": {
"os": target_os,
"arch": target_arch,
+170 -12
View File
@@ -15,6 +15,8 @@ from api.dependencies import is_loopback, require_admin, require_admin_action
from fastapi.responses import FileResponse, StreamingResponse
import torch
import shutil
import subprocess
import shlex
from core.config import OUTPUTS_DIR, DATA_DIR, CRASH_LOG_PATH, LOG_PATH, IDLE_TIMEOUT_SECONDS
from core.version import APP_VERSION
@@ -38,6 +40,10 @@ logger = logging.getLogger("omnivoice.api")
# Cache device checks at module load — they don't change at runtime
_is_mac = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
_is_cuda = torch.cuda.is_available()
try:
_is_xpu = hasattr(torch, "xpu") and torch.xpu.is_available()
except Exception:
_is_xpu = False
# Prime psutil's internal CPU counter so the first non-blocking call returns useful data
psutil.cpu_percent(interval=None)
@@ -51,6 +57,14 @@ def _detect_cpu_model() -> str:
for line in f:
if line.lower().startswith("model name"):
return line.split(":", 1)[1].strip()
if sys.platform == "win32":
import winreg
with winreg.OpenKey(
winreg.HKEY_LOCAL_MACHINE,
r"HARDWARE\DESCRIPTION\System\CentralProcessor\0",
) as key:
return str(winreg.QueryValueEx(key, "ProcessorNameString")[0]).strip()
if sys.platform == "darwin":
import subprocess
return subprocess.check_output(
@@ -61,6 +75,77 @@ def _detect_cpu_model() -> str:
return platform.processor() or ""
def _gpu_name_priority(name: str) -> tuple[int, int]:
lowered = name.lower()
if any(token in lowered for token in ("remote", "virtual", "basic display")):
return (-1, len(name))
if any(token in lowered for token in ("nvidia", "radeon", "amd", "intel arc")):
return (2, len(name))
return (1, len(name))
def _detect_os_gpu_name() -> str:
"""Best-effort display-adapter identity when the active torch build is CPU-only."""
try:
if sys.platform == "win32":
executable = shutil.which("powershell.exe") or shutil.which("powershell")
if not executable:
return ""
result = subprocess.run(
[
executable,
"-NoProfile",
"-NonInteractive",
"-Command",
"Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name",
],
capture_output=True,
text=True,
timeout=3,
check=False,
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
)
names = [line.strip() for line in result.stdout.splitlines() if line.strip()]
return max(names, key=_gpu_name_priority, default="")
if sys.platform.startswith("linux"):
executable = shutil.which("lspci")
if not executable:
return ""
result = subprocess.run(
[executable, "-mm"],
capture_output=True,
text=True,
timeout=2,
check=False,
)
names = []
for line in result.stdout.splitlines():
parts = shlex.split(line)
if len(parts) >= 4 and parts[1] in {"VGA compatible controller", "3D controller"}:
names.append(" ".join(parts[2:4]))
return max(names, key=_gpu_name_priority, default="")
if sys.platform == "darwin":
executable = shutil.which("system_profiler")
if not executable:
return ""
result = subprocess.run(
[executable, "SPDisplaysDataType"],
capture_output=True,
text=True,
timeout=3,
check=False,
)
names = [
line.split(":", 1)[1].strip()
for line in result.stdout.splitlines()
if "Chipset Model:" in line
]
return max(names, key=_gpu_name_priority, default="")
except (OSError, ValueError, subprocess.SubprocessError):
return ""
return ""
def _detect_gpu() -> tuple[str, float]:
"""(gpu_name, vram_total_gb) — static for the process lifetime.
@@ -71,11 +156,15 @@ def _detect_gpu() -> tuple[str, float]:
if _is_cuda:
props = torch.cuda.get_device_properties(0)
return torch.cuda.get_device_name(0), round(props.total_memory / (1024 ** 3), 1)
if _is_xpu:
props = torch.xpu.get_device_properties(0)
total_memory = float(getattr(props, "total_memory", 0.0))
return torch.xpu.get_device_name(0), round(total_memory / (1024 ** 3), 1)
if _is_mac:
return "Apple Silicon (MPS)", 0.0
except Exception:
pass
return "", 0.0
return _detect_os_gpu_name(), 0.0
# Static hardware facts, captured once — /system/info is hit on every
@@ -93,6 +182,37 @@ def _disk_free_gb() -> float:
return 0.0
def _nvidia_live_stats() -> tuple[float, float, float] | None:
"""Return GPU%, used VRAM GiB, total VRAM GiB without an optional Python dependency."""
executable = shutil.which("nvidia-smi")
if not executable:
return None
try:
creationflags = subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0
result = subprocess.run(
[
executable,
"--query-gpu=utilization.gpu,memory.used,memory.total",
"--format=csv,noheader,nounits",
"--id=0",
],
capture_output=True,
text=True,
timeout=1.5,
check=False,
creationflags=creationflags,
)
if result.returncode != 0:
return None
values = [float(value.strip()) for value in result.stdout.splitlines()[0].split(",")]
if len(values) != 3:
return None
utilization, used_mib, total_mib = values
return utilization, used_mib / 1024, total_mib / 1024
except (OSError, ValueError, IndexError, subprocess.SubprocessError):
return None
def _ui_port() -> int:
"""The Vite UI dev-server port, single-sourced from OMNIVOICE_UI_PORT.
@@ -185,8 +305,8 @@ def loaded_models():
@router.post("/model/unload/{model_id}")
async def unload_model(model_id: str):
"""Unload a specific model by id (MM2-04). Delegates to model_lifecycle;
an unknown id maps to HTTP 400. ``tts`` | ``diarization`` |
``sidecar:<id>`` | ``sidecars``."""
an unknown id maps to HTTP 400. Supports every id returned by
``GET /model/loaded`` plus the aggregate ``sidecars`` id."""
from services import model_lifecycle
try:
return await model_lifecycle.unload(model_id)
@@ -204,7 +324,18 @@ def system_info():
try:
_ffmpeg = find_ffmpeg()
from services import model_manager as _mm
from services import asr_backend as _asr_backend
from core import prefs as _prefs_mod
_asr_engine = _asr_backend.active_backend_id()
_asr_model = (
_asr_backend._offline_asr_repo(_asr_engine)
or os.environ.get("ASR_MODEL")
or _asr_engine
)
_translation_provider = (
os.environ.get("TRANSLATE_PROVIDER")
or _prefs_mod.get("translation_backend", "argos")
)
return {
"app_version": APP_VERSION,
"generate_timeout_s": _mm.GPU_JOB_TIMEOUT_S,
@@ -224,8 +355,8 @@ def system_info():
"crash_log_path": CRASH_LOG_PATH,
"idle_timeout_seconds": IDLE_TIMEOUT_SECONDS,
"model_checkpoint": resolve_omnivoice_checkpoint(), # #693: show the effective checkpoint, not a leaked raw value
"asr_model": os.environ.get("ASR_MODEL", "Systran/faster-whisper-large-v3"),
"translate_provider": os.environ.get("TRANSLATE_PROVIDER", "google"),
"asr_model": _asr_model,
"translate_provider": _translation_provider,
"has_hf_token": _has_hf_token(),
"fast_download": _fast_download_status(),
"device": get_best_device(),
@@ -669,6 +800,7 @@ async def clear_tauri_logs():
@router.get("/sysinfo", response_model=SysinfoResponse)
def get_sys_info():
vram = 0.0
total_vram = 0.0
gpu_active = False
try:
@@ -681,18 +813,38 @@ def get_sys_info():
vram = alloc() / (1024**3)
elif _is_cuda:
vram = torch.cuda.memory_allocated() / (1024**3)
total_vram = torch.cuda.get_device_properties(torch.cuda.current_device()).total_memory / (1024**3)
elif _is_xpu:
vram = torch.xpu.memory_allocated() / (1024**3)
total_vram = float(
getattr(torch.xpu.get_device_properties(0), "total_memory", 0.0)
) / (1024**3)
except Exception:
pass
if vram > 0.01:
gpu_active = True
gpu_utilization = None
nvidia_stats = _nvidia_live_stats() if _is_cuda else None
if nvidia_stats:
gpu_utilization, vram, total_vram = nvidia_stats
gpu_active = gpu_active or gpu_utilization > 0 or vram > 0.01
vm = psutil.virtual_memory()
cpu_frequency = psutil.cpu_freq()
return {
"cpu": psutil.cpu_percent(interval=None),
"cpu_model": _CPU_MODEL,
"cpu_physical_cores": psutil.cpu_count(logical=False) or 0,
"cpu_logical_cores": psutil.cpu_count(logical=True) or 0,
"cpu_frequency_ghz": round((cpu_frequency.current if cpu_frequency else 0.0) / 1000, 2),
"ram": vm.used / (1024**3),
"total_ram": vm.total / (1024**3),
"gpu_name": _GPU_NAME,
"gpu_utilization": gpu_utilization,
"vram": round(vram, 2),
"total_vram": round(total_vram, 2),
"gpu_active": gpu_active
}
@@ -708,12 +860,18 @@ async def flush_memory(unload_model: bool = False):
freed_model = False
if unload_model:
import services.model_manager as mm
async with mm._model_lock:
# Also drops the clone-prompt side cache, which this path used to
# leave resident — an "unload" that kept the encoded reference
# tensors belonging to the model it just released (#1495).
freed_model = mm.unload_shared_model()
from services import model_lifecycle
# The user-facing action has always promised "Unload all". Route it
# through the lifecycle facade so alternate TTS engines, dictation,
# diarisation, translation and sidecars are released as well as the
# shared OmniVoice model. Individual runtimes still decline while
# leased by active work.
released = await model_lifecycle.unload_all()
freed_model = any(
bool(result.get("success"))
for result in released.get("results", {}).values()
)
# Multi-pass GC to break reference cycles
gc.collect(generation=2)
@@ -883,7 +1041,7 @@ def system_notifications():
from core import run_sentinel
rec = run_sentinel.newest_record()
if rec is not None and not rec[1]:
if rec is not None and not rec[1] and run_sentinel.warrants_user_notice(rec[0]):
record = rec[0]
last = record.get("last_activity") or {}
doing = f" Last activity: {last.get('kind')}." if last.get("kind") else ""
+5 -3
View File
@@ -217,7 +217,7 @@ async def convert_speech(
# clone-less engine with the actionable switch-engine message (→ 400),
# and a backend mid-shutdown raises ModelLoadInterruptedByShutdown out
# of the model load → the global 503 [shutting_down] handler.
from services.tts_backend import resolve_generation_backend
from services.tts_backend import active_backend_id, resolve_generation_backend
try:
backend = await resolve_generation_backend(
require_cloning=True, cloning_purpose="voice conversion",
@@ -319,14 +319,16 @@ async def convert_speech(
)
start_time = time.time()
from services.performance_profiles import tts_defaults
_profile_defaults = tts_defaults(active_backend_id())
_render = functools.partial(
_run_backend_inference,
backend, text, language, cond["ref_audio_path"], cond["ref_text"],
cond["instruct"],
None, # duration — the model picks; match_duration owns pacing
16, 2.0, # num_step / guidance_scale (the /generate defaults)
_profile_defaults.get("num_step", 16), 2.0,
1.0, # speed
True, True, # denoise / postprocess_output
True, _profile_defaults.get("postprocess_output", True),
used_seed,
)
try:
+12
View File
@@ -116,6 +116,18 @@ def get_target(op: str = "") -> dict:
return routing.status(op=op.strip() or None)
@router.get("/runtime")
async def get_runtime(engine: str = "", op: str = "tts") -> dict:
"""Runtime/model facts for the machine that will execute this operation."""
from services import gpu_gateway # noqa: PLC0415
return await gpu_gateway.status(
engine=engine.strip() or None,
op=op.strip() or "tts",
control_plane=service.control_plane,
)
@router.post("/target")
def set_target(request: TargetRequest) -> dict:
"""Choose where work runs. Exactly one target is active at a time."""
+8 -1
View File
@@ -15,9 +15,16 @@ class SysinfoResponse(BaseModel):
model_config = ConfigDict(extra="allow")
cpu: float = Field(description="CPU usage percentage (0100)")
cpu_model: str = ""
cpu_physical_cores: int = 0
cpu_logical_cores: int = 0
cpu_frequency_ghz: float = 0.0
ram: float = Field(description="Used RAM in GiB")
total_ram: float = Field(description="Total RAM in GiB")
gpu_name: str = ""
gpu_utilization: float | None = None
vram: float = Field(0.0, description="Used VRAM in GiB")
total_vram: float = Field(0.0, description="Total VRAM in GiB when reported by the runtime")
gpu_active: bool = Field(False, description="Whether a GPU is actively used")
@@ -82,7 +89,7 @@ class ModelStatusResponse(BaseModel):
status: str = Field(description="idle | loading | ready")
checkpoint: str | None = None
loaded_at: str | None = None
sub_stage: str | None = Field(None, description="Current loading sub-stage: importing | loading_weights | loading_asr | compiling | ready | error")
sub_stage: str | None = Field(None, description="Current TTS loading sub-stage: importing | loading_weights | compiling | ready | error")
detail: str | None = Field(None, description="Human-readable detail of current loading phase")
error: str | None = Field(None, description="Error message if loading failed")
+32 -5
View File
@@ -10,7 +10,7 @@
# repo_id (required) — HuggingFace repository ID
# engines (required) — backend ids that load this repo; [] for pipeline weights no single engine owns (they list under "Other weights")
# label (required) — Human-readable display name
# role (required) — TTS | ASR | Diarisation
# role (required) — TTS | ASR | Translation | Diarisation
# size_gb (required) — Approximate download size in GiB
# required (optional) — true if the app needs this model to function.
# Only the TTS model is required: the app boots and
@@ -46,13 +46,18 @@ models:
curated_on: [all]
- repo_id: "audio-cpp/audio.cpp-gguf"
label: "Breeze-TTS-2 Q8_0 for audio.cpp (English + Chinese, clone + design)"
label: "audio.cpp native bundle (Breeze-TTS-2 + Sortformer diarisation)"
role: TTS
engines: [audiocpp]
size_gb: 4.73
families: [tts, diarisation]
size_gb: 4.98
required_files:
- "Breeze-TTS-2-GGUF/breeze-tts-2-q8_0.gguf"
- "Sortformer-Diar-4spk-v1-GGUF/sortformer-diar-4spk-v1-q8_0.gguf"
allow_patterns:
- "Breeze-TTS-2-GGUF/breeze-tts-2-q8_0.gguf"
note: "Optional audio.cpp model. Research/non-commercial weights and self-hosted outputs; install only after reviewing the license."
- "Sortformer-Diar-4spk-v1-GGUF/sortformer-diar-4spk-v1-q8_0.gguf"
note: "Optional audio.cpp bundle for native voice cloning and up-to-four-speaker diarisation. Research/non-commercial weights and self-hosted outputs; install only after reviewing the licenses."
# ── ASR (optional — curated per platform) ─────────────────────────────
# No ASR model is required to boot: TTS-only installs work. Dubbing,
@@ -256,6 +261,14 @@ models:
curated_on: [all]
note: "Recommended cross-platform dictation default (auto-detect). CPU, int8 ONNX. Requires sherpa-onnx."
# ── Translation ──────────────────────────────────────────────────────
- repo_id: "facebook/nllb-200-distilled-600M"
label: "NLLB-200 distilled 600M (local, 200 languages)"
role: Translation
size_gb: 2.4
note: "Best fully-local translation quality. Install explicitly before selecting NLLB; translation never downloads these weights in the background."
# ── Diarisation ───────────────────────────────────────────────────────
- repo_id: "pyannote/speaker-diarization-3.1"
@@ -264,7 +277,21 @@ models:
engines: []
size_gb: 0.8
config_only: true # pipeline repo; real weights live in referenced sub-repos
note: "Needs an HF_TOKEN with license accepted."
config_required_files: ["config.yaml"]
dependencies:
- repo_id: "pyannote/segmentation-3.0"
required_files: ["pytorch_model.bin"]
allow_patterns: ["config.yaml", "pytorch_model.bin"]
- repo_id: "pyannote/wespeaker-voxceleb-resnet34-LM"
required_files: ["pytorch_model.bin"]
allow_patterns: ["config.yaml", "pytorch_model.bin"]
gated: true
requires_hf_token: true
access_url: "https://huggingface.co/pyannote/speaker-diarization-3.1"
prerequisite_repo_id: "pyannote/segmentation-3.0"
prerequisite_access_url: "https://huggingface.co/pyannote/segmentation-3.0"
failure_topic: "PYANNOTE_LICENSE_REQUIRED"
note: "Requires access to both pyannote repositories and an HF token."
# ── Optional TTS ──────────────────────────────────────────────────────
+4 -2
View File
@@ -73,7 +73,7 @@ def _origin_tuple(value: str | None) -> tuple[str, str, int | None] | None:
):
return None
scheme = parsed.scheme.lower()
if scheme not in {"http", "https", "tauri"}:
if scheme not in {"http", "https", "tauri", "app"}:
return None
if port is None:
if scheme == "http":
@@ -83,6 +83,8 @@ def _origin_tuple(value: str | None) -> tuple[str, str, int | None] | None:
return scheme, parsed.hostname.lower(), port
DEFAULT_DESKTOP_ORIGINS = ("tauri://localhost", "http://tauri.localhost", "app://voicestudio")
def configured_allowed_origins() -> frozenset[tuple[str, str, int | None]]:
raw_port = os.environ.get("OMNIVOICE_UI_PORT", "3901")
try:
@@ -92,7 +94,7 @@ def configured_allowed_origins() -> frozenset[tuple[str, str, int | None]]:
values = os.environ.get(
"OMNIVOICE_ALLOWED_ORIGINS",
f"http://localhost:{ui_port},http://127.0.0.1:{ui_port},"
"tauri://localhost,http://tauri.localhost",
+ ",".join(DEFAULT_DESKTOP_ORIGINS),
).split(",")
return frozenset(
origin
+17
View File
@@ -51,6 +51,23 @@ _DIALECTS = set(_VD._INSTRUCT_CATEGORIES[5]) # the 12 Chinese dialect tokens
# the archetype ``attrs`` shape, so the response drops straight into vdStates.
CATEGORY_ORDER = ("Gender", "Age", "Pitch", "Style", "EnglishAccent", "ChineseDialect")
def instruct_to_vd_states(instruct: str | None) -> dict[str, str]:
"""Project a saved validator-token instruct onto the complete UI recipe."""
attrs = {category: "Auto" for category in CATEGORY_ORDER}
sanitized = _VD.sanitize_instruct(instruct)
if not sanitized:
return attrs
for token in sanitized.split(", "):
category_index = _VD._instruct_category_index(token)
if category_index < 0 or category_index >= len(CATEGORY_ORDER):
continue
# The first four frontend categories use the English canonical token;
# dialects and accents already use their engine-native form.
canonical = _VD._INSTRUCT_ZH_TO_EN.get(token, token)
attrs[CATEGORY_ORDER[category_index]] = canonical
return attrs
# ── Pinyin / romanized names → Chinese-dialect tokens (functional vocabulary) ─
DIALECT_PINYIN = {
"henan": "河南话",
+3
View File
@@ -0,0 +1,3 @@
"""Stable engine IDs whose first use requires local license acceptance."""
LICENSE_GATED_ENGINES: frozenset[str] = frozenset({"supertonic3", "pockettts"})
+3 -1
View File
@@ -4,7 +4,7 @@ Used by the React ErrorBoundary's "Open docs for this error" button (via the
TypeScript mirror at `frontend/src/utils/errorDocsMap.ts`) and by the Phase 5
bug-reporter for "this error has a docs page" links.
The 5-class taxonomy below is the contract Phase 5 reporter consumes it,
The error taxonomy below is the contract Phase 5 reporter consumes it,
the TS map mirrors it, and `test_error_docs_map.test_keys_match_taxonomy`
locks the key set. To add a new class:
@@ -20,6 +20,8 @@ from core import links
_BASE = links.PROJECT_REPO_BLOB_MAIN
ERROR_DOCS: dict[str, str] = {
"DIARIZATION_LOAD_FAILED": f"{_BASE}/docs/features/diarization.md#troubleshooting",
"DIARIZATION_MODEL_MISSING": f"{_BASE}/docs/features/diarization.md#local-installation-and-repair",
"GATEKEEPER_QUARANTINE": f"{_BASE}/docs/install/macos.md#gatekeeper-quarantine",
"APPIMAGE_WEBKIT_WHITESCREEN": f"{_BASE}/docs/install/linux.md#appimage-white-screen-on-fedora-44--ubuntu-2404",
"PKG_RESOURCES_MISSING": f"{_BASE}/docs/install/troubleshooting.md#pkg_resources-missing",
+18 -1
View File
@@ -85,6 +85,8 @@ _HINTS: dict[str, str] = {
"GATEKEEPER_QUARANTINE": "Clear the macOS quarantine flag (xattr -cr the app), then reopen.",
"APPIMAGE_WEBKIT_WHITESCREEN": "Launch with WEBKIT_DISABLE_DMABUF_RENDERER=1 set.",
"HF_AUTH_FAILED": "Set a valid HF_TOKEN in Settings → Hugging Face and retry.",
"DIARIZATION_MODEL_MISSING": "Install or repair the selected diarisation model in Settings > Models > Diarisation, then retry transcription.",
"DIARIZATION_LOAD_FAILED": "Open Settings > Logs > Backend for the model load error, then retry transcription after correcting it.",
"PYANNOTE_LICENSE_REQUIRED": "Accept the pyannote model licenses on Hugging Face, then retry.",
"POCKETTTS_GATED_WEIGHTS": "PocketTTS weights are gated on HuggingFace. Accept the access agreement at huggingface.co/kyutai/pocket-tts, then set HF_TOKEN in Settings → Hugging Face and retry.",
"COMPUTE_TYPE_UNSUPPORTED": "Your GPU doesn't support float16 — VoiceStudio retried on int8. If transcription still fails, set OMNIVOICE/ASR_COMPUTE_TYPE=int8 or use CPU.",
@@ -404,7 +406,22 @@ def classify(reason: str) -> str:
or "access conditions" in low
) and ("pocket" in low or "kyutai" in low):
return "POCKETTTS_GATED_WEIGHTS"
if "pyannote" in low or ("gated" in low and "model" in low) or "accept the" in low:
diarisation = any(marker in low for marker in (
"pyannote", "diarization", "diarisation", "sortformer",
))
access_failure = any(marker in low for marker in (
"gated", "unauthorized", "forbidden", "401", "403",
"accept the", "license", "user conditions",
))
if diarisation and not access_failure:
if any(marker in low for marker in (
"files are missing", "files are missing or incomplete",
"filenotfounderror", "localentrynotfounderror", "model is missing",
)):
return "DIARIZATION_MODEL_MISSING"
if any(marker in low for marker in ("failed to load", "load failed", "runtime failed")):
return "DIARIZATION_LOAD_FAILED"
if (diarisation and access_failure) or ("gated" in low and "model" in low) or "accept the" in low:
return "PYANNOTE_LICENSE_REQUIRED"
# ASR robustness (#551 / #549): name the class so the no-segments toast is
# actionable. Place before the generic returns so a compute-type/transformers
+56
View File
@@ -57,6 +57,48 @@ class HFTokenRedactor(logging.Filter):
return True
class RoutineHealthAccessFilter(logging.Filter):
"""Drop only successful routine liveness access lines.
The desktop supervisor probes every two seconds. Startup/not-ready responses
and every other request remain visible, while the steady-state 200 line no
longer consumes the small rotating diagnostic log.
"""
def filter(self, record: logging.LogRecord) -> bool:
try:
args = record.args
if not isinstance(args, tuple) or len(args) < 5:
return True
_client, method, path, _http_version, status = args[:5]
return not (
method == "GET"
and str(path).partition("?")[0] == "/health"
and int(status) == 200
)
except (TypeError, ValueError):
return True
class RoutineAsyncioTransportFilter(logging.Filter):
"""Drop only expected socket-close noise from asyncio's transport layer."""
def filter(self, record: logging.LogRecord) -> bool:
try:
message = record.getMessage()
if record.levelno == logging.WARNING and "socket.send() raised exception" in message:
return False
exception = record.exc_info[1] if record.exc_info else None
return not (
message.startswith(
"Exception in callback _ProactorBasePipeTransport._call_connection_lost"
)
and isinstance(exception, (BrokenPipeError, ConnectionResetError))
)
except Exception:
return True
def install_redaction_filter(root_logger: logging.Logger | None = None) -> None:
"""Attach a single HFTokenRedactor to the root logger and to every
existing handler. Idempotent repeated calls do not stack up duplicate
@@ -69,3 +111,17 @@ def install_redaction_filter(root_logger: logging.Logger | None = None) -> None:
for handler in list(target.handlers):
if not any(isinstance(f, HFTokenRedactor) for f in handler.filters):
handler.addFilter(HFTokenRedactor())
def install_access_log_filter(logger: logging.Logger | None = None) -> None:
"""Install the routine-health filter on Uvicorn's access logger once."""
target = logger or logging.getLogger("uvicorn.access")
if not any(isinstance(item, RoutineHealthAccessFilter) for item in target.filters):
target.addFilter(RoutineHealthAccessFilter())
def install_asyncio_transport_filter(logger: logging.Logger | None = None) -> None:
"""Install the expected transport-close filter on asyncio once."""
target = logger or logging.getLogger("asyncio")
if not any(isinstance(item, RoutineAsyncioTransportFilter) for item in target.filters):
target.addFilter(RoutineAsyncioTransportFilter())
+19 -2
View File
@@ -13,6 +13,23 @@ from typing import Any, BinaryIO, Callable, Optional
WINDOWS_PIPE_POLL_INTERVAL_S = 0.25
_FILE_TYPE_PIPE = 3 # winbase.h FILE_TYPE_PIPE
def _exit_after_parent_loss(code: int) -> None:
"""Retire backend-only crash forensics before the desktop-owned exit.
Losing the containment pipe means the desktop process ended, including an
Electron development reload. That is not a backend crash: the shell owns
this child and the watchdog is deliberately terminating it. ``os._exit``
skips FastAPI lifespan cleanup, so clear the run sentinel here first. A
real backend abort/OOM never reaches this callback and remains detectable
on the next start.
"""
try:
from core import run_sentinel
run_sentinel.clear_sentinel()
except Exception:
pass
os._exit(code)
def _watch_parent_pipe(reader: BinaryIO, exit_process: Callable[[int], None]) -> None:
"""Block until the desktop-owned stdin pipe closes, then exit immediately."""
@@ -91,12 +108,12 @@ def arm_desktop_parent_watchdog() -> bool:
if reader is None:
return False
target: Callable[..., None] = _watch_parent_pipe
args: tuple = (reader, os._exit)
args: tuple = (reader, _exit_after_parent_loss)
if os.name == "nt":
handle = _windows_pipe_handle(reader)
if handle is not None:
target = _watch_parent_pipe_handle
args = (handle, os._exit)
args = (handle, _exit_after_parent_loss)
# A non-pipe stdin (file, NUL) cannot have a read pending against a
# pipe file object, so the blocking reader stays correct there.
threading.Thread(
+11
View File
@@ -75,6 +75,17 @@ def set_(key: str, value: Any) -> None:
_save(data)
def update_mapping(key: str, changes: dict, *, replace: bool = False) -> None:
"""Atomically update one preference object without losing concurrent edits."""
with _MUTATE_LOCK:
data = _load()
current = data.get(key)
value = dict(current) if isinstance(current, dict) and not replace else {}
value.update(changes)
data[key] = value
_save(data)
def delete(key: str) -> None:
"""Remove *key* from prefs.json if present."""
with _MUTATE_LOCK:
+27
View File
@@ -0,0 +1,27 @@
"""Small, metadata-free local profile portraits."""
import io
import warnings
from fastapi import HTTPException
from PIL import Image, ImageOps, UnidentifiedImageError
MAX_IMAGE_BYTES = 5 * 1024 * 1024
def normalize_portrait(data: bytes) -> bytes:
if len(data) > MAX_IMAGE_BYTES:
raise HTTPException(413, "Profile image exceeds 5 MB")
try:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(io.BytesIO(data)) as source:
if source.format not in {"JPEG", "PNG", "WEBP"}:
raise ValueError("unsupported image format")
if source.width * source.height > 16_000_000:
raise ValueError("image dimensions too large")
portrait = ImageOps.fit(ImageOps.exif_transpose(source).convert("RGB"), (256, 256))
output = io.BytesIO()
portrait.save(output, format="JPEG", quality=88)
return output.getvalue()
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombWarning, Image.DecompressionBombError) as exc:
raise HTTPException(422, "Use a valid JPEG, PNG or WebP image up to 16 megapixels") from exc
+32
View File
@@ -70,6 +70,20 @@ LOG_TAIL_LINES = 40
#: burst instead of one per request.
ACTIVITY_THROTTLE_S = 2.0
# An idle desktop process can disappear with its owning shell during an OS
# shutdown, package replacement, or a forced development relaunch. Keep that
# forensic record, but do not nag the user unless there is evidence that work
# was interrupted or the backend itself logged a fatal failure.
_ACTIONABLE_LOG_MARKERS = (
"traceback (most recent call last)",
"critical",
"fatal error",
"out of memory",
"memoryerror",
"segmentation fault",
"access violation",
)
# In-memory run state. `owns` guards clear_sentinel()/touch_activity() so an
# instance that skipped writing (another live instance holds the sentinel)
# can never clobber or delete the other instance's sentinel.
@@ -298,6 +312,24 @@ def _build_crash_record(sentinel: dict, now: float) -> dict:
}
def warrants_user_notice(record: dict) -> bool:
"""Whether an unclean record is actionable enough to interrupt the user.
The record remains available to diagnostics either way. A meaningful
activity marker means a generation/transcription/task may have been lost;
a strict fatal-log marker catches startup/native crashes that happened
before an activity could be recorded. Idle shell-owned exits stay quiet.
"""
activity = record.get("last_activity")
if isinstance(activity, dict) and str(activity.get("kind") or "").strip():
return True
tail = record.get("log_tail")
if not isinstance(tail, list):
return False
joined = "\n".join(str(line).lower() for line in tail[-LOG_TAIL_LINES:])
return any(marker in joined for marker in _ACTIONABLE_LOG_MARKERS)
def _load_store() -> dict:
store = _read_json(CRASH_RECORD_PATH) or {}
records = store.get("records")
+10 -3
View File
@@ -124,9 +124,16 @@ def _get_model():
return _model
def _transcribe(audio_path, word_timestamps):
def _transcribe(audio_path, word_timestamps, decode_options=None):
options = decode_options or {}
if not isinstance(options, dict) or any(
key not in {"beam_size", "best_of"}
or type(value) is not int or not 1 <= value <= 8
for key, value in options.items()
):
raise ValueError("Invalid ASR decoding options")
model = _get_model()
segments, info = model.transcribe(audio_path, word_timestamps=word_timestamps)
segments, info = model.transcribe(audio_path, word_timestamps=word_timestamps, **options)
out = []
for s in segments:
seg = {"start": float(s.start), "end": float(s.end), "text": s.text}
@@ -178,7 +185,7 @@ def main() -> int:
if op == "ping":
_send(stdout, {"op": "pong"})
elif op == "transcribe":
result = _transcribe(msg.get("audio_path"), bool(msg.get("word_timestamps", True)))
result = _transcribe(msg.get("audio_path"), bool(msg.get("word_timestamps", True)), msg.get("decode_options"))
_send(stdout, {"op": "segments", "result": result})
elif op == "shutdown":
return 0
+41 -14
View File
@@ -35,9 +35,9 @@ from pathlib import Path
logger = logging.getLogger("omnivoice.audiocpp.bootstrap")
#: Pinned audio.cpp release. BreezeTTS-2 support landed in 0.7.2 — older
#: binaries have no ``breeze_tts`` family, so the floor is also the pin.
VERSION = "v0.7.2"
#: Pinned audio.cpp release. BreezeTTS-2 support landed in 0.7.2; 0.7.4 adds
#: the current native fixes and Sortformer v2.1 streaming runtime.
VERSION = "v0.7.4"
#: GitHub repo serving the prebuilt binaries.
GH_REPO = "0xShug0/audio.cpp"
@@ -45,7 +45,7 @@ GH_REPO = "0xShug0/audio.cpp"
#: HuggingFace repo serving the GGUF model packages (not gated).
HF_MODEL_REPO = "audio-cpp/audio.cpp-gguf"
# Immutable repository revision used for the v0.7.2 Breeze-TTS-2 package.
# Immutable repository revision used for the Breeze-TTS-2 package.
# Pinning prevents a later upstream file replacement from silently changing
# the model exercised by this backend.
HF_MODEL_REVISION = "dc6fecccc2b0c6bdda0a8b2f38fa61394fee0b9c"
@@ -87,28 +87,34 @@ DEFAULT_PORT = 17860
#: This package's owned binary dir (probe 3).
_PKG_BIN_DIR: Path = Path(__file__).parent / "bin"
# Recommended (asset filename, sha256) per platform slug, from the v0.7.2
# Recommended (asset filename, sha256) per platform slug, from the v0.7.4
# release. Windows and Linux use the vendor-neutral Vulkan build, which also
# exposes the native CPU backend. Upstream publishes the macOS builds under
# the Metal package name. No linux-aarch64 prebuilt exists in v0.7.2.
# the Metal package name. No linux-aarch64 prebuilt exists in v0.7.4.
_ASSETS: dict[str, tuple[str, str]] = {
"windows-x64": (
"audio-v0.7.2-bin-windows-x64-vulkan.zip",
"15b8232eae740e21e507d87f827a89966de9451b085a45932d9e214e032962c1",
"audio-v0.7.4-bin-windows-x64-vulkan.zip",
"057332f9e3fb37706a8ecb5075ac1797efcd85fdccd739f7b65761a5920f2828",
),
"linux-x64": (
"audio-v0.7.2-bin-ubuntu-x64-vulkan.tar.gz",
"fee1f978cee76453cf17f00196554bc2ee294645739538af0726a143b6a69a23",
"audio-v0.7.4-bin-ubuntu-x64-vulkan.tar.gz",
"e0ef3123a9f94e130ad463db0db5a69b65485ef8db1b46edead00c03a86fa787",
),
"darwin-arm64": (
"audio-v0.7.2-bin-macos-arm64-metal.tar.gz",
"c01e4f82971bedbe341697e63a9cebd5a5d1f72d5a9bcb51a3191f95ddab7a95",
"audio-v0.7.4-bin-macos-arm64-metal.tar.gz",
"639926715b1cb537f82aa31656aabbae5d9a85ac36568c402026968f3072e2b3",
),
"darwin-x64": (
"audio-v0.7.2-bin-macos-x64-metal.tar.gz",
"3862270f33439077225324169313f727064f727305b54d8ce920244d75ddcc24",
"audio-v0.7.4-bin-macos-x64-metal.tar.gz",
"bdb797d54dcf8416bd5ac0fac282ce5500dd08843f8f22e20e9fc378ebc24c1f",
),
}
_ASSET_SIZES = {
"windows-x64": 56_818_905,
"linux-x64": 71_551_673,
"darwin-arm64": 25_270_657,
"darwin-x64": 26_718_959,
}
#: Binary filename per platform.
_BINARY_NAMES = {"windows-x64": "audiocpp_server.exe"}
@@ -291,10 +297,23 @@ def _probe_paths() -> list[Path]:
user_dir = os.environ.get(DIR_ENV, "").strip()
if user_dir:
out.append(Path(user_dir) / binary_name())
out.append(managed_runtime_dir() / binary_name())
out.append(_PKG_BIN_DIR / binary_name())
return out
def platform_slug() -> str:
"""Stable release-platform key used by the managed runtime installer."""
return _platform_slug()
def managed_runtime_dir() -> Path:
"""Update-surviving location for the checksummed app-managed runtime."""
from core.config import DATA_DIR
return Path(DATA_DIR) / "engines" / "audio-cpp" / VERSION.lstrip("v") / _platform_slug()
def is_installed() -> bool:
"""Cheap precedence-aware check for a usable server binary."""
try:
@@ -576,6 +595,11 @@ def default_asset() -> tuple[str, str] | None:
return _ASSETS.get(_platform_slug())
def default_asset_size() -> int | None:
"""Published byte size of this host's pinned release archive."""
return _ASSET_SIZES.get(_platform_slug())
def server_port() -> int:
"""Loopback port for the managed server (env override or default)."""
raw = os.environ.get(PORT_ENV, "").strip()
@@ -725,9 +749,12 @@ __all__ = [
"_materialize_gguf_cache_path",
"binary_name",
"default_asset",
"default_asset_size",
"invalidate",
"is_installed",
"managed_runtime_dir",
"package_filename",
"platform_slug",
"parse_device_list",
"probe_devices",
"resolve_compute_selection",
+34 -4
View File
@@ -34,6 +34,7 @@ import json
import os
import struct
import sys
import threading
import traceback
# Mirrors backend/services/subprocess_backend.py::MAX_FRAME_BYTES (T-02-01).
@@ -55,6 +56,8 @@ _GEN_KW_ALLOWLIST = (
)
_model = None
_SEND_LOCK = threading.Lock()
_LOAD_HEARTBEAT_S = 5.0
# ── wire protocol ─────────────────────────────────────────────────────────
@@ -62,9 +65,12 @@ _model = None
def _send(stream, obj: dict) -> None:
body = json.dumps(obj, separators=(",", ":")).encode("utf-8")
stream.write(struct.pack("!I", len(body)))
stream.write(body)
stream.flush()
# Progress callbacks and the cold-load heartbeat can write from different
# threads. Keep each frame atomic or their header/body pairs can interleave.
with _SEND_LOCK:
stream.write(struct.pack("!I", len(body)))
stream.write(body)
stream.flush()
def _recv(stream):
@@ -133,11 +139,27 @@ def _load_model(stdout):
# Forward real HF download/weight progress so the parent's recv loop keeps
# its watchdog alive across a slow cold load (the parent consumes these
# {"op": "progress"} frames and re-arms its deadline on each one).
progress = {"percent": 0}
def _on_progress(ev):
pct = ev.get("pct", 0.0)
if pct:
progress["percent"] = min(round(pct * 100), 99)
_send(stdout, {"op": "progress", "stage": "loading_model",
"percent": min(round(pct * 100), 99)})
"percent": progress["percent"]})
# Cached checkpoints produce no download callbacks. Loading and moving a
# model onto MPS can still exceed the normal generation budget, so keep
# both bounded parent watchdogs informed that the child remains alive.
stop_heartbeat = threading.Event()
def _heartbeat():
while not stop_heartbeat.wait(_LOAD_HEARTBEAT_S):
_send(stdout, {
"op": "progress",
"stage": "loading_model",
"percent": progress["percent"],
})
torch = _lazy_torch()
OmniVoice = _lazy_omnivoice()
@@ -146,11 +168,19 @@ def _load_model(stdout):
preload_asr = should_preload_tts_asr()
lid = register_listener(_on_progress)
heartbeat = threading.Thread(
target=_heartbeat,
name="omnivoice-load-heartbeat",
daemon=True,
)
heartbeat.start()
try:
_model = OmniVoice.from_pretrained(
checkpoint, device_map=device, dtype=torch.float16, load_asr=preload_asr,
)
finally:
stop_heartbeat.set()
heartbeat.join()
unregister_listener(lid)
_send(stdout, {"op": "progress", "stage": "loading_model", "percent": 100})
return _model
+66 -40
View File
@@ -274,16 +274,15 @@ logging.basicConfig(
# inherits the filter, so even handler-formatted output (file, stream,
# JSON) strips real HF tokens. Cheap (regex on each record) and
# idempotent — extra calls are no-ops.
from core.logging_filter import install_redaction_filter # noqa: E402
from core.logging_filter import ( # noqa: E402
install_access_log_filter,
install_asyncio_transport_filter,
install_redaction_filter,
)
install_redaction_filter()
class AsyncioExceptionFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
if record.levelno == logging.WARNING and "socket.send() raised exception" in record.getMessage():
return False
return True
logging.getLogger("asyncio").addFilter(AsyncioExceptionFilter())
install_access_log_filter()
install_asyncio_transport_filter()
# Silence HF Hub unauthenticated warnings unless specifically requested.
logging.getLogger("huggingface_hub.utils._http").setLevel(logging.ERROR)
@@ -537,8 +536,8 @@ async def _cancel_and_await_tasks(*tasks, timeout: float = 3.0) -> None:
# mutation, runs in an executor thread (deferred) or inline (eager).
# Phase A finalize: router/mount registration — mutates the app, so it runs
# ON the event loop (deferred) with no awaits inside, making it atomic with
# respect to in-flight requests; the StartupGate keeps everything but
# /health + /startup/progress out until ready regardless.
# respect to in-flight requests; the StartupGate keeps work routes out until
# ready while retaining readiness probes and deliberate desktop shutdown.
# Phase B: the old lifespan startup body (DB, background services).
_phase_a_built = False
@@ -664,6 +663,7 @@ def _phase_a_build_inner() -> None:
from api.routers import (
system,
profiles,
profile_images,
exports,
generation,
dub_core,
@@ -703,7 +703,7 @@ def _phase_a_build_inner() -> None:
from api.routers import mcp_bindings as _mcp_bindings_router # noqa: E402
from api.routers import workers as workers_router # noqa: E402
_router_modules.extend([
system, profiles, exports, generation, voice_convert, dub_core, dub_generate,
system, profiles, profile_images, exports, generation, voice_convert, dub_core, dub_generate,
dub_export, dub_translate, projects, glossary, engines, tools,
stories, setup, gallery, archetypes, describe_voice, community,
batch, watermark, events, capture, capture_ws, speech_platform, dictation,
@@ -877,6 +877,18 @@ async def _phase_b(app: FastAPI) -> None:
logger.exception("Startup job-sweep failed (non-fatal).")
_startup_progress.begin_step("services_start")
# Reapply an explicitly saved speed/quality profile after the local model
# inventory is available. Older builds saved the slider but could leave
# ASR/Dictation pointing at missing models even when compatible weights
# were already installed. This path is local-cache-only and download-free.
try:
from services.performance_profiles import reconcile_active_profile
recovered = reconcile_active_profile()
if recovered:
logger.info("Startup performance selections reconciled: %s", recovered)
except Exception:
logger.exception("Performance-profile reconciliation failed (non-fatal).")
# Phase 1 Wave 3 — macOS Gatekeeper quarantine probe (#54). Informational
# only; we never auto-run `xattr -cr`.
try:
@@ -918,12 +930,8 @@ async def _phase_b(app: FastAPI) -> None:
"Capture ASR preload skipped: <4GB free RAM; "
"dictation ASR will load on first use.")
return
loading_detail = None
prev_loading_detail = None
try:
from services.model_manager import _gpu_pool, _loading_detail
loading_detail = _loading_detail
prev_loading_detail = dict(loading_detail)
from services.model_manager import _gpu_pool
loop = asyncio.get_running_loop()
def _warm():
from services.asr_backend import (
@@ -937,20 +945,12 @@ async def _phase_b(app: FastAPI) -> None:
"Capture ASR preload skipped: no ASR model installed; "
"dictation will offer a download on first use.")
return
loading_detail["sub_stage"] = "loading_asr"
loading_detail["detail"] = "Warming up ASR engine…"
backend = get_capture_asr_backend()
logger.info("Capture ASR backend selected: %s", backend.id)
if hasattr(backend, 'warmup'):
loading_detail["detail"] = f"Loading {backend.display_name}"
backend.warmup()
loading_detail["sub_stage"] = "ready"
loading_detail["detail"] = "ASR engine ready"
await loop.run_in_executor(_gpu_pool, _warm)
except Exception as e:
if loading_detail is not None and loading_detail.get("sub_stage") == "loading_asr":
loading_detail.clear()
loading_detail.update(prev_loading_detail or {})
logger.warning("Capture ASR preload skipped: %s", e)
app.state.capture_preload_task = asyncio.create_task(_preload_capture_asr())
else:
@@ -1104,13 +1104,10 @@ async def lifespan(app: FastAPI):
# correctness independent of how much of that tail runs, instead of
# depending on the shell-side deadline being long enough to cover it.
#
# SCOPE, explicitly: this only helps platforms where lifespan teardown
# actually BEGINS. On Windows it does not — tools.rs terminates the job
# object with no graceful phase at all, so this line is never reached and
# a deliberate quit is still misreported as a crash there. That needs the
# shell to signal deliberate intent before the hard kill, which is a
# separate Rust-side change and is tracked separately; nothing here
# should be read as fixing Windows.
# Desktop shells that must hard-kill a Windows process tree retire the
# sentinel through /system/shutdown-intent before termination. This
# remains the graceful-path fallback for every platform and direct server
# runs.
#
# sentinel_cleared feeds the truthful "Shutdown: done."/degraded log at
# the end of this function; nothing below re-clears the sentinel, so a
@@ -1219,11 +1216,17 @@ async def lifespan(app: FastAPI):
# Best-effort drain: a failure here must not abort the remaining
# shutdown steps (model unload, MCP teardown) below.
logger.warning("Watermark pool drain failed at shutdown", exc_info=True)
# Unload the model and free GPU memory
# Release every runtime that can retain model memory, then free allocator
# caches. This includes alternate TTS engines, dictation and translation;
# limiting shutdown to the shared OmniVoice model left those runtimes to
# process-exit cleanup and made graceful restarts look like crashes.
try:
import services.model_manager as mm
if mm.unload_shared_model():
logger.info("Shutdown: model unloaded.")
from services import model_lifecycle
released = await model_lifecycle.unload_all()
if any(result.get("success") for result in released["results"].values()):
logger.info("Shutdown: model runtimes unloaded.")
# Still unconditional: there are allocator caches to hand back even when
# no model was resident.
mm.free_vram()
@@ -1265,6 +1268,23 @@ app = FastAPI(
)
@app.post("/system/shutdown-intent", include_in_schema=False)
def prepare_deliberate_shutdown_during_startup(request: Request):
"""Retire crash forensics even while deferred startup is still gated.
Electron must hard-kill a Windows process tree after a bounded wait. The
ordinary system router is registered only after native/ML imports finish,
so a quit during those imports previously received the startup 503 and left
a false crash sentinel behind. Keep this one tiny control route available
from socket bind; its authorization remains identical to the system router.
"""
from api.dependencies import require_admin
from core import run_sentinel
require_admin(request)
return {"prepared": run_sentinel.clear_sentinel()}
@app.get("/docs", include_in_schema=False)
async def scalar_docs():
"""Interactive API documentation powered by Scalar."""
@@ -1454,13 +1474,17 @@ async def global_exception_handler(request: Request, exc: Exception):
_SHELL_PATHS = {"/", "/index.html", "/favicon.ico", "/health"}
# Paths that answer while the deferred startup is still running.
_STARTUP_EXEMPT = {"/health", "/startup/progress"}
# Paths that answer while deferred startup is still running. The shutdown
# signal must exist before the ordinary system router so a bounded Windows
# process-tree stop cannot leave a false crash sentinel.
_STARTUP_EXEMPT = {"/health", "/startup/progress", "/system/shutdown-intent"}
class StartupGateMiddleware:
"""503 everything except /health + /startup/progress until the deferred
startup completes. Two jobs: honest not-ready signaling (the [starting]
"""503 work routes until deferred startup completes.
Readiness probes and deliberate desktop shutdown remain live. Two jobs:
honest not-ready signaling (the [starting]
marker keeps the UI from offering "Report" for it, same convention as
[shutting_down]), and route-mutation safety no request can reach the
router while _phase_a_finalize is still adding routes, because the ready
@@ -1693,10 +1717,12 @@ def _ui_port() -> int:
return 3901
from core.csrf import DEFAULT_DESKTOP_ORIGINS
_ui = _ui_port()
_allowed = os.environ.get(
"OMNIVOICE_ALLOWED_ORIGINS",
f"http://localhost:{_ui},http://127.0.0.1:{_ui},tauri://localhost,http://tauri.localhost",
f"http://localhost:{_ui},http://127.0.0.1:{_ui}," + ",".join(DEFAULT_DESKTOP_ORIGINS),
).split(",")
# Registered FIRST → innermost: the startup gate holds every request except
+25 -3
View File
@@ -60,7 +60,9 @@ class DubRequest(BaseModel):
language: str = "Auto"
language_code: str = "und" # ISO 639-1 for ffmpeg metadata (e.g. "es", "fr", "de")
instruct: str = ""
num_step: int = 16
# None means "use the shared performance profile". An explicit value is
# still authoritative for Production overrides and existing API clients.
num_step: Optional[int] = None
guidance_scale: float = 2.0
speed: float = 1.0
# Phase 4.1 — partial regen. Parallel lists by index with `segments`.
@@ -71,7 +73,8 @@ class DubRequest(BaseModel):
regen_only: Optional[List[str]] = None
# Fast-preview mode for interactive edits. When true, TTS runs at
# num_step=8 (~2× faster, ~10-20% quality drop). Client is responsible
# for re-rendering preview segs at full quality before final export.
# for re-rendering preview segs with the explicit override or shared
# performance profile before final export.
preview: Optional[bool] = False
# How to handle segs whose TTS audio is longer than its slot (the
# "ghost lang" overlap bug otherwise). Options:
@@ -150,7 +153,7 @@ class TranslateRequest(BaseModel):
provider: Optional[str] = None
source_lang: Optional[str] = None # ISO 639-1; overrides job detection
job_id: Optional[str] = None # Dub job id, used to resolve detected source_lang
quality: Optional[str] = "fast" # "fast" (one-shot) | "cinematic" (reflect→adapt) | "autofit" (cinematic + strict fit-to-slot)
quality: Optional[str] = "fast" # fast | cinematic | autofit | agent (measured render/rewrite loop)
glossary: Optional[List[dict]] = None # [{"source": "...", "target": "...", "note": "..."}]
# Optional regional dialect (BCP-47, e.g. "es-AR", "pt-BR") — #280 item 2.
# Applied by LLM-backed paths (provider="openai" or quality="cinematic"):
@@ -175,6 +178,25 @@ class TranslateRequest(BaseModel):
# No LLM configured / LLM failure → silently no suggestion.
condense: Optional[bool] = False
class AgentFitSegment(BaseModel):
"""One rendered translation and its measured timing evidence."""
id: str
text: str
source_text: Optional[str] = None
context_before: Optional[str] = None
context_after: Optional[str] = None
slot_seconds: float
measured_seconds: float
class AgentFitRequest(BaseModel):
"""Revise only rendered lines that missed their exact timeline slot."""
segments: List[AgentFitSegment]
target_lang: str
class ParseSubtitleTextRequest(BaseModel):
"""Raw pasted subtitle text (SRT/VTT-ish) to be parsed into timed cues.
+213 -48
View File
@@ -1005,13 +1005,11 @@ class FasterWhisperBackend(ASRBackend):
# CTranslate2: CUDA or CPU (no upstream ROCm/HIP build — see WhisperX note).
gpu_compat = ("cuda", "cpu")
def __init__(self):
def __init__(self, model_name: str | None = None):
# Defaulting to the CTranslate2-converted large-v3 repo. Matches
# KNOWN_MODELS in api/routers/setup.py so the first-run wizard
# downloads what the backend will actually load.
self._model_name = os.environ.get(
"ASR_MODEL_FASTER", "Systran/faster-whisper-large-v3"
)
self._model_name = model_name or faster_whisper_model_id()
self._model = None # lazy — first transcribe() loads weights
# Set by _ensure_model() to the device/compute_type that actually loaded
# (after the #551 compute_type / #255 OOM→CPU fallback chain).
@@ -1114,10 +1112,13 @@ class FasterWhisperBackend(ASRBackend):
# faster-whisper returns a generator of Segment objects + an Info
# struct. Materialise the generator so downstream consumers can
# index / re-iterate.
from services.performance_profiles import asr_decode_defaults
segments_iter, info = self._model.transcribe(
audio_path,
word_timestamps=word_timestamps,
vad_filter=True, # built-in Silero VAD — cleaner segment starts
**asr_decode_defaults(),
)
segments = list(segments_iter)
# Normalise to the shape segment_transcript(...) expects: a dict with
@@ -1864,6 +1865,8 @@ class SherpaDictationBackend(ASRBackend):
f"{[s.id for s in _sd.list_specs()]}"
)
self._spec = spec
from services.performance_profiles import requested_tier
self.performance_tier = requested_tier("dictation")
self._rec = None # lazy OfflineRecognizer / OnlineRecognizer
# One backend is shared across live-dictation WS sessions (see
# get_sherpa_dictation_backend), so guard the one-time recognizer build
@@ -2823,7 +2826,7 @@ class ASRModelMissingError(RuntimeError):
super().__init__(asr_model_missing_detail(payload))
def load_active_asr_backend(*, asr_pipe=None) -> ASRBackend:
def load_active_asr_backend(*, asr_pipe=None, require_installed: bool = False) -> ASRBackend:
""":func:`get_active_asr_backend` + eager ``ensure_loaded()``, degrading
past backends whose deep import chain is broken (#1185).
@@ -2852,11 +2855,11 @@ def load_active_asr_backend(*, asr_pipe=None) -> ASRBackend:
while True:
backend = get_active_asr_backend(asr_pipe=asr_pipe)
bid = getattr(backend, "id", "?")
if tried:
if tried or require_installed:
# Preflight the SPECIFIC candidate about to load — not the global
# selection, which can disagree when an asr_pipe steers
# get_active_asr_backend (Greptile review, #1198).
missing = asr_model_missing_error(backend_id=bid)
missing = asr_model_missing_error(backend_id=bid, require_installed=require_installed)
if missing is not None:
raise ASRModelMissingError(missing)
try:
@@ -2931,6 +2934,109 @@ def _ref_audio_fingerprint(audio_path: str) -> str | None:
return None
def _installed_reference_fallbacks(
selected: list[ASRBackend],
) -> list[ASRBackend]:
"""Return the strongest compatible local fallbacks without changing prefs."""
fallbacks: list[ASRBackend] = []
selected_repos = {
_fw_repo(str(getattr(item, "_model_name", "")))
for item in selected
if isinstance(item, FasterWhisperBackend)
}
try:
from api.routers.setup.models import (
KNOWN_MODELS,
_model_supported,
_snapshot_dirs,
snapshot_is_complete,
)
available, _reason = FasterWhisperBackend.is_available()
if available:
compatible = sorted(
(
model
for model in KNOWN_MODELS
if str(model.get("role", "")).lower() == "asr"
and not model.get("dictation_id")
and (
str(model.get("repo_id", "")).startswith("Systran/faster-")
or model.get("repo_id")
== "deepdml/faster-whisper-large-v3-turbo-ct2"
)
and _model_supported(model)
and model.get("repo_id") not in selected_repos
),
key=lambda model: float(model.get("size_gb") or 0),
reverse=True,
)
for model in compatible:
snapshots = [
path
for path in _snapshot_dirs(str(model["repo_id"]))
if snapshot_is_complete(model, path)
]
if not snapshots:
continue
# A concrete complete snapshot cannot trigger a Hub download.
snapshot = max(snapshots, key=lambda path: os.path.getmtime(path))
backend = FasterWhisperBackend(model_name=snapshot)
setattr(backend, "_reference_ephemeral", True)
fallbacks.append(backend)
break
except Exception as exc: # noqa: BLE001 - optional local fallback
logger.warning("reference ASR cache fallback unavailable (%s)", exc)
try:
from services import sherpa_dictation
selected_sherpa = {
item.spec.id
for item in selected
if isinstance(item, SherpaDictationBackend)
}
installed = sorted(
(
spec
for spec in sherpa_dictation.list_specs()
if spec.id not in selected_sherpa
and sherpa_dictation.is_installed(spec)
),
key=lambda spec: float(spec.size_gb or 0),
reverse=True,
)
if installed:
fallbacks.append(get_sherpa_dictation_backend(installed[0].id))
except Exception as exc: # noqa: BLE001 - optional local fallback
logger.warning("reference dictation cache fallback unavailable (%s)", exc)
return fallbacks
def _transcribe_reference_candidates(
candidates: list[ASRBackend], audio_path: str,
) -> str:
for backend in candidates:
try:
result = backend.transcribe(audio_path, word_timestamps=False) or {}
candidate_text = result.get("text") or " ".join(
(seg.get("text") or "").strip()
for seg in result.get("segments", [])
)
candidate_text = (candidate_text or "").strip()
if candidate_text:
return candidate_text
except Exception as exc: # noqa: BLE001 - try the next local engine
logger.warning("transcribe_reference: %s failed (%s)", backend.id, exc)
finally:
if getattr(backend, "_reference_ephemeral", False):
try:
backend.unload()
except Exception: # noqa: BLE001 - release is best-effort
logger.warning("reference ASR fallback unload failed", exc_info=True)
return ""
def transcribe_reference(audio_path: str) -> str | None:
"""Transcribe a voice-clone reference clip with the active ASR backend.
@@ -2952,42 +3058,55 @@ def transcribe_reference(audio_path: str) -> str | None:
if cached is not None:
_ref_transcript_cache.move_to_end(fingerprint)
return cached
# No ASR model installed (TTS-only install): skip quietly instead of
# letting the backend auto-download multi-GB weights mid-/generate — this
# path is best-effort by contract (the engine's built-in fallback applies).
if asr_model_missing_error() is not None:
logger.info("transcribe_reference: no ASR model installed — skipping "
"reference auto-transcription (no silent download).")
# Prefer the selected offline ASR engine. When its selected weights are not
# installed, reuse the selected dictation engine if that model is already
# local. Short clone references need plain transcription, which dictation
# engines provide well. Falling straight through to the TTS model's bundled
# fallback produced incomplete reference conditioning for longer clips and
# introduced spurious words at the start of short generations. Neither
# branch may download weights implicitly.
candidates: list[ASRBackend] = []
offline_missing = asr_model_missing_error()
if offline_missing is None:
try:
# `load_*`, not `get_*`: a backend whose shallow probe passes but
# whose deep import chain is broken must fall through cleanly.
backend = load_active_asr_backend()
if not isinstance(backend, PyTorchWhisperBackend):
candidates.append(backend)
except Exception as e: # noqa: BLE001 — reference ASR is best-effort
logger.warning("transcribe_reference: offline ASR unavailable (%s)", e)
capture_missing = asr_model_missing_error(purpose="dictation")
if capture_missing is None:
try:
capture = get_capture_asr_backend()
if not isinstance(capture, PyTorchWhisperBackend) and not any(
type(item) is type(capture) and item.id == capture.id
for item in candidates
):
candidates.append(capture)
except Exception as e: # noqa: BLE001 — reference ASR is best-effort
logger.warning("transcribe_reference: dictation ASR unavailable (%s)", e)
text = _transcribe_reference_candidates(candidates, audio_path)
fallbacks: list[ASRBackend] = []
if not text:
fallbacks = _installed_reference_fallbacks(candidates)
text = _transcribe_reference_candidates(fallbacks, audio_path)
if not candidates and not fallbacks:
logger.info(
"transcribe_reference: no installed ASR model available — skipping "
"reference auto-transcription (no silent download)."
)
return None
try:
# `load_*`, not `get_*`: a backend whose shallow probe passes but whose
# deep import chain is broken would otherwise be handed back here and
# fail at `.transcribe()` below, costing every clone-without-transcript
# its reference text even with a healthy engine next in line (#1185).
# This path is best-effort, so a genuinely exhausted chain still just
# returns None and defers to the model's built-in fallback.
backend = load_active_asr_backend()
except Exception as e: # noqa: BLE001 — never let ASR break generation
logger.warning("transcribe_reference: no ASR backend available (%s)", e)
return None
if isinstance(backend, PyTorchWhisperBackend):
# The registry fell through to the model-attached pipeline; let the
# model load it lazily rather than constructing a second copy here.
return None
try:
result = backend.transcribe(audio_path, word_timestamps=False)
except Exception as e: # noqa: BLE001 — degrade to the model fallback
if not text:
logger.warning(
"transcribe_reference: %s failed (%s) — deferring to the model's "
"built-in ASR fallback",
backend.id, e,
"transcribe_reference: installed ASR engines returned no transcript "
"— deferring to the model's built-in ASR fallback"
)
return None
result = result or {}
text = result.get("text") or " ".join(
(seg.get("text") or "").strip() for seg in result.get("segments", [])
)
text = (text or "").strip()
if text and fingerprint is not None:
with _ref_transcript_lock:
_ref_transcript_cache[fingerprint] = text
@@ -3089,10 +3208,14 @@ def get_sherpa_dictation_backend(model_id: str) -> "SherpaDictationBackend":
:func:`get_capture_asr_backend`. Thread-safe: the recognizer is shared;
each session creates its own decode stream (see capture_ws)."""
global _capture_backend, _capture_backend_key
from services.performance_profiles import requested_tier
performance_tier = requested_tier("dictation")
_touch_capture() # any handout resets the idle clock
with _capture_backend_lock:
if (isinstance(_capture_backend, SherpaDictationBackend)
and _capture_backend_key == model_id):
and _capture_backend_key == model_id
and _capture_backend.performance_tier == performance_tier):
return _capture_backend
backend = SherpaDictationBackend(model_id=model_id)
_capture_backend = backend
@@ -3270,8 +3393,11 @@ def get_capture_asr_backend(*, skip_sherpa: bool = False) -> ASRBackend:
if sherpa_id:
ok, _ = SherpaDictationBackend.is_available()
if ok:
from services.performance_profiles import requested_tier
performance_tier = requested_tier("dictation")
if not (isinstance(_capture_backend, SherpaDictationBackend)
and _capture_backend_key == sherpa_id):
and _capture_backend_key == sherpa_id
and _capture_backend.performance_tier == performance_tier):
try:
_capture_backend = SherpaDictationBackend(model_id=sherpa_id)
_capture_backend_key = sherpa_id
@@ -3346,6 +3472,34 @@ ASR_MODEL_MISSING = "asr_model_missing"
_PYTORCH_ASR_DEFAULT = "openai/whisper-large-v3-turbo"
_FASTER_WHISPER_DEFAULT = "Systran/faster-whisper-large-v3"
def faster_whisper_model_id() -> str:
"""Resolve the UI-selected CTranslate2 model, with env pins authoritative."""
from core import prefs
return str(
prefs.resolve(
"asr_model_faster",
env="ASR_MODEL_FASTER",
default=_FASTER_WHISPER_DEFAULT,
)
)
def select_faster_whisper_model(repo_id: str) -> None:
"""Persist and apply a CTranslate2 model selection for this process."""
from core import prefs
if prefs.is_env_shadowed("ASR_MODEL_FASTER"):
raise ValueError("ASR_MODEL_FASTER is set outside VoiceStudio")
prefs.set_("asr_model_faster", repo_id)
# Sidecars inherit the process environment. Updating it here makes the
# selection effective immediately as well as after the next app launch.
os.environ["ASR_MODEL_FASTER"] = repo_id
instance = _ISOLATED_INSTANCES.pop("faster-whisper-isolated", None)
if instance is not None:
instance.shutdown()
# faster-whisper / WhisperX short model aliases → the HF repo they download.
# Covers our own defaults plus the documented size aliases; an unrecognized
# alias returns None and the preflight stays out of the way (never blocks).
@@ -3378,7 +3532,7 @@ def _offline_asr_repo(backend_id: str | None = None) -> str | None:
if bid == "whisperx":
return _fw_repo(os.environ.get("ASR_MODEL_WHISPERX", "large-v3"))
if bid == "faster-whisper":
return _fw_repo(os.environ.get("ASR_MODEL_FASTER", _FASTER_WHISPER_DEFAULT))
return _fw_repo(faster_whisper_model_id())
if bid == "faster-whisper-isolated":
# Mirror the sidecar's own resolution (_asr_sidecar/main.py):
# ASR_MODEL_FW is a sidecar-only override, otherwise the shared
@@ -3386,7 +3540,7 @@ def _offline_asr_repo(backend_id: str | None = None) -> str | None:
# download a different repo than the sidecar will load.
return _fw_repo(
os.environ.get("ASR_MODEL_FW")
or os.environ.get("ASR_MODEL_FASTER")
or faster_whisper_model_id()
or _FASTER_WHISPER_DEFAULT
)
if bid == "mlx-whisper":
@@ -3427,7 +3581,7 @@ def _capture_whisper_repo() -> str | None:
# resolve but our alias table doesn't know) yields None here — FAIL
# OPEN rather than coerce to the default repo and demand a download
# of a model the user never picked.
return _fw_repo(os.environ.get("ASR_MODEL_FASTER", _FASTER_WHISPER_DEFAULT))
return _fw_repo(faster_whisper_model_id())
return os.environ.get("OMNIVOICE_PYTORCH_ASR_MODEL", _PYTORCH_ASR_DEFAULT)
@@ -3499,12 +3653,12 @@ def _recommended_asr_model(
_INSTALLED_REPO_MEMO: set[str] = set()
def _repo_installed(repo: str) -> bool:
def _repo_installed(repo: str, *, refresh: bool = False) -> bool:
"""``is_cached`` + ``cache_is_complete`` with a positive-only session memo.
Installed state comes from the same HF-cache helpers the model store uses,
so the answer matches the Model Catalogue's install badges."""
if repo in _INSTALLED_REPO_MEMO:
so the answer matches the Model Catalogue Models install badges."""
if not refresh and repo in _INSTALLED_REPO_MEMO:
return True
from api.routers.setup.models import cache_is_complete, get_model_catalog, is_cached
meta = get_model_catalog().get(repo) or {"repo_id": repo}
@@ -3582,7 +3736,7 @@ def asr_model_missing_error(*, purpose: str = "transcribe",
return None # explicit opt-in engine — can't (and shouldn't) preflight
from api.routers.setup.models import get_model_catalog
if require_installed:
if _repo_installed(repo):
if _repo_installed(repo, refresh=True):
return None
return {
"error": ASR_MODEL_MISSING,
@@ -3607,6 +3761,14 @@ def asr_model_missing_error(*, purpose: str = "transcribe",
),
}
except Exception: # noqa: BLE001 — preflight is best-effort, never a blocker
if require_installed:
logger.warning("ASR install preflight failed; refusing implicit download", exc_info=True)
return {
"error": ASR_MODEL_MISSING,
"missing_repo_id": "unverified-local-model",
"reason": "verification_failed",
"recommended": None,
}
logger.warning("ASR install preflight failed — proceeding without it",
exc_info=True)
return None
@@ -3615,6 +3777,9 @@ def asr_model_missing_error(*, purpose: str = "transcribe",
def asr_model_missing_detail(payload: dict) -> str:
"""Human-readable (English) fallback message for the typed payload —
what legacy clients / logs see; the frontend renders its own i18n copy."""
if payload.get("reason") == "verification_failed":
return ("Could not verify the local speech-to-text model. "
"Check Settings > Logs > Backend, then retry. No model was downloaded.")
rec = payload.get("recommended") or {}
if rec.get("label"):
return (
+10 -1
View File
@@ -66,6 +66,12 @@ logger = logging.getLogger("omnivoice.audio_io")
PathOrBuf = Union[str, "os.PathLike[str]", BinaryIO, io.IOBase]
def _ensure_audio_parent(path_or_buf: PathOrBuf) -> None:
"""Recover app output folders removed after backend initialization."""
if isinstance(path_or_buf, (str, os.PathLike)):
os.makedirs(os.path.dirname(os.path.abspath(path_or_buf)), exist_ok=True)
def _safe_torchaudio_save(
path_or_buf: PathOrBuf,
tensor: torch.Tensor,
@@ -156,6 +162,7 @@ def _safe_torchaudio_save(
fmt = (format or "wav").lower()
try:
_ensure_audio_parent(path_or_buf)
if fmt == "wav":
torchaudio.save(
path_or_buf,
@@ -313,6 +320,7 @@ def _safe_soundfile_write(
else:
samples = np.ascontiguousarray(samples)
_ensure_audio_parent(path)
sf.write(path, samples, sample_rate, subtype=subtype)
@@ -334,7 +342,7 @@ def atomic_save_wav(
publication AND audited tensor normalization.
Args:
target_path: Final destination. Parent directory must already exist.
target_path: Final destination. Missing parent directories are recreated.
audio: ``(channels, samples)`` or ``(samples,)`` tensor.
sample_rate: WAV sample rate in Hz.
**kwargs: Forwarded to ``_safe_torchaudio_save`` (``format``,
@@ -346,6 +354,7 @@ def atomic_save_wav(
unlinked on failure so we do not leak ``.tmp`` files in
``DUB_DIR``.
"""
_ensure_audio_parent(target_path)
target_dir = os.path.dirname(target_path) or "."
target_base = os.path.basename(target_path)
# The temp file must end in ``.wav`` even though it is conceptually a
@@ -0,0 +1,197 @@
"""Checksummed, user-triggered installer for the native audio.cpp runtime."""
from __future__ import annotations
import hashlib
import logging
import os
from pathlib import Path
import shutil
import subprocess
import tarfile
import tempfile
import threading
import urllib.request
import zipfile
from engines.audiocpp import bootstrap
logger = logging.getLogger("omnivoice.audiocpp.install")
_CHUNK = 256 * 1024
_lock = threading.Lock()
_job = {"state": "idle", "progress": 0.0, "error": None}
def _snapshot() -> dict:
with _lock:
return dict(_job)
def _update(**fields) -> None:
with _lock:
_job.update(fields)
def _runtime_paths() -> tuple[Path | None, Path | None]:
try:
server = bootstrap.resolve_server_binary()
except (OSError, RuntimeError):
return None, None
cli = server.with_name("audiocpp_cli.exe" if os.name == "nt" else "audiocpp_cli")
if not cli.is_file() or (os.name != "nt" and not os.access(cli, os.X_OK)):
return server, None
return server, cli
def status() -> dict:
server, cli = _runtime_paths()
managed = bootstrap.managed_runtime_dir()
is_managed = bool(
server
and server.resolve() == (managed / bootstrap.binary_name()).resolve()
)
return {
"supported": bootstrap.default_asset() is not None,
"installed": server is not None and cli is not None,
"managed": is_managed,
"version": bootstrap.VERSION if server and cli and is_managed else None,
"platform": bootstrap.platform_slug(),
"job": _snapshot(),
}
def _download(url: str, destination: Path, digest: str, expected_size: int) -> None:
if not url.startswith("https://github.com/"):
raise ValueError("audio.cpp downloads require the pinned GitHub release")
request = urllib.request.Request(url, headers={"User-Agent": "VoiceStudio"})
hasher = hashlib.sha256()
received = 0
with urllib.request.urlopen(request, timeout=30) as response, destination.open("wb") as out:
total = expected_size or int(response.headers.get("Content-Length") or 0)
while chunk := response.read(_CHUNK):
out.write(chunk)
hasher.update(chunk)
received += len(chunk)
if total:
_update(progress=min(received / total, 0.9))
if received != expected_size:
raise RuntimeError("The audio.cpp runtime download size did not match the release")
if hasher.hexdigest() != digest:
raise RuntimeError("The audio.cpp runtime checksum did not match the release")
def _safe_destination(root: Path, name: str) -> Path:
destination = (root / name.replace("\\", "/")).resolve()
if destination != root and root not in destination.parents:
raise RuntimeError("The audio.cpp archive contains an unsafe path")
return destination
def _extract(archive: Path, destination: Path) -> None:
destination.mkdir(parents=True)
root = destination.resolve()
if archive.suffix.lower() == ".zip":
with zipfile.ZipFile(archive) as bundle:
for member in bundle.infolist():
target = _safe_destination(root, member.filename)
if member.is_dir():
target.mkdir(parents=True, exist_ok=True)
continue
target.parent.mkdir(parents=True, exist_ok=True)
with bundle.open(member) as source, target.open("wb") as out:
shutil.copyfileobj(source, out)
return
with tarfile.open(archive, "r:*") as bundle:
for member in bundle.getmembers():
target = _safe_destination(root, member.name)
if member.isdir():
target.mkdir(parents=True, exist_ok=True)
continue
if not member.isfile():
raise RuntimeError("The audio.cpp archive contains an unsupported link")
target.parent.mkdir(parents=True, exist_ok=True)
source = bundle.extractfile(member)
if source is None:
raise RuntimeError("The audio.cpp archive contains an unreadable file")
with source, target.open("wb") as out:
shutil.copyfileobj(source, out)
target.chmod(member.mode & 0o700)
def _install() -> None:
asset = bootstrap.default_asset()
expected_size = bootstrap.default_asset_size()
if asset is None or expected_size is None:
raise RuntimeError("No audio.cpp runtime is published for this platform")
filename, digest = asset
url = (
f"https://github.com/{bootstrap.GH_REPO}/releases/download/"
f"{bootstrap.VERSION}/{filename}"
)
target = bootstrap.managed_runtime_dir()
target.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="audiocpp-install-", dir=target.parent) as temp:
temp_path = Path(temp)
archive = temp_path / filename
_download(url, archive, digest, expected_size)
extracted = temp_path / "extracted"
_extract(archive, extracted)
candidates = sorted(
extracted.rglob(bootstrap.binary_name()), key=lambda path: len(path.parts)
)
if not candidates:
raise RuntimeError("The audio.cpp release does not contain its server")
source_dir = candidates[0].parent
cli_name = "audiocpp_cli.exe" if os.name == "nt" else "audiocpp_cli"
if not (source_dir / cli_name).is_file():
raise RuntimeError("The audio.cpp release does not contain its CLI")
prepared = temp_path / "prepared"
shutil.copytree(source_dir, prepared)
for executable in (prepared / bootstrap.binary_name(), prepared / cli_name):
executable.chmod(0o700)
probe = subprocess.run( # nosec B603 -- checksummed fixed release binary
[str(prepared / bootstrap.binary_name()), "--list-devices"],
capture_output=True,
timeout=20,
check=False,
)
if probe.returncode != 0:
raise RuntimeError("The downloaded audio.cpp runtime failed its device check")
if target.exists():
shutil.rmtree(target)
os.replace(prepared, target)
bootstrap.invalidate()
def start_install(*, wait: bool = False) -> dict:
current = status()
if current["installed"]:
return {"status": "already_installed", **current}
if not current["supported"]:
raise RuntimeError("No audio.cpp runtime is published for this platform")
with _lock:
running = _job["state"] == "running"
if not running:
_job.update(state="running", progress=0.0, error=None)
if running:
return {"status": "already_running", **status()}
def worker() -> None:
try:
_install()
_update(state="done", progress=1.0, error=None)
except Exception:
logger.exception("audio.cpp runtime installation failed")
_update(
state="error",
error="The audio.cpp runtime could not be installed. Check the backend log.",
)
if wait:
worker()
else:
threading.Thread(target=worker, name="audiocpp-install", daemon=True).start()
return {"status": "started", **status()}
def reset_job_for_tests() -> None:
_update(state="idle", progress=0.0, error=None)
+37
View File
@@ -0,0 +1,37 @@
"""Resolve the installed pyannote bundle without network access at job time."""
from contextlib import contextmanager
from pathlib import Path
from tempfile import TemporaryDirectory
@contextmanager
def local_pipeline_config():
import yaml
from huggingface_hub import hf_hub_download
from huggingface_hub.constants import HF_HUB_CACHE
from services.hf_revisions import installed_revision
def cached(repo: str, filename: str) -> str:
return hf_hub_download(
repo_id=repo,
filename=filename,
revision=installed_revision(repo, HF_HUB_CACHE),
local_files_only=True,
)
config_path = cached("pyannote/speaker-diarization-3.1", "config.yaml")
config = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
params = config["pipeline"]["params"]
# The reviewed pipeline references these two checkpoints. Local checkpoint
# paths prevent pyannote's nested Model.from_pretrained calls fetching them.
for key, repo in (
("segmentation", "pyannote/segmentation-3.0"),
("embedding", "pyannote/wespeaker-voxceleb-resnet34-LM"),
):
if params.get(key) != repo:
raise ValueError(f"Unexpected diarisation {key} repository; repair the installed pipeline")
params[key] = cached(repo, "pytorch_model.bin")
with TemporaryDirectory(prefix="voicestudio-pyannote-") as directory:
path = Path(directory) / "config.yaml"
path.write_text(yaml.safe_dump(config), encoding="utf-8")
yield str(path)
+163
View File
@@ -0,0 +1,163 @@
"""Explicit local audio.cpp Sortformer adapter for the shared diarisation flow."""
from __future__ import annotations
import json
import logging
import os
from pathlib import Path
import subprocess
import time
import threading
from tempfile import TemporaryDirectory
logger = logging.getLogger("omnivoice.diarisation.native")
_process_lock = threading.Lock()
_processes: set = set()
MAX_V1_AUDIO_SECONDS = 120.0
SORTFORMER_FRAME_SAMPLES = 1280 # 80 ms at the required 16 kHz input rate.
def _sortformer_command(binary: Path, model: Path, device, source: Path, output: Path):
return [
str(binary), "--task", "diar", "--family", "sortformer_diar",
"--model", str(model), "--backend", device.backend,
"--device", str(device.index), "--audio", str(source),
"--turns-out", str(output),
# Accelerator builds otherwise keep the default 20-second fixed graph
# and reject ordinary clips. Grow remains bounded by the v1 limit below.
"--session-option", "graph_capacity_mode=grow",
]
def _validated_turn(turn: dict, audio_frames: int) -> tuple[int, int, str]:
start, end = turn.get("start_sample"), turn.get("end_sample")
speaker = turn.get("speaker_id")
if (
type(start) is not int
or type(end) is not int
or start < 0
or start >= end
or end > audio_frames + SORTFORMER_FRAME_SAMPLES
or not isinstance(speaker, str)
or not speaker
):
raise ValueError("Invalid native speaker-turn boundaries")
# The decoder works in 80 ms frames and can pad its final turn one frame
# beyond a non-aligned WAV boundary. Keep the timeline inside the media.
return start, min(end, audio_frames), speaker
def is_running() -> bool:
with _process_lock:
return bool(_processes)
class NativeSortformer:
"""Stateless native invocation; the GGUF is never downloaded implicitly."""
def __init__(self):
from engines.audiocpp.bootstrap import resolve_server_binary
from services.diarization_runtime import sortformer_model_path
try:
self.model = sortformer_model_path()
except Exception as exc:
raise FileNotFoundError(
"Install the audio.cpp Sortformer model in Settings > Models > Diarisation"
) from exc
if not self.model.is_file() or self.model.suffix.lower() != ".gguf":
raise FileNotFoundError("The configured Sortformer GGUF is missing")
with self.model.open("rb") as model_file:
if model_file.read(4) != b"GGUF":
raise ValueError("The configured Sortformer model is not a GGUF file")
server = resolve_server_binary()
self.binary = server.with_name("audiocpp_cli.exe" if os.name == "nt" else "audiocpp_cli")
if not self.binary.is_file():
raise FileNotFoundError("The installed audio.cpp directory has no audiocpp_cli")
def __call__(self, audio_path, *, num_speakers=None, job_id=None, cancel_check=None):
if num_speakers is not None:
raise ValueError("Sortformer v1 detects up to four speakers but cannot enforce an exact speaker count")
import soundfile as sf
from pyannote.core import Annotation, Segment
from core.contained_subprocess import spawn_owned
from engines.audiocpp.bootstrap import resolve_compute_selection
from services.proc_registry import register_proc, unregister_proc
def check_cancelled():
if cancel_check is not None and cancel_check():
raise RuntimeError("Native diarisation cancelled")
check_cancelled()
audio_info = sf.info(str(audio_path))
if audio_info.duration > MAX_V1_AUDIO_SECONDS:
raise ValueError(
"Sortformer v1 supports recordings up to 120 seconds; "
"select pyannote for longer recordings"
)
device = resolve_compute_selection().device
with TemporaryDirectory(prefix="voicestudio-sortformer-") as directory:
source = Path(audio_path).resolve()
output = Path(directory) / "turns.json"
command = _sortformer_command(
self.binary, self.model, device, source, output
)
def run_owned(command, log_name):
with (Path(directory) / log_name).open("wb") as log:
check_cancelled()
process = spawn_owned(command, stdout=log, stderr=subprocess.STDOUT)
with _process_lock:
_processes.add(process)
try:
if job_id is not None:
register_proc(job_id, process)
deadline = time.monotonic() + 600
while True:
check_cancelled()
remaining = deadline - time.monotonic()
if remaining <= 0:
raise subprocess.TimeoutExpired(command, 600)
try:
code = process.wait(timeout=min(0.25, remaining))
break
except subprocess.TimeoutExpired:
continue
except BaseException:
process.kill()
process.wait()
raise
finally:
with _process_lock:
_processes.discard(process)
if job_id is not None:
unregister_proc(job_id, process)
check_cancelled()
if code != 0:
with (Path(directory) / log_name).open("rb") as diagnostic:
diagnostic.seek(0, 2)
diagnostic.seek(max(0, diagnostic.tell() - 8192))
tail = diagnostic.read().decode("utf-8", errors="replace")
logger.error("Sortformer exited with %s; native log tail:\n%s", code, tail)
raise RuntimeError(f"Native Sortformer failed (exit {code})")
if (audio_info.samplerate != 16000 or audio_info.channels != 1
or audio_info.format != "WAV" or audio_info.subtype != "PCM_16"):
from services.ffmpeg_utils import find_ffmpeg
normalized = Path(directory) / "input.wav"
run_owned([
find_ffmpeg(), "-nostdin", "-hide_banner", "-loglevel", "error", "-y",
"-i", str(source), "-vn", "-ac", "1", "-ar", "16000",
"-c:a", "pcm_s16le", str(normalized),
], "normalize.log")
source = normalized
audio_info = sf.info(str(source))
command[command.index("--audio") + 1] = str(source)
run_owned(command, "native.log")
turns = json.loads(output.read_text(encoding="utf-8"))
if not isinstance(turns, list):
raise ValueError("Invalid native speaker-turn output")
annotation = Annotation()
for index, turn in enumerate(turns):
start, end, speaker = _validated_turn(turn, audio_info.frames)
annotation[Segment(start / 16000, end / 16000), index] = speaker
return annotation
+118
View File
@@ -0,0 +1,118 @@
"""Persisted selection and installed-only resolution for diarisation runtimes."""
from __future__ import annotations
import os
from pathlib import Path
from core import prefs
PYANNOTE = "pyannote"
SORTFORMER = "audiocpp-sortformer"
SORTFORMER_REPO = "audio-cpp/audio.cpp-gguf"
SORTFORMER_FILE = "Sortformer-Diar-4spk-v1-GGUF/sortformer-diar-4spk-v1-q8_0.gguf"
_SORTFORMER_MODEL_MISSING = "Install the Sortformer model bundle"
_SORTFORMER_MODEL_BROKEN = "Repair the installed Sortformer model bundle"
_SORTFORMER_RUNTIME_MISSING = (
"The Sortformer model is installed. Install the audio.cpp runtime to use it"
)
_SORTFORMER_CLI_MISSING = (
"The installed audio.cpp runtime does not include speaker diarisation"
)
def selected_backend() -> str:
value = str(
prefs.resolve(
"diarization_backend",
env="OMNIVOICE_DIARIZATION_BACKEND",
default=PYANNOTE,
)
).strip()
return value if value in {PYANNOTE, SORTFORMER} else PYANNOTE
def sortformer_model_path() -> Path:
configured = os.environ.get("OMNIVOICE_DIARIZATION_MODEL", "").strip()
if configured:
return Path(configured).expanduser().resolve()
# An installed-only lookup never reaches the network. Installation remains
# an explicit Model Library action through the reviewed audio.cpp bundle.
from huggingface_hub import hf_hub_download
from services.hf_revisions import revision_for
return Path(
hf_hub_download(
repo_id=SORTFORMER_REPO,
filename=SORTFORMER_FILE,
revision=revision_for(SORTFORMER_REPO),
local_files_only=True,
)
).resolve()
def select_backend(backend: str) -> None:
if backend not in {PYANNOTE, SORTFORMER}:
raise ValueError("Unknown diarisation engine")
prefs.set_("diarization_backend", backend)
def sortformer_status() -> dict:
"""Return path-free readiness for the model and its native executable."""
status = {
"model": SORTFORMER_REPO,
"model_installed": False,
"runtime_installed": False,
"installed": False,
"reason": _SORTFORMER_MODEL_MISSING,
}
try:
model = sortformer_model_path()
except Exception:
return status
try:
if not model.is_file() or model.suffix.lower() != ".gguf":
return status
with model.open("rb") as model_file:
if model_file.read(4) != b"GGUF":
status["reason"] = _SORTFORMER_MODEL_BROKEN
return status
except OSError:
status["reason"] = _SORTFORMER_MODEL_BROKEN
return status
status["model_installed"] = True
status["reason"] = _SORTFORMER_RUNTIME_MISSING
try:
from engines.audiocpp.bootstrap import resolve_server_binary
server = resolve_server_binary()
except (OSError, RuntimeError):
return status
cli = server.with_name("audiocpp_cli.exe" if os.name == "nt" else "audiocpp_cli")
if not cli.is_file() or (os.name != "nt" and not os.access(cli, os.X_OK)):
status["reason"] = _SORTFORMER_CLI_MISSING
return status
status.update(runtime_installed=True, installed=True, reason=None)
return status
def installed_backends() -> set[str]:
"""Return complete local runtimes without loading weights or downloading."""
installed: set[str] = set()
from api.routers.setup.models import KNOWN_MODELS, cache_is_complete, is_cached
repo_id = "pyannote/speaker-diarization-3.1"
spec = next(model for model in KNOWN_MODELS if model["repo_id"] == repo_id)
if is_cached(repo_id) and cache_is_complete(spec):
installed.add(PYANNOTE)
try:
native = sortformer_status()
if not native["installed"]:
return installed
installed.add(SORTFORMER)
except (OSError, RuntimeError, ValueError):
pass
return installed
+56
View File
@@ -0,0 +1,56 @@
"""Shared native-TTS batching policy for interactive and queued dubbing."""
from __future__ import annotations
import logging
import os
logger = logging.getLogger("omnivoice.dub_batching")
BATCH_WIDTH_ENV = "OMNIVOICE_DUB_BATCH_WIDTH"
_MAX_BATCH_WIDTH = 16
def native_batch_width(backend) -> int:
"""Return a host-safe native batch width for ``backend``."""
override = os.environ.get(BATCH_WIDTH_ENV, "").strip()
if override:
try:
return max(1, min(_MAX_BATCH_WIDTH, int(override)))
except (TypeError, ValueError):
logger.warning(
"%s=%r is not an integer; deriving the batch width from the host",
BATCH_WIDTH_ENV,
override,
)
try:
from core.device_caps import detect_host_caps
caps = detect_host_caps()
except Exception: # noqa: BLE001 - an unprobeable host takes the safe path
return 1
if caps.family == "cpu" or not caps.vram_gb:
return 1
headroom = caps.vram_gb - float(getattr(backend, "min_vram_gb", 0.0) or 0.0)
if headroom < 2.0:
return 1
if headroom < 6.0:
return 2
if headroom < 12.0:
return 4
return 8
def batch_timeout_s(texts: list[str], backend) -> float:
"""Bound one native batch without multiplying the executor base timeout."""
from services.model_manager import generate_timeout_s
floor = generate_timeout_s("", engine=backend)
overage = sum(
max(0.0, generate_timeout_s(text, engine=backend) - floor)
for text in texts
)
return floor + overage
__all__ = ["BATCH_WIDTH_ENV", "batch_timeout_s", "native_batch_width"]
+43 -3
View File
@@ -1084,9 +1084,7 @@ def yt_download_sync(
if sub_langs:
langs = list(sub_langs)
else:
orig = (info.get("language") or "").strip()
manual = list((info.get("subtitles") or {}).keys())
langs = sorted({*manual, *([orig] if orig else [])})
langs = _default_caption_languages(info)
if not langs:
logger.info("No captions available on %s (skipping subtitle pass)", log_safe(url))
else:
@@ -1115,6 +1113,48 @@ def yt_download_sync(
return video_path, title, sub_files
def _default_caption_languages(info: dict) -> list[str]:
"""Return original-language caption tracks without translated auto-captions.
Some extractors omit ``language`` even though yt-dlp exposes an original
automatic-caption track such as ``en-orig``. Treat that explicit suffix as
source metadata so caption-first ingest still works instead of needlessly
loading ASR. Manual tracks remain eligible because they are authored source
material and yt-dlp's ``skip=translated_subs`` guard still applies.
"""
original = str(info.get("language") or "").strip()
manual = {
str(language).strip()
for language in (info.get("subtitles") or {})
if str(language).strip()
}
automatic = {
str(language).strip()
for language in (info.get("automatic_captions") or {})
if str(language).strip()
}
selected = set(manual)
if original:
primary = original.split("-", 1)[0]
has_source_manual = any(
language == original or language.split("-", 1)[0] == primary
for language in manual
)
if not has_source_manual:
for candidate in (
f"{original}-orig",
f"{primary}-orig",
original,
primary,
):
if candidate in automatic:
selected.add(candidate)
break
else:
selected.update(language for language in automatic if language.endswith("-orig"))
return sorted(selected)
def parse_vtt_segments(vtt_path: str) -> list[dict]:
"""Very small WEBVTT parser → list of {start, end, text}.
+214 -30
View File
@@ -62,6 +62,8 @@ from __future__ import annotations
import asyncio
import logging
import os
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Callable, Optional
@@ -76,6 +78,87 @@ logger = logging.getLogger("omnivoice.gateway")
# life of the process.
_POLL_SECONDS = 0.5
# Remote model installs run as worker prewarms rather than scheduler tasks. Keep
# their progress on the control plane so Models can reconnect without pretending
# the download stopped as soon as POST /models/install returned.
_REMOTE_DOWNLOAD_RETENTION_SECONDS = 60.0
_remote_download_lock = threading.Lock()
_remote_downloads: dict[tuple[str, str], dict[str, Any]] = {}
def begin_remote_download(target: str, repo_id: str) -> None:
with _remote_download_lock:
_remote_downloads[(target, repo_id)] = {
"repo_id": repo_id,
"target": target,
"state": "downloading",
"phase": "starting",
"updated_at": time.monotonic(),
}
def record_remote_download_progress(target: str, event: dict) -> None:
"""Persist authenticated worker progress for reconnectable model rows."""
repo_id = str(event.get("repo_id") or "").strip()
if not repo_id:
return
phase = str(event.get("phase") or "progress")
state = (
"done"
if phase == "install_done"
else "failed"
if phase == "install_error"
else "install_cancelled"
if phase in {"cancelled", "install_cancelled"}
else "cancelling"
if phase == "cancelling"
else "downloading"
)
normalized = {
"repo_id": repo_id,
"target": target,
"state": state,
"phase": phase,
"updated_at": time.monotonic(),
}
for source, destination in (
("bytes_done", "bytes_done"),
("downloaded", "bytes_done"),
("total_bytes", "total_bytes"),
("total", "total_bytes"),
("rate", "rate"),
("eta_seconds", "eta_seconds"),
("files_done", "files_done"),
("files_total", "files_total"),
("error", "error"),
("docs_topic", "docs_topic"),
("failed_at", "failed_at"),
("retry_after_seconds", "retry_after_seconds"),
):
value = event.get(source)
if value is not None:
normalized[destination] = value
with _remote_download_lock:
previous = _remote_downloads.get((target, repo_id), {})
_remote_downloads[(target, repo_id)] = {**previous, **normalized}
def remote_download_jobs() -> list[dict[str, Any]]:
now = time.monotonic()
with _remote_download_lock:
expired = [
key
for key, job in _remote_downloads.items()
if job.get("state") in {"done", "failed", "cancelled", "install_cancelled"}
and now - float(job.get("updated_at") or 0) >= _REMOTE_DOWNLOAD_RETENTION_SECONDS
]
for key in expired:
_remote_downloads.pop(key, None)
return [
{key: value for key, value in job.items() if key != "updated_at"}
for job in _remote_downloads.values()
]
# Consecutive remote failures a multi-unit job tolerates before it stops trying
# the remote worker. One is a blip (a dropped stream, a worker restart); two in
# a row is a machine that has gone away, and the remaining 160 chapters should
@@ -352,7 +435,7 @@ async def prewarm(
"""
decision = decision or decide(op, control_plane=control_plane)
if decision.remote:
await preflight(engine, decision, control_plane=control_plane)
await preflight(engine, decision, operation=op, control_plane=control_plane)
plane = _plane(control_plane)
if engine and plane is not None and getattr(plane, "servicer", None) is not None:
await plane.servicer.prewarm(decision.worker_id, engine=engine)
@@ -495,12 +578,22 @@ async def _run_remote(
if scheduler is None:
raise _NotDispatched("the control plane has no scheduler")
await preflight(call.engine, decision, call.model_id, control_plane=plane)
await preflight(
call.engine,
decision,
call.model_id,
operation=call.operation,
control_plane=plane,
)
params = dict(call.params or {})
deadline = call.deadline_seconds
if deadline is None:
deadline = _default_deadline(call.operation, params.get("text"))
deadline = _default_deadline(
call.operation,
params.get("text"),
input_seconds=float(params.get("input_seconds") or 0.0),
)
try:
submit = getattr(scheduler, "submit_async", None)
@@ -547,6 +640,7 @@ async def preflight(
decision: Decision,
model_id: str = "",
*,
operation: str = "tts",
control_plane=None,
) -> None:
"""Refuse a positively absent remote model before scheduler admission.
@@ -558,7 +652,12 @@ async def preflight(
"""
if not engine:
return
target = await status(engine, decision=decision, control_plane=control_plane)
target = await status(
engine,
decision=decision,
op=operation,
control_plane=control_plane,
)
for cap in target["models"]:
if model_id and cap.get("model_id") not in (model_id, "", None):
continue
@@ -819,7 +918,7 @@ async def status(
"remote": False,
"label": decision.label,
"reason": decision.reason,
"models": _filtered(_local_capabilities(), engine),
"models": _filtered(_local_capabilities(), engine, op),
}
plane = _plane(control_plane)
@@ -834,14 +933,27 @@ async def status(
"remote": False,
"label": decision.label,
"reason": "the chosen worker is not connected",
"models": _filtered(_local_capabilities(), engine),
"models": _filtered(_local_capabilities(), engine, op),
}
models = []
capacity = getattr(worker, "capacity", None)
for advertised in worker.record.capabilities or []:
model = dict(advertised)
if capacity is not None:
engine_id = str(model.get("engine") or "")
model_id = str(model.get("model_id") or "")
if engine_id and model_id:
# Heartbeats are authoritative for residency. Capability
# discovery is refreshed less often and can otherwise leave
# Engine Ready green after a worker unloads a model.
model["resident"] = capacity.is_resident(engine_id, model_id)
models.append(model)
return {
"target": decision.worker_id,
"remote": True,
"label": decision.label,
"reason": decision.reason,
"models": _filtered(list(worker.record.capabilities or []), engine),
"models": _filtered(models, engine, op),
}
@@ -851,15 +963,39 @@ def _local_capabilities() -> list[dict]:
return capabilities.discover(include_unavailable=True)
def _filtered(models: list[dict], engine: Optional[str]) -> list[dict]:
if not engine:
return models
return [m for m in models if m.get("engine") == engine]
def _filtered(models: list[dict], engine: Optional[str], operation: str = "tts") -> list[dict]:
worker_operation = {
"batch": "batch_segments",
"dub": "dub_segments",
"longform": "audiobook",
}.get(operation, operation)
return [
model
for model in models
if (not engine or model.get("engine") == engine)
and (
not model.get("operations")
or worker_operation in model.get("operations", ())
)
]
# ── Download: weights, onto the machine that needs them ────────────────────
def _remote_model_capability(plane, worker_id: str, repo_id: str) -> Optional[dict]:
pool = getattr(plane, "pool", None)
worker = pool.get(worker_id) if pool is not None else None
return next(
(
capability
for capability in (worker.record.capabilities if worker is not None else [])
if repo_id in (capability.get("repo_ids") or [])
),
None,
)
async def download(
repo_id: str,
*,
@@ -869,23 +1005,17 @@ async def download(
) -> dict:
"""Fetch a catalog model onto the target machine.
Remote downloads are not implemented yet, and this refuses rather than
falling back: downloading onto *this* machine when the user asked for the
weights on the 4090 leaves the remote box exactly as unprepared, having
reported success.
Remote downloads are sent to the selected worker and never fall back to
this machine: fetching weights onto the wrong host would report success
while leaving the selected GPU exactly as unprepared.
"""
decision = decision or decide(op, control_plane=control_plane)
if decision.remote:
plane = _plane(control_plane)
if plane is None or getattr(plane, "servicer", None) is None:
raise RemoteUnsupported(f"{decision.label} is not connected.")
live = plane.pool.get(decision.worker_id) if plane.pool is not None else None
capability = next(
(
cap for cap in (live.record.capabilities if live is not None else [])
if repo_id in (cap.get("repo_ids") or [])
),
None,
capability = _remote_model_capability(
plane, decision.worker_id, repo_id
)
if capability is None:
raise GatewayError(f"Unknown model for {decision.label}: {repo_id!r}.")
@@ -898,13 +1028,29 @@ async def download(
f"{repo_id!r} must be installed directly on {decision.label}; "
"remote sidecar installation is disabled."
)
sent = await plane.servicer.prewarm(
decision.worker_id,
engine=str(capability.get("engine") or ""),
model_id=str(capability.get("model_id") or ""),
download_if_missing=True,
)
begin_remote_download(decision.worker_id, repo_id)
try:
sent = await plane.servicer.prewarm(
decision.worker_id,
engine=str(capability.get("engine") or ""),
model_id=str(capability.get("model_id") or ""),
download_if_missing=True,
)
except Exception as exc:
record_remote_download_progress(
decision.worker_id,
{"repo_id": repo_id, "phase": "install_error", "error": str(exc)},
)
raise
if not sent:
record_remote_download_progress(
decision.worker_id,
{
"repo_id": repo_id,
"phase": "install_error",
"error": f"{decision.label} is not connected.",
},
)
raise RemoteUnsupported(f"{decision.label} is not connected.")
return {"status": "started", "repo_id": repo_id, "target": decision.worker_id}
@@ -921,6 +1067,33 @@ async def download(
return await install_model(InstallModelRequest(repo_id=repo_id, target="local"))
async def cancel_download(
repo_id: str,
*,
target: str,
control_plane=None,
) -> dict:
"""Cancel an explicit model install on its authenticated remote worker."""
plane = _plane(control_plane)
servicer = getattr(plane, "servicer", None) if plane is not None else None
if servicer is None:
raise RemoteUnsupported(f"{target} is not connected.")
capability = _remote_model_capability(plane, target, repo_id)
if capability is None:
raise GatewayError(f"Unknown model for {target}: {repo_id!r}.")
model_id = str(capability.get("model_id") or "")
if not model_id:
raise GatewayError(f"{repo_id!r} has no worker model identifier.")
sent = await servicer.cancel_model_install(target, model_id=model_id)
if not sent:
raise RemoteUnsupported(f"{target} is not connected.")
record_remote_download_progress(
target,
{"repo_id": repo_id, "phase": "cancelling"},
)
return {"cancelling": repo_id, "target": target}
# ── Plumbing ───────────────────────────────────────────────────────────────
@@ -936,7 +1109,12 @@ def _plane(control_plane=None):
return None
def _default_deadline(operation: str, text: Optional[str]) -> float:
def _default_deadline(
operation: str,
text: Optional[str],
*,
input_seconds: float = 0.0,
) -> float:
"""Worst-case wall time for one attempt, from the shared deadline policy.
Same budget the assignment itself carries, so the awaiting side cannot give
@@ -944,7 +1122,13 @@ def _default_deadline(operation: str, text: Optional[str]) -> float:
"""
from worker import deadlines # noqa: PLC0415
return float(deadlines.for_task(operation, text=text).total_seconds)
return float(
deadlines.for_task(
operation,
text=text,
input_seconds=max(0.0, float(input_seconds)),
).total_seconds
)
def _model_load_timeout() -> float:
+2
View File
@@ -39,6 +39,8 @@ CURATED_REVISIONS: dict[str, str] = {
"csukuangfj/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23": "204ad334e2e683fd295359930cc16fc0432a23ac",
"csukuangfj/sherpa-onnx-whisper-tiny": "65176e2deb88badc814a94058666cadccc29b61c",
"pyannote/speaker-diarization-3.1": "84fd25912480287da0247647c3d2b4853cb3ee5d",
"pyannote/segmentation-3.0": "e66f3d3b9eb0873085418a7b813d3b369bf160bb",
"pyannote/wespeaker-voxceleb-resnet34-LM": "837717ddb9ff5507820346191109dc79c958d614",
"OpenMOSS-Team/MOSS-TTS-Nano-100M": "44502f80dbf9743528fa921cc544d662c685ebec",
"KittenML/kitten-tts-mini-0.8": "c02725660cea441db4c383af69f1f26f5cd00947",
"openbmb/VoxCPM2": "bffb3df5a29440629464e5e839f4d214c8714c3d",
+108 -5
View File
@@ -48,6 +48,27 @@ def _asr_device() -> str:
return "cpu"
def _backend_device(backend: object) -> str:
"""Report the device this backend can actually use.
The host's best device is not evidence that a CPU-only runtime uses it.
Prefer load-time facts, then constrain the fallback by the backend's
declared compatibility contract.
"""
for attr in ("_device", "device", "execution_device"):
value = getattr(backend, attr, None)
if value is not None and not callable(value):
text = str(value).strip()
if text:
return text
compat = tuple(getattr(type(backend), "gpu_compat", ("cpu",)))
preferred = get_best_device()
family = preferred.split(":", 1)[0]
if family in compat:
return preferred
return "cpu" if "cpu" in compat else (compat[0] if compat else "unknown")
def _active_tts_id() -> Optional[str]:
"""Configured TTS engine id, or None if it can't be resolved. Attribution
is advisory a prefs/import hiccup must never break /model/loaded."""
@@ -169,6 +190,40 @@ def list_loaded() -> dict:
logger.warning("Loaded-model inventory unavailable for in-process engines")
degraded_sources.append("engines")
# The local generation path keeps its selected backend in a separate
# active-instance slot. Non-OmniVoice models held there must be visible as
# well; otherwise Model Settings can report an empty runtime while an
# alternate TTS model still occupies memory.
try:
import services.tts_backend as tb
from services.subprocess_backend import SubprocessBackend
inst = getattr(tb, "_active_instance", None)
eid = getattr(tb, "_active_instance_id", None)
if (
inst is not None
and eid
and eid != "omnivoice"
and not isinstance(inst, SubprocessBackend)
and any(
getattr(inst, attr, None) is not None
for attr in getattr(inst, "_MODEL_ATTRS", ("_model", "_tts"))
)
):
identity = getattr(inst, "model_identity", None)
models.append({
"id": f"active-engine:{eid}",
"name": getattr(inst, "display_name", None) or f"{eid} (engine)",
"checkpoint": identity() if callable(identity) else eid,
"device": _backend_device(inst),
"vram_mb": 0,
"unloadable": True,
**_tts_attribution(eid, active_tts),
})
except Exception:
logger.warning("Loaded-model inventory unavailable for active TTS engine")
degraded_sources.append("active-engine")
# 6. The warm capture/dictation ASR singleton — resident until idle-released
# (#1101 class). Held separately from the co-loaded WhisperX ASR above.
try:
@@ -176,11 +231,12 @@ def list_loaded() -> dict:
cap = getattr(ab, "_capture_backend", None)
if cap is not None:
model_label = getattr(getattr(cap, "spec", None), "label", None)
models.append({
"id": "capture-asr",
"name": f"{type(cap).__name__} (dictation)",
"name": f"{model_label or getattr(cap, 'display_name', type(cap).__name__)} (dictation)",
"checkpoint": getattr(ab, "_capture_backend_key", None) or type(cap).__name__,
"device": get_best_device(),
"device": _backend_device(cap),
"vram_mb": 0,
"unloadable": True,
"note": "released after the idle timeout",
@@ -189,6 +245,25 @@ def list_loaded() -> dict:
logger.warning("Loaded-model inventory unavailable for dictation")
degraded_sources.append("dictation")
# 7. Offline translation can remain resident when the user opts out of the
# default post-job release. Keep it visible and manually unloadable.
try:
from api.routers import dub_translate as dt
if getattr(dt, "_nllb_model", None) is not None:
models.append({
"id": "translation:nllb",
"name": "NLLB-200 Translation",
"checkpoint": dt._NLLB_REPO_ID,
"device": str(getattr(dt, "_nllb_device", None) or "cpu"),
"vram_mb": 0,
"unloadable": True,
"note": "released after translation by default",
})
except Exception:
logger.warning("Loaded-model inventory unavailable for translation")
degraded_sources.append("translation")
# System memory snapshot — free/total RAM (and VRAM on a dedicated GPU) plus
# a low-memory advisory, so the panel can show pressure instead of leaving
# the 16 GB-Mac OOM class invisible until the backend dies.
@@ -247,6 +322,14 @@ async def unload(model_id: str) -> dict:
return {"unloaded": model_id, "success": True}
return {"unloaded": model_id, "success": False, "reason": "in use by dictation"}
if model_id == "translation:nllb":
from api.routers import dub_translate as dt
if getattr(dt, "_nllb_model", None) is None:
return {"unloaded": model_id, "success": False, "reason": "not loaded"}
dt._unload_nllb()
return {"unloaded": model_id, "success": True}
# In-process engines (#1247). `list_loaded_models` has advertised these as
# `engine:<id>` with `"unloadable": True` since they were made visible in
# the panel — but this dispatcher never grew a branch for them, so pressing
@@ -270,14 +353,34 @@ async def unload(model_id: str) -> dict:
return {"unloaded": model_id, "success": True}
return {"unloaded": model_id, "success": False, "reason": "not loaded"}
if model_id.startswith("active-engine:"):
engine_id = model_id.split(":", 1)[1]
import services.tts_backend as tb
if (
getattr(tb, "_active_instance", None) is None
or getattr(tb, "_active_instance_id", None) != engine_id
):
return {"unloaded": model_id, "success": False, "reason": "not loaded"}
tb.reset_active_backend()
return {"unloaded": model_id, "success": True}
raise ValueError(f"Unknown model id: {model_id}")
async def unload_all() -> dict:
"""Release every releasable model — in-process TTS + diarization + all
sidecars. Convenience for app shutdown / a global flush."""
"""Release shared/alternate TTS, diarisation, sidecars, dictation and translation."""
results = {}
for mid in ("tts", "diarization", "sidecars"):
model_ids = ["tts", "diarization", "sidecars", "capture-asr", "translation:nllb"]
try:
model_ids.extend(
entry["id"]
for entry in list_loaded()["models"]
if entry["id"].startswith(("engine:", "active-engine:"))
)
except Exception:
logger.warning("Could not enumerate optional engines during global unload")
for mid in dict.fromkeys(model_ids):
try:
results[mid] = await unload(mid)
except Exception as exc: # noqa: BLE001
+87 -10
View File
@@ -1372,7 +1372,7 @@ _last_used = time.time()
# Updated by _load_model_sync() so get_model_status() can report
# granular progress to the frontend pill.
_loading_detail: dict = {
"sub_stage": None, # importing | loading_weights | loading_asr | compiling | ready | error
"sub_stage": None, # importing | loading_weights | compiling | ready | error
"detail": "", # human-readable description
"error": None, # error message string if failed
"progress": None, # 0-100 percentage (None = indeterminate)
@@ -1851,6 +1851,15 @@ def _set_loading(sub_stage: str, detail: str = "", error: str | None = None, pro
_loading_detail["detail"] = detail
_loading_detail["error"] = error
_loading_detail["progress"] = progress
# Model state is a declared real-time event. Emit only at these explicit
# lifecycle transitions; high-frequency Hugging Face byte progress updates
# write the dict directly and remain covered by the active one-second poll.
try:
from core import event_bus
event_bus.emit("model_status", {"sub_stage": sub_stage})
except Exception:
logger.debug("Could not publish model status", exc_info=True)
def _env_flag(name: str, default: bool = False) -> bool:
@@ -2612,6 +2621,24 @@ def _load_model_sync():
except Exception as e:
logger.info("torch.compile skipped: %s", e)
# Bind status identity to the object that actually finished loading.
# Resolving preferences later can name a newly-selected checkpoint
# while the previous one is still resident, and process-global load
# metadata can be overwritten by a loader that completed after its
# caller timed out. Instance metadata keeps /model/status honest.
try:
setattr(_model, "_voicestudio_checkpoint", checkpoint)
setattr(
_model,
"_voicestudio_loaded_at",
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
)
except Exception:
# OmniVoice is an ordinary nn.Module and accepts attributes, but
# a future slotted/proxied model must still be usable. Status falls
# back to the effective configured checkpoint below.
logger.debug("Could not attach resident model identity", exc_info=True)
_set_loading("ready", "Model ready", progress=100)
logger.info("VoiceStudio model loaded successfully.")
return _model
@@ -3030,20 +3057,39 @@ def get_model_status():
is_loading = False
status = "loading" if is_loading else ("ready" if is_loaded else "idle")
checkpoint = None
loaded_at = None
if is_loaded:
checkpoint = getattr(model, "_voicestudio_checkpoint", None)
loaded_at = getattr(model, "_voicestudio_loaded_at", None)
if not checkpoint:
try:
checkpoint = resolve_omnivoice_checkpoint()
except Exception:
# Status is a recovery surface. A broken preferences layer
# must not turn a resident-model query into a 500.
logger.debug("Could not resolve resident model identity", exc_info=True)
result = {
"loaded": is_loaded,
"loading": is_loading,
"status": status,
"checkpoint": checkpoint,
"loaded_at": loaded_at,
}
# Attach sub-stage detail when loading or after an error
# Attach sub-stage detail only while it describes the current resident/load
# state, or when a failure must remain actionable. A completed model can be
# unloaded while the last successful "ready" detail remains in memory.
# Publishing that stale detail alongside status=idle/loaded=false gives
# clients two contradictory readiness states.
sub = _loading_detail.get("sub_stage")
if sub:
err = _loading_detail.get("error")
if sub and (is_loading or is_loaded or err):
result["sub_stage"] = sub
result["detail"] = _loading_detail.get("detail", "")
progress = _loading_detail.get("progress")
if progress is not None:
result["progress"] = progress
err = _loading_detail.get("error")
if err:
result["error"] = err
return result
@@ -3430,6 +3476,7 @@ _diar_pipeline = None
DIARIZATION_ERR_NO_TOKEN = "NO_TOKEN"
DIARIZATION_ERR_LICENSE = "PYANNOTE_LICENSE_REQUIRED"
DIARIZATION_ERR_LOAD = "LOAD_FAILED"
DIARIZATION_ERR_MISSING = "MODEL_MISSING"
def _classify_diarization_error(exc: BaseException) -> str:
@@ -3446,13 +3493,14 @@ def _classify_diarization_error(exc: BaseException) -> str:
"""
name = type(exc).__name__.lower()
msg = str(exc).lower()
if "localentrynotfounderror" in name or isinstance(exc, FileNotFoundError):
return DIARIZATION_ERR_MISSING
if (
"401" in msg
or "403" in msg
or "unauthorized" in msg
or "gated" in msg
or "accept" in msg and ("license" in msg or "terms" in msg or "user conditions" in msg)
or "hfhubhttperror" in name
or "gatedrepoerror" in name
or "repositorynotfounderror" in name and "gated" in msg
):
@@ -3513,6 +3561,15 @@ def get_diarization_pipeline(return_error: bool = False):
a docs deeplink issue #78.
"""
global _diar_pipeline
from services.diarization_runtime import SORTFORMER, selected_backend
if selected_backend() == SORTFORMER:
try:
from services.diarization_native import NativeSortformer
pipeline = NativeSortformer()
return (pipeline, None) if return_error else pipeline
except Exception as exc:
logger.exception("Could not prepare native Sortformer")
return (None, _classify_diarization_error(exc)) if return_error else None
if _diar_pipeline is not None:
return (_diar_pipeline, None) if return_error else _diar_pipeline
@@ -3521,9 +3578,9 @@ def get_diarization_pipeline(return_error: bool = False):
# reads HF tokens, and that place is `token_resolver.resolve()`.
from services import token_resolver
resolved = token_resolver.resolve()
if not resolved:
return (None, DIARIZATION_ERR_NO_TOKEN) if return_error else None
hf_token = resolved.token
# Access is checked during explicit installation. An already-installed
# local bundle remains usable after a token expires or is removed.
hf_token = resolved.token if resolved else False
try:
torch = _lazy_torch()
_ensure_pyannote_hf_token_compat() # #167: use_auth_token -> token
@@ -3541,11 +3598,16 @@ def get_diarization_pipeline(return_error: bool = False):
logger.debug("pyannote safe-globals allowlist skipped: %s", _glob_e)
from pyannote.audio import Pipeline
logger.info("Loading Pyannote Diarization Pipeline...")
_diar_pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", use_auth_token=hf_token)
from services.diarization_local import local_pipeline_config
with local_pipeline_config() as config_path:
pipeline = Pipeline.from_pretrained(config_path, use_auth_token=hf_token)
if pipeline is None:
raise RuntimeError("The installed diarisation pipeline could not be loaded")
device = get_best_device()
# Pyannote supports CUDA and CPU; route XPU/DirectML to CPU
if device in ("cuda",):
_diar_pipeline.to(torch.device(device))
pipeline.to(torch.device(device))
_diar_pipeline = pipeline
logger.info("Pyannote Diarization Pipeline loaded on %s.", device)
return (_diar_pipeline, None) if return_error else _diar_pipeline
except Exception as e:
@@ -3554,3 +3616,18 @@ def get_diarization_pipeline(return_error: bool = False):
"Failed to load Pyannote pipeline (class=%s)", err_class,
)
return (None, err_class) if return_error else None
def unload_diarization_pipeline() -> bool:
"""Release a resident pyannote pipeline after the runtime changes."""
global _diar_pipeline
pipeline = _diar_pipeline
_diar_pipeline = None
if pipeline is None:
return False
del pipeline
try:
free_vram()
except Exception:
logger.debug("Could not clear accelerator cache after diarisation unload", exc_info=True)
return True
+463
View File
@@ -0,0 +1,463 @@
"""Shared local performance preferences and runtime defaults. No model downloads."""
from __future__ import annotations
import math
import os
_PERFORMANCE_PROFILE_KEY = "performance_profile"
_PERFORMANCE_TIERS = ("fast", "balanced", "quality", "max")
_PERFORMANCE_FAMILIES = (
"tts",
"asr",
"dictation",
"diarisation",
"translation",
"llm",
)
# Advertise only implemented runtime controls, never speculative model switches.
_PERFORMANCE_TARGETS = {
"tts": {
"fast": {"steps": 8, "postprocess": False},
"balanced": {"steps": 16, "postprocess": True},
"quality": {"steps": 32, "postprocess": True},
"max": {"steps": 64, "postprocess": True, "model_policy": "largest-installed-compatible"},
},
"asr": {
tier: {"beam_size": width, "best_of": width, "engine": "faster-whisper"}
for tier, width in zip(_PERFORMANCE_TIERS, (1, 3, 5, 8))
},
"dictation": {
"fast": {"decoding_method": "greedy_search", "max_active_paths": 1, "engine": "sherpa-onnx"},
"balanced": {"decoding_method": "greedy_search", "max_active_paths": 4, "engine": "sherpa-onnx"},
"quality": {"decoding_method": "modified_beam_search", "max_active_paths": 4, "engine": "sherpa-onnx"},
"max": {"decoding_method": "modified_beam_search", "max_active_paths": 8, "engine": "sherpa-onnx"},
},
"diarisation": {
"fast": {"engine": "audiocpp-sortformer"},
"balanced": {"engine": "audiocpp-sortformer"},
"quality": {"engine": "pyannote"},
"max": {"engine": "pyannote"},
},
"translation": {
"fast": {"num_beams": 1, "engine": "argos"},
"balanced": {"num_beams": 3, "engine": "argos"},
"quality": {"num_beams": 5, "engine": "nllb"},
"max": {"num_beams": 8, "engine": "nllb"},
},
}
_TIER_POSITION = {"fast": 0.0, "balanced": 0.5, "quality": 0.8, "max": 1.0}
def _tier_choice(items: list, tier: str, *, size) -> object | None:
"""Pick an installed model along the user's speed/quality continuum."""
if not items:
return None
ordered = sorted(items, key=lambda item: (float(size(item) or 0), str(item)))
position = _TIER_POSITION.get(tier, _TIER_POSITION["balanced"])
index = math.floor(position * (len(ordered) - 1) + 0.5)
return ordered[index]
def _installed_ct2_models() -> list[dict]:
"""Installed CTranslate2 Whisper models usable by the shared ASR runtime."""
from api.routers.setup.models import (
KNOWN_MODELS,
_model_supported,
cache_is_complete,
is_cached,
)
return [
model
for model in KNOWN_MODELS
if str(model.get("role", "")).lower() == "asr"
and not model.get("dictation_id")
and (
str(model.get("repo_id", "")).startswith("Systran/faster-")
or model.get("repo_id") == "deepdml/faster-whisper-large-v3-turbo-ct2"
)
and _model_supported(model)
and is_cached(model["repo_id"])
and cache_is_complete(model)
]
def _faster_whisper_backend() -> str | None:
from services import asr_backend
if asr_backend._probe_available(asr_backend.FasterWhisperBackend):
return "faster-whisper"
row = next(
(
item
for item in asr_backend.list_backends()
if item["id"] == "faster-whisper-isolated"
),
None,
)
return (
"faster-whisper-isolated"
if row and row.get("available") and row.get("routing_status") != "unavailable"
else None
)
def _dictation_supports_locale(spec, language: str | None) -> bool:
if not language or spec.id == "sherpa-whisper-tiny":
return True
if spec.id == "sherpa-parakeet-tdt-v3":
from services.asr_backend import _PARAKEET_MLX_LANGS
return language in _PARAKEET_MLX_LANGS
if spec.id in {"sherpa-parakeet-tdt-v2", "sherpa-zipformer-en-20m"}:
return language == "en"
if spec.id == "sherpa-zipformer-zh-14m":
return language == "zh"
if spec.id in {
"sherpa-zipformer-bilingual-zh-en",
"sherpa-paraformer-bilingual-zh-en",
}:
return language in {"en", "zh"}
return True
def _installed_dictation_models() -> list:
from services import asr_backend, sherpa_dictation
language = asr_backend._locale_language()
installed = [
spec
for spec in sherpa_dictation.list_specs()
if sherpa_dictation.is_installed(spec)
and not sherpa_dictation.is_demoted(spec.id)
]
compatible = [
spec for spec in installed if _dictation_supports_locale(spec, language)
]
# A machine without a usable locale should still recover to an explicitly
# installed model instead of claiming no speech model exists.
return compatible or installed
def _activate_asr_model(tier: str) -> dict | None:
from core import prefs
from services import asr_backend
if os.environ.get("OMNIVOICE_ASR_BACKEND") or prefs.is_env_shadowed(
"ASR_MODEL_FASTER"
):
return None
model = _tier_choice(
_installed_ct2_models(), tier, size=lambda item: item.get("size_gb")
)
backend_id = _faster_whisper_backend()
if model is None or backend_id is None:
return None
repo_id = str(model["repo_id"])
if asr_backend.faster_whisper_model_id() != repo_id:
asr_backend.select_faster_whisper_model(repo_id)
if asr_backend.active_backend_id() != backend_id:
prefs.set_("asr_backend", backend_id)
return {"engine": backend_id, "model": repo_id}
def _activate_dictation_model(tier: str) -> dict | None:
from core import prefs
from services import asr_backend, sherpa_dictation
if os.environ.get("OMNIVOICE_SHERPA_ASR_MODEL"):
return None
available, _ = sherpa_dictation.sherpa_available()
if not available:
return None
model = _tier_choice(
_installed_dictation_models(), tier, size=lambda item: item.size_gb
)
if model is None:
return None
if prefs.get("dictation.model_id") != model.id:
prefs.set_("dictation.model_id", model.id)
asr_backend._capture_backend = None
asr_backend._capture_backend_key = None
return {"engine": model.kind, "model": model.id}
def _activate_translation_model(tier: str) -> dict | None:
from core import prefs
from services import translation_engines
current = str(prefs.get("translation_backend", "argos"))
# Keep a usable explicitly chosen network provider. A stale provider whose
# package/key disappeared must not strand Dubbing while an installed local
# translator is ready.
if current not in {"argos", "nllb"} and translation_engines.is_ready(current):
return None
target = str(_PERFORMANCE_TARGETS["translation"][tier]["engine"])
if not translation_engines.is_ready(target):
target = next(
(
candidate
for candidate in ("argos", "nllb")
if translation_engines.is_ready(candidate)
),
"",
)
if not target:
return None
if current != target:
prefs.set_("translation_backend", target)
return {
"engine": target,
"model": "facebook/nllb-200-distilled-600M" if target == "nllb" else target,
}
def _installed_selectable_families() -> set[str]:
from services import sherpa_dictation, translation_engines
families: set[str] = set()
if _installed_ct2_models() and _faster_whisper_backend():
families.add("asr")
sherpa_available, _ = sherpa_dictation.sherpa_available()
if sherpa_available and _installed_dictation_models():
families.add("dictation")
if translation_engines.is_ready("nllb"):
families.add("translation")
return families
def _activate_installed_models(tier: str, family: str | None) -> dict[str, dict]:
requested = set(_PERFORMANCE_FAMILIES if family is None else (family,))
activated: dict[str, dict] = {}
selectors = {
"asr": _activate_asr_model,
"dictation": _activate_dictation_model,
"translation": _activate_translation_model,
}
for name, select in selectors.items():
if name in requested:
result = select(tier)
if result:
activated[name] = result
return activated
def profile_state() -> dict:
from core import prefs
from services import asr_backend, diarization_runtime
from services.sherpa_dictation import get_spec as dictation_spec
from services.tts_backend import active_backend_id as active_tts
selected_dictation = (
dictation_spec(str(prefs.get("dictation.model_id", "")))
if prefs.get("dictation.enabled", True)
else None
)
diarisation_choices = diarization_runtime.installed_backends()
tts_engine = active_tts()
asr_engine = asr_backend.active_backend_id()
translation_engine = str(prefs.get("translation_backend", "argos"))
active_engines = {
"tts": tts_engine,
"asr": asr_engine,
"translation": translation_engine,
"dictation": selected_dictation.kind if selected_dictation else "inactive",
}
supported_engines = {
"tts": {"omnivoice", "omnivoice-isolated"},
"asr": {"faster-whisper", "faster-whisper-isolated"},
"translation": {"nllb"},
"dictation": {"offline-transducer", "online-transducer"},
}
stored = prefs.get(_PERFORMANCE_PROFILE_KEY, {})
raw = stored if isinstance(stored, dict) else {}
global_tier = str(raw.get("global", "balanced")).lower()
if global_tier not in _PERFORMANCE_TIERS:
global_tier = "balanced"
overrides = {
str(family): str(tier)
for family, tier in (raw.items() if isinstance(raw, dict) else [])
if family in _PERFORMANCE_FAMILIES and tier in _PERFORMANCE_TIERS
}
effective = {
family: overrides.get(family, global_tier) for family in _PERFORMANCE_FAMILIES
}
applicable_families = [
family
for family, engines in supported_engines.items()
if active_engines[family] in engines
]
for family in _installed_selectable_families():
if family not in applicable_families:
applicable_families.append(family)
if len(diarisation_choices) > 1:
applicable_families.append("diarisation")
selections = {
"tts": {
"engine": tts_engine,
# OmniVoice has one checkpoint family today; its performance tiers
# tune sampling rather than silently changing voice capabilities.
"model": "k2-fsa/OmniVoice"
if tts_engine in {"omnivoice", "omnivoice-isolated", "omnivoice-subprocess"}
else tts_engine,
},
"asr": {
"engine": asr_engine,
"model": asr_backend.faster_whisper_model_id()
if asr_engine in {"faster-whisper", "faster-whisper-isolated"}
else asr_engine,
},
"dictation": {
"engine": selected_dictation.kind if selected_dictation else "inactive",
"model": selected_dictation.id if selected_dictation else None,
"label": selected_dictation.label if selected_dictation else None,
},
"diarisation": {
"engine": diarization_runtime.selected_backend()
if diarisation_choices
else "inactive",
"model": (
diarization_runtime.SORTFORMER_REPO
if diarization_runtime.selected_backend() == diarization_runtime.SORTFORMER
else "pyannote/speaker-diarization-3.1"
)
if diarisation_choices
else None,
},
"translation": {
"engine": translation_engine,
"model": "facebook/nllb-200-distilled-600M"
if translation_engine == "nllb"
else translation_engine,
},
"llm": {"engine": "inactive", "model": None},
}
return {
"global": global_tier,
"overrides": overrides,
"effective": effective,
"tiers": list(_PERFORMANCE_TIERS),
"families": list(_PERFORMANCE_FAMILIES),
"implemented_families": list(_PERFORMANCE_TARGETS),
"applicable_families": applicable_families,
"targets": {
family: _PERFORMANCE_TARGETS[family][effective[family]]
for family in _PERFORMANCE_TARGETS
},
"selections": selections,
"downloads_started": False,
}
def requested_tier(family: str) -> str | None:
"""None preserves existing workflow defaults until a user picks a preset."""
from core import prefs
stored = prefs.get(_PERFORMANCE_PROFILE_KEY, {})
if not isinstance(stored, dict):
return None
tier = stored.get(family, stored.get("global"))
return tier if tier in _PERFORMANCE_TIERS else None
def activate_maximum_capacity_models(family: str | None = None) -> dict:
"""Select the strongest already-installed compatible local models.
This is intentionally download-free. Choosing Max is explicit permission to
change model selections, but model installation remains its own reviewable
action in the catalogue.
"""
return _activate_installed_models("max", family)
def activate_performance_tier(tier: str, family: str | None = None) -> dict:
"""Apply installed-only model/runtime selections implied by a preset."""
requested = set(_PERFORMANCE_FAMILIES if family is None else (family,))
activated = _activate_installed_models(tier, family)
if "diarisation" in requested and not os.environ.get(
"OMNIVOICE_DIARIZATION_BACKEND"
):
from services import diarization_runtime
installed = diarization_runtime.installed_backends()
if len(installed) > 1:
engine = _PERFORMANCE_TARGETS["diarisation"][tier]["engine"]
if engine in installed:
diarization_runtime.select_backend(engine)
if engine == diarization_runtime.SORTFORMER:
from services import model_manager
model_manager.unload_diarization_pipeline()
activated["diarisation"] = {"engine": engine}
return activated
def reconcile_active_profile() -> dict[str, dict]:
"""Reapply a persisted profile after installs or an app restart.
Older builds persisted the slider but selected models only for Max. That
left installed ASR/Dictation models stranded behind stale missing choices.
Reconciliation is startup-only, installed-only, and never downloads.
"""
from core import prefs
# The UI presents Balanced as the selected initial value, so the runtime
# must honor it even before the user changes the control for the first time.
stored = prefs.get(_PERFORMANCE_PROFILE_KEY, {})
if not isinstance(stored, dict):
return {}
global_tier = str(stored.get("global", "balanced")).lower()
if global_tier not in _PERFORMANCE_TIERS:
global_tier = "balanced"
activated: dict[str, dict] = {}
for family in _PERFORMANCE_TARGETS:
tier = str(stored.get(family, global_tier)).lower()
if tier not in _PERFORMANCE_TIERS:
tier = global_tier
activated.update(activate_performance_tier(tier, family))
return activated
def tts_defaults(engine: str = "omnivoice") -> dict:
"""Only map sampling controls verified for the selected engine family."""
tier = requested_tier("tts")
if tier is None or engine not in {"omnivoice", "omnivoice-isolated"}:
return {}
target = _PERFORMANCE_TARGETS["tts"][tier]
return {"num_step": target["steps"], "postprocess_output": target["postprocess"]}
def asr_decode_defaults() -> dict:
"""Bound Faster-Whisper's search effort without changing language coverage."""
tier = requested_tier("asr")
if tier is None:
return {}
target = _PERFORMANCE_TARGETS["asr"][tier]
return {"beam_size": target["beam_size"], "best_of": target["best_of"]}
def translation_decode_defaults() -> dict:
"""Adjust local NLLB search effort without changing the chosen provider."""
tier = requested_tier("translation")
if tier is None:
return {}
return {"num_beams": _PERFORMANCE_TARGETS["translation"][tier]["num_beams"]}
def dictation_decode_defaults() -> dict:
"""Tune Sherpa transducer search without changing the selected language model."""
tier = requested_tier("dictation")
if tier is None:
return {}
target = _PERFORMANCE_TARGETS["dictation"][tier]
return {
"decoding_method": target["decoding_method"],
"max_active_paths": target["max_active_paths"],
}
+78
View File
@@ -0,0 +1,78 @@
"""Safe extraction for remote multi-segment WAV results."""
from __future__ import annotations
import os
import re
import shutil
import zipfile
_MEMBER = re.compile(r"segments/(\d+)\.wav")
def extract_segment_wavs(artifact_path: str, target_dir: str) -> dict[int, str]:
"""Extract an exact ``segments/<index>.wav`` bundle atomically.
The worker controls the ZIP member names, so accept only the protocol's
flat numeric namespace. Streaming each member into a locally minted name
also avoids ZipFile.extract() path traversal and symlink behaviour.
"""
if not artifact_path or not os.path.isfile(artifact_path):
raise ValueError("the segment bundle is missing")
os.makedirs(target_dir, exist_ok=True)
paths: dict[int, str] = {}
partials: list[str] = []
try:
with zipfile.ZipFile(artifact_path) as archive:
for member in archive.infolist():
match = _MEMBER.fullmatch(member.filename)
if not match:
raise ValueError(
f"unexpected segment artifact member: {member.filename}"
)
index = int(match.group(1))
if index in paths:
raise ValueError(f"duplicate segment artifact index: {index}")
destination = os.path.join(target_dir, f"{index}.wav")
partial = f"{destination}.part"
partials.append(partial)
with archive.open(member) as source, open(partial, "wb") as output:
shutil.copyfileobj(source, output)
os.replace(partial, destination)
partials.remove(partial)
paths[index] = destination
if not paths:
raise ValueError("the segment bundle is empty")
return paths
except BaseException:
for path in (*partials, *paths.values()):
try:
os.unlink(path)
except FileNotFoundError:
pass
try:
os.rmdir(target_dir)
except OSError:
pass
raise
def remove_segment_wavs(paths: dict[int, str]) -> None:
"""Remove files minted by :func:`extract_segment_wavs`, then empty dirs."""
directories = set()
for path in paths.values():
directories.add(os.path.dirname(path))
try:
os.unlink(path)
except FileNotFoundError:
pass
for directory in sorted(directories, key=len, reverse=True):
try:
os.rmdir(directory)
except OSError:
pass
__all__ = ["extract_segment_wavs", "remove_segment_wavs"]
+6 -2
View File
@@ -411,6 +411,8 @@ def build_offline_recognizer(spec: SherpaModelSpec, *, download: bool = True):
return os.path.join(d, spec.files[role])
if spec.kind == "offline-transducer":
from services.performance_profiles import dictation_decode_defaults
return sherpa_onnx.OfflineRecognizer.from_transducer(
encoder=p("encoder"),
decoder=p("decoder"),
@@ -418,7 +420,7 @@ def build_offline_recognizer(spec: SherpaModelSpec, *, download: bool = True):
tokens=p("tokens"),
num_threads=_threads_for(spec),
provider=_PROVIDER,
decoding_method="greedy_search",
**dictation_decode_defaults(),
model_type=spec.model_type or "nemo_transducer",
)
if spec.kind == "offline-whisper":
@@ -450,6 +452,8 @@ def build_online_recognizer(spec: SherpaModelSpec, *, download: bool = True):
return os.path.join(d, spec.files[role])
if spec.kind == "online-transducer":
from services.performance_profiles import dictation_decode_defaults
return sherpa_onnx.OnlineRecognizer.from_transducer(
tokens=p("tokens"),
encoder=p("encoder"),
@@ -457,7 +461,7 @@ def build_online_recognizer(spec: SherpaModelSpec, *, download: bool = True):
joiner=p("joiner"),
num_threads=_threads_for(spec),
provider=_PROVIDER,
decoding_method="greedy_search",
**dictation_decode_defaults(),
enable_endpoint_detection=True,
rule1_min_trailing_silence=rule1,
rule2_min_trailing_silence=rule2,
+117
View File
@@ -71,6 +71,11 @@ TOL_HIGH = 1.08
# Max LLM attempts per segment. Past this we just return the best we got.
MAX_ATTEMPTS = 3
# Acceptance window for real TTS measurements. One rewrite is made between
# renders; repeating guesses inside a call would discard the useful evidence.
MEASURED_TOL_LOW = 0.9
MEASURED_TOL_HIGH = 1.04
def expected_duration(text: str, lang: str = "en") -> float:
"""Rough CPS-based duration estimate. Returns seconds."""
@@ -106,6 +111,118 @@ Reply with ONLY the new line. No quotes, no commentary."""
_MIN_EXPANDABLE_RATIO = 0.15
_MEASURED_PROMPT = """\
You are a dialogue adaptation agent for precise dubbing. Rewrite the translated
line so the SAME voice can speak it inside the exact target duration. The user
provides the duration measured from a real render, so use the requested length
change as a concrete constraint. Preserve meaning, tone, names, numbers,
technical terms, and the target language. Shorten natural phrasing when long;
gently expand only when short without inventing facts or dialogue.
Reply with ONLY the revised line. No quotes or commentary."""
def adjust_for_measured_slot(
text: str,
*,
slot_seconds: float,
measured_seconds: float,
target_lang: str,
source_text: Optional[str] = None,
context_before: Optional[str] = None,
context_after: Optional[str] = None,
) -> dict:
"""Make one evidence-based rewrite between real TTS measurements."""
text = (text or "").strip()
slot = max(0.0, float(slot_seconds or 0.0))
measured = max(0.0, float(measured_seconds or 0.0))
ratio = measured / slot if slot else 1.0
base = {
"text": text,
"measured_seconds": round(measured, 3),
"target_seconds": round(slot, 3),
"measured_ratio": round(ratio, 3),
"changed": False,
}
if not text or slot <= 0 or measured <= 0:
return {**base, "error": "invalid-timing"}
if MEASURED_TOL_LOW <= ratio <= MEASURED_TOL_HIGH:
return {**base, "error": "already-fits"}
# Leave honest silence for extremely short dialogue instead of inventing
# speech merely to fill a long shot.
if ratio < 0.45:
return {**base, "error": "fit-skip-short"}
from services import llm_skills
llm = llm_skills.skill_backend(_SKILL_ID, active=lambda: get_active_llm_backend())
if isinstance(llm, OffBackend):
return {**base, "error": "no-llm"}
desired = max(0.2, min(2.0, slot / measured))
user_lines = [
f"Target language: {target_lang}",
f"Exact target duration: {slot:.2f}s",
f"Measured duration of this line: {measured:.2f}s",
f"Measured ratio: {ratio:.3f} (1.000 is exact)",
f"Requested text-length factor: about {desired:.3f}x",
f"Current translated line: {text}",
]
if source_text:
user_lines.append(f"Source line (meaning authority): {source_text}")
if context_before:
user_lines.append(f"Previous source line (context only): {context_before}")
if context_after:
user_lines.append(f"Next source line (context only): {context_after}")
try:
reply = llm.chat(
system=_MEASURED_PROMPT,
user="\n".join(user_lines),
temperature=0.15,
)
except Exception:
logger.warning("measured slot-fit provider failed")
return {**base, "error": "fit-provider-failed"}
candidate = (reply or "").strip()
if not candidate or candidate == text:
return {**base, "error": "fit-unchanged"}
ok, reason = refine_output_ok(text, candidate, target_lang)
if not ok:
logger.warning("measured slot-fit reply rejected (%s)", log_safe(reason))
return {**base, "error": "fit-diverged"}
return {**base, "text": candidate, "changed": True}
async def adjust_for_measured_slot_many(
items: Iterable[tuple], *, executor=None, concurrency: Optional[int] = None,
) -> dict:
"""Run one bounded measured rewrite per segment, keyed by segment id."""
import asyncio
import os
rows = list(items)
if not rows:
return {}
loop = asyncio.get_running_loop()
sem = asyncio.Semaphore(concurrency or int(os.environ.get("OMNIVOICE_LLM_CONCURRENCY", "6")))
async def _one(key, line, slot, measured, lang, source, before, after):
async with sem:
result = await loop.run_in_executor(
executor,
lambda: adjust_for_measured_slot(
line,
slot_seconds=slot,
measured_seconds=measured,
target_lang=lang,
source_text=source,
context_before=before,
context_after=after,
),
)
return key, result
return dict(await asyncio.gather(*(_one(*row) for row in rows)))
def adjust_for_slot(
text: str,
*,
+10
View File
@@ -60,6 +60,11 @@ class SubprocessASRBackend(SubprocessBackend):
def generate(self, text: str, **kw): # pragma: no cover - unused
raise NotImplementedError("ASR sidecar does not synthesize speech")
def ensure_loaded(self) -> None:
"""Prove the lazy ASR sidecar is ready for the shared loader."""
with self._lock:
self._spawn()
# ── ASR surface ────────────────────────────────────────────────────────
@staticmethod
def _device() -> str:
@@ -109,13 +114,18 @@ class SubprocessASRBackend(SubprocessBackend):
raise TimeoutError("timed out waiting for a free GPU worker")
with self._lock:
self._spawn()
from services.performance_profiles import asr_decode_defaults
self._send({
"op": "transcribe",
"audio_path": str(audio_path),
"word_timestamps": bool(word_timestamps),
"decode_options": asr_decode_defaults(),
})
reply = self._recv_with_timeout(ASR_RECV_TIMEOUT_S)
if not reply:
# EOF can arrive before Windows updates poll(); retire the
# stale handle so an immediate retry respawns the sidecar.
self.shutdown()
# Pipe closed mid-transcription → the child crashed.
raise RuntimeError(
f"{self.id} ASR sidecar crashed mid-transcription "
+54 -6
View File
@@ -747,6 +747,9 @@ class SubprocessBackend(TTSBackend):
with self._lock:
self._validate_generate_authorization()
self._spawn()
# getattr keeps lightweight protocol-loop test doubles valid;
# real instances always initialise _proc in __init__.
proc = getattr(self, "_proc", None)
msg = {"op": "synthesize", "text": text}
# Filter kwargs to JSON-safe primitives. Tensor / Path / etc.
# don't survive json.dumps and are silently dropped — the
@@ -754,8 +757,15 @@ class SubprocessBackend(TTSBackend):
for k, v in kw.items():
if _is_jsonable(v):
msg[k] = v
self._send(msg)
reply = self._recv_with_timeout(self.recv_timeout_s)
try:
self._send(msg)
reply = self._recv_with_timeout(self.recv_timeout_s)
except (RuntimeError, OSError):
# A broken or malformed protocol stream cannot be reused.
# Reap it before releasing the request lock so an immediate
# retry cannot race poll() and write to the same dead pipe.
self._reap_unusable_process(proc)
raise
# A cold sidecar may emit non-terminal {"op": "progress"} frames
# (during a model load, etc.) before the terminal audio frame.
# Each recv re-arms the watchdog, so a long-but-active load
@@ -779,12 +789,27 @@ class SubprocessBackend(TTSBackend):
report_model_load_activity()
except Exception:
pass # the heartbeat is best-effort; never fail a synth over it
reply = self._recv_with_timeout(self.recv_timeout_s)
if not reply:
raise RuntimeError(f"{self.id} sidecar closed pipe mid-generate")
try:
reply = self._recv_with_timeout(self.recv_timeout_s)
except (RuntimeError, OSError):
self._reap_unusable_process(proc)
raise
if not reply:
self._reap_unusable_process(proc)
raise RuntimeError(f"{self.id} sidecar closed pipe mid-generate")
if reply.get("op") == "error":
stage = str(reply.get("stage") or "unknown")
message = str(reply.get("message") or "unknown sidecar error")
traceback_text = str(reply.get("traceback") or "").strip()
logger.error(
"[%s] sidecar %s error: %s%s",
self.id,
stage,
message,
f"\n{traceback_text[:20_000]}" if traceback_text else "",
)
raise RuntimeError(
f"{self.id} sidecar error: {reply.get('message')!r}"
f"{self.id} sidecar {stage} error: {message}"
)
if reply.get("op") != "audio":
raise RuntimeError(
@@ -909,6 +934,29 @@ class SubprocessBackend(TTSBackend):
item for item in self._timeout_quarantine if item is not proc
]
def _reap_unusable_process(self, proc: Optional[subprocess.Popen]) -> None:
"""Synchronously retire a sidecar whose protocol pipe is unusable."""
if proc is None:
return
try:
proc.wait(timeout=0.25)
except Exception:
try:
proc.kill()
except Exception:
pass
try:
proc.wait(timeout=2)
except Exception:
with self._timeout_quarantine_lock:
if not any(item is proc for item in self._timeout_quarantine):
self._timeout_quarantine.append(proc)
return
with self._timeout_quarantine_lock:
self._timeout_quarantine = [
item for item in self._timeout_quarantine if item is not proc
]
def _retry_timeout_cleanup(self) -> bool:
"""Retry bounded cleanup, retaining every owner that could still be live."""
with self._timeout_quarantine_lock:
+131 -9
View File
@@ -20,12 +20,18 @@ import logging
import functools
import re
import os
import re
import shutil
import subprocess
import sys
import threading
logger = logging.getLogger("omnivoice.translation_engines")
_NLLB_REPO_ID = "facebook/nllb-200-distilled-600M"
_ARGOS_INSTALL_LOCK = threading.Lock()
_ARGOS_LANG_ALIASES = {"cmn": "zh"}
# Engine ID → registry entry. Keyed by the `provider` string sent from the
# frontend (must match the values of `translateProvider` in the store).
@@ -38,7 +44,7 @@ REGISTRY: dict[str, dict] = {
"category": "offline",
"needs_key": False,
"builtin": True,
"notes": "Pure-CPU offline translator. Downloads a ~50MB language pack on first use per pair.",
"notes": "Pure-CPU offline translator. Install the required language pack explicitly for each pair.",
},
"nllb": {
"id": "nllb",
@@ -115,13 +121,25 @@ def is_frozen() -> bool:
return bool(getattr(sys, "frozen", False) or os.environ.get("OMNIVOICE_FROZEN"))
def _probe(entry: dict) -> tuple[bool, str]:
def _probe(entry: dict) -> tuple[bool, str | None]:
mod = entry.get("probe_module")
if not mod:
return True, "no module required"
return True, None
try:
importlib.import_module(mod)
return True, "ready"
if entry.get("id") == "nllb":
# Transformers being importable only proves the runtime exists.
# The weights are a separate explicit model install; do not report
# NLLB ready and let from_pretrained download 2.4 GB silently.
from api.routers.setup.models import cache_is_complete, is_cached
model = {"repo_id": _NLLB_REPO_ID}
if not is_cached(_NLLB_REPO_ID) or not cache_is_complete(model):
return False, "NLLB model weights are not installed"
# availability_reason is failure-only metadata. Returning a success
# label here made every healthy provider look unavailable after the
# public diagnostic scrubber intentionally replaced non-null details.
return True, None
except ImportError as e:
return False, f"import {mod!r} failed: {e}"
@@ -164,23 +182,38 @@ def _llm_configured() -> tuple[bool, "str | None"]:
return False, None
def _configured(entry: dict) -> tuple[bool, str | None]:
"""Whether an installed engine has the configuration needed to run."""
engine_id = entry.get("id")
if engine_id == "openai":
return _llm_configured()
if engine_id == "deepl":
return bool(os.environ.get("DEEPL_API_KEY") or os.environ.get("TRANSLATE_API_KEY")), None
if engine_id == "microsoft":
return bool(os.environ.get("MICROSOFT_API_KEY") or os.environ.get("TRANSLATE_API_KEY")), None
return True, None
def list_engines() -> list[dict]:
"""Return a UI-ready list with per-engine availability stamped in."""
out = []
for e in REGISTRY.values():
installed, reason = _probe(e)
configured, via = _configured(e)
ready = installed and configured
entry = {
**e,
"installed": installed,
"availability_reason": reason,
"configured": configured,
"configured_via": via,
"ready": ready,
"availability_reason": reason or (
None if configured else "Translation provider is not configured"
),
"install_command": install_command(e),
}
# LLM engines additionally need a provider/key — surface configured-ness
# so the UI can distinguish "importable" from "actually ready to call".
if e.get("category") == "llm":
configured, via = _llm_configured()
entry["configured"] = configured
entry["configured_via"] = via
out.append(entry)
return out
@@ -256,6 +289,95 @@ def is_installed(engine_id: str) -> bool:
return ok
def is_ready(engine_id: str) -> bool:
"""True only when both runtime/model and required configuration exist."""
entry = REGISTRY.get(engine_id)
if not entry:
return False
installed, _ = _probe(entry)
configured, _ = _configured(entry)
return installed and configured
def argos_lang_code(value: str) -> str:
"""Return the base language token used by Argos package metadata."""
code = str(value or "").strip().lower().split("-", 1)[0]
code = _ARGOS_LANG_ALIASES.get(code, code)
if not re.fullmatch(r"[a-z]{2,3}", code):
raise ValueError("Choose a valid source and target language")
return code
def _configure_argos_cache() -> None:
cache_dir = os.environ.get("OMNIVOICE_CACHE_DIR")
if not cache_dir:
return
argos_cache = os.path.join(cache_dir, "argos-translate")
os.makedirs(argos_cache, exist_ok=True)
os.environ.setdefault("ARGOS_PACKAGES_DIR", argos_cache)
os.environ.setdefault("ARGOS_DATA_DIR", argos_cache)
def argos_pack_status(source_lang: str, target_langs: list[str]) -> dict:
"""Report installed Argos pairs without refreshing the remote index."""
_configure_argos_cache()
import argostranslate.package
source = argos_lang_code(source_lang)
targets = list(dict.fromkeys(argos_lang_code(code) for code in target_langs))
installed = {
(package.from_code, package.to_code)
for package in argostranslate.package.get_installed_packages()
}
return {
"source_lang": source,
"pairs": [
{
"source_lang": source,
"target_lang": target,
"installed": source == target or (source, target) in installed,
}
for target in targets
],
}
def install_argos_packs(source_lang: str, target_langs: list[str]) -> dict:
"""Explicitly download and install the requested Argos language pairs."""
_configure_argos_cache()
import argostranslate.package
source = argos_lang_code(source_lang)
targets = list(dict.fromkeys(argos_lang_code(code) for code in target_langs))
with _ARGOS_INSTALL_LOCK:
status = argos_pack_status(source, targets)
missing = {
pair["target_lang"]
for pair in status["pairs"]
if not pair["installed"]
}
if missing:
argostranslate.package.update_package_index()
available = argostranslate.package.get_available_packages()
for target in targets:
if target not in missing:
continue
package = next(
(
item
for item in available
if item.from_code == source and item.to_code == target
),
None,
)
if package is None:
raise ValueError(
f"No Argos language pack is available for {source}{target}"
)
argostranslate.package.install_from_path(package.download())
return argos_pack_status(source, targets)
def _in_virtualenv() -> bool:
"""True if the current interpreter is inside a venv/virtualenv."""
return getattr(sys, "base_prefix", sys.prefix) != sys.prefix or hasattr(sys, "real_prefix")
+77 -1
View File
@@ -542,6 +542,26 @@ def _prompt_disk_load(key: tuple):
return None
def _prompt_cache_evict(key: tuple) -> None:
"""Discard one prompt from both cache layers. Never raises.
Transcript-free prompts use a different identity from fully conditioned
prompts. Once ASR resolves the transcript, the former must not remain as a
viable stale fallback for the same reference clip.
"""
with _prompt_cache_lock:
_prompt_cache.pop(key, None)
cache_dir = _prompt_disk_dir()
if cache_dir is None:
return
try:
os.remove(_prompt_disk_path(cache_dir, key))
except FileNotFoundError:
pass
except OSError as exc:
logger.debug("could not evict stale voice prompt: %s", exc)
def _prompt_disk_save(key: tuple, prompt) -> None:
"""Persist ``prompt`` under ``key`` and prune old entries. Never raises."""
cache_dir = _prompt_disk_dir()
@@ -605,6 +625,29 @@ def _get_clone_prompt(
Every short segment falling back to its speaker ref then re-encodes it
(~0.4 s each, measured). Scan-resistance, not a second cache policy.
"""
# Resolve transcript-free references through an already-installed ASR
# before deriving the cache key. This protects every native OmniVoice
# caller (generate, streaming, batch, dub, audiobook and OpenAI-compatible
# speech), including routes that do not have a profile row on which to
# persist the transcript. Incomplete reference conditioning can destabilize
# the reference/target boundary and introduce words in the generated prefix.
unresolved_key = None
if ref_audio and not ref_text:
try:
unresolved_key = _clone_prompt_key(
ref_audio, None, preprocess_prompt
)
except Exception:
pass
try:
from services.asr_backend import transcribe_reference
ref_text = transcribe_reference(ref_audio)
except Exception as e: # noqa: BLE001 — model fallback remains available
logger.warning("reference transcript resolution failed: %s", e)
if ref_text and unresolved_key is not None:
_prompt_cache_evict(unresolved_key)
try:
key = _clone_prompt_key(ref_audio, ref_text, preprocess_prompt)
except Exception:
@@ -755,6 +798,27 @@ class OmniVoiceBackend(TTSBackend):
# model_manager so memory isn't doubled.
self._model = model
@property
def execution_device(self) -> str | None:
"""Actual device of the shared model, for live engine diagnostics."""
if self._model is None:
return None
try:
return str(next(self._model.parameters()).device)
except Exception: # noqa: BLE001 - third-party model wrappers vary
device = getattr(self._model, "device", None)
return str(device) if device is not None else None
@property
def dtype(self) -> str | None:
"""Actual parameter precision of the shared model when resident."""
if self._model is None:
return None
try:
return str(next(self._model.parameters()).dtype)
except Exception: # noqa: BLE001 - diagnostics must remain best effort
return None
@classmethod
def is_available(cls) -> tuple[bool, str]:
try:
@@ -2371,7 +2435,7 @@ _INSTALL_HINTS: dict[str, str] = {
"moss-tts-v15": "git clone OpenMOSS/MOSS-TTS + set OMNIVOICE_MOSS_TTS_V15_DIR (own venv, transformers==5.0; 8B, ~16 GB weights; CUDA/ROCm/XPU/NPU/CPU, no MPS; Apache-2.0)",
"dots-tts": "git clone rednote-hilab/dots.tts + set OMNIVOICE_DOTS_TTS_DIR (own venv, transformers==4.57; 2B, ~9 GB weights; CUDA/CPU, Linux/macOS only — no Windows; Apache-2.0)",
"confucius4-tts":"git clone netease-youdao/Confucius4-TTS + set OMNIVOICE_CONFUCIUS4_TTS_DIR (own Python 3.10 venv; 14-lang cross-lingual zero-shot clone; ~5 GB weights auto-download; CUDA/ROCm/XPU/NPU/CPU, no MPS; Apache-2.0)",
"audiocpp": "download the matching audio.cpp v0.7.2 prebuilt + set OMNIVOICE_AUDIOCPP_BIN, then explicitly install Breeze-TTS-2 in the engine's Weights list in Model Catalogue (native CPU/Vulkan/CUDA/Metal GGUF server, no Python; en+zh clone+design; ~4.73 GiB; weights research/non-commercial only)",
"audiocpp": "download the matching audio.cpp v0.7.4 prebuilt + set OMNIVOICE_AUDIOCPP_BIN, then explicitly install Breeze-TTS-2 in Model Catalogue → Models (native CPU/Vulkan/CUDA/Metal GGUF server, no Python; en+zh clone+design; ~4.73 GiB; weights research/non-commercial only)",
}
@@ -2598,6 +2662,18 @@ def list_backends(*, include_hidden: bool = False) -> list[dict]:
loaded_instance = _active_instance
if loaded_instance is None:
loaded_instance = _ENGINE_INSTANCES.get(cls)
if loaded_instance is None and bid == "omnivoice":
# Startup preloads OmniVoice through model_manager directly, before
# any generation route needs an adapter instance. Reflect that
# shared resident model here instead of contradicting
# /model/loaded with a stale `not_loaded` engine state.
try:
from services import model_manager
if model_manager.model is not None:
loaded_instance = OmniVoiceBackend(model=model_manager.model)
except Exception: # noqa: BLE001 - catalogue reads never fail on diagnostics
pass
out.append({
"id": bid,
"display_name": cls.display_name,
+59 -10
View File
@@ -67,6 +67,7 @@ OMNI_MESSAGE = [0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1]
# 30 s bounds each call to tens of MB; the 16-bit message repeats throughout
# the audio, so per-chunk embedding/detection is equivalent.
_CHUNK_SECONDS = 30
_AUDIOSEAL_SAMPLE_RATE = 16_000
# AudioSeal vendors moshi's ``@torch_compile_lazy`` on SEANetEncoder.forward,
@@ -163,6 +164,49 @@ def _iter_chunks(audio: torch.Tensor, sample_rate: int):
yield audio[..., start:end]
def _audioseal_input(audio: torch.Tensor, sample_rate: int) -> torch.Tensor:
"""Return audio at the one rate supported by AudioSeal 0.2.
AudioSeal keeps a legacy ``sample_rate`` argument but deliberately ignores
it. Resampling here prevents 24/44.1/48 kHz audio from being interpreted as
16 kHz while keeping every model invocation on its native contract.
"""
if sample_rate == _AUDIOSEAL_SAMPLE_RATE:
return audio
from torchaudio.functional import resample
return resample(audio, sample_rate, _AUDIOSEAL_SAMPLE_RATE)
def _restore_watermark_rate(
original: torch.Tensor,
native_audio: torch.Tensor,
native_marked: torch.Tensor,
sample_rate: int,
) -> torch.Tensor:
"""Move only AudioSeal's residual back to the source rate.
A full 16 kHz round trip would unnecessarily band-limit generated speech.
Resampling the watermark residual preserves the original waveform and its
bandwidth while embedding the signal produced by AudioSeal.
"""
if sample_rate == _AUDIOSEAL_SAMPLE_RATE:
return native_marked
from torchaudio.functional import resample
residual = resample(
native_marked - native_audio,
_AUDIOSEAL_SAMPLE_RATE,
sample_rate,
)
target = original.shape[-1]
if residual.shape[-1] < target:
residual = torch.nn.functional.pad(residual, (0, target - residual.shape[-1]))
elif residual.shape[-1] > target:
residual = residual[..., :target]
return original + residual
def _check_available() -> bool:
"""Check if AudioSeal is installed and importable."""
global _audioseal_available
@@ -464,16 +508,18 @@ def embed_watermark(
else:
audio = waveform
# AudioSeal operates at 16kHz internally; it handles resampling, but
# we need to inform it of the source rate for correct embedding.
# AudioSeal 0.2 accepts a legacy sample_rate argument but ignores it.
# Run each bounded chunk at its native 16 kHz, then move only the
# watermark residual back so higher-rate speech keeps its bandwidth.
with _eager_audioseal():
watermarked = torch.cat(
[
generator(seg, sample_rate=sample_rate, message=msg)
for seg in _iter_chunks(audio, sample_rate)
],
dim=-1,
)
chunks = []
for seg in _iter_chunks(audio, sample_rate):
native = _audioseal_input(seg, sample_rate)
marked = generator(native, message=msg)
chunks.append(
_restore_watermark_rate(seg, native, marked, sample_rate)
)
watermarked = torch.cat(chunks, dim=-1)
# Restore original shape
if len(original_shape) == 2:
@@ -533,7 +579,10 @@ def detect_watermark(
best_conf, decoded_msg = -1.0, None
with _eager_audioseal():
for seg in _iter_chunks(audio, sample_rate):
result = detector.detect_watermark(seg, sample_rate=sample_rate, message_threshold=0.5)
result = detector.detect_watermark(
_audioseal_input(seg, sample_rate),
message_threshold=0.5,
)
seg_conf = float(result[0]) if isinstance(result, tuple) else 0.0
if seg_conf > best_conf:
best_conf = seg_conf
@@ -0,0 +1,39 @@
from services.dub_pipeline import _default_caption_languages
def test_caption_languages_include_manual_and_declared_source_tracks():
assert _default_caption_languages(
{
"language": "en",
"subtitles": {"fr": [{}]},
"automatic_captions": {"en": [{}], "en-orig": [{}], "es": [{}]},
}
) == ["en-orig", "fr"]
def test_caption_languages_prefer_matching_manual_source_over_automatic_duplicate():
assert _default_caption_languages(
{
"language": "en-US",
"subtitles": {"en": [{}], "fr": [{}]},
"automatic_captions": {"en-orig": [{}], "en-US": [{}]},
}
) == ["en", "fr"]
def test_caption_languages_recover_original_auto_track_without_language_metadata():
assert _default_caption_languages(
{
"automatic_captions": {
"de": [{}],
"ja-orig": [{}],
"es": [{}],
}
}
) == ["ja-orig"]
def test_caption_languages_do_not_guess_from_translated_automatic_tracks():
assert _default_caption_languages(
{"automatic_captions": {"de": [{}], "es": [{}]}}
) == []
@@ -73,6 +73,10 @@ while True:
sys.exit(0)
elif op == "synthesize":
t = m.get("text", "")
if t == "ERROR":
_send({"op": "error", "stage": "synthesize", "message": "bad model",
"traceback": "Traceback: useful child frame"})
continue
if t == "CRASH":
os._exit(137)
if t == "HANG":
@@ -534,6 +538,55 @@ def test_generate_does_not_deadlock_when_called_on_gpu_pool_worker(stub_sidecar,
b.shutdown()
def test_sidecar_traceback_is_preserved_in_backend_log(stub_sidecar, monkeypatch, caplog):
_use_stub(monkeypatch, stub_sidecar)
b = OmniVoiceSubprocessBackend()
try:
with caplog.at_level("ERROR"), pytest.raises(
RuntimeError, match="sidecar synthesize error: bad model"
):
b.generate("ERROR")
assert "Traceback: useful child frame" in caplog.text
finally:
b.shutdown()
def test_cached_model_load_emits_periodic_heartbeats(monkeypatch):
"""A slow cached MPS load has no HF progress events but is still alive."""
from engines.omnivoice_subprocess import main as sidecar
from services import model_manager
from utils import hf_progress
frames = []
class FakeTorch:
float16 = object()
class FakeOmniVoice:
@classmethod
def from_pretrained(cls, *_args, **_kwargs):
time.sleep(0.06)
return object()
monkeypatch.setattr(sidecar, "_model", None)
monkeypatch.setattr(sidecar, "_LOAD_HEARTBEAT_S", 0.01)
monkeypatch.setattr(sidecar, "_send", lambda _stream, frame: frames.append(frame))
monkeypatch.setattr(model_manager, "_lazy_torch", lambda: FakeTorch())
monkeypatch.setattr(model_manager, "_lazy_omnivoice", lambda: FakeOmniVoice)
monkeypatch.setattr(model_manager, "resolve_omnivoice_checkpoint", lambda: "cached")
monkeypatch.setattr(model_manager, "get_best_device", lambda: "mps")
monkeypatch.setattr(model_manager, "should_preload_tts_asr", lambda: False)
monkeypatch.setattr(hf_progress, "register_listener", lambda _listener: 1)
monkeypatch.setattr(hf_progress, "unregister_listener", lambda _listener_id: None)
sidecar._load_model(object())
loading = [frame for frame in frames if frame.get("stage") == "loading_model"]
assert loading[0]["percent"] == 0
assert loading[-1]["percent"] == 100
assert len(loading) >= 3, "cached model load went silent between 0% and 100%"
def test_sidecar_forwards_native_controls_and_applies_seed(monkeypatch):
import torch
from engines.omnivoice_subprocess import main as sidecar
+36 -1
View File
@@ -284,17 +284,40 @@ def test_touch_activity_without_ownership_never_writes(sentinel_env):
assert not os.path.exists(run_sentinel.SENTINEL_PATH)
def test_idle_shell_exit_is_retained_without_becoming_a_user_warning(sentinel_env):
record = {
"last_activity": None,
"log_tail": [
"INFO VoiceStudio model loaded successfully.",
"INFO Preload complete - model ready.",
],
}
assert run_sentinel.warrants_user_notice(record) is False
def test_interrupted_work_or_fatal_startup_still_warrants_a_warning(sentinel_env):
assert run_sentinel.warrants_user_notice(
{"last_activity": {"kind": "generate"}, "log_tail": []}
) is True
assert run_sentinel.warrants_user_notice(
{"last_activity": None, "log_tail": ["CRITICAL: native runtime failed"]}
) is True
# ── Record store semantics (mirrors crash.rs) ──────────────────────────────
def _crash_once(kind="generate"):
last_activity = None
if kind is not None:
last_activity = {"ts": time.time() - 5, "kind": kind, "detail": None}
with open(run_sentinel.SENTINEL_PATH, "w", encoding="utf-8") as f:
json.dump(
{
"pid": _dead_pid(),
"started_at": time.time() - 60,
"version": run_sentinel.APP_VERSION,
"last_activity": {"ts": time.time() - 5, "kind": kind, "detail": None},
"last_activity": last_activity,
},
f,
)
@@ -421,3 +444,15 @@ def test_notification_surfaces_unacked_crash_and_reack(client):
fresh = [n for n in notes if n["id"].startswith("last-run-crash-")]
assert len(fresh) == 1
assert fresh[0]["id"] != crash_notes[0]["id"]
def test_notification_keeps_idle_exit_forensics_without_repeated_warning(client):
record = _crash_once(kind=None)
assert record is not None
notes = client.get("/system/notifications").json()["notifications"]
assert not [n for n in notes if n["id"].startswith("last-run-crash-")]
details = client.get("/system/last-run-crash").json()
assert details["record"]["detected_at"] == record["detected_at"]
assert details["acknowledged"] is False
+76 -11
View File
@@ -21,6 +21,9 @@ divergent notion of what a worker can do.
from __future__ import annotations
import logging
import os
import shutil
import subprocess
from typing import Optional
from worker.capacity import derive_concurrency
@@ -33,8 +36,66 @@ _CPU_ONLY = {"cpu"}
def _free_memory_bytes(caps) -> int:
vram_gb = float(getattr(caps, "vram_gb", 0) or 0)
return int(vram_gb * 1024**3)
return _accelerator_memory_bytes(caps)[0]
def _accelerator_memory_bytes(caps) -> tuple[int, int]:
"""Return live free/total accelerator memory, falling back to static VRAM."""
fallback = int(float(getattr(caps, "vram_gb", 0) or 0) * 1024**3)
family = getattr(caps, "family", "") or ""
if family not in {"cuda", "rocm"}:
return fallback, fallback
try:
import torch # noqa: PLC0415
free_bytes, total_bytes = torch.cuda.mem_get_info()
return max(0, int(free_bytes)), max(0, int(total_bytes))
except Exception:
logger.debug("Live accelerator memory probe failed", exc_info=True)
return fallback, fallback
def _nvidia_driver_version() -> str:
executable = shutil.which("nvidia-smi")
if not executable and os.path.isfile("/usr/lib/wsl/lib/nvidia-smi"):
executable = "/usr/lib/wsl/lib/nvidia-smi"
try:
result = subprocess.run(
[
executable or "nvidia-smi",
"--query-gpu=driver_version",
"--format=csv,noheader,nounits",
],
capture_output=True,
text=True,
timeout=2,
check=False,
)
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
return ""
if result.returncode != 0:
return ""
return next((line.strip() for line in result.stdout.splitlines() if line.strip()), "")
def _accelerator_details(caps) -> tuple[str, str]:
"""Return driver and architecture details without making registration brittle."""
family = getattr(caps, "family", "") or ""
driver = str(getattr(caps, "driver", "") or "")
compute = ""
try:
import torch # noqa: PLC0415
if family == "cuda":
major, minor = torch.cuda.get_device_capability(0)
compute = f"{major}.{minor}"
elif family == "rocm":
compute = str(getattr(torch.cuda.get_device_properties(0), "gcnArchName", "") or "")
except Exception:
logger.debug("Accelerator architecture probe failed", exc_info=True)
if family == "cuda" and not driver:
driver = _nvidia_driver_version()
return driver, compute
def discover(*, include_unavailable: bool = False) -> list[dict]:
@@ -228,17 +289,18 @@ def model_id_for(entry: dict) -> str:
def _operations_for(entry: dict) -> list[str]:
"""Which task kinds this engine can serve.
Cloning is the one genuine split an engine that cannot clone must never
be handed a clone task, and ``supports_cloning`` is ``None`` when the
answer depends on the loaded model, which we treat as "no" rather than
risk a task that fails at the last moment.
Cloning is the one genuine split: an engine that cannot clone must never
be handed a clone or Dubbing task. ``supports_cloning`` is ``None`` when
the answer depends on the loaded model; that is treated as "no" instead of
risking a task that fails at the last moment.
"""
# Audiobook chapters use the same TTS engine, but are advertised as their
# own schedulable operation so an older worker cannot accept a task whose
# chapter assembler it does not implement.
operations = ["audiobook", "dub_segments", "tts"]
operations = ["audiobook", "batch_segments", "tts"]
if entry.get("supports_cloning") is True:
operations.append("clone")
operations.extend(("clone", "dub_segments"))
return operations
@@ -276,14 +338,17 @@ def describe_gpus() -> list[dict]:
if caps is None:
return []
family = getattr(caps, "family", "") or ""
free_bytes, total_bytes = _accelerator_memory_bytes(caps)
driver, compute = _accelerator_details(caps)
return [
{
"vendor": _vendor_for(family),
"model": getattr(caps, "device_name", "") or "",
"backend": family,
"memory_bytes": _free_memory_bytes(caps),
"free_memory_bytes": _free_memory_bytes(caps),
"driver_version": getattr(caps, "driver", "") or "",
"memory_bytes": total_bytes,
"free_memory_bytes": free_bytes,
"driver_version": driver,
"compute_capability": compute,
}
]
+2
View File
@@ -70,6 +70,7 @@ class Operation(str, enum.Enum):
CLONE = "clone"
ASR = "asr"
DUB = "dub"
BATCH_SEGMENTS = "batch_segments"
AUDIOBOOK = "audiobook"
@classmethod
@@ -90,6 +91,7 @@ _PROFILE: dict[Operation, tuple[float, int]] = {
Operation.TTS: (1.0, 75),
Operation.CLONE: (2.0, 75),
Operation.DUB: (12.0, 90),
Operation.BATCH_SEGMENTS: (12.0, 90),
Operation.AUDIOBOOK: (24.0, 90),
}
+22 -1
View File
@@ -188,7 +188,28 @@ def from_reason(reason: str, *, code: Optional[str] = None) -> WorkerError:
def from_exception(exc: BaseException, *, code: Optional[str] = None) -> WorkerError:
return from_reason(failure.describe_exception(exc), code=code)
reason = failure.describe_exception(exc)
if code is None and _is_invalid_generation_input(reason):
# OmniVoice validates its closed voice-direction vocabulary inside
# inference. The local route already turns these signatures into a
# 400; a worker used to call the same deterministic ValueError
# UNKNOWN/TRANSIENT and spend the task's retry budget on identical
# renders. Keep the message (it contains the accepted vocabulary),
# but stop the fleet after the first attempt.
code = "INVALID_TASK_PARAMS"
return from_reason(reason, code=code)
def _is_invalid_generation_input(reason: str) -> bool:
low = (reason or "").lower()
return any(
signature in low
for signature in (
"unsupported instruct items",
"conflicting instruct items",
"in a single instruct",
)
)
def _hint_for(taxonomy_key: str) -> str:
+146 -12
View File
@@ -135,6 +135,7 @@ class TaskExecutor:
"tts": self._run_tts,
"clone": self._run_tts,
"audiobook": self._run_audiobook,
"batch_segments": self._run_dub_segments,
"dub_segments": self._run_dub_segments,
}.get(operation)
if handler is None:
@@ -176,20 +177,95 @@ class TaskExecutor:
)
await report.loading(1.0, "model ready")
rendered: list[tuple[int, bytes]] = []
for index, row in enumerate(rows):
row = dict(row)
prepared_rows = []
for index, source in enumerate(rows):
row = dict(source)
row["ref_audio"] = refs[index] if index < len(refs) else None
audio = await self._bounded_thread(
self._synthesize_dub_segment,
backend,
row,
timeout=run_budget, code="EXECUTION_TIMEOUT", what=f"Dubbing segment {index + 1}",
prepared_rows.append(row)
# Remote dubbing used to hold one coarse GPU lease but still call the
# engine once per line. That bypassed the same native variable-length
# batch path used by local dubbing, leaving large GPUs mostly idle.
# Group only rows whose scalar generation contract is compatible;
# language, reference, duration, speed and instruct remain per-row.
try:
from services.dub_batching import batch_timeout_s, native_batch_width
from services.tts_backend import TTSBackend
has_native_batch = (
getattr(type(backend), "generate_batch", TTSBackend.generate_batch)
is not TTSBackend.generate_batch
)
payload, _meta = await self._thread_call(
self._encode, audio, row, backend
batch_width = native_batch_width(backend) if has_native_batch else 1
except Exception: # noqa: BLE001 - capability probing takes the safe path
batch_timeout_s = None
batch_width = 1
index = 0
while index < len(prepared_rows):
row = prepared_rows[index]
batch = [row]
if batch_width > 1 and row.get("seed") is None:
compatibility = self._dub_batch_compatibility(row)
for candidate in prepared_rows[index + 1 : index + batch_width]:
if (
candidate.get("seed") is not None
or self._dub_batch_compatibility(candidate) != compatibility
):
break
batch.append(candidate)
audios = None
if len(batch) > 1:
try:
timeout = (
batch_timeout_s([str(item.get("text") or "") for item in batch], backend)
if batch_timeout_s is not None
else run_budget
)
audios = await self._bounded_thread(
self._synthesize_dub_batch,
backend,
batch,
timeout=min(run_budget, timeout),
code="EXECUTION_TIMEOUT",
what=f"Dubbing segments {index + 1}-{index + len(batch)}",
)
except TaskFailure:
raise
except Exception as exc: # native batching is an optimization
logger.warning(
"Native remote dub batch failed for segments %s-%s; falling back: %s",
index + 1,
index + len(batch),
exc,
)
if audios is None:
batch = [row]
audios = [
await self._bounded_thread(
self._synthesize_dub_segment,
backend,
row,
timeout=run_budget,
code="EXECUTION_TIMEOUT",
what=f"Dubbing segment {index + 1}",
)
]
encoded = await asyncio.gather(
*(self._thread_call(self._encode, audio, item, backend)
for audio, item in zip(audios, batch))
)
rendered.append((int(row.get("index", index)), payload))
await report.progress((index + 1) / len(rows), f"segment {index + 1} of {len(rows)}")
for offset, (item, (payload, _meta)) in enumerate(zip(batch, encoded), 1):
rendered.append((int(item.get("index", index + offset - 1)), payload))
completed = index + offset
await report.progress(
completed / len(prepared_rows),
f"segment {completed} of {len(prepared_rows)}",
)
index += len(batch)
bundle = io.BytesIO()
with zipfile.ZipFile(bundle, "w", compression=zipfile.ZIP_STORED) as archive:
@@ -221,7 +297,7 @@ class TaskExecutor:
"num_step": int(row.get("num_step") or 16),
"guidance_scale": float(row.get("guidance_scale") or 2.0),
"speed": float(row.get("speed") or 1.0), "denoise": True,
"postprocess_output": True,
"postprocess_output": bool(row.get("postprocess_output", True)),
}
if (
getattr(backend, "supports_native_omnivoice_controls", False)
@@ -239,6 +315,64 @@ class TaskExecutor:
audio = normalize_audio(audio, target_dBFS=-2.0)
return audio
@staticmethod
def _dub_batch_compatibility(row: dict) -> tuple:
"""Scalar options that a native backend requires to match in a batch."""
return (
not bool(row.get("ref_single_use")),
bool(row.get("ref_audio")),
int(row.get("num_step") or 16),
float(row.get("guidance_scale") or 2.0),
bool(row.get("postprocess_output", True)),
)
@staticmethod
def _synthesize_dub_batch(backend, rows: list[dict]):
"""Worker-side equivalent of local dubbing's native batch path."""
from services.audio_dsp import (
apply_effects_chain,
apply_mastering,
get_effect_chain,
normalize_audio,
)
from services.text_normalization import normalize_for_tts
texts = [normalize_for_tts(row.get("text") or "", row.get("language")) for row in rows]
outputs = backend.generate_batch(
texts,
language=[
row.get("language") if row.get("language") != "Auto" else None for row in rows
],
ref_audio=[row.get("ref_audio") for row in rows],
ref_text=[row.get("ref_text") for row in rows],
cache_ref=not bool(rows[0].get("ref_single_use")),
instruct=[row.get("instruct") or None for row in rows],
duration=[row.get("duration") for row in rows],
num_step=int(rows[0].get("num_step") or 16),
guidance_scale=float(rows[0].get("guidance_scale") or 2.0),
speed=[float(row.get("speed") or 1.0) for row in rows],
denoise=True,
postprocess_output=bool(rows[0].get("postprocess_output", True)),
)
if len(outputs) != len(rows):
raise RuntimeError(
f"native batch returned {len(outputs)} outputs for {len(rows)} segments"
)
rendered = []
for output, row in zip(outputs, rows):
preset = row.get("effect_preset") or "broadcast"
if preset != "raw":
if not getattr(backend, "applies_own_mastering", False):
output = apply_mastering(output, sample_rate=backend.sample_rate)
chain = get_effect_chain(preset)
if chain:
output = apply_effects_chain(
output, sample_rate=backend.sample_rate, chain=chain
)
output = normalize_audio(output, target_dBFS=-2.0)
rendered.append(output)
return rendered
# ── Operations ────────────────────────────────────────────────────────
async def _run_tts(self, assignment, params: dict, report: "_Reporters") -> dict:
+17 -4
View File
@@ -91,7 +91,17 @@ _TASK_TRANSITIONS: dict[TaskState, frozenset[TaskState]] = {
}
),
TaskState.MODEL_LOADING: frozenset(
{TaskState.RUNNING, TaskState.QUEUED, TaskState.CANCELLED, TaskState.TIMEOUT, TaskState.FAILED}
{
TaskState.RUNNING,
# A worker reports ``started`` before loading. If the executor's
# final phase report is model loading, a completed render can move
# directly into bulk result delivery without another started frame.
TaskState.RESULT_UPLOADING,
TaskState.QUEUED,
TaskState.CANCELLED,
TaskState.TIMEOUT,
TaskState.FAILED,
}
),
TaskState.RUNNING: frozenset(
{
@@ -406,12 +416,15 @@ class Task:
raise LifecycleError("stale session epoch")
if attempt.state.terminal:
raise LifecycleError(f"attempt {attempt_id} already terminal ({attempt.state.value})")
if new is not attempt.state:
attempt.phase_started_at = resolve(now)
attempt.state = new
implied = _ATTEMPT_TO_TASK.get(new)
if implied is not None:
self._set_state(implied, now=now)
# Mutate the attempt only after the task transition succeeds. Otherwise
# one rejected transition leaves the pair contradictory (for example
# task=model_loading with attempt=uploading) and every retry is refused.
if new is not attempt.state:
attempt.phase_started_at = resolve(now)
attempt.state = new
return attempt
def accept(self, attempt_id: str, **kw) -> Attempt:
File diff suppressed because one or more lines are too long
+12 -2
View File
@@ -480,6 +480,14 @@ class PrewarmRequest(_message.Message):
download_if_missing: bool
def __init__(self, envelope: _Optional[_Union[Envelope, _Mapping]] = ..., engine: _Optional[str] = ..., model_id: _Optional[str] = ..., download_if_missing: _Optional[bool] = ...) -> None: ...
class ModelInstallCancelRequest(_message.Message):
__slots__ = ("envelope", "model_id")
ENVELOPE_FIELD_NUMBER: _ClassVar[int]
MODEL_ID_FIELD_NUMBER: _ClassVar[int]
envelope: Envelope
model_id: str
def __init__(self, envelope: _Optional[_Union[Envelope, _Mapping]] = ..., model_id: _Optional[str] = ...) -> None: ...
class Ping(_message.Message):
__slots__ = ("envelope", "nonce")
ENVELOPE_FIELD_NUMBER: _ClassVar[int]
@@ -507,7 +515,7 @@ class Shutdown(_message.Message):
def __init__(self, envelope: _Optional[_Union[Envelope, _Mapping]] = ..., reason: _Optional[str] = ...) -> None: ...
class ServerMessage(_message.Message):
__slots__ = ("assignment", "cancel", "result_ack", "config", "ping", "drain", "shutdown", "prewarm", "registered")
__slots__ = ("assignment", "cancel", "result_ack", "config", "ping", "drain", "shutdown", "prewarm", "registered", "model_install_cancel")
ASSIGNMENT_FIELD_NUMBER: _ClassVar[int]
CANCEL_FIELD_NUMBER: _ClassVar[int]
RESULT_ACK_FIELD_NUMBER: _ClassVar[int]
@@ -517,6 +525,7 @@ class ServerMessage(_message.Message):
SHUTDOWN_FIELD_NUMBER: _ClassVar[int]
PREWARM_FIELD_NUMBER: _ClassVar[int]
REGISTERED_FIELD_NUMBER: _ClassVar[int]
MODEL_INSTALL_CANCEL_FIELD_NUMBER: _ClassVar[int]
assignment: TaskAssignment
cancel: TaskCancel
result_ack: ResultAckMessage
@@ -526,7 +535,8 @@ class ServerMessage(_message.Message):
shutdown: Shutdown
prewarm: PrewarmRequest
registered: RegisterResponse
def __init__(self, assignment: _Optional[_Union[TaskAssignment, _Mapping]] = ..., cancel: _Optional[_Union[TaskCancel, _Mapping]] = ..., result_ack: _Optional[_Union[ResultAckMessage, _Mapping]] = ..., config: _Optional[_Union[ConfigUpdate, _Mapping]] = ..., ping: _Optional[_Union[Ping, _Mapping]] = ..., drain: _Optional[_Union[Drain, _Mapping]] = ..., shutdown: _Optional[_Union[Shutdown, _Mapping]] = ..., prewarm: _Optional[_Union[PrewarmRequest, _Mapping]] = ..., registered: _Optional[_Union[RegisterResponse, _Mapping]] = ...) -> None: ...
model_install_cancel: ModelInstallCancelRequest
def __init__(self, assignment: _Optional[_Union[TaskAssignment, _Mapping]] = ..., cancel: _Optional[_Union[TaskCancel, _Mapping]] = ..., result_ack: _Optional[_Union[ResultAckMessage, _Mapping]] = ..., config: _Optional[_Union[ConfigUpdate, _Mapping]] = ..., ping: _Optional[_Union[Ping, _Mapping]] = ..., drain: _Optional[_Union[Drain, _Mapping]] = ..., shutdown: _Optional[_Union[Shutdown, _Mapping]] = ..., prewarm: _Optional[_Union[PrewarmRequest, _Mapping]] = ..., registered: _Optional[_Union[RegisterResponse, _Mapping]] = ..., model_install_cancel: _Optional[_Union[ModelInstallCancelRequest, _Mapping]] = ...) -> None: ...
class ArtifactRef(_message.Message):
__slots__ = ("artifact_id", "task_id", "attempt_id", "filename", "content_type", "size_bytes", "sha256", "session_token")
+9
View File
@@ -423,6 +423,14 @@ message PrewarmRequest {
bool download_if_missing = 4;
}
// Cancel the explicit catalogue install attached to a pre-warm. The opaque
// model id is resolved against the worker's advertised capabilities; repository
// paths and arbitrary install targets never cross this boundary.
message ModelInstallCancelRequest {
Envelope envelope = 1;
string model_id = 2;
}
message Ping { Envelope envelope = 1; uint64 nonce = 2; }
// Fleet operations: stop taking work, finish what you have, then reconnect.
@@ -452,6 +460,7 @@ message ServerMessage {
// never in a frame: a credential in the stream would be copied into every
// protocol trace and every debug log that dumps one.
RegisterResponse registered = 9;
ModelInstallCancelRequest model_install_cancel = 10;
}
}
+19 -5
View File
@@ -42,8 +42,8 @@ producer for it — so ``decide(op=...)`` answers for the surface the user is
looking at rather than for the machine, and the badge cannot read
"gpu2 ● ready" on a tab whose work is 100% local.
Speech synthesis, chapter-at-a-time audiobook rendering, and coarse
``dub_segments`` synthesis have remote producers. Dub assembly, ASR,
Speech synthesis, chapter-at-a-time audiobook and Stories rendering, and
coarse ``dub_segments`` synthesis have remote producers. Dub assembly, ASR,
diarization, translation and RVC remain local. Dictation is
intentionally local regardless of the selected target because its latency is
the feature.
@@ -63,24 +63,38 @@ LOCAL = "local"
# Operations with a remote producer today. Ports land one at a time, and this
# set is what keeps the picker honest about which ones have arrived.
#
# `dub` is the surface the user picks; `dub_segments` is the coarse worker op
# it dispatches (dub_generate.py). Both belong here because the GPU work does
# `dub` / `batch` are the surfaces the user picks; `dub_segments` and
# `batch_segments` are their coarse worker ops. Both pairs belong here because the GPU work does
# leave this machine — listing only the worker op would make the Dub tab read
# "Local" while a remote card renders it.
#
# Dictation is deliberately absent and should stay that way: it runs ASR per
# utterance inside a live WebSocket loop, where a round trip per utterance
# would spend the one thing that route exists for.
REMOTE_OPERATIONS = frozenset({"audiobook", "dub", "dub_segments", "tts"})
REMOTE_OPERATIONS = frozenset(
{
"audiobook",
"batch",
"batch_segments",
"clone",
"dub",
"dub_segments",
"longform",
"tts",
}
)
# Only for the sentence the user reads; an unknown op falls back to its id
# rather than inventing a name for it.
_OP_LABELS = {
"tts": "speech synthesis",
"clone": "voice cloning",
"batch": "batch dubbing",
"batch_segments": "batch dubbing",
"dub": "dubbing",
"dub_segments": "dubbing",
"audiobook": "audiobook rendering",
"longform": "story rendering",
"dictation": "dictation",
"asr": "transcription",
}
+117 -1
View File
@@ -274,12 +274,19 @@ def describe_host() -> dict:
from core.version import APP_VERSION # noqa: PLC0415
except Exception:
APP_VERSION = ""
try:
import psutil # noqa: PLC0415
system_memory_bytes = int(psutil.virtual_memory().total)
except Exception:
system_memory_bytes = 0
return {
"hostname": socket.gethostname(),
"os": {"darwin": "darwin", "win32": "windows"}.get(sys.platform, "linux"),
"arch": platform.machine(),
"worker_version": APP_VERSION,
"cpu_count": os.cpu_count() or 0,
"system_memory_bytes": system_memory_bytes,
}
@@ -332,6 +339,8 @@ class WorkerClient:
self._running: dict[str, asyncio.Task] = {}
self._keepalives: dict[str, asyncio.Task] = {}
self._maintenance: set[asyncio.Task] = set()
self._prewarms: dict[str, asyncio.Task] = {}
self._prewarm_cancellations: dict[str, asyncio.Task] = {}
self._epoch = 0
self._session_token = ""
# Negotiated by ConfigUpdate; None means "use the executor's own
@@ -435,6 +444,8 @@ class WorkerClient:
*draining, return_exceptions=True
)
self._maintenance.clear()
self._prewarms.clear()
self._prewarm_cancellations.clear()
for key, task in running:
if self._running.get(key) is task:
self._running.pop(key, None)
@@ -801,16 +812,105 @@ class WorkerClient:
elif kind == "prewarm":
if not self._accepting_assignments:
return
model_id = message.prewarm.model_id
existing = self._prewarms.get(model_id)
if model_id and existing is not None and not existing.done():
return
task = asyncio.create_task(
self._on_prewarm(message.prewarm), name="worker-prewarm"
)
self._maintenance.add(task)
if model_id:
self._prewarms[model_id] = task
task.add_done_callback(self._maintenance_finished)
elif kind == "model_install_cancel":
await self._cancel_model_install(message.model_install_cancel)
def _maintenance_finished(self, task: asyncio.Task) -> None:
self._maintenance.discard(task)
for tasks in (self._prewarms, self._prewarm_cancellations):
for model_id, current in tuple(tasks.items()):
if current is task:
tasks.pop(model_id, None)
self._maybe_finish_drain()
async def _cancel_model_install(
self, request: pb.ModelInstallCancelRequest
) -> None:
"""Cancel one explicit catalogue install without blocking control I/O."""
model_id = request.model_id.strip()
capability = next(
(
cap
for cap in (self.config.capabilities or [])
if cap.get("model_id") == model_id
),
None,
)
repo_ids = list((capability or {}).get("repo_ids") or [])
if len(repo_ids) != 1:
logger.warning("Ignoring model cancellation for unknown model %s", model_id)
return
repo_id = repo_ids[0]
task = self._prewarms.get(model_id)
if task is None or task.done():
await self._send_model_install_terminal(
repo_id,
"install_done"
if bool((capability or {}).get("downloaded"))
else "install_cancelled",
)
return
existing = self._prewarm_cancellations.get(model_id)
if existing is not None and not existing.done():
return
task.cancel()
confirmation = asyncio.create_task(
self._confirm_model_install_cancel(task, repo_id),
name="worker-model-install-cancel",
)
self._maintenance.add(confirmation)
self._prewarm_cancellations[model_id] = confirmation
confirmation.add_done_callback(self._maintenance_finished)
async def _confirm_model_install_cancel(
self, task: asyncio.Task, repo_id: str
) -> None:
await asyncio.gather(task, return_exceptions=True)
capability = next(
(
cap
for cap in (self.config.capabilities or [])
if repo_id in (cap.get("repo_ids") or [])
),
None,
)
await self._send_model_install_terminal(
repo_id,
"install_done"
if bool((capability or {}).get("downloaded"))
else "install_cancelled",
)
async def _send_model_install_terminal(self, repo_id: str, phase: str) -> None:
event = {
"repo_id": repo_id,
"filename": repo_id,
"downloaded": 0,
"total": 0,
"pct": 0.0,
"phase": phase,
}
await self._send(
pb.WorkerMessage(
download_progress=pb.DownloadProgress(
event_json=json.dumps(
event, separators=(",", ":"), ensure_ascii=False
)
)
)
)
def _maybe_finish_drain(self) -> None:
if (
self._draining
@@ -906,6 +1006,21 @@ class WorkerClient:
async def _on_assignment(self, assignment: pb.TaskAssignment) -> None:
key = self._key(assignment.ref)
# Assignment delivery is at-least-once. A reconnect or a control-stream
# retry may repeat the exact same attempt while it is still running or
# waiting for its result acknowledgement. Treating that repeat as a
# capacity rejection terminalizes the original attempt underneath its
# result upload; starting it again spends the GPU twice. Reaffirm the
# live claim, or redeliver the result we already hold.
if key in self._running:
await self._send(
pb.WorkerMessage(accepted=pb.TaskAccepted(ref=assignment.ref))
)
return
pending = self._pending.get(key)
if pending is not None:
await self._send(_result_message(pending), bulk=True)
return
if not self._accepting_assignments or self._stop.is_set():
await self._send(
pb.WorkerMessage(
@@ -1164,7 +1279,8 @@ class WorkerClient:
break
resumed = int(ack.bytes_received)
if ack.error.code and ack.error.code != "OFFSET_MISMATCH":
raise RuntimeError(ack.error.message or "the control plane refused the upload")
detail = ack.error.message or "the control plane refused the upload"
raise RuntimeError(f"{ack.error.code}: {detail}")
if resumed < 0 or resumed > len(payload) or resumed == offset:
raise RuntimeError(ack.error.message or "the control plane could not resume the upload")
offset = resumed
+68 -4
View File
@@ -71,6 +71,7 @@ REQUIRED_FEATURES = frozenset({
"task_progress_v1",
"task_inputs_v1",
"remote_model_download_v1",
"remote_model_cancel_v1",
# A generic backend.generate() call accepts the same wire shape but drops
# profile conditioning controls. Require the canonical worker render path
# so an older peer cannot successfully return a different voice.
@@ -456,10 +457,22 @@ class _Upload:
)
await to_thread_and_drain_on_cancel(_write_all, self._handle, data)
if self._discarded or self.session.revoked or self.attempt.state.terminal:
logger.warning(
"Refusing result upload for task %s attempt %s "
"(attempt=%s, session_revoked=%s, discarded=%s, error=%s)",
self.attempt.task_id,
self.attempt.attempt_id,
self.attempt.state.value,
self.session.revoked,
self._discarded,
getattr(self.attempt.error, "code", None),
)
await self.discard_async()
return _upload_refused(
"ATTEMPT_NOT_LIVE",
"This attempt stopped accepting a result during upload.",
"This attempt stopped accepting a result during upload "
f"(attempt={self.attempt.state.value}, "
f"error={getattr(self.attempt.error, 'code', None)}).",
error_class=pb.ERROR_CLASS_TRANSIENT,
)
self._digest.update(data)
@@ -486,10 +499,22 @@ class _Upload:
error_class=pb.ERROR_CLASS_TRANSIENT,
)
if self._discarded or self.session.revoked or self.attempt.state.terminal:
logger.warning(
"Refusing result commit for task %s attempt %s "
"(attempt=%s, session_revoked=%s, discarded=%s, error=%s)",
self.attempt.task_id,
self.attempt.attempt_id,
self.attempt.state.value,
self.session.revoked,
self._discarded,
getattr(self.attempt.error, "code", None),
)
await self.discard_async()
return _upload_refused(
"ATTEMPT_NOT_LIVE",
"This attempt is no longer accepting a result.",
"This attempt is no longer accepting a result "
f"(attempt={self.attempt.state.value}, "
f"error={getattr(self.attempt.error, 'code', None)}).",
error_class=pb.ERROR_CLASS_TRANSIENT,
)
try:
@@ -509,6 +534,16 @@ class _Upload:
# running in its thread. It must win before the commit callback spends
# budget or this RPC licenses the worker to forget its only copy.
if self._discarded or self.session.revoked or self.attempt.state.terminal:
logger.warning(
"Refusing result after durable write for task %s attempt %s "
"(attempt=%s, session_revoked=%s, discarded=%s, error=%s)",
self.attempt.task_id,
self.attempt.attempt_id,
self.attempt.state.value,
self.session.revoked,
self._discarded,
getattr(self.attempt.error, "code", None),
)
try:
os.remove(self.final)
except OSError:
@@ -517,7 +552,9 @@ class _Upload:
self._on_finished(self)
return _upload_refused(
"ATTEMPT_NOT_LIVE",
"This attempt stopped accepting a result during commit.",
"This attempt stopped accepting a result during commit "
f"(attempt={self.attempt.state.value}, "
f"error={getattr(self.attempt.error, 'code', None)}).",
error_class=pb.ERROR_CLASS_TRANSIENT,
)
self._on_finished(self)
@@ -2065,8 +2102,10 @@ class WorkerServicer(pb_grpc.WorkerServiceServicer):
# The authenticated session, never the worker payload, is the
# authoritative target identity.
event["target"] = session.worker_id
from services import gpu_gateway # noqa: PLC0415
from utils import hf_progress # noqa: PLC0415
gpu_gateway.record_remote_download_progress(session.worker_id, event)
hf_progress.emit(event)
except (TypeError, ValueError, json.JSONDecodeError):
logger.warning("Worker %s sent malformed download progress", session.worker_id)
@@ -2723,6 +2762,17 @@ class WorkerServicer(pb_grpc.WorkerServiceServicer):
)))
return True
async def cancel_model_install(self, worker_id: str, *, model_id: str) -> bool:
session = self._sessions.get(worker_id)
if session is None:
return False
await session.send(
pb.ServerMessage(
model_install_cancel=pb.ModelInstallCancelRequest(model_id=model_id)
)
)
return True
def revoke_worker_sessions(self, worker_id: str) -> int:
"""Invalidate every transport generation for a durably revoked worker."""
sessions = {
@@ -3104,11 +3154,16 @@ class WorkerServicer(pb_grpc.WorkerServiceServicer):
upload: Optional[_Upload] = None
try:
if not self._begin_uploading(attempt):
task = self.scheduler.get(attempt.task_id)
self._release_artifact_reservation(
final, owner=reservation_owner
)
return None, _upload_refused(
"ATTEMPT_NOT_LIVE", "This attempt is no longer accepting a result."
"ATTEMPT_NOT_LIVE",
"This attempt is no longer accepting a result "
f"(task={getattr(getattr(task, 'state', None), 'value', 'missing')}, "
f"attempt={attempt.state.value}, "
f"error={getattr(attempt.error, 'code', None)}).",
), session
upload = _Upload(
session=session,
@@ -3167,6 +3222,15 @@ class WorkerServicer(pb_grpc.WorkerServiceServicer):
"""
task = self.scheduler.get(attempt.task_id)
if task is None or task.state.terminal or attempt.state.terminal:
logger.warning(
"Refusing result upload admission for task %s attempt %s "
"(task=%s, attempt=%s, error=%s)",
attempt.task_id,
attempt.attempt_id,
getattr(getattr(task, "state", None), "value", "missing"),
attempt.state.value,
getattr(attempt.error, "code", None),
)
return False
try:
task.uploading(attempt.attempt_id, session_epoch=attempt.session_epoch)
+1445 -16
View File
File diff suppressed because it is too large Load Diff
+8 -1
View File
@@ -11,7 +11,9 @@ spark identifies creation. The product voice is clear, calm, and direct.
| README mark | `docs/logo.png` and `docs/logo-256.png` |
| Browser icon | `frontend/public/favicon.svg` |
| In-app mark | `frontend/src/components/brand/VoiceStudioMark.jsx` |
| Desktop/platform icons | `frontend/src-tauri/icons/` |
| Desktop/platform icons (Tauri and Electron) | `frontend/src-tauri/icons/` |
| Electron sidebar and browser icon | `frontend/public/favicon.svg` via `electron/src/renderer/src/lib/brand.ts` |
| Shared sidebar/launchpad artwork | `frontend/src/assets/signal-field.webp` |
Regenerate every desktop icon from the canonical vector after changing the
mark:
@@ -59,3 +61,8 @@ a separately tested migration exists:
Visible copy can explain those compatibility names, but must not silently rename
them on disk or over the wire.
Electron uses the shared multi-resolution ICO for Windows window/taskbar and tray
icons, the shared PNG for Linux and macOS runtime icons, and the ICNS for the macOS
bundle. The tray icon restores the window; closing the app retains its existing
quit behavior. Installed executable icons are applied when building the installer.
+27
View File
@@ -0,0 +1,27 @@
# Electron batch dubbing
Open Batch dubbing from the cloning sidebar or command search. Add video files,
choose one or more target languages, optionally select a saved voice, and choose
whether to preserve background audio. Add to Queue submits through the existing
backend. Successfully submitted files leave the upload list; failed files remain.
The backend enforces ASR readiness and owns translation, generation and mixing.
The Electron setup sidebar accepts file picking or drag-and-drop, groups media,
languages and voice/audio choices into stable cards, and exposes the same Add Videos
action from the empty job view.
Active, Completed and Failed views poll backend jobs. Progress shows backend stages
and percentages. Active jobs can be cancelled; finished records require an inline
confirmation before deletion. Completed language outputs use the native save dialog.
Reloading the renderer reads existing jobs and never re-enqueues them. Backend
process restart recovery is not established by this behavior: the batch queue is
in memory. Native watch folders detect settled files through a capability-scoped
directory handle and stream multipart uploads directly to the selected local or
HTTPS remote backend. Remote uploads receive the scoped session from Electron main;
the renderer never handles it.
Verification: node electron/tests/batch-smoke.mjs uses mocked jobs to check file
selection, enqueue, progress, renderer reload, cancellation, export availability
and confirmed deletion. The enqueue unit regression covers partial failure and
language/voice fields. The native helper smoke verifies confined streaming,
authorization replacement, rename detection and revocation. No test yet establishes
real model-backed batch completion or a separate remote-machine transfer.
+50
View File
@@ -0,0 +1,50 @@
# Electron network and credentials
Settings now exposes Network and Credentials in the shared settings shell. Sidebar search includes proxy, Hugging Face, DeepL and Microsoft labels.
Network reads the configured proxy from `/system/info`. Save and Clear update all six upper/lowercase HTTP, HTTPS and ALL proxy variables using the same helper as Tauri. Writes run sequentially; failures stop the sequence and do not show success. A partial failure can be retried or cleared. Saving does not modify the audio-tool executable setting; its link opens Audio tools.
Credentials shows the Hugging Face resolver sources (App, environment, CLI), masked values, active source and validation status. Ordinary reads use local state only. Test now explicitly requests fresh validation. Save clears the password input after success. Clearing asks inline and clears only the app token by default; CLI-file removal requires its separate opt-in switch. Environment credentials remain managed outside the app. Secret inputs never enter draft persistence or browser storage.
DeepL and Microsoft keys/base URLs use the existing persisted `/system/set-env` contract. Field definitions are shared with Tauri. Blank inputs cannot overwrite stored credentials; successful saves clear the input. Provider connectivity is not inferred from a successful settings write.
`electron/tests/connection-settings-smoke.mjs` checks proxy save/reload/clear, partial failure, local token reads, explicit validation, duplicate submission, app-only/CLI clearing, and provider-key submission against mocked routes. The live backend token-state read returned the expected source schema. No real proxy or credential values were changed for verification. Tauri Network and Translation regression tests pass after helper extraction.
Model settings now include Hugging Face mirror selection: automatic routing,
backend-advertised presets and a custom URL. Settings reads use cached endpoint
status; the network test runs only on explicit click. Writes use the existing
backend and honor its restart-required response. Browser fixtures verify saves,
reload, failed writes and Auto reset. The live read-only schema was verified;
no real mirror preference was changed or probe triggered during verification.
Model downloads follow the compute target selected when the request starts. The
control plane retains authenticated remote-worker progress so Models can recover
it after navigation or renderer reconnect. Jobs are keyed by both repository and
target, preventing a local download of the same repository from appearing on a
remote model card. Cancel sends only the worker's advertised opaque model ID,
keeps polling while the worker drains the install, and completes after the worker
returns a terminal progress event.
Engine Ready resolves TTS against the selected execution target. For a remote
device it shows that worker's advertised engine/model, install and download
readiness, execution backend, and heartbeat-confirmed residency; ASR,
translation, dictation and diarisation remain attributed to the local device
because those operations are not remotely routed. Clone, Design, Stories,
Audiobook, profile preview and comparison actions use the same
operation-scoped readiness, including cloning-capability filtering for Clone,
so a model available only on the selected worker is usable without a duplicate
local installation. If the selected worker is
offline or an operation is local-only, readiness follows the backend's real
local fallback instead of blocking on the remote choice. Batch validates the
selected target before files or watched-folder items enter the queue, sends one
coarse segment bundle per target language to a capable worker, and keeps ASR,
translation, timeline assembly and muxing local. Dubbing and Batch preflight the
selected worker and load local TTS lazily only if dispatch falls back, so a
remote-only installation does not require duplicate weights on the control
device.
Sharing exposes the backend only after an inline confirmation. It shows the active PIN and LAN addresses, generates QR links locally, supports a configurable share port, and controls the backend's Tailscale serve integration. These actions remain explicit and do not run during settings reads.
Remote backend configuration is owned by Electron main. Connection tests validate a VoiceStudio health response and exchange an optional server master key once for a scoped, expiring session. The renderer clears the key immediately; only the URL is persisted. Main injects the session into production and development HTTP proxy traffic, native watch-folder uploads, and path-bound dictation WebSocket tickets. Switching back to the local backend is always available.
Remote-worker routing was exercised against an Ubuntu 26.04 WSL worker with an RTX 4090. A real profile-backed TTS request returned a WAV with `X-OmniVoice-Routing: remote`; stopping the worker changed the same selected target to an explicit local fallback, and a second request returned `X-OmniVoice-Routing: local_fallback`. Restarting the worker restored remote readiness without re-enrollment. Dubbing and Batch also completed real multi-segment worker tasks: the Batch proof returned two exact indexed, non-silent mono WAVs at 24 kHz in one committed bundle. The selected target was returned to Local after verification.
+201
View File
@@ -0,0 +1,201 @@
# Electron dubbing workspace
Open Dub from the cloning sidebar or command search. Upload or drop audio/video, or explicitly submit a video URL;
preparation completes before transcription starts. The editor shows source text,
editable translated text, and per-segment voice/timing controls. Translation uses
the selected Settings > Models > Translation provider. Choose a target language,
translate, review the text, then generate. Completed tracks can be previewed and
exported through the native save dialog.
Segment rows scan as compact source/translation pairs: speaker, voice, fit state,
selection and timestamp stay visible, while row actions reveal on hover or keyboard
focus. Inset hairline separators preserve the reading rhythm; source text and metadata
stay dimmed until the row is active, while the translation remains the visual lead.
Clicking a translation turns only that row into a growing editor. Advanced
voice and timing controls remain folded behind the speaker header. When a workspace
also opens local controls, constrained or scaled windows automatically use the main
navigation rail so the transcript keeps the available width; expanding it remains an
explicit temporary override.
When several targets are selected, progress tabs above the transcript switch the
active language in one click while preserving every target for Translate All and
Generate. Persisted provider error pages are discarded and restored to the source
dialogue with a retryable error state.
Long projects virtualize transcript rows, so only the visible editors are mounted.
Timeline waveform peaks and onsets are computed once by the backend and cached as a
small JSON payload; Chromium never decodes the full separated-vocals WAV to draw the
timeline. Video previews are written atomically with MP4 fast-start metadata and are
immutable per generated-track revision. The renderer warms the current target in the
background, reuses one Vidstack player while switching tracks, preserves the playhead
and playing state across Original/Dub changes, and uses byte ranges on subsequent
playback. Extension-derived native MIME hints select Vidstack's native provider
immediately for common MP4, WebM, Ogg, MOV and MKV sources; URL imports use the
backend's normalized MP4 type even when their display name has no extension. The media endpoints
also answer metadata-only `HEAD` requests, including after backend restart, without
reading the source body or starting a preview mux. Local NLLB batches scale with available accelerator memory while
bounding batch multiplied by beam count.
The backend remains responsible for separation, ASR, speaker cloning, translation,
TTS, fitting, mixing and export. Electron reuses Tauri's speaker binding and
segment generation helpers. A stream close without a terminal event is a failure,
not success. Cancellation aborts the HTTP stream and requests backend task/job
cancellation. Edits, target language, track metadata and task IDs persist locally across reloads.
Interrupted preparation/generation offers Resume, which reads the existing task
and replays its stream; generation is never resubmitted just because the UI reloaded.
Interrupted transcription offers an explicit Retry against the existing prepared
media, without uploading or preparing the source again. ASR restarts from the
beginning because its backend stream is request-scoped, not a replayable task. Batch language runs and advanced QC controls remain in `electron/PARITY.md`.
Verification: `node electron/tests/dub-smoke.mjs` against the development renderer.
The test mocks backend jobs and never uploads or generates user media. Optionally
set `VOICESTUDIO_TEST_VIDEO` to a local MP4 fixture to verify native video transport.
These checks establish UI wiring, not a completed real model-backed dubbing run.
`node electron/tests/native-translation-agent-smoke.mjs <agent>` separately launches the packaged
app with isolated data and verifies that the detected CLI returns complete, ordered translations
for real time-budgeted segments without a backend or source checkout. Codex, Claude Code and
OpenCode pass on the current Windows host; Pi remains gated on a host where it is installed.
`uv run python scripts/smoke_dub_url_captions.py` separately verifies the live
public downloader without loading speech models or touching app data. The current
smoke downloaded a browser-safe MP4 and one original-language caption track, then
parsed 165 usable cues.
A disconnected preparation or generation stream retains its existing task for Resume or Cancel. Editing and new jobs remain disabled until that task finishes or cancellation is confirmed; reconnecting never creates a replacement generation. A task already absent from the backend counts as cancelled. If the backend cannot confirm cancellation, Change file explicitly abandons the unreachable local recovery record so the workspace cannot become permanently blocked.
The setup sidebar groups the source, target language and translation engine, timing,
production overrides and export choices into stable sections. Advanced controls stay
collapsed until requested. Before media is loaded, the main workspace presents the
three actual steps—upload and transcribe, translate, generate—and hides inactive
generation actions.
Import .srt replaces the current segment text and timings after source preparation.
The backend retains voice references only where their timing overlaps the new cues.
Malformed, overlapping, or duration-clamped cue counts remain visible in the sidebar.
Failed imports preserve the current edits. Generated track buttons clear on successful
replacement to avoid presenting older audio as the new subtitles' output.
URL import runs only after clicking Ingest; it uses the backend's existing yt-dlp
pipeline. Explicit cookies.txt selection is available under URL sign-in options; optional caption downloads are available.
Translation quality uses the existing backend Fast, Autofit and Cinematic modes.
The choice persists in the working draft and saved project (`translateQuality`),
including legacy project imports. New media preserves the user's quality choice.
If the backend reports that no LLM is configured, the UI selects Fast and shows
an inline explanation with a link to LLM settings. It does not silently claim
that Cinematic or Autofit completed. Browser fixtures cover this fallback and
reload persistence; real LLM-backed quality passes remain unverified.
**Translate with Agent** detects the installed Codex, Claude Code, OpenCode and Pi CLIs. Dubbing
presents Agent and the active Google, Argos, NLLB or API engine as separate translator choices;
Fast, Autofit and Cinematic remain quality choices for engine translation. The selected agent receives the complete ordered
dialogue, glossary, dialect and per-segment speech budget as untrusted data, returns a strict
id-preserving translation, and never receives repository or filesystem write access. Google,
Argos, NLLB and configured API translators remain available through the ordinary Translate All
action. Every translate entry point honors the chosen translator. Agent translations use the same reviewable segment fields and, during generation, reuse the
bounded measured-speech loop to rewrite only timing misses before rerendering. Timing-fit provenance
is stored per target language, so every translated track keeps that behavior after language switches
and draft reloads. Cancelling Dubbing also terminates the local agent process tree.
Export options expand inside the existing sidebar. Users can select included video
tracks and the default track, background mixing, burned subtitles, dual layout and
karaoke (disabled with dual layout). Audio supports WAV or MP3 with bitrate choice;
SRT/VTT/ASS sidecars and per-language stem/segment ZIPs use the existing backend.
Each download is explicit and targets the selected language. Export errors retain
all choices for retry. Native save filters match the encoded file format.
Browser fixtures verify MP3/SRT downloads, query options and failed-export retry;
unit tests cover video/package parameters. Real rendered exports, batch presets
and native save dialogs remain unverified or incomplete.
Timing options now match Tauri: Concise, Smart Fit, Stretch Video and Lip sync.
Voice matching offers per-line references or one consistent reference per speaker.
These choices persist in working drafts and projects and are sent to generation;
existing defaults remain Strict slot and Per line. Interrupted generation retains
its submitted timing strategy. Completed output retains that strategy separately
from current controls, so changing the next render's settings does not change the
export capability of existing audio. Stretch Video output disables incompatible
subtitle burn-in and directs users to sidecar exports instead. Browser fixtures
verify persisted settings, request values and the export guard; actual timing and
voice consistency across a real multi-speaker render remain unverified.
ASR retry browser coverage includes stream interruption, reload with zero automatic
requests and successful explicit retry. Cancel releases recovery only after the
backend acknowledges it. A real mid-ASR disconnect remains unverified.
Spoken-language and speaker-count hints are available in a collapsed sidebar
section. Automatic detection remains the default. Explicit language hints reach
both local uploads and URL ingest; speaker counts (1?20) reach transcription and
persist through retries, drafts and projects. Browser checks exercise selection,
reload and retry; unit tests inspect both file and URL request bodies. These are
hints to the existing backend, not a guarantee of diarization accuracy.
Cookie exports are selected explicitly for one import, limited to 1 MB and sent
only over HTTPS or the local desktop transport. Selection clears on submission;
contents never enter the persisted dubbing session. Tauri and Electron share size
and transport validation constants/helpers. Browser fixtures verify oversized
rejection, request contents and absence from local storage; the Tauri cookie tests
remain green. No real authenticated website download was performed.
URL Advanced options include downloading available captions through the existing
yt-dlp pipeline. When usable cues are returned, Electron chooses the closest
source-language track, normalizes its timing and opens it directly in the editor.
Missing or malformed tracks fall back to the normal ASR path without another user
decision. The downloader skips automatic translations. Real caption downloads
are verified by the isolated public-URL smoke above. Authenticated sites still
require a user-owned cookies export for native acceptance.
Production overrides expose steps, guidance, speed and global voice direction,
matching the existing Tauri generation request. Defaults remain 16 / 2 / 1 with
no direction. Values persist in drafts and projects; Reset clears only these
overrides. Browser checks verify reload followed by the exact generation values,
including zero guidance. Engine-specific audible effects remain unverified.
Saved Smart Fit override values now survive project import and draft recovery and
reach generation only when Smart Fit is selected. Other timing strategies omit
those parameters. Unit tests cover round-trip preservation and request routing;
there is no new tuning panel (Tauri exposes these as stored preferences).
Real export verification: `tests/test_smart_fit_export.py` passes all 43 tests
with the installed app-managed FFmpeg/ffprobe explicitly supplied. Its seven
integration cases render synthetic video/audio through retiming and the backend
export endpoint, then probe output durations and fitted subtitle bounds. This
proves those backend export paths with real codecs; it does not prove a complete
Electron upload ? ASR ? multi-speaker synthesis run. The broader targeted export
suite passed 73 tests before the codec paths were supplied (seven skipped then).
Export format, bitrate, track selection, background mixing and subtitle choices now
persist in the working draft and saved project. Legacy Tauri export preferences
populate the panel rather than merely being retained as unused data. Browser
checks verify format/bitrate/background/dual-layout recovery after reload; project
tests verify legacy disabled tracks and background preferences.
Each segment now exposes volume (0?2), optional speed and a direction note within
its existing voice disclosure. Empty speed follows global speed. Segment gain
reaches generation separately from shared TTS fingerprint inputs, matching Tauri;
zero is preserved for muted segments. Browser checks cover edits, reload and
request values; backend audible mixing is not established by these UI fixtures.
Segment rows support insertion, deletion, cursor-aware splitting and merging with
either neighbor. Merge/split uses Tauri's shared attribution bookkeeping, so speaker,
voice, direction, gain and target language survive moving words across a boundary.
Undo and redo retain the latest 50 edit states and reset when a new source, subtitle
file, translation or project becomes the editing baseline. The toolbar and
Cmd/Ctrl shortcuts expose the same operations. Unit and browser regressions cover
the full insert/delete/merge/split/undo/redo sequence and merge-to-split attribution.
The editor includes a proportional timing lane above the segment rows. Segment
blocks select their matching row, drag to move, expose edge handles for resizing,
and support keyboard nudging or deletion. Timing changes use Tauri's shared clamp
and speed-recalculation helpers. Adjacent overlaps are highlighted in the lane with
an explicit warning; the media duration reported by preparation defines its scale.
Browser coverage verifies timeline rendering, keyboard timing edits and overlap QC.
Segment checkboxes now enable one bulk edit surface without changing the active
timeline segment. A single operation applies a saved voice, per-segment target
language reset/override, or deletion to the selection and records one undo step.
Select all and Clear keep large transcripts manageable while generation is idle.
Dubbing uses a wider responsive secondary sidebar (up to 32rem) and a wider transcript workspace. Once media is prepared, the upload form becomes a compact source row; source, target and casting remain a visible three-step sequence. Automatic cast matching is the default, so its overrides start folded. Voice chips wrap instead of requiring horizontal scrolling. Segment rows show the original text only when it differs from the editable text, retaining source comparison after translation without duplicating every unmodified transcript.
Below 40rem of workspace width, Dubbing stacks its controls above the editor with controls limited to 40% of the available height. Both regions retain independent scrolling.
Global speed/quality changes preserve explicit Dubbing production steps, including settings restored from projects. Unset steps continue to follow the backend's current preset.
NLLB resolves explicit FLORES language/script codes, unambiguous ISO-639-3 codes and common short aliases, including Traditional Chinese. Unsupported or script-ambiguous source, target or per-segment languages are rejected before model loading instead of silently translating to English.
+30
View File
@@ -0,0 +1,30 @@
# Electron voice gallery
Open VoiceStudio Gallery from the cloning sidebar or command search. Browse the
existing local archetype catalogue by category, search it, and load more results.
Age, gender, pitch, accent, language and whisper filters reuse the Tauri taxonomy.
Their labels reuse the existing translated voice-design vocabulary, so no internal
translation keys appear in the filter sidebar.
Star voices to keep local favorites; Favorites searches all matching catalogue
pages, including voices beyond the currently loaded cards.
Preview playback starts only on click and uses the shared Vidstack player.
Use voice asks the existing backend to materialize a reusable design profile,
then opens Voice Design with that profile's attributes and seed. The current
script stays intact. Navigating away cancels the frontend request and prevents
a late response from redirecting the user; the backend may still finish saving.
The catalogue and profile generation remain owned by the existing backend.
API wire types are shared with Tauri. No new required network service is added.
Community voices, local/search imports, inline trimming, and portable persona
bundles use the same backend contracts and keep network actions explicit.
Verification: node electron/tests/gallery-smoke.mjs exercises categories, search,
pagination, real native playback of synthetic audio, saved-profile handoff, and
navigation during saving using mocked API responses. The running backend also
returned 1,126 catalogue entries and seven categories during development. The
live Community acceptance forces a temporary catalogue outage, retries in place,
then favorites and previews a real item before handing its attributes to Designer.
Separate real smokes cover upload, trim, persona import/export, and profile
materialization without retaining disposable profiles. The packaged Ubuntu app
also completed profile creation, native `.ovsvoice` Save As, bundle inspection
and cleanup against a reused installed runtime without downloading anything.
+5
View File
@@ -0,0 +1,5 @@
# Electron Launchpad
The root route is VoiceStudio's Launchpad. The app logo returns to it from the full or compact sidebar without changing any workspace route. It provides direct entry to cloning, design, dubbing, Stories, Audiobook, Gallery, Transcriptions and Tools, plus the unified Projects library.
Saved voices and recent takes provide short continuation lists. Choosing a voice restores the clone profile before navigation; choosing a take opens that take in the existing cloning workspace. The Launchpad uses the shared brand artwork, theme tokens, profile/history queries and responsive card grid, with no new network dependency.
+26
View File
@@ -0,0 +1,26 @@
# Electron LLM provider settings
Settings > Models > LLM includes the existing provider catalogue. Translation
settings links to it. Configure an endpoint/model and, where needed, an API key
or account ID. Save preserves a stored key when the key input is blank. Keys
are sent only to the existing backend credential storage, never localStorage.
Environment-pinned fields and activation remain read-only with an explanation.
Save & use for translation explicitly activates the provider. Saving or testing
alone does not imply activation. Test and Fetch models first save the current
form, stop if saving fails, and then call the backend probe. They never run
on page load. Provider calls may use the network only when explicitly requested;
local endpoints remain supported. Failed probes use classified localized messages.
The browser smoke `node electron/tests/llm-providers-smoke.mjs` mocks credentials
and provider responses. It verifies blank-key preservation, environment pinning,
save-before-test, no probe after failed save, model choice and activation. A live
catalogue read returned 17 provider descriptors without key material. Actual
external credentials and remote-provider calls are not verified by those mocks.
Per-skill routing is available beneath providers. Each backend capability can be
disabled or assigned a configured provider, with an option to follow the active
provider. Existing unavailable overrides stay visible. Readiness comes from the
backend; it is not proof of a successful network probe. Non-LLM translation-provider
credentials for DeepL and Microsoft are available under Settings > Credentials and are
written through the backend environment-setting endpoint. The skills browser smoke
verifies routing and disable behavior against mocked API responses.
+17
View File
@@ -0,0 +1,17 @@
# Electron diagnostics
Settings > Logs provides Backend and Frontend sources. Backend logs use the existing
rolling-log API, with the shell startup tail as a fallback when the API is unavailable.
Frontend logs reuse the existing bounded console buffer. Nothing is sent remotely.
Search filters the visible lines; Copy copies that view. Refresh fetches immediately,
and the view refreshes every three seconds. Scrolling up stops automatic scrolling
until the reader returns to the bottom. Clear requires an explicit confirmation.
The backend log file can be revealed with the native file manager when available.
Synthesis error details link directly to this settings view. Error and failure lines
are red, warnings amber, successful readiness events green, and debug output muted.
**Repair with an agent** opens the footer dock without covering the log and preloads
the visible failure and warning lines as repair context.
Run `node electron/tests/logs-smoke.mjs` against the development renderer. It mocks
backend reads/clears and checks frontend capture without clearing any real log files.
+31
View File
@@ -0,0 +1,31 @@
# Electron Stories and Audiobooks
Stories and Audiobook are available from the sidebar and command search. Each keeps a separate draft. Choose a saved default voice, enter or import text, then render. Stories supports per-line voice overrides; chapter headings and pause markup use the same `storyToSpans` compiler as Tauri. Audiobook submits the original script to the existing backend parser.
Both editors use the shared longform backend, real chapter progress, assembly status, and an explicit Stop action. Stop aborts the HTTP stream so the backend stops scheduling chapters. A truncated stream cannot replace the last successful output. Interrupted server manifests can be resumed explicitly without sending the edited manuscript again. Navigation leaves a render running; a renderer restart disconnects it and recovery uses the server manifest inventory.
Playback and export remain independent of generation. Vidstack plays the output; long-form audio uses a seek bar without decoding an entire book into a waveform. Play waits until the native media provider is ready. MP3 and M4B are supported by the existing backend. The native export dialog saves a local copy.
Draft writes reuse the coalesced persistence helper and flush on lifecycle events. Working drafts currently use browser storage. Named Stories and Audiobook projects use the shared native IndexedDB adapter in a separate Electron database. Saves await the committed transaction, concurrent writes are serialized, and opening or deleting a project requires confirmation. Projects include script/lines, voices, settings and the last output reference. Storage failures are surfaced. Book details include author, narrator, year, genre, description and an uploaded JPEG/PNG cover. Loudness offers off, ACX and podcast presets. Audiobook pronunciation rows reach the existing lexicon engine; duplicate words are flagged before rendering. These settings persist with the draft. Full cast management and other advanced controls remain tracked in `electron/PARITY.md`.
`electron/tests/longform-smoke.mjs` verifies the main flows with mocked generation and real audio playback. Runtime tests cover stop, duplicate prevention, truncated responses, explicit resume and shared Stories compilation. Real installed-model verification passed for one English chapter: MP3 rendering, cached WAV chapter audition, and Electron-driven M4B rendering/playback. Multi-voice, multi-chapter, cancellation/recovery and other engines still require real runtime coverage.
Script voice tags now expose saved-profile assignments in the Cast section. Only names present in the current script are sent as `voice_map`; assignments persist for reuse if a tag is reintroduced. An assignment to a removed profile blocks rendering until changed or reset to Default. Chapter, word and estimated runtime counts reuse the Tauri script helpers.
Audiobook Preview plan uses the backend parser and exposes per-chapter auditions. Auditions send the same voice, cast, language and pronunciation inputs as full renders, warming the shared cache. Editing those inputs clears stale auditions. Preview requests are aborted on navigation, never replace a full render, and expose their own Vidstack playback.
Production overrides now expose synthesis steps, guidance, sampling temperatures, postprocessing, seed and repeat variation. Emotion controls appear when the active engine advertises support. Reset restores the shared Tauri defaults; untouched fields are omitted from requests. The extracted `longformOverrides` helper is used by both apps, and chapter auditions carry the same overrides as full renders.
Run `electron/tests/longform-live.mjs` with `VOICESTUDIO_LIVE_PROFILE` set to a local saved profile for an explicit real-render check. It verifies the active model is already installed before synthesis. The real run exposed missing JSON request headers in the shared client and array-shaped failed-chapter results; regression tests now cover both.
Stories now includes named characters with saved voice assignments, character selection per line, explicit per-line voice overrides, shared/global speed and per-line speed overrides. Voice resolution and chapter compilation reuse the Tauri helpers. Lines move up/down with keyboard-accessible buttons; chapter insertion uses the same heading grammar. A story with every spoken span assigned can render without an unnecessary default-voice selection. Removed profiles block rendering rather than silently changing the voice.
Stories Auto-cast accepts tagged dialogue, screenplay text and attributed prose. The shared local parser appends lines, reuses existing characters/voices, and assigns available profiles to new speakers. Distinct names that normalize to the same identifier remain separate characters. No network or LLM call is involved.
Stories imports TXT, Markdown and SRT locally; EPUB/PDF still use the shared backend importer. Imported text goes into a persisted review buffer, then Auto-cast or Split into lines appends to existing work. The sentence-aware splitter is shared with Tauri and offers the same 40-2000 character limit.
The Electron workspace keeps title, default voice, language and output format together in a fixed setup card. Advanced cast, project, production and book controls use separate collapsed cards. The editor keeps its full working width; an empty Story shows one Add First Line action, while an empty Audiobook shows the supported chapter and voice markup directly in the script field.
Stories line auditions reuse the shared WAV assembler and canonical voice/pause/speed parser. Auditions resolve inline names with the same fallback as full renders, expose Stop, and play through Vidstack independently. Changing synthesis inputs clears stale playback; navigation aborts pending auditions. Browser coverage verifies actual playback with mocked generation.
The optional Stories Stems section renders one WAV per character with the shared Tauri assembler. It reports completed character groups, supports cancellation, and offers explicit individual downloads after completion. No automatic multi-file download is triggered. Input changes discard stale download links. Browser regression checks all character groups and the downloaded WAV bytes. The shared native Save As bridge writes renderer-generated bytes in packaged Windows and Linux builds; macOS remains the native export gate.
+67
View File
@@ -0,0 +1,67 @@
# Electron and Tauri behavior parity
This inventory tracks user-visible behavior. A row is **implemented** when Electron exposes the
same task and backend contract. Platform-specific performance may differ. A native row stays
**verification pending** until its real macOS, Windows, and Linux smoke gates pass.
| Area | Electron state | Evidence | Remaining gate |
|---|---|---|---|
| First run and managed backend | Implemented | `docs/electron-runtime.md`, installed Debian/AppImage consent smokes, and fully uncached packaged Windows and Ubuntu 26.04 install-to-paused-first-sound runs | Full packaged uncached install-to-first-sound on macOS |
| Backend crash recovery | Implemented | crash/health smokes plus packaged Windows deliberate quit during deferred startup | Signed-package OS matrix |
| Voice cloning and saved profiles | Implemented | live Faster-Whisper uploaded-reference transcription plus atomic audio/transcript/portrait profile save, clone demo and profile image tests | Real engine smoke on each accelerator family |
| Voice design | Implemented | `docs/electron-voice-design.md`, design and persona smoke tests | Real model generation matrix |
| Stories and Audiobook | Implemented | `docs/electron-longform.md`, a real two-profile Stories cast/render/playback, real chaptered M4B render/playback, longform smokes, and packaged Linux native Save As | Native export smoke on macOS |
| Dubbing and batch dubbing | Implemented | `docs/electron-dubbing.md`, `docs/electron-batch.md`, live public URL/caption ingest, a 15-minute 195-segment Windows project, real interactive/batch model-backed exports, opt-in live-as-you-edit CAST playback, the shared bounded Agent Fit loop, and packaged timing-budgeted translation through Codex, Claude Code and OpenCode while Pi retains contract coverage and ordinary providers remain available | Repeat the long-media run on native macOS/Linux |
| Profiles and Gallery | Implemented | `docs/electron-gallery.md`, gallery and persona round-trip smokes, explicit no-key five-result portrait search, packaged Ubuntu native persona export/inspection, plus live Community outage/retry/favorite/preview/Designer handoff | None |
| Transcription and dictation | Implemented | `docs/electron-transcriptions.md`, transcription and native dictation smoke tests, plus trusted-origin audio-only Chromium permission regressions | Native hotkey/input permission matrix |
| Projects and Tools | Implemented | `docs/electron-projects.md`, `docs/electron-tools.md`, project and tool smoke tests, plus real Windows Faster-Whisper/OmniVoice conversion and Vidstack playback | Native reveal/export matrix |
| Model catalogue and engine selection | Implemented | model-library component tests, recommendations smoke, live install/cancel/delete/unload verification, current in-process/isolated Faster-Whisper selection and restoration, and repeatable app-managed audio.cpp v0.7.4 Sortformer inference/cancellation on Windows Vulkan | Native install matrix across model families and desktop OSes |
| Local and remote compute target | Implemented | worker API tests, compute-target UI, remote-target recommendation smoke, Ubuntu 26.04 WSL real TTS plus disconnect/reconnect fallback | Native-machine remote run outside WSL |
| Settings, logs, API reference and support | Implemented | settings smoke tests, responsive live runtime/hardware identity, 640 px layout smoke, packaged Linux 150% DPR/no-overflow visual smoke and `docs/electron-logs.md` | Native visual capture on macOS |
| Updates, notifications, app/tray/taskbar branding | Implemented; published-update verification pending | updater IPC lifecycle/authorization regression, platform-specific feed names, bounded channel-aware release history, byte/SHA-512 release contract, release-matrix validation of each unpacked Electron app, branded Windows NSIS/app icon, launched AppImage and installed Debian package with shared brand assets/native helper | First signed Stable/Preview update and installed-shell visual check on macOS/Linux |
| Watch folders and global capture | Verification pending | Windows native watch lifecycle/global shortcut/insertion plus Linux AppImage Wayland tray/watch and clipboard-fallback smokes | macOS permissions, Linux portal global shortcut and physical microphone matrix |
| T3-inspired shell, themes, scale and keyboard navigation | Implemented | shared resizable secondary panes, sidebar, locale-layout, workspace-shell and all-route accessibility smoke tests, including 6401920 px secondary-pane geometry, 38 routes at 1440/640 px, packaged Linux 150% DPR and main-process empty-renderer recovery with a localized asset-independent fallback | Native screen-reader pass on macOS/Linux and high-DPI pass on macOS |
| In-app agent repair dock | Electron extension | `docs/electron-repair.md`, native smoke, packaged Codex/Claude/OpenCode acceptance on Windows, and packaged Linux host-shim rejection | Pi acceptance when installed; native macOS/Linux CLI matrix |
Latest local verification (2026-09-14): the final unpacked Windows build launched against its
managed runtime and reported backend 0.5.2 healthy on an RTX 4090. Its responsive System Preflight
resolved the live CPU, GPU/VRAM, Python runtime, compute device, selected engines and storage paths.
OmniVoice was resident on CUDA,
all required setup models were ready, seven saved profiles loaded, and Parakeet TDT v3 was installed
and active for dictation. A saved 17-second clone reference also completed a live Parakeet
transcription in 0.66 seconds with the expected text. Focused model/runtime/profile/transcription coverage passed 41 renderer
checks plus 150 backend checks with `HF_HUB_OFFLINE=1` and an empty Hugging Face cache. The shared
secondary-sidebar smoke also passed every workspace from 640 to 1920 px.
The complete Electron component suite passes 462 checks across 119 files; first-run browser
acceptance additionally covers partial preflight recovery, recommended-model disclosure and the
compact Settings layout, while all 38 routes fit at 1440 and 640 px and 37 routes pass the DOM
accessibility audit at both widths.
The refreshed packaged backend also reported the resident `k2-fsa/OmniVoice` checkpoint with a
non-empty UTC load timestamp, `cuda:0` execution and 1937.2 MB allocated VRAM, closing the anonymous
ready-state gap across renderer refreshes.
The current production renderer additionally switched the live ASR backend out and back without
losing its selected model, completed a real Faster-Whisper upload through timed segments, export,
Clone handoff and cleanup, and round-tripped a generated take into a disposable saved profile.
Its live route sweep remained free of renderer warnings, exceptions and API failures after
updater, dictation, QR, locale loading, translation-file and local-agent discovery rejection paths
were contained at their initiating UI boundaries.
The inventory guard maps every current Tauri page, every static Electron route and every model
family to its maintained capability or responsive-layout evidence; adding an unmapped page or route
now fails the required test gate.
An isolated clean Ubuntu 26.04 workspace also completed the Electron typecheck and production
package, then passed the unpacked Linux artifact contract with its x64 app, bundled `uv`, branded
resources and native dictation helper present.
The inventory records behavior and verification only. It does not reopen layouts or mechanics that
already satisfy their task. New parity work should update the relevant row and add the narrowest
regression evidence that proves it.
Electron also consumes the backend's `/ws/events` invalidation stream for profiles, generation and
dub history, projects, exports and model state. Local development waits for backend health before
opening the proxied socket; packaged and authenticated remote connections use the main-owned,
path-bound WebSocket URL and reconnect without exposing credentials.
`bun run smoke:live-routes` from `electron/` exercises all 37 concrete route views through the
running hash router and real backend. It fails on an error boundary, missing heading, uncaught
exception, console error or warning, failed load or HTTP error response. The 2026-09-13 live run passed every
route with no failures.
+33
View File
@@ -0,0 +1,33 @@
# Electron compute and performance settings
Settings > Compute device exposes the existing device override, Windows torch.compile workaround, generation time budgets, and hardware readouts.
Device choices come from the backend's detected families plus Auto. The chosen preference and currently active family are displayed separately. Environment-pinned choices are disabled, an ignored unavailable override is explained, and a changed preference shows its actual restart requirement. Failed saves keep the last confirmed state. Nothing automatically restarts the backend or changes the active model.
The torch.compile workaround uses the same platform gate as Tauri: Windows can opt in; other platforms retain their working optimization. Generation budgets preserve separate GPU and CPU limits, validate the existing positive/21600-second range, and keep edits during refetches. An externally overridden budget reports that fact instead of implying the saved value will take effect after restart. Hardware RAM/VRAM readouts poll only while this view is mounted.
During synthesis, the fixed-width primary action polls the existing model-status contract and names the active runtime phase: starting the AI runtime, loading weights, warming speech recognition, optimizing the model, generating, or receiving audio. Model-load percentage and elapsed time share the reserved status line, and the progress track switches from model loading to streamed audio delivery without moving the controls.
The backend publishes explicit model lifecycle transitions to the renderer event stream. Engine Ready refreshes immediately from those events and keeps one-second polling only while work is active; idle model, worker, batch, performance-profile and diarisation checks back off to bounded 1530 second recovery intervals.
`electron/tests/performance-settings-smoke.mjs` checks saved/active separation, failed saves, environment pinning, unavailable devices, the platform guard, budget validation and external overrides with mocked contracts. Live read-only checks verified the compute, compile and hardware response schemas. Tests did not change the user's device, optimization or timeout preferences. Native hardware behavior on macOS/Linux still requires platform verification.
## Speed and quality presets
The engine sidebar and Performance settings expose Fast, Balanced, Quality and Max. A global choice resets family overrides; a family choice overrides the global preference. Changes are blocked while foreground or batch work is active. Choosing a preset never downloads weights or activates cloud providers. A ready network translator remains authoritative when explicitly selected; if that provider becomes unavailable, profile reconciliation recovers to the tier-appropriate installed local translator instead of leaving translation unusable.
Currently connected controls are OmniVoice sampling (8/16/32/64 steps), Faster-Whisper decoding search (1/3/5/8), Sherpa transducer dictation search (greedy through 8-path modified beam search), local NLLB beam search (1/3/5/8), and the installed diarisation runtimes. Clone, Dubbing, Batch, Voice Conversion, Stories and Audiobook all resolve untouched TTS controls through this shared contract; an explicit Production override still wins. Dictation keeps the selected language model and rebuilds its warm recognizer after a tier change. When both diarisation choices are installed, Fast/Balanced select native audio.cpp Sortformer and Quality/Max select pyannote; with only one runtime, its family control stays unavailable rather than accepting a no-op preference. Max selects the strongest already-installed compatible Faster-Whisper, dictation and NLLB choices without downloading anything. LLM remains unavailable until the selected runtime exposes a meaningful comparable effort control. The API reports only implemented targets.
Performance Settings presents each family as a discrete Fast/Balanced/Quality/Max control and identifies the effective local model, runtime, and decoding effort beneath it. A choice is persisted before any optional renderer-side synchronization and receives explicit applied or failed feedback, so a cold engine catalogue cannot make the control appear inert. Families without an installed compatible target stay disabled, name the required engine, and link to its Models view; an engine with no comparable effort contract explains that limitation. The compact sidebar keeps the slider form of the same setting.
Settings > Models turns those tiers into one-click, target-aware model packs. Each pack previews its exact compatible models, installed size, remaining download and aggregate progress before starting the existing resumable installer. Fast installs the smallest local ASR and dictation set; Balanced selects the faster Whisper Turbo and Parakeet set; Quality and Max add Whisper large-v3 and local NLLB. The saved tier is reconciled after every successful model download, so newly available engines become active without another selection or restart. Diarisation stays explicit because native audio.cpp setup and gated pyannote access require separate consent; LLM stays explicit because it has no common local performance target.
Backend startup was checked live after the lifecycle changes: OmniVoice loaded successfully and the performance-profile endpoint responded. This does not establish the cause of historical native crashes or verify recovery from every stall.
Crash-isolated Faster-Whisper receives the same ASR decoding preset with each transcription request. The parent snapshots the selected beam/best-of values; the child validates them before loading the model. Existing callers without decoding options retain their original defaults, and changing a preset does not require restarting the child.
## Reference transcription
Uploaded voice references use the selected ASR engine through the shared transcribe endpoint's reference mode. This skips word alignment, checks locally installed models before loading, and never enables LLM refinement. Dictation selection remains independent. Missing models leave the optional transcript editable and retryable; asynchronous results do not overwrite manual edits or a subsequently selected saved voice.
Reference mode fails closed if local installation cannot be verified, including unknown model selections and preflight errors. The loader rechecks the actual selected engine and every fallback immediately before loading, bypassing stale positive cache entries.
+33
View File
@@ -0,0 +1,33 @@
# Electron playback
Existing Electron preview, reference, history and generated-output playback use
Vidstack through `components/media-player.tsx`. Generation stores the result and
its autoplay intent; only the visible output player starts playback. The shared
playback owner pauses another preview before taking ownership.
WaveSurfer uses the Vidstack-owned audio element for waveform visualization.
It does not own a second player. Blob and extensionless API sources explicitly
use native MIME sniffing through `audioSource`; otherwise Vidstack infers blobs
as video. Seek input processes the input event once, avoiding a second change
event restoring the previous controlled value before the provider reports seeked.
Run `node electron/tests/playback-smoke.mjs` with the development renderer running.
The test uses synthetic WAV/profile/generation responses and never saves user data.
Set `PLAYWRIGHT_CHANNEL` or `VOICESTUDIO_UI_URL` for another installed browser/server.
HLS and DASH libraries are bundled and lazy-loaded. The shared provider includes
Vidstack's native audio/video, HLS, DASH, YouTube and Vimeo selection plus the
Remotion loader. Gallery search previews exercise the embedded YouTube provider;
Dubbing uses the custom Vidstack video controls for local and normalized URL imports.
Native video MIME hints select a provider before its `<video>` element connects, and
extensionless Dubbing routes use the backend's normalized MP4 type. Those endpoints
also support cheap metadata-only `HEAD` requests. Workflows can pass any other
supported source through the same shared player without creating another playback
stack.
Video previews use a glass gradient overlay and expose play/pause, 10-second seek,
mute, volume, playback rate and enter/exit fullscreen controls. Dubbing keeps one
Vidstack instance while changing preview sources, adds revisioned byte-range URLs,
and displays the source thumbnail while a first preview is prepared. Audio preview
buttons show pending/buffering and unavailable states instead of silently swallowing
playback failures. Playback labels are translated in all 21 locales.
+11
View File
@@ -0,0 +1,11 @@
# Electron privacy and retention
Settings > Privacy exposes the existing invisible-watermark setting, analytics consent and generation-history retention.
Watermark controls appear when the backend reports AudioSeal available and affect new audio through the existing marking path. No audio producer bypasses `mark_synthetic`. Analytics consent uses the existing backend opt-in endpoint and remains unchanged until the user explicitly switches it. The Electron renderer does not initialize an additional analytics SDK; the setting controls backend analytics. Unavailable features do not display inert toggles, and failed saves preserve confirmed state.
Translation privacy classification is shared with Tauri. Unknown or unavailable backend data does not claim offline operation. The existing `libretranslate` identifier is an alias for the backend's local Argos branch, not an assumption about an external LibreTranslate service. The Translation link opens engine settings.
Retention supports 0 (unlimited), validates whole-number limits up to 100000 and explains that cleanup removes old unstarred takes and their audio after generation. A more restrictive cap requires inline confirmation. Raising the cap or disabling cleanup saves directly. No records or files are deleted by simply opening settings or editing the input.
`electron/tests/privacy-settings-smoke.mjs` covers consent, watermark changes, failed writes, availability, retention confirmation/cancel, unlimited retention and reload with mocked endpoints. Tauri privacy regressions pass after sharing classification. Live read-only checks verified the watermark, analytics and retention schemas without changing the user's privacy preferences or deleting data.
+26
View File
@@ -0,0 +1,26 @@
# Electron projects
Open Projects from the sidebar, command search, or Dubbing header. Current dubbing work can be saved with a name or saved as a new copy. Projects use the existing backend `/projects` database and Tauri state format.
Opening a project asks before replacing the current draft. Active dubbing work and unresolved task recovery prevent switching. Deletion requires confirmation, removes only the saved project record, and detaches it from the current draft. It does not delete source media or generated audio.
Segment edits, selected language, source job, and available tracks are restored. Unexposed legacy options and per-segment metadata are retained when saving. Renderer reload preserves the active project identity. This does not recreate missing source media or imply an interrupted backend job is running.
The library combines backend dubbing projects, local IndexedDB Stories and Audiobooks, saved
voice profiles, transcription history, generated takes, export history and completed longform
renders. A stable secondary sidebar filters each source and shows live counts; one search and the
list/grid switch operate without moving the workspace origin. At compact widths the global
sidebar becomes its icon rail, matching the other secondary-sidebar workspaces.
Opening a book or story restores its saved draft after confirmation; deletion detaches the draft
without removing generated media. Inline rename changes only the saved name: dubbing uses the
lightweight PATCH endpoint, and local books rename the latest durable record inside the serialized
write queue. Profile rows select the voice in cloning, transcript and take rows reuse their script,
take and completed-render rows preview through Vidstack, and export rows reveal the destination.
Verification: `electron/tests/projects-smoke.mjs` exercises the UI with a mocked project API; project-format unit tests check legacy round trips and malformed segments. A separate disposable fixture passed create/read/update/delete against the running backend.
Browser regressions verify all-source counts and profile/transcript filters alongside local book
save, unified-list rename, reload, confirmed open with original script intact, and deletion; dubbing
rename preserves unexposed options and updates the current save name. A disposable real backend
rename/read/delete round trip confirmed state preservation.
+113
View File
@@ -0,0 +1,113 @@
# Repair with an agent
The Electron app exposes a repair launcher centered on the right edge of every workspace and
Settings view. Opening it reserves footer space and resizes the active view instead of covering
content. Opening it on Logs automatically collects the visible backend and frontend failures,
deduplicates them, fills the report, and identifies common Hugging Face access, memory, port and
broken-runtime causes before an agent runs. The evidence remains expandable in the dock. It detects
supported command-line agents already installed on the machine:
- Codex
- Claude Code
- OpenCode
- Pi
**Diagnose** gives the selected agent read-only access where the CLI supports it. **Fix** allows
workspace edits under that agent's normal permission model. Each run receives the user's report,
the current route, recent renderer and backend logs, the local system diagnostic report, and any
durable Electron main-process fatal-error record. Manual
runs start only after the user clicks an action. A repeated renderer crash may start the available
default agent only after the user has explicitly selected that default on the first failure; the
diagnostic context stays local to the selected command-line agent.
Each run also receives a temporary connection file for an app-owned loopback API bridge. The bridge
targets the backend currently attached to VoiceStudio, injects remote authentication inside the
Electron main process, and gives the agent only a random per-run capability. Diagnose bridges reject
mutating methods. Authentication routes are blocked in both modes, and the bridge, capability and
connection file are removed when the run stops. This lets an authorized Fix select engines, install
models and verify live state without putting the real backend address or bearer token in the prompt,
agent output or child environment.
The connection file also lists capability-scoped Electron controls for reading supervisor status,
restarting the backend, and resuming runtime setup. These controls remain reachable when Python is
down, so the backend failure screen keeps the repair dock mounted and its **Fix** action can restore
the runtime before using normal backend APIs. Clean reinstall is intentionally absent because it
removes the owned runtime and remains an explicit **Clean Retry** decision.
Setup blockers use supported app actions before invoking an agent. For example, **Fix** on a
missing text-to-speech notice selects an installed compatible engine or starts the device-aware
required-model installer, follows that local or remote job to completion, refreshes every mounted
workflow, and activates the engine. An installer or activation failure becomes an `ACTION_REQUEST`
for the user's opted-in default agent with the current route and logs already attached. Clicking
**Fix** is the explicit authorization for that recovery; merely viewing the notice changes nothing.
Unresolved ASR setup in Dubbing, Dictation and Voice Conversion exposes the same agent action while
keeping prepared media, selected voices and retry state intact. Voice Design, voice comparison and
saved-profile previews use the same compact one-click TTS recovery instead of sending the user to a
Settings page.
Reference-audio ASR, accurate file transcription and live-dictation model failures expose the same
checkout-free action while preserving the selected media, editable transcript and retry state.
For checkout-free app operations, the agent discovers supported actions from the bridge OpenAPI
document, chooses ordinary model and engine settings from current hardware and recommendations, and
verifies the final state. The triggering **Fix** click authorizes required model downloads and engine
selection. Licenses, credentials, privacy or telemetry consent, data deletion and remote-device
connections remain explicit user decisions; the agent stops at that one decision instead of filling
it in silently.
Explicit `ACTION_REQUEST` recovery runs can operate a packaged app without a source checkout. The
agent starts in an isolated temporary directory, can reach only the session bridge, cannot use
authentication routes, and is instructed to inspect, perform and verify the requested app operation
without editing user files. Codex keeps its workspace sandbox for these sessions and enables network
access so it can reach the loopback bridge. Its automatic-review mode owns that workspace-write
sandbox; VoiceStudio does not also pass Codex's mutually exclusive explicit sandbox flag. Claude
Code and OpenCode receive a generated per-run MCP configuration whose default permission is deny;
their checkout-free sessions can call only VoiceStudio's scoped API tool and cannot invoke shell or
filesystem tools. The MCP process reads the capability from the protected connection file, so the
token never appears in agent arguments, configuration or output.
Diagnosis and code repair still require a writable VoiceStudio source checkout and must follow its
`AGENTS.md`, `CLAUDE.md`, `CONTEXT.md`, and installed skills. Source builds select the current
checkout automatically. Packaged builds ask the user to choose a checkout only when source work is
actually required and remember that location. The picker rejects read-only folders and unrelated
repositories.
The repair prompt prohibits opening an issue, pushing, publishing, merging, or creating a pull
request. A successful fix ends with a review prompt so the user can inspect the diff and targeted
tests before deciding whether to propose a PR. An app-only recovery instead reports the verified
application state and any remaining user input; it never presents branch or pull-request guidance.
A renderer crash opens the dock with its sanitized error and stack. The first crash asks which
installed agent should become the default. Starting that fix stores the local choice; later renderer
crashes start the available default agent automatically. If a code repair lacks a source checkout or
the selected agent is missing, the dock remains open and asks for that prerequisite instead of
discarding the failure. App-only recovery does not show or require the source picker.
Transient dynamic-module fetch failures perform one controlled renderer reload so the module loader
actually refetches the chunk. A session marker survives that reload, so only the same repeated
failure invokes agent recovery and no reload loop is possible.
Only one repair process runs at a time. **Stop** terminates its process tree. Output is retained for
the current app session and streamed into the dock; JSON event formats from supported CLIs are
reduced to readable agent and tool output.
Verification: `electron/src/main/repair-api-bridge.test.ts` proves that the real remote credential
never enters the agent connection file, forged credentials are replaced inside main, Diagnose stays
read-only, authentication routes stay blocked, Electron restart/setup controls work without Python,
clean reinstall stays unavailable, and teardown removes the capability. The
`electron/tests/native-repair-agent-smoke.mjs` smoke launches an isolated Electron shell,
checks the native bridge and source checkout, verifies detected-agent controls, asserts that the
closed launcher stays centered on the right edge, and confirms that localized update-channel labels
render without raw keys. `electron/tests/error-recovery-smoke.mjs` verifies in-place render recovery,
the one-time chunk reload, scrubbed repeated-failure evidence and automatic repair handoff. Neither
smoke launches an agent or modifies the checkout.
`electron/src/shared/repair-request.test.ts` pins the boundary between checkout-free app operations
and source-gated diagnosis or code repair.
`node electron/tests/packaged-repair-agent-smoke.mjs` launches the packaged application with a clean
profile, confirms no source checkout is present, keeps explicit app-operation Diagnose/Fix enabled,
and verifies Electron main still rejects an ordinary source-repair request.
`node electron/tests/packaged-repair-operation-acceptance.mjs <agent>` is the opt-in live acceptance
for an installed CLI and an already-running packaged backend. It performs a read-only TTS readiness
check through the scoped bridge and starts no download. Windows packaged runs pass for Codex, Claude
Code and OpenCode; Pi was not installed on the verification host. The harness selects the native
package and backend port per platform. A packaged Ubuntu 26.04 run also proves that a Windows
OpenCode npm shim injected into WSL's PATH is rejected instead of being exposed as a usable Linux
agent; acceptance with a native Linux CLI remains required.
+75
View File
@@ -0,0 +1,75 @@
# Electron runtime setup
The development shell keeps DevTools closed by default. Open it with the normal
Electron shortcut, or set `VOICESTUDIO_OPEN_DEVTOOLS=1` before `bun run dev`.
This prevents Chromium's detached performance monitor from injecting failing
timers into the app execution context during route transitions.
Packaged startup first checks for a running VoiceStudio backend. If one answers, the shell attaches without creating or modifying a Python environment.
Otherwise, an installation matching the bundled `pyproject.toml` and `uv.lock` is required. The setup view explains the dependency download and waits for **Install local runtime**. Startup and Retry do not install automatically. Model downloads remain separate engine actions.
The setup view shows the runtime destination before downloading. The user can keep the default location, choose a folder with the native directory picker, or restore the default. The install action first checks that target filesystem for 9 GiB of free space (the same environment allowance as Tauri) and verifies write access. It then copies bundled backend/package sources into the selected `VoiceStudio/project` directory, preserving the interpreter and unrelated files. Release packages include the same pinned official Astral uv 0.12.13 executable as Tauri, so first launch skips the uv network bootstrap; development builds fall back to an installed uv or an app-private download. `uv sync --frozen --no-dev --python 3.11` installs dependencies. On a CUDA host, setup also installs the pinned cuDNN 8 compatibility wheel required by CTranslate2; Linux setup clears the obsolete executable-stack request from the installed CTranslate2 library for hardened kernels. The runtime is marked ready only after the compatibility files and imports validate. No environment is created under the read-only application bundle.
Setup content is anchored below the fixed brand header with fluid top padding. Its primary action therefore stays stationary while native status and font metrics settle, including Wayland and short-window launches.
The branded setup view reports environment checks, uv download, dependency installation and final verification from real runtime phases. During dependency installation it shows the resolved package count, announced transfer bytes, remaining data, current rate, ETA and the largest unfinished package; stale rate and ETA values disappear when the transfer pauses. Once the last announced artifact arrives, the view switches to package installation instead of leaving that artifact displayed as if its download stalled. It also streams the latest setup activity while preserving the full log disclosure. Retry and Clean & Retry preserve uv's verified download cache outside the replaceable Python project, so rebuilding a broken environment does not repeat the multi-gigabyte transfer.
Cancel stops the active process tree or download and returns to setup. Failure preserves logs and offers Retry plus a confirmed Clean & Retry. Clean recovery removes only the dedicated Python project in the default runtime or a custom runtime Electron has claimed after beginning installation; compatible Tauri environments and unowned custom locations are refused. Incomplete environments or a changed dependency graph require setup again. Source updates replace old bundled modules while preserving the venv.
Verification: runtime regression tests cover consent gating, dependency changes, incomplete environments, failed repair, cancellation before download, and source replacement. `node electron/tests/packaged-smoke.mjs --setup` verifies the first-run view without initiating an installation. `--install` performs the explicit isolated runtime installation and verifies the packaged renderer's same-origin connection to its managed backend.
The first-run browser gate also survives a partial preflight payload without blanking, keeps navigation disabled until a complete passing report arrives, and presents required, installed and curated recommended models before the optional catalogue. Its compact 640 px layout leaves the model cards usable without horizontal overflow.
A subsequent packaged launch also reused that installed runtime and started a managed backend. Electron discovers compatible Tauri default, custom and portable runtime locations and can reuse them without a download when the interpreter and both frozen dependency manifests match. Its location record distinguishes reused environments from custom runtimes Electron creates. Uninstall includes only an Electron-owned custom runtime; it never claims or removes a reused Tauri runtime.
While the native supervisor is attaching, starting, restarting or reporting a crash, Electron pauses
renderer queries through TanStack Query's online state. Active queries resume when the supervisor
publishes `ready`; this avoids a burst of predictable 502/503 requests while retaining the backend
failure screen, restart action and logs.
Source-mode shutdown closes active renderer-proxy streams as well as its listening socket. A live
backend response or keep-alive connection therefore cannot strand Quit, Restart, or an isolated
native lifecycle check.
The long-running supervisor probes the canonical compact `/health` contract. Full `/system/info`
hardware, storage and settings data is fetched only by views that use it, avoiding repeated payload
work while Dubbing or another model-heavy workflow is active.
`/model/status` binds its checkpoint and UTC load timestamp to the model instance that actually
became resident. Engine Ready and diagnostics therefore retain the loaded model identity across
renderer refreshes, even if the configured checkpoint changes before the resident model unloads.
Idle status clears both fields, and failures resolving preferences cannot break the recovery surface.
If another VoiceStudio shell replaces Electron's managed backend, Electron pauses renderer requests
for a bounded handoff window and attaches to the healthy replacement. The intentional ownership
transfer does not create a crash record or strand the app on its recovery screen.
If no replacement appears during that ten-second window, the owned exit becomes a persisted crash
with its exit code, timestamp, app version, uptime and bounded log tail. The recovery screen exposes
those details, and the next Electron launch restores them while an intentional later quit stays out
of the journal. Opening the details marks that crash seen and clears its repair-attention indicator,
while retaining the evidence for bug reports; a later crash starts unacknowledged. `node electron/tests/native-crash-journal-smoke.mjs` exercises that complete native
process/relaunch path with an isolated profile.
A fresh Windows environment has also completed a real frozen dependency installation and import check. The current packaged app repaired the isolated runtime after its dependency manifest changed, started backend 0.5.2, then relaunched against the same runtime without setup; renderer, preload bridge and same-origin API passed, and deliberate shutdown removed the run sentinel. A separate empty-runtime, empty-uv-cache and empty-Hugging-Face-cache run completed dependency setup, required-model download, onboarding and a paused 341804-byte first-sound WAV before shutting down cleanly. The segmented model downloader keeps both the canonical cache blob and snapshot pointer on Windows, using a hard link when symlinks are unavailable, so model loading does not download the same multi-gigabyte weight again.
Ubuntu 26.04 under WSL completed the current frozen dependency graph from an empty uv cache: 227 packages and a 7.9 GiB runtime. The current unpacked Linux package used its bundled uv 0.12.13, completed that explicit runtime installation from empty dependency and model caches, downloaded the 3,267,470,260-byte required OmniVoice model to a host-backed cache after preflight correctly rejected the space-constrained WSL disk, loaded it on the RTX 4090, and produced a paused 341,804-byte first-sound WAV before shutting down cleanly. A clean AppImage profile also completed the visible explicit setup action from the dependency cache and connected its packaged renderer to backend 0.5.2 under WSLg Wayland. The same profile relaunched without installation, reused its managed runtime, and cleared the backend run sentinel on both deliberate exits. These runs exposed and fixed Electron's missing POSIX nested-operation ownership descriptor.
Remaining verification: fully uncached packaged installation through first generated sound on macOS, plus native directory-picker interaction. Use `--install` only for an explicit integration run: it installs real dependencies into an isolated profile. `--first-sound` continues through required-model setup and verifies a valid paused WAV. Set `VOICESTUDIO_TEST_PROFILE` to that profile to verify a subsequent launch.
Deliberate shutdown remains available during deferred native and ML imports. Electron can therefore retire the backend run sentinel before Windows performs its bounded process-tree termination, so closing during “Preparing GPU libraries” does not become a false crash warning on the next launch. A packaged Windows acceptance quits inside that gated startup phase and verifies the sentinel is gone.
The backend's default desktop origins include `app://voicestudio` alongside the Tauri origins. HTTP CORS and WebSocket checks share this list. An explicit `OMNIVOICE_ALLOWED_ORIGINS` overrides the defaults; deployments using it must include the Electron origin to allow native live dictation. Non-loopback WebSocket connections still require remote authorization.
Clean recovery serializes cleanup against install, region and location actions. Filesystem failures return to setup with current logs and access guidance. Closing or restarting the app during cleanup prevents that stale action from starting a new installation.
An installation-in-progress marker persists through interruption or verification failure. Both readiness and legacy-environment compatibility reject marked projects until a successful import check completes, so Retry cannot bypass a partial installation merely because its copied dependency manifests match.
macOS packaging includes the microphone purpose description shared with Tauri and the audio-input entitlement for the app and helper processes, alongside Electron's runtime entitlements. This is checked by the packaging contract; an actual signed macOS microphone run remains required.
Setup reserves its active operation before asynchronous compatibility checks. Repeated install clicks cannot race those checks, cancellation invalidates preflight, and each explicit attempt starts with a fresh elapsed timer and activity log.
If the renderer remains empty after three bounded reloads, Electron paints an asset-independent localized recovery page. Its retry clears only Chromium's display cache, schedules a relaunch, and still shuts the managed backend down cleanly; voices, projects, models and preferences remain untouched.
Branding remains in a fixed native title row while onboarding status loads, installation runs or recovery needs retry. Runtime details scroll independently below it; installer phases wrap into two columns on narrow windows, so a growing progress/log surface cannot clip the wordmark or window controls.
+17
View File
@@ -0,0 +1,17 @@
# Electron storage settings
Settings > Storage reads the existing cached disk report, shows volume use/free space, model cache, application data, engine environments and temporary files, and marks incomplete scans explicitly. Largest models and data subtotals expand inline. Warning formatting and byte formatting are shared with Tauri.
Open folder uses Electron's native reveal bridge, with the existing backend reveal route for browser development. Model and log links open their existing management views. Temporary-file cleanup requires explicit confirmation with the running-job warning. A partial deletion reports failure instead of claiming all files were cleared, and refreshes usage. Opening the page never deletes anything.
Database backup status displays the latest pre-migration snapshot and date. This is database backup information, not a claim that source media and generated files are backed up. History retention reuses the confirmed cap editor from Privacy.
The model cache location uses Electron's native directory picker. Main verifies the directory is writable, stores a one-shot `models_dir` capability in the backend data directory, and returns only its token to the renderer. The backend consumes that token when persisting `OMNIVOICE_CACHE_DIR`; raw host paths never cross the HTTP boundary. Reset uses the same capability flow with an empty path, and either change takes effect after restart.
Reset & remove provides four common presets and an advanced per-scope list with measured disk sizes. The renderer owns UI preferences, history, drafts and IndexedDB projects. Main owns settings, generated content, engines, tools, models, caches and logs; it accepts only known scopes from the trusted main frame, rejects remote-backend resets and unsafe roots, stops the local backend before deletion and restarts it afterward. Removing voices/projects/audio requires typing the localized confirmation word. Shared Hugging Face caches carry a separate warning. Pending draft writes are suspended before reload so deleted work cannot be recreated by pagehide persistence.
The application data location uses Electron's native folder picker and a main-process one-shot authorization. Relocation stops the managed local backend, copies and verifies every file into an empty destination, atomically persists `OMNIVOICE_DATA_DIR` in the shared durable environment, restarts the backend and confirms `/system/info` advertises the new path before removing the old copy. A failed copy or activation restores the previous setting, deletes only the verified destination and restarts from the original folder. If final old-folder cleanup fails, the new location stays active and the UI tells the user that the old copy can be removed manually. Remote and separately started backends are rejected because Electron cannot freeze their writes safely.
Remove all data scans the backend data root, Electron runtime/configuration, logs, durable environment and model cache with real sizes. Shared Hugging Face caches remain an explicit opt-in. After typed confirmation, main rescans and validates every root, stops the backend and hands the exact plan to the signed desktop helper. The helper canonicalizes every path, waits for Electron to exit, then removes the locked Chromium/runtime tree without following a path alias outside VoiceStudio-owned data. A failed helper launch restores the backend and keeps the app open; a successful handoff quits immediately. Removing the installed application binary remains the operating system's normal uninstall step.
`electron/tests/storage-settings-smoke.mjs` verifies warning/partial-scan display, folder reveal, cancel/confirm cleanup, partial cleanup failure and backup status with mocked mutations. Live read-only reports returned all four categories, one volume and an existing backup. No actual files were deleted. Tauri storage regression tests pass after shared-helper extraction. Connection, performance and privacy browser tests also pass after the settings-navigation refactor.
+31
View File
@@ -0,0 +1,31 @@
# Electron tools workspace
Tools in the cloning sidebar or command search opens three utility panes:
- Directorial AI parses direction into instruction, translation hint, rate bias,
tokens and taxonomy using the existing direction service.
- Speech-rate fit submits translated text, a positive time slot and target
language to the existing fitting service. Invalid durations block submission.
- Probe file submits an absolute path to the existing ffprobe metadata endpoint.
Each operation starts only on explicit submission. Switching tools aborts the
frontend request and prevents stale results from replacing another tool's content.
LLM-backed operations follow the configured backend routing. Errors stay compact
with expandable diagnostics. No new processing engine or network dependency is added.
The tools browser smoke verifies request fields and results with mocked responses.
Settings > Audio tools exposes the existing app-managed media bundle installer,
status/progress, system-copy selection, restore, and yt-dlp update/restore actions.
Downloads start only through an explicit action. Custom executable selection still
needs the Electron native path-authorization bridge.
The existing checksummed installer successfully installed ffprobe in the development
app data directory. A real probe then read two streams and a four-second duration
from an MP4 fixture. No system-wide binary or application version was changed.
Custom FFmpeg/FFprobe executables can be selected from Media tools in the desktop app. The native picker validates the selected executable with `-version`, then issues the same one-use path capability used by Tauri in the backend-advertised data directory. Cancelling leaves the current tool unchanged. Browser-only sessions do not expose the native picker.
The live Windows conversion smoke now exercises the complete Tools > Convert path with the selected
Faster-Whisper and OmniVoice engines: upload, ASR, saved-profile conversion, returned waveform and
Vidstack playback-clock advancement. The fixture uses an existing clone profile and does not change
the user's engine selections.
+127
View File
@@ -0,0 +1,127 @@
# Electron transcriptions
Open Transcriptions above the engine list, or through command search. Upload an
audio file or start dictation, then stop recording to transcribe. The workspace
checks dictation readiness before recording/upload and again before transcription.
When no model is active, the workspace offers an installed model first or the
backend-recommended model to download after an explicit click. It shows install
progress, supports cancellation, and unlocks recording/upload when ready. If the
dictation engine is unavailable, use Settings > Models > Dictation to recover it.
Results use the existing Tauri `omni_transcriptions` history format, shared reader
and writer, with the same newest-first 200-entry bound. Electron and Tauri have
separate browser storage origins; sharing the format does not migrate old data
between applications. You can search results, inspect available segment timings,
copy or export text, delete an entry, and send text to the cloning script.
The secondary pane is labeled **History** so its saved-result list stays distinct
from the Transcribe workspace and its recording actions.
The empty workspace now leads with Start dictation and Upload audio. History-only
search/export/clear controls stay hidden until a transcription exists, and backend
failures appear once with direct Retry and Dismiss actions.
Recordings use the existing capture/cleanup helpers. Playback uses Vidstack.
Pending HTTP requests abort when leaving the workspace. Refinement settings,
native dictation and target-app delivery are described below. Cross-application
history migration remains separate parity work.
Electron installs Chromium permission-check and request handlers before either
the main window or recorder loads. Only the trusted VoiceStudio top-level origin
receives audio capture; camera, embedded-frame and foreign-origin requests are
denied. The operating system's microphone permission remains authoritative.
Dictation model settings now expose optional LLM cleanup, filler removal, self-corrections and technical-term preservation. Upload/record transcription reads the saved master setting; if settings are unavailable, it safely keeps raw transcription. Raw text remains in history, with any refined text stored separately and available in an expandable section with Copy. Reference-audio ASR continues to use raw transcription. Cleanup failure notes reuse the shared Tauri status mapping.
Transcriptions now exposes microphone and mono/stereo selection using the exact
same RecordingInputs component as reference recording. Controls are disabled
during capture/processing and feed the existing recorder constraints. Browser
checks cover device/channel choice with a simulated device list; physical device
recording remains a native macOS/Linux verification gate. Windows shortcut and
target-insertion coverage is described below.
Recorder lifecycle guards now release microphone access granted after navigation
and cancel pending cleanup on unmount. A late cleanup result cannot start a
transcription or load a reference in a closed view. Two fail-before/pass-after
regressions cover delayed permission and delayed cleanup.
Transcription timing labels and text-export formatting are shared with Tauri.
Exports retain date/language context, use the same dated filename and confirm a
successful save; segment labels show both timing bounds
when available without inventing missing values. Shared Tauri tests and browser
checks of the downloaded file verify this behavior.
History supports Clear all with inline confirmation and cancellation. A failed
storage write preserves visible and stored history and leaves confirmation open
for retry. Individual deletion reads the latest stored entries before writing.
Browser checks cover cancel, failed storage and successful confirmed clearing.
Live Dictation now streams mono 16 kHz PCM through the shared Tauri AudioWorklet
and anti-alias capture graph. Record remains the separate clip-preview workflow.
Dictation checks the selected model is enabled and installed before requesting
a microphone; pause, stop and cancel preserve explicit session boundaries.
Live utterances enter history once, including repeated speech, and EOF summaries
avoid duplicate entries. Legacy fallback finals also finish normally. Failed
history writes retain visible text for copying. Leaving the page releases capture.
Verification includes eight controller tests, mocked browser controls and a real
installed Parakeet session using Chromium's prerecorded test input (no physical
microphone). The backend returned live speech and final history successfully.
The real Faster-Whisper upload check also verifies timed segments, text export,
Clone handoff, persisted history and deletion against the running local backend.
The production build emits the exact shared worklet asset. The recorder widget,
global shortcuts and target delivery are implemented; native Windows acceptance
passes, while physical microphone and macOS/Linux interaction remain release gates.
The tray now offers localized Start dictation and Stop recording actions backed
by a dedicated, non-activating recorder window. It captures the output target
before revealing the recorder, queues startup/stop events until registration,
and uses the shared native delivery helper. The main app frame cannot invoke
recorder-only output IPC. Cancellation/navigation/crash invalidate pending work;
sequence numbers reject duplicate deliveries. Clipboard fallback retains the
complete transcript and is labeled as copied, not inserted.
Native Windows smoke checks cover helper acceptance, pause/resume, no-speech,
cancel/reopen, main-frame output denial and saved transcript history. A separate
disposable target-app smoke verifies actual ordered insertion and clipboard
restoration. Native macOS recorder/insertion and Linux portal shortcut behavior
with a physical microphone remain unverified; WSLg verifies tray/watch behavior
and clipboard fallback where no desktop portal is available.
Windows cross-application insertion is now verified against a disposable second
Electron process: two utterances arrive in order and the clipboard is restored.
The test uses deterministic ASR and a fake microphone, but the real native target,
activation and paste path. It found a shared Tauri/Electron race between queued
paste consumption and the next utterance. The native operation now holds its lock
through the existing 300 ms clipboard-consumption window. The integration failed
before the fix and passed twice afterward; 19 helper and 16 Tauri tests pass.
macOS/Linux insertion and global shortcut/hold/portal behavior remain native
acceptance gates.
Global dictation now follows the saved shared backend mode: Hold stops on key
release (including release before microphone startup), while Toggle stops on the
next press. Settings > Models > Dictation exposes mode, key recording, save and
reset. Key recording reuses Tauri's parser and modifier/cancel behavior. The
accelerator is stored atomically in Electron userData; a failed save restores the
previous native binding. Disabled dictation unregisters the shortcut. Repeated
preferences refreshes do not reopen a declined portal permission request.
Windows native integration verifies the settings recording/save path, persisted
accelerator, real global key presses, early hold release, toggle and disable. It
uses a fake microphone and a no-speech response, so it does not type into arbitrary
apps. Native session and settings regressions total 19; Tauri's seven existing
key-recording tests still pass. Native macOS/Wayland interaction and restarting
capture while a previous session is still transcribing remain native verification
work. Windows behavior parity is verified with deterministic ASR and a fake mic.
Capture restart now matches Tauri's actual state rules: transcription remains
exclusive, while a visible completed/no-speech/error result accepts the next
shortcut immediately. A session-tagged phase report enables that decision in
main. Replacement retains an early hold release while old output cleanup finishes;
stale cancellation cannot dismiss the new session. The Windows restart integration
failed before this change and passes after it, alongside 17 session regressions.
Electron development now uses the renderer's same-origin WebSocket proxy even
when the preload supplies a backend URL; the packaged app scheme still uses the
trusted backend URL directly. Both paths have targeted coverage. A real native
Electron development run, with prerecorded microphone input and installed Parakeet,
returned live text and saved the transcript through the proxy successfully.
+15
View File
@@ -0,0 +1,15 @@
# Electron voice design
Open Design from the cloning sidebar or command search. Choose a preset or adjust
voice traits, write a script, and synthesize. The existing Tauri category, conflict
resolution and seed helpers build the request; clone references are never forwarded.
A seed stays fixed while adjusting traits, and New seed creates another identity.
Save as voice profile stores the seed and validated attribute state using the existing
backend profile endpoint. Saved design profiles can be restored from the Design sidebar.
Generation uses the shared lifecycle, cancellation, progress and Vidstack output player.
Local natural-language trait extraction calls the existing deterministic mapper;
manual choices cancel queued or running mappings. Personality and demo starting points,
the collapsed recipe summary, language and production controls, profile editing and persona
export use the same established controls as the Tauri workflow. Native verification and any
remaining release gates are recorded in `electron/PARITY.md`.
+24 -14
View File
@@ -42,30 +42,38 @@ instead of that dedicated-VRAM floor.
## Install
1. Download the v0.7.2 prebuilt for your platform from
[audio.cpp releases](https://github.com/0xShug0/audio.cpp/releases/tag/v0.7.2)
On a supported desktop, install the model from **Settings → Models**, then use
**Install runtime** on the audio.cpp/Sortformer engine row. VoiceStudio fetches
the pinned archive, verifies its published size and SHA-256, rejects unsafe
archive paths, probes its device list, and installs it under the update-surviving
app-data engine directory. The action is explicit; generation never downloads
or updates executable code.
For a user-managed runtime:
1. Download the v0.7.4 prebuilt for your platform from
[audio.cpp releases](https://github.com/0xShug0/audio.cpp/releases/tag/v0.7.4)
and extract it. Use the Vulkan archive on Windows or Linux for broad GPU
support, the CPU archive when Vulkan is unavailable, or the matching CUDA
archive on Windows for NVIDIA. A Windows CUDA install needs both the
`bin-…-cuda…` and matching `cudart-…-cuda…` archives extracted into the
same directory, as required by upstream. VoiceStudio does not download
executable code for this engine. Linux archives do not preserve the
same directory, as required by upstream. Linux archives do not preserve the
executable bit, so run `chmod +x audiocpp_server` after extracting one.
Verify the archive before extracting it. The pinned SHA-256 checksums are:
| Archive | SHA-256 |
|---|---|
| `audio-v0.7.2-bin-windows-x64-cpu-portable.zip` | `0b1f4bd78c5226ee3fa0eb24d95d603a429439cdf5dab45872d44a87412dd8c1` |
| `audio-v0.7.2-bin-windows-x64-vulkan.zip` | `15b8232eae740e21e507d87f827a89966de9451b085a45932d9e214e032962c1` |
| `audio-v0.7.2-bin-windows-x64-cuda12.4.zip` | `06c426095008022a2984ff1c75de4c9fab463c4201c0ff0a5dc4e14043f52326` |
| `audio-v0.7.2-cudart-windows-x64-cuda12.4.zip` | `7115be4d462817ad293f7932a8ac436d51023128e6728af09bba92a85593f393` |
| `audio-v0.7.2-bin-windows-x64-cuda13.3.zip` | `f975fec52745807b8c787e826c110acc3424a45156092455e1635604b68ec832` |
| `audio-v0.7.2-cudart-windows-x64-cuda13.3.zip` | `9b508f702636a9cdf3bf4dd8e75a86c20a0b87bdc39ca07e714c82f748efc1fa` |
| `audio-v0.7.2-bin-ubuntu-x64-cpu.tar.gz` | `6f5e43dd7b80e8ddf688ef84b411fadcd1f934d2c83963178bc4e2d9c4f07736` |
| `audio-v0.7.2-bin-ubuntu-x64-vulkan.tar.gz` | `fee1f978cee76453cf17f00196554bc2ee294645739538af0726a143b6a69a23` |
| `audio-v0.7.2-bin-macos-arm64-metal.tar.gz` | `c01e4f82971bedbe341697e63a9cebd5a5d1f72d5a9bcb51a3191f95ddab7a95` |
| `audio-v0.7.2-bin-macos-x64-metal.tar.gz` | `3862270f33439077225324169313f727064f727305b54d8ce920244d75ddcc24` |
| `audio-v0.7.4-bin-windows-x64-cpu-portable.zip` | `d241c56ba78fd3c1b28bf289792fb8ec258d36586b4e0c8d667080ec248c0d2f` |
| `audio-v0.7.4-bin-windows-x64-vulkan.zip` | `057332f9e3fb37706a8ecb5075ac1797efcd85fdccd739f7b65761a5920f2828` |
| `audio-v0.7.4-bin-windows-x64-cuda12.4.zip` | `83fdd5b6e7bd4362604c10cc88d7d3564ef82030dc1d21c693a62cdcbe2e5e38` |
| `audio-v0.7.4-cudart-windows-x64-cuda12.4.zip` | `88d8943a2a8011f02c2a4efa7dbbe258608362615cce51e7f0e0e3a0c62f5a43` |
| `audio-v0.7.4-bin-windows-x64-cuda13.3.zip` | `af56012969bcb68f54e6ea14a123e5c6ecd62830c1b2816080eb7f99d985a779` |
| `audio-v0.7.4-cudart-windows-x64-cuda13.3.zip` | `c20793d8cc9b7c66ab28ab335aa908c726f2df15832330ddcfdbc837c7670145` |
| `audio-v0.7.4-bin-ubuntu-x64-cpu.tar.gz` | `638e6114550c5ea02b96907de400379c8f31bd13325525083f98d158027acc40` |
| `audio-v0.7.4-bin-ubuntu-x64-vulkan.tar.gz` | `e0ef3123a9f94e130ad463db0db5a69b65485ef8db1b46edead00c03a86fa787` |
| `audio-v0.7.4-bin-macos-arm64-metal.tar.gz` | `639926715b1cb537f82aa31656aabbae5d9a85ac36568c402026968f3072e2b3` |
| `audio-v0.7.4-bin-macos-x64-metal.tar.gz` | `bdb797d54dcf8416bd5ac0fac282ce5500dd08843f8f22e20e9fc378ebc24c1f` |
Run `sha256sum <archive>` on Linux, `shasum -a 256 <archive>` on macOS,
or `Get-FileHash <archive> -Algorithm SHA256` in PowerShell and compare the
@@ -140,3 +148,5 @@ complete reinstall, file an issue with the package listing.
audio.cpp runs as a managed native server (no Python venv, no
`transformers` conflict). Only the downloaded GGUF counts toward
[sidecar disk usage](disk-usage.md).
The catalogue checks the exact Breeze Q8_0 package inside the shared GGUF repository. Other audio.cpp packages do not make Breeze appear installed. Required weight files must also satisfy the normal weight-size floor; empty or truncated placeholders remain incomplete. Broader native model discovery and execution are still pending integration.
+37 -21
View File
@@ -26,24 +26,19 @@ the model's license once.
- https://huggingface.co/pyannote/speaker-diarization-3.1 → **"Agree and
access repository"**.
- https://huggingface.co/pyannote/segmentation-3.0 → same.
4. Restart the dub job. The first run downloads ~600 MB of model weights.
4. Install the model in **Settings > Models > Diarisation**. Wait for installation
to finish, then retry transcription. Jobs only load locally installed files.
If you skip the license acceptance, the HF API returns `401 Unauthorized` for
If you skip the license acceptance, the HF API returns `401 Unauthorized` or `403 Forbidden` for
the download — the same error class the in-app **"Open docs for this error"**
button deeplinks to.
## Fallback behaviour
When diarization is unavailable (no HF token, license not accepted, model
download failed mid-run), VoiceStudio's dub pipeline falls back to a
**silence-gap heuristic** that splits speakers on long quiet stretches.
You'll see a warning toast and the `dub_core.py` reason string surfaces in
the job log:
- `"diarization_skipped:no_token"` — no token resolved from the cascade.
- `"diarization_skipped:401"` — token present but unauthorised on the gated
model (license not accepted).
- `"diarization_skipped:network"` — model download interrupted.
When diarization files are missing or the installed runtime cannot load,
VoiceStudio falls back to a **silence-gap heuristic**. The job warning explains
whether to install/repair the model or inspect the backend log. Jobs never
download missing weights.
The heuristic is not as accurate as pyannote — speakers with similar pitch
or rapid turn-taking conversation get merged — but it lets the dub finish
@@ -51,15 +46,36 @@ end-to-end instead of erroring.
## HF token requirement
Diarization is the one VoiceStudio feature where a HF token is **required**, not
just recommended. See
[docs/setup/huggingface-token.md](../setup/huggingface-token.md) for the
three-source cascade and how the in-app **Settings → API Keys** panel works.
An HF token and accepted repository access are required to install the gated
pyannote bundle. Once its pipeline, segmentation and embedding files are cached,
inference works offline without retaining the token. See
[Hugging Face token setup](../setup/huggingface-token.md).
## Troubleshooting
- HF 401 → see [troubleshooting.md#2-hf-401--pyannote-license-not-accepted](../install/troubleshooting.md).
- Model download stuck → check `~/.cache/huggingface/hub/models--pyannote--*`
size grows during the dub; if it stalls at 0 bytes, your token isn't being
read — confirm in **Settings → API Keys** that the active source has a
green checkmark.
- Install returns 401/403: verify that the token belongs to the account with
access to both gated repositories.
- Missing files: install or repair from Settings > Models > Diarisation.
- Installed model fails to load: inspect Settings > Logs > Backend. A runtime
failure does not by itself indicate a licence or token problem.
## Local installation and repair
Install pyannote from Settings > Models > Diarisation after completing the access steps above. The install includes its segmentation and speaker-embedding checkpoints. A pipeline configuration alone is not a complete installation; repair also retrieves missing dependencies.
Dubbing resolves all three files from the local cache and does not download models during a job. Once installed, the bundle can run without retaining an HF token. Missing files prompt installation or repair; transcription continues with the existing silence-gap fallback and its accuracy caveat. A runtime load failure is reported separately from missing files.
## Native Sortformer adapter
The shared diarisation path also supports audio.cpp Sortformer v1. Install the reviewed model and its checksummed native runtime in **Settings → Models → Diarisation**, then select Sortformer on that page. VoiceStudio resolves the installed Q8 GGUF locally and persists the choice; it never downloads a model while transcribing. `OMNIVOICE_DIARIZATION_BACKEND` and `OMNIVOICE_DIARIZATION_MODEL` remain authoritative overrides for managed deployments.
The adapter normalizes input to mono 16 kHz, converts sample-based turns to the shared annotation format, and contains the native process with a ten-minute timeout. Sortformer supports up to four speakers but cannot enforce the existing exact-speaker-count setting; requesting that setting follows the existing fallback/warning path. Its accelerator graph grows from the default 20-second context for clips up to 120 seconds. Longer recordings require pyannote until locally converted Sortformer v2.1 streaming weights can be distributed. No binary or model is downloaded implicitly. The model's CC-BY-NC-4.0 restrictions apply; see the upstream audio.cpp model card. Native process status and cancellation are wired into the shared job lifecycle.
Run `uv run --no-sync python scripts/smoke_sortformer.py <local-media> --min-speakers 2`
to verify an installed runtime against local media. Add `--seconds 120 --cancel-after 0.5`
to verify process cancellation. The smoke writes only a normalized temporary clip and never
downloads models or changes app data. On the maintained Windows RTX 4090 fixture, v0.7.4
processed a real 60-second Dubbing source through Vulkan in 3.27 seconds, returned 16 bounded
turns, and cancelled the 120-second variant in 1.2 seconds without leaving a registered process.
audio.cpp v0.7.4 contains the newer Sortformer v2.1 streaming runtime. Its NVIDIA Open Model License checkpoint currently has no redistributable GGUF package, so VoiceStudio does not advertise a download that upstream cannot legally supply. A locally converted mixed F16/F32 checkpoint is the planned long-recording path once the model can be offered through an explicit local-package workflow.
+4
View File
@@ -15,6 +15,10 @@ and ships a script that finds and removes them for you (with a dry-run first).
owns with its real size, lets you opt in (separately) to the shared Hugging Face
model cache, asks you to type `DELETE`, then removes everything and quits.
The Electron build finishes deletion through its signed desktop helper after the
window exits, because Chromium keeps parts of its profile directory locked while
the app is open. The same ownership checks and model-cache opt-in still apply.
This is the right path if you installed the **.dmg / .msi / AppImage** — you
don't have the repo, so the script below isn't available to you.
+10 -3
View File
@@ -268,9 +268,12 @@ but with **operation-count budgets** in
makes **zero** TTS calls. The zero-decode / zero-rewrite budget activates
with the natural-rate cached fast path (each cache is then decoded exactly
once, by the final assembly).
- **Batch dubbing (native batches)**: N renderable segments at batch width W
- **Dubbing synthesis (native batches)**: N renderable segments at batch width W
cost exactly ⌈N/W⌉ `generate_batch` calls and zero per-segment `generate`
calls when native batching is enabled.
calls when native batching is enabled, in both interactive and queued jobs.
- **NLLB dubbing translation**: rows sharing a target language render in
bounded batches instead of one model forward per subtitle. Mixed targets
retain their request order, and a failed batch retries per row.
Updating a budget is a deliberate act: if a change legitimately adds an
operation to a guarded path, change the expected count in the same PR with a
@@ -279,7 +282,7 @@ comment justifying the new floor. Never loosen a budget just to make CI pass
## Batch and streaming behavior
Batch dubbing renders several segments in one native forward pass when the
Interactive and Batch Dubbing render several segments in one native forward pass when the
selected engine supports it. The width is derived from the host rather than
fixed, because a wider forward pass needs proportionally more device memory:
CPU hosts and cards with less than ~2 GB of headroom above the engine's
@@ -288,6 +291,10 @@ and 8 as headroom allows. `OMNIVOICE_DUB_BATCH_WIDTH` overrides it (1 disables
batching, 16 is the ceiling). Engines without native batching inherit a
compatibility fallback that preserves the one-segment behavior.
NLLB similarly groups subtitles by target language and translates four rows
per forward pass on CPU/MPS or eight on CUDA by default. Set
`OMNIVOICE_NLLB_BATCH_SIZE=1` to disable it or choose up to 32 explicitly.
Streaming clients also receive measured latency in the `/ws/tts` terminal
`done` frame: `ttfa_ms` is request-to-first-audio, `gen_time_s` is the
end-to-end wall clock including delivery, and `rtf` is *synthesis* time
+95
View File
@@ -0,0 +1,95 @@
# Profile portraits and saving
The Electron cloning workspace shows a name-and-portrait form immediately after
an audio upload or recording. Saving selects the new profile; **Use without
saving** proceeds with the temporary reference. **Change voice** opens the saved
voice chooser without discarding the script.
Saved voices use compact selectable rows, with separate preview/delete controls.
The script editor fills the available workspace; generation controls and the
compact latest-take player stay anchored at the bottom. Preserve this established
layout when polishing visuals rather than moving the primary action. The reference pane includes saved-reference playback.
Language search measures its virtual list after the popover mounts.
Latest take spans the full content pane. Closing it stops its playback and clears
the current player without deleting the saved take from history.
Initials are generated locally from the first and last words of the profile name.
Optional JPEG, PNG and WebP uploads are limited to 5 MB and 16 megapixels,
cropped to 256 × 256, and re-encoded without source metadata. Portraits live next
to profile audio as `<id>.portrait.jpg`; deleting a profile removes its portrait.
The reference pane and saved-profile editor allow replacing a portrait through either upload or the same explicit five-result image search used while creating a profile.
After a new clone profile is saved, Electron releases the temporary upload and
switches the composer to the returned profile as one atomic state change. The
returned language, style and server-generated local-ASR transcript become the
composer metadata, so deleting or changing that profile cannot revive a stale
reference file from before the save.
`POST /profiles` accepts an optional `image` multipart field and returns the full
profile, including `image_url`. `PUT /profiles/{id}/image` replaces the portrait;
`GET /profiles/{id}/image` serves it. No database migration is required.
## Optional image search
Search runs only when the user clicks **Search images**. It sends the entered
name to the ordinary Google Images search page. No API key is required.
The search requests SafeSearch and Google's JPEG file-type filter, then returns
up to five valid thumbnails. Thumbnails are decoded, normalized and saved locally
through the same profile-image flow as uploads; remote originals are never fetched.
Only Google's HTTPS thumbnail proxy hosts and embedded JPEG previews are accepted from that page.
If Google requires browser JavaScript, consent, or a CAPTCHA, VoiceStudio falls back to Openverse's
public search with mature content excluded and reuse-friendly licensing filters. It never bypasses
Google challenges. Initials and file uploads remain available if neither source responds.
Filtering does not guarantee that every result is appropriate or licensed for reuse.
Live verification: `node electron/tests/profile-image-search-real-smoke.mjs` creates a disposable
local profile, opens its Electron editor, requests the explicit no-key search, verifies five
selectable portraits, persists one through the real image endpoint, verifies the normalized JPEG
and refreshed avatar, then removes the profile.
## Recoverable generation failures
Generation failures appear once in the composer, with technical details collapsed.
The script is preserved for retry. Shared audio writers recreate missing parent
directories before writing, including folders removed after backend startup.
Playback belongs to the latest-take player. Synthesize always starts generation;
playing a take never replaces that button with a playback control.
The synthesis button reserves fixed space for its label and cancellation control.
Elapsed time uses tabular digits below the button. During an active request,
`/model/status` supplies the localized runtime sub-stage and model-load percentage;
response-body progress takes over when audio delivery begins.
The voice selector labels the active voice explicitly. Voice sample opens its
reference pane, and an empty script prompts with the selected voice name.
Paste and Insert remain secondary actions; Insert explains expression tokens
in its accessible label and tooltip. These refinements preserve the anchored layout.
## Reference transcription and editing
The save-profile view uses the editor's full content width. New uploaded or
recorded references use the selected local dictation pipeline (including its
configured Parakeet/Whisper fallback). Installed-model readiness is checked before
transcription; no model downloads are initiated. Missing models leave manual
transcript entry available. ASR never overwrites text edited while it runs.
Saving waits for transcription; using the reference without saving remains possible.
Saved-voice pencil buttons open a profile editor for name, transcript, style,
and portrait. Editing does not select a different voice. The voice-sample pane's
chevron collapses it; the Voice sample toolbar control reopens it.
The sidebar lists selected TTS, ASR and LLM engines, resolved model identities
where available, and the selected installed dictation model. These are selections,
not a claim that model weights are currently loaded.
Engine change controls open Settings / Models with separate TTS, ASR, dictation,
and LLM pages. Lists scroll independently of the settings sidebar; unavailable
engines and undownloaded dictation models cannot be selected. Engine selection
uses the shared backend preferences and refreshes sidebar metadata.
Script import supports TXT, Markdown, DOC, DOCX, PDF and EPUB (text documents,
not scanned-image OCR). Paste inserts at the caret; Replace script offers Undo.
Clicking in the script shows the expression picker near the caret without taking
typing focus. The voice chooser and save-profile form share the editor width.

Some files were not shown because too many files have changed in this diff Show More