fix(dub): restore cast voices from source audio (#1484)

* fix(dub): restore cast voices from source audio

* docs: record source-audio cast repair

* fix(dub): honor explicit cross-speaker cast

* fix(dub): sanitize restored cast metadata

* fix(dub): preserve legacy per-line cast refs
This commit is contained in:
Palash Debnath
2026-08-10 23:46:37 +00:00
committed by GitHub
parent 7cde2fcd06
commit 581c51662b
15 changed files with 281 additions and 44 deletions
+1
View File
@@ -43,6 +43,7 @@ The bundled TTS model package (`pyproject.toml`) is versioned independently.
### Fixed
- Dubbing's **From video** cast now uses available source-audio samples for every speaker and short line, including jobs without a pooled diarization clone. (#1484)
- Basic Dubbing translation remains available without an LLM; Cinematic and Autofit now degrade through the existing Fast translation path instead of blocking the quality choice. (#1481)
- Linux microphone recording now falls back to WAV when WebKit cannot encode MediaRecorder audio, and desktop scaling/titlebar controls remain responsive at every UI scale. (#1481)
- Dubbing can install a missing ASR model and retry the same job, navigate back through completed stages, and finish transcription under low GPU memory without producing an empty transcript. (#1481)
+16 -12
View File
@@ -1384,7 +1384,11 @@ async def dub_transcribe_stream(
# new target language and have the ORIGINAL speaker speak it — the
# central pro-grade dubbing promise.
try:
from services.speaker_clone import extract_speaker_clones, auto_profile_id
from services.speaker_clone import (
auto_profile_id,
build_cast_sources,
extract_speaker_clones,
)
vocals_for_clone = job.get("vocals_path") or asr_audio_target
clones = {}
if labels_source == "heuristic":
@@ -1486,7 +1490,9 @@ async def dub_transcribe_stream(
except Exception as e:
logger.warning("per-segment clone refs skipped: %s", e)
if clones or seg_clones:
cast_sources = build_cast_sources(final_segs, clones, seg_clones)
job["cast_sources"] = cast_sources
if cast_sources:
if clones:
job["speaker_clones"] = clones
# Default each segment's profile_id to its detected speaker's
@@ -1508,16 +1514,11 @@ async def dub_transcribe_stream(
if s.get("profile_id"):
continue
spk = s.get("speaker_id") or "Speaker 1"
if spk in clones:
if spk in cast_sources:
# Keep one UI-visible value for pooled and per-segment
# sources. Generation resolves this line's own clip
# first and falls back to the speaker's best clip.
s["profile_id"] = auto_profile_id(spk)
continue
# No per-speaker clone for this speaker (too little usable
# audio overall) but this single line was long enough for
# its own ref — fall back to the per-segment id. The editor
# can't render it, but generation still clones correctly.
sid = str(s.get("id", ""))
if sid and sid in seg_clones:
s["profile_id"] = f"auto-seg:{sid}"
except Exception as e:
logger.warning("speaker_clone extraction skipped: %s", e)
@@ -1550,7 +1551,10 @@ async def dub_transcribe_stream(
"segments": final_segs,
"source_lang": job["source_lang"],
"full_transcript": job["full_transcript"],
"speaker_clones": job.get("speaker_clones", {}),
# The client only needs labels and durations. Never send host
# paths or reference transcripts through this public event.
"speaker_clones": job.get("cast_sources", {}),
"cast_sources": job.get("cast_sources", {}),
})
yield _sse_event("done", {})
+34 -10
View File
@@ -29,6 +29,7 @@ from services.rvc import apply_rvc, is_enabled as rvc_is_enabled
from services.incremental import segment_fingerprint, fit_fingerprint
from services.fit_planner import UNDERRUN_TOLERANCE, FitParams, plan_fit
from services.watermark import mark_synthetic
from services.speaker_clone import auto_profile_id
from api.routers.dub_core import _get_job, _save_job
from omnivoice.utils.voice_design import heal_design_instruct
@@ -174,7 +175,7 @@ CONSISTENT_MIN_REF_S = 3.0
def _speaker_key_matches(speaker_id: str, key: str) -> bool:
"""Same matching rule the `auto:` branch has always used: the safe-name
slug first (`auto_profile_id`), the raw speaker id as fallback."""
return speaker_id.lower().replace(" ", "_") == key or speaker_id == key
return auto_profile_id(speaker_id) == f"auto:{key}" or speaker_id == key
def _find_speaker_clone(clones: dict, key: str):
@@ -199,7 +200,7 @@ def _speaker_key_for_segment(job: dict, sid) -> str | None:
for row in job.get("segments") or []:
if isinstance(row, dict) and str(row.get("id", "")) == str(sid):
spk = row.get("speaker_id") or "Speaker 1"
return spk.lower().replace(" ", "_")
return auto_profile_id(spk)[len("auto:"):]
return None
@@ -689,7 +690,19 @@ async def dub_generate(job_id: str, req: DubRequest):
# editor's Voice dropdown can actually render ("From
# Video → Speaker N"). `seg_id` is closed over from
# the per-segment loop below.
seg_ref = (job.get("segment_clones") or {}).get(str(seg_id))
segment_speaker_key = _speaker_key_for_segment(job, seg_id)
# Legacy jobs may not persist diarized segment rows.
# Preserve their established per-line preference; only
# suppress it when current metadata proves the user
# explicitly selected a different speaker.
selected_is_segment_speaker = (
segment_speaker_key is None or segment_speaker_key == key
)
seg_ref = (
(job.get("segment_clones") or {}).get(str(seg_id))
if selected_is_segment_speaker
else None
)
if seg_ref:
ref_audio = seg_ref.get("ref_audio")
ref_text = seg_ref.get("ref_text")
@@ -698,6 +711,13 @@ async def dub_generate(job_id: str, req: DubRequest):
auto = _find_speaker_clone(
job.get("speaker_clones") or {}, key
)
if auto is None:
# Short lines may have no line-specific clip.
# Reuse this speaker's best source instead of
# silently reverting to the engine default.
auto = resolve_consistent_ref(
job, key, _consistent_ref_memo
)
if auto:
ref_audio = auto.get("ref_audio")
ref_text = auto.get("ref_text")
@@ -1528,12 +1548,17 @@ async def preview_segment(job_id: str, req: SegmentPreviewRequest):
pid = req.profile_id
if pid and pid.startswith("auto:"):
key = pid[len("auto:"):]
clones = job.get("speaker_clones") or {}
for spk, info in clones.items():
if spk.lower().replace(" ", "_") == key or spk == key:
ref_audio = info.get("ref_audio")
ref_text = info.get("ref_text")
break
info = None
if (
req.segment_id is not None
and _speaker_key_for_segment(job, req.segment_id) == key
):
info = (job.get("segment_clones") or {}).get(str(req.segment_id))
if info is None:
info = resolve_consistent_ref(job, key)
if info:
ref_audio = info.get("ref_audio")
ref_text = info.get("ref_text")
pid = None
instruct_str = req.instruct
@@ -1602,4 +1627,3 @@ async def preview_segment(job_id: str, req: SegmentPreviewRequest):
"X-Audio-Duration": str(round(audio_tensor.shape[-1] / sr, 2)),
},
)
+42
View File
@@ -415,3 +415,45 @@ def auto_profile_id(speaker_id: str) -> str:
"""Stable profile id prefix so `_gen` can tell auto-clones apart from
persistent voice-profile ids."""
return f"auto:{_safe_name(speaker_id)}"
def build_cast_sources(
segments: list[dict],
speaker_clones: dict[str, dict] | None,
segment_clones: dict[str, dict] | None,
) -> dict[str, dict]:
"""Return path-free metadata for every usable ``From video`` voice.
A trusted diarizer can produce a pooled per-speaker clone. When it
cannot, the pipeline still extracts clean per-segment references; those
references are valid voice prompts even though they are not reliable
evidence for grouping identities. The cast UI needs to know that a
speaker label has at least one usable source without receiving host paths
or transcript text.
"""
sources: dict[str, dict] = {}
for speaker_id, info in (speaker_clones or {}).items():
sources[speaker_id] = {
"duration": float(info.get("duration") or 0.0),
"source_count": int(info.get("source_count") or 1),
"kind": "speaker",
}
for segment in segments or []:
if not isinstance(segment, dict):
continue
speaker_id = segment.get("speaker_id") or "Speaker 1"
current = sources.get(speaker_id)
if current and current.get("kind") == "speaker":
continue
info = (segment_clones or {}).get(str(segment.get("id", "")))
if not info or not info.get("ref_audio"):
continue
duration = float(info.get("duration") or 0.0)
if current is None or duration > current["duration"]:
sources[speaker_id] = {
"duration": duration,
"source_count": 1,
"kind": "segment",
}
return sources
+4 -5
View File
@@ -84,6 +84,7 @@ import {
} from './utils/constants';
import { LANG_CODES } from './utils/languages';
import { restoreProjectExtras } from './utils/projectState';
import { castSourcesFromJob } from './utils/segments';
import { API, apiFetch, apiJson } from './api/client';
import { flushMemory as apiFlushMemory } from './api/system';
import {
@@ -1078,11 +1079,9 @@ function App() {
item.language_code || job.language_code || 'und',
);
}
// Rehydrate the auto-extracted speaker clones so the CAST dropdown's
// "🎤 From video" option reappears after a reload. Projects that
// predate the speaker-clone feature have an empty map; the Extract
// Voices button in the CAST strip handles those.
setSpeakerClones(job.speaker_clones || {});
// Rehydrate path-free cast sources. Legacy heuristic jobs may have only
// per-segment references; castSourcesFromJob recovers those too.
setSpeakerClones(castSourcesFromJob(job));
} catch (e) {
console.error('Failed to restore job_data', e);
}
+2 -2
View File
@@ -6,6 +6,7 @@ import SearchableSelect from './SearchableSelect';
import { ArchetypeIcon } from '../utils/archetypeIcons';
import { PRESETS } from '../utils/constants';
import { useArchetypes } from '../api/hooks';
import { autoProfileId } from '../utils/segments';
import { useArchetypeAsProfile } from '../api/archetypes';
import { useAppStore } from '../store';
@@ -149,9 +150,8 @@ export default function VoiceSelector({
// 2. fromVideo (dub only) slug rule byte-identical to DubSegmentRow.
const speakers = speakerClones ? Object.keys(speakerClones) : [];
for (const spk of speakers) {
const slug = (spk || '').toLowerCase().replace(/\s+/g, '_');
list.push({
value: `auto:${slug}`,
value: autoProfileId(spk),
label: `🎤 ${spk}`,
group: 'fromVideo',
groupLabel: t('voiceSelector.fromVideo'),
@@ -27,6 +27,7 @@ import { dubSegmentsText } from '../../api/dub';
import { copyText } from '../../utils/copyText';
import { openExternal } from '../../api/external';
import { TRANSLATION_ENGINES_DOCS } from '../../utils/errorDocsMap';
import { autoProfileId } from '../../utils/segments';
import toast from 'react-hot-toast';
// Translation-settings bar utility class clusters
@@ -364,7 +365,7 @@ export default function DubLeftColumn({
{t('dub.cast')}
</span>
{[...new Set(dubSegments.map((s) => s.speaker_id).filter(Boolean))].map((spk) => {
const autoId = `auto:${(spk || '').toLowerCase().replace(/\s+/g, '_')}`;
const autoId = autoProfileId(spk);
const clone = speakerClones[spk];
return (
<div key={spk} className="dub-cast__pair">
@@ -4,6 +4,7 @@ import { Button, Segmented } from '../../ui';
import GlossaryPanel from '../GlossaryPanel';
import CheckpointBanner from '../CheckpointBanner';
import { LANG_CODES } from '../../utils/languages';
import { autoProfileId } from '../../utils/segments';
const DubSegmentTable = lazy(() => import('../DubSegmentTable'));
const DubPasteTranslationDialog = lazy(() => import('./DubPasteTranslationDialog'));
@@ -261,7 +262,7 @@ export default function DubRightColumn({
{speakerClones && Object.keys(speakerClones).length > 0 && (
<optgroup label={t('dub.cast')}>
{Object.keys(speakerClones).map((spk) => {
const autoId = `auto:${(spk || '').toLowerCase().replace(/\s+/g, '_')}`;
const autoId = autoProfileId(spk);
return (
<option key={autoId} value={autoId}>
🎤 {spk}
+4 -3
View File
@@ -281,10 +281,11 @@ export default function useDubWorkflow({
}));
// #486: bind each segment to its detected speaker's clone up front, so
// a 2-speaker dub doesn't land every row on "Default".
setDubSegments(applySpeakerCloneDefaults(normalized, m.speaker_clones));
const castSources = m.cast_sources || m.speaker_clones || {};
setDubSegments(applySpeakerCloneDefaults(normalized, castSources));
setDubTranscript(m.full_transcript || '');
if (m.speaker_clones && typeof m.speaker_clones === 'object') {
setSpeakerClones(m.speaker_clones);
if (castSources && typeof castSources === 'object') {
setSpeakerClones(castSources);
}
} catch (err) {
console.warn('Transcribe SSE handler failed:', err);
+5 -6
View File
@@ -127,17 +127,16 @@ export interface DubSlice {
// The client re-renders these at full quality before final export.
previewSegIds: string[];
// Per-speaker auto-clones extracted from the source video's vocals. Keys
// are speaker_id (e.g. "Speaker 1"), values are {ref_audio, ref_text,
// duration, source_count}. Enables the cross-lingual "same voice in a
// new language" dubbing flow.
// Path-free metadata for usable source-video voices. A source can be a
// pooled per-speaker clone or a clean per-segment reference.
speakerClones: Record<
string,
{
ref_audio: string;
ref_text: string;
ref_audio?: string;
ref_text?: string;
duration: number;
source_count: number;
kind?: 'speaker' | 'segment';
}
>;
+8
View File
@@ -91,6 +91,14 @@ describe('VoiceSelector', () => {
expect(onChange).toHaveBeenCalledWith('auto:speaker_1');
});
it('uses the canonical backend slug for raw diarizer speaker ids', () => {
const onChange = vi.fn();
renderVS({ value: '', onChange, profiles: [], speakerClones: { SPEAKER_00: {} } });
open();
fireEvent.mouseDown(screen.getByText('🎤 SPEAKER_00'));
expect(onChange).toHaveBeenCalledWith('auto:speaker00');
});
it('renders a ghost row (does NOT auto-clear) for a deleted-but-referenced voice', () => {
const onChange = vi.fn();
renderVS({ value: 'p_gone', onChange, profiles: PROFILES });
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
import { applySpeakerCloneDefaults, autoProfileId } from '../utils/segments';
import { applySpeakerCloneDefaults, autoProfileId, castSourcesFromJob } from '../utils/segments';
// #486: multi-speaker dub must auto-bind each segment to its detected
// speaker's cloned voice instead of leaving every row on "Default".
@@ -17,6 +17,48 @@ describe('applySpeakerCloneDefaults (#486)', () => {
expect(autoProfileId('Speaker 2')).toBe('auto:speaker_2');
});
it('uses the backend-compatible portable slug for raw diarizer labels', () => {
expect(autoProfileId('SPEAKER_00')).toBe('auto:speaker00');
expect(autoProfileId('Guest-2!')).toBe('auto:guest_2');
});
it('recovers From video choices from legacy per-segment references', () => {
const sources = castSourcesFromJob({
segments: [
{ id: 'a', speaker_id: 'Speaker 1' },
{ id: 'b', speaker_id: 'Speaker 2' },
],
speaker_clones: {},
segment_clones: {
a: { ref_audio: '/private/a.wav', ref_text: 'hidden', duration: 3.4 },
b: { ref_audio: '/private/b.wav', ref_text: 'hidden', duration: 4.2 },
},
});
expect(sources).toEqual({
'Speaker 1': { duration: 3.4, source_count: 1, kind: 'segment' },
'Speaker 2': { duration: 4.2, source_count: 1, kind: 'segment' },
});
expect(JSON.stringify(sources)).not.toContain('/private');
});
it('strips paths and transcript text from legacy pooled clone metadata', () => {
const sources = castSourcesFromJob({
speaker_clones: {
'Speaker 1': {
ref_audio: '/home/person/private.wav',
ref_text: 'private transcript',
duration: 7.5,
source_count: 2,
},
},
});
expect(sources).toEqual({
'Speaker 1': { duration: 7.5, source_count: 2, kind: 'speaker' },
});
expect(JSON.stringify(sources)).not.toContain('/home/person');
expect(JSON.stringify(sources)).not.toContain('private transcript');
});
it('never clobbers a profile_id the user already chose', () => {
const segs = [{ id: '0', speaker_id: 'Speaker 1', profile_id: 'preset:narrator' }];
expect(applySpeakerCloneDefaults(segs, clones)[0].profile_id).toBe('preset:narrator');
+33 -3
View File
@@ -36,10 +36,40 @@ export function segmentGenInputs(s) {
}
/** The `auto:<safe>` profile id for a diarized speaker. Mirrors the backend's
* clone-resolution key (`speaker_id.lower().replace(" ", "_")`) and the
* Voice-dropdown option value in DubTab, so the three always agree. */
* portable filename/profile slug and is shared by every voice picker. */
export function autoProfileId(speakerId) {
return `auto:${(speakerId || '').toLowerCase().replace(/\s+/g, '_')}`;
const cleaned = [];
for (const char of String(speakerId || '').toLowerCase()) {
if (/^[\p{L}\p{N}]$/u.test(char)) cleaned.push(char);
else if (char === ' ' || char === '-') cleaned.push('_');
}
return `auto:${cleaned.join('') || 'speaker'}`;
}
/** Recover path-free cast metadata from current or legacy job payloads. */
export function castSourcesFromJob(job) {
if (!job || typeof job !== 'object') return {};
const sources = {};
for (const [speaker, info] of Object.entries(job.cast_sources || job.speaker_clones || {})) {
if (!info || typeof info !== 'object') continue;
sources[speaker] = {
duration: Number(info.duration) || 0,
source_count: Number(info.source_count) || 1,
kind: info.kind === 'segment' ? 'segment' : 'speaker',
};
}
for (const segment of job.segments || []) {
const speaker = segment?.speaker_id || 'Speaker 1';
const current = sources[speaker];
if (current && current.kind !== 'segment') continue;
const info = (job.segment_clones || {})[String(segment?.id ?? '')];
if (!info?.ref_audio) continue;
const duration = Number(info.duration) || 0;
if (!current || duration > current.duration) {
sources[speaker] = { duration, source_count: 1, kind: 'segment' };
}
}
return sources;
}
/**
+47
View File
@@ -137,6 +137,14 @@ def test_consistent_pick_unknown_speaker_returns_none():
assert resolve_consistent_ref(job, "speaker_9") is None
def test_clone_lookup_uses_the_same_portable_speaker_slug_as_profile_ids():
from api.routers.dub_generate import _find_speaker_clone
clone = {"ref_audio": "/v/speaker.wav"}
assert _find_speaker_clone({"SPEAKER_00": clone}, "speaker00") is clone
assert _find_speaker_clone({"Guest-2!": clone}, "guest_2") is clone
# ── Schema: validation + default ────────────────────────────────────────────
@@ -392,6 +400,45 @@ def test_per_line_auto_binding_still_prefers_segment_clip(patched_generate):
assert model.refs[0] == ("/v/seg0.wav", "seg0 ref", False)
def test_per_line_cross_speaker_cast_uses_the_selected_speaker(patched_generate):
"""Choosing Speaker 2 on Speaker 1's row must not silently keep that
row's Speaker 1 segment reference."""
body = {
"segments": [
{
"start": 0.0,
"end": 3.0,
"text": "hola",
"profile_id": "auto:speaker_2",
},
],
"segment_ids": ["0"],
"language": "Auto",
"language_code": "es",
"num_step": 4,
"timing_strategy": "concise",
"voice_match": "per_line",
}
model = patched_generate(_DIARIZED_JOB, body)
assert model.refs[0] == ("/v/spk2.wav", "spk2 ref", True)
def test_per_line_auto_binding_short_line_uses_speakers_best_video_clip(patched_generate):
"""A cast choice applies to short lines too; they must not fall back to
the engine default merely because that line has no segment reference."""
job = {
**_HEURISTIC_JOB,
"segment_clones": {
"0": _HEURISTIC_JOB["segment_clones"]["0"],
"1": _HEURISTIC_JOB["segment_clones"]["1"],
},
}
body = _heuristic_body()
body["segments"][2]["profile_id"] = "auto:speaker_1"
model = patched_generate(job, body)
assert model.refs[2][0:2] == ("/v/seg1.wav", "seg1 ref")
def test_consistent_explicit_cross_auto_seg_binding_is_honoured(patched_generate):
"""An auto-seg binding to ANOTHER segment's clip can only come from an
explicit request consistent mode must not override it. Only the
+38
View File
@@ -26,6 +26,7 @@ from services.speaker_clone import (
MIN_REF_DURATION_S,
MIN_SLICE_DURATION_S,
_pick_reference_slices,
build_cast_sources,
extract_speaker_clones,
refine_ref_text,
refine_ref_texts,
@@ -134,6 +135,43 @@ class TestExtractSpeakerClones:
assert 0 < ADJACENT_TURN_GUARD_S < SPEAKER_GAP
class TestBuildCastSources:
def test_exposes_segment_reference_when_no_pooled_clone_exists(self):
segments = [
{"id": "a", "speaker_id": "Speaker 1"},
{"id": "b", "speaker_id": "Speaker 1"},
]
sources = build_cast_sources(
segments,
{},
{
"a": {"ref_audio": "/private/a.wav", "duration": 3.1},
"b": {"ref_audio": "/private/b.wav", "duration": 6.4},
},
)
assert sources == {
"Speaker 1": {"duration": 6.4, "source_count": 1, "kind": "segment"}
}
assert "/private" not in repr(sources)
def test_pooled_clone_wins_and_private_fields_do_not_cross_api(self):
sources = build_cast_sources(
[{"id": "a", "speaker_id": "Speaker 1"}],
{
"Speaker 1": {
"ref_audio": "/private/speaker.wav",
"ref_text": "secret transcript",
"duration": 8.2,
"source_count": 2,
}
},
{"a": {"ref_audio": "/private/a.wav", "duration": 12.0}},
)
assert sources == {
"Speaker 1": {"duration": 8.2, "source_count": 2, "kind": "speaker"}
}
class _FakeASR:
"""Stands in for the active ASR backend's .transcribe() — no model, no
network. `chunks_by_path` maps a ref_audio path to the canned chunk list