Improve dubbing speaker detection, timing, and responsive UI
This commit is contained in:
victordonat0
@@ -912,6 +912,12 @@ async def dub_transcribe_stream(
|
||||
# Words (global-timeline) retained so diarization can re-split a segment
|
||||
# that spans two speakers' turns at the word boundary (#486).
|
||||
all_words: list = []
|
||||
# Preserve the ASR backend's natural phrase boundaries before
|
||||
# segment_transcript merges short neighboring phrases. Pyannote 3.1
|
||||
# occasionally collapses rapid exchanges into one dominant speaker; in
|
||||
# that narrow case these phrase spans give its own WeSpeaker embedding
|
||||
# model clean candidate utterances for a conservative recovery pass.
|
||||
asr_phrase_segments: list[dict] = []
|
||||
detected_lang = None
|
||||
next_seg_id = 0
|
||||
chunk_errors: list[str] = []
|
||||
@@ -1048,6 +1054,17 @@ async def dub_transcribe_stream(
|
||||
if detected_lang is None and part.get("language"):
|
||||
detected_lang = part["language"]
|
||||
asr_speaker_turns.extend(part.get("speaker_turns") or [])
|
||||
for _phrase in part.get("chunks", []) or []:
|
||||
_pts = _phrase.get("timestamp") or (None, None)
|
||||
_ptext = (_phrase.get("text") or "").strip()
|
||||
try:
|
||||
_ps, _pe = float(_pts[0]), float(_pts[1])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
continue
|
||||
if _ptext and _pe > _ps:
|
||||
asr_phrase_segments.append({
|
||||
"start": _ps, "end": _pe, "text": _ptext,
|
||||
})
|
||||
chunk_segs = segment_transcript(part, duration=t1, scene_cuts=scene_cuts)
|
||||
# Same word source segment_transcript used (already global-timeline),
|
||||
# kept for the post-diarization speaker re-split (#486).
|
||||
@@ -1213,6 +1230,104 @@ async def dub_transcribe_stream(
|
||||
"speaker_hint": {"requested": num_speakers, "status": "ignored"},
|
||||
}, "turns"
|
||||
|
||||
def _recover_from_phrase_embeddings(diar_pipe, diarized_segments):
|
||||
"""Recover rapid speaker turns when pyannote's final labels collapse.
|
||||
|
||||
Uses only components already loaded by speaker-diarization-3.1:
|
||||
the ASR phrase boundaries and pyannote's WeSpeaker embedding.
|
||||
Automatic two-speaker recovery is intentionally conservative;
|
||||
weak/imbalanced clusters are rejected so single-speaker audio is
|
||||
left untouched. Returns ``(segments, score)`` or ``None``.
|
||||
"""
|
||||
present = {
|
||||
str(seg.get("speaker_id")) for seg in diarized_segments
|
||||
if seg.get("speaker_id")
|
||||
}
|
||||
if len(present) > 1:
|
||||
return None
|
||||
phrases = [
|
||||
q for q in asr_phrase_segments
|
||||
if q.get("text") and float(q.get("end", 0.0)) - float(q.get("start", 0.0)) >= 0.75
|
||||
]
|
||||
if len(phrases) < 4:
|
||||
return None
|
||||
requested = int(num_speakers) if num_speakers else 2
|
||||
if requested != 2:
|
||||
return None
|
||||
embedding = getattr(diar_pipe, "_embedding", None)
|
||||
audio = getattr(diar_pipe, "_audio", None)
|
||||
if embedding is None or audio is None:
|
||||
return None
|
||||
try:
|
||||
import numpy as np
|
||||
from pyannote.core import Segment as _PyannoteSegment
|
||||
from sklearn.cluster import AgglomerativeClustering
|
||||
|
||||
vectors = []
|
||||
durations = []
|
||||
for q in phrases:
|
||||
qs, qe = float(q["start"]), float(q["end"])
|
||||
dur = qe - qs
|
||||
waveform, _ = audio.crop(
|
||||
asr_audio_target, _PyannoteSegment(qs, qe),
|
||||
duration=dur, mode="pad",
|
||||
)
|
||||
vec = np.asarray(embedding(waveform[None])).reshape(-1)
|
||||
if not np.isfinite(vec).all():
|
||||
return None
|
||||
vectors.append(vec)
|
||||
durations.append(dur)
|
||||
matrix = np.vstack(vectors)
|
||||
labels = AgglomerativeClustering(
|
||||
n_clusters=2, metric="cosine", linkage="average",
|
||||
).fit_predict(matrix)
|
||||
labels = np.asarray(labels)
|
||||
if len(set(labels.tolist())) != 2:
|
||||
return None
|
||||
|
||||
counts = [int(np.sum(labels == k)) for k in (0, 1)]
|
||||
durs = [float(sum(d for d, lab in zip(durations, labels) if lab == k)) for k in (0, 1)]
|
||||
if min(counts) < 2 or min(durs) < 1.5:
|
||||
return None
|
||||
|
||||
norm = matrix / np.maximum(np.linalg.norm(matrix, axis=1, keepdims=True), 1e-8)
|
||||
sims = norm @ norm.T
|
||||
within, cross = [], []
|
||||
for a in range(len(labels)):
|
||||
for b in range(a + 1, len(labels)):
|
||||
(within if labels[a] == labels[b] else cross).append(float(sims[a, b]))
|
||||
if not within or not cross:
|
||||
return None
|
||||
separation = float(np.mean(within) - np.mean(cross))
|
||||
min_sep = 0.12 if num_speakers == 2 else 0.18
|
||||
if separation < min_sep:
|
||||
logger.info(
|
||||
"phrase-embedding speaker recovery rejected (separation=%.3f < %.3f)",
|
||||
separation, min_sep,
|
||||
)
|
||||
return None
|
||||
|
||||
speaker_map = {}
|
||||
next_id = 1
|
||||
turns = []
|
||||
for q, lab in zip(phrases, labels.tolist()):
|
||||
if lab not in speaker_map:
|
||||
speaker_map[lab] = f"Speaker {next_id}"
|
||||
next_id += 1
|
||||
turns.append({
|
||||
"start": float(q["start"]),
|
||||
"end": float(q["end"]),
|
||||
"speaker": speaker_map[lab],
|
||||
})
|
||||
assigned = assign_speakers_from_turns(all_segments, turns)
|
||||
recovered = resplit_segments_by_turns(assigned, all_words, turns)
|
||||
if len({x.get("speaker_id") for x in recovered if x.get("speaker_id")}) < 2:
|
||||
return None
|
||||
return recovered, separation
|
||||
except Exception:
|
||||
logger.exception("phrase-embedding speaker recovery failed")
|
||||
return None
|
||||
|
||||
# The active ASR backend already diarized inline (FunASR cam++):
|
||||
# its turns are the fast path and skip pyannote entirely (#182) —
|
||||
# but ONLY when the user didn't set an explicit speaker count.
|
||||
@@ -1313,7 +1428,17 @@ async def dub_transcribe_stream(
|
||||
assigned = assign_speakers_from_diarization(all_segments, diar)
|
||||
# #486: split any segment that spans two speakers' turns at the
|
||||
# word boundary (single-speaker segments pass through unchanged).
|
||||
return resplit_segments_by_diarization(assigned, all_words, diar), None, "pyannote"
|
||||
resplit = resplit_segments_by_diarization(assigned, all_words, diar)
|
||||
recovered = _recover_from_phrase_embeddings(diar_pipe, resplit)
|
||||
if recovered is not None:
|
||||
recovered_segments, separation = recovered
|
||||
logger.info(
|
||||
"Recovered rapid two-speaker exchange from ASR phrase embeddings "
|
||||
"(phrases=%d, separation=%.3f).",
|
||||
len(asr_phrase_segments), separation,
|
||||
)
|
||||
return recovered_segments, None, "phrase_embeddings"
|
||||
return resplit, None, "pyannote"
|
||||
except Exception as e:
|
||||
logger.exception("Diarization failed")
|
||||
# Inline ASR turns beat the silence-gap heuristic as a crash
|
||||
|
||||
@@ -25,6 +25,10 @@ const ONSET_STRIP_H = 8; // px — non-interactive onset tick strip
|
||||
const KB_STEP_S = 0.01; // ←/→ nudge
|
||||
const KB_STEP_BIG_S = 0.1; // Ctrl+←/→ nudge
|
||||
const DRAG_DEADZONE_PX = 3;
|
||||
// Small/medium dubbing jobs are cheap enough to render in full. Avoid tying
|
||||
// visibility to transient WaveSurfer resize/zoom metrics until the transcript
|
||||
// is genuinely large; this also keeps short multi-speaker clips complete.
|
||||
const VIRTUALIZE_THRESHOLD = 200;
|
||||
|
||||
const fmt = (t) => {
|
||||
const m = Math.floor(t / 60);
|
||||
@@ -478,7 +482,8 @@ export default function SegmentTrack({
|
||||
|
||||
const innerWidth = Math.max(viewWidth, Math.ceil(duration * pxPerSec));
|
||||
const playheadX = currentTime * pxPerSec - effScroll;
|
||||
const windowed = effSegments.slice(lo, hi);
|
||||
const windowed =
|
||||
effSegments.length <= VIRTUALIZE_THRESHOLD ? effSegments : effSegments.slice(lo, hi);
|
||||
// Scroll offset baked into each box's `left` (viewport coordinates) instead
|
||||
// of a `translateX` on the lane — an animated lane transform is composited
|
||||
// by Chromium and flashes on some Windows GPU/WebView2 drivers (#373). In
|
||||
@@ -489,7 +494,7 @@ export default function SegmentTrack({
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`seg-track relative w-full select-none mt-[2px] ${disabled ? 'is-disabled' : ''}`}
|
||||
className={`seg-track relative w-full select-none mt-[2px] shrink-0 flex-none min-h-[50px] ${disabled ? 'is-disabled' : ''}`}
|
||||
ref={hostRef}
|
||||
>
|
||||
<canvas
|
||||
|
||||
@@ -235,7 +235,7 @@ export default function DubLeftColumn({
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="studio-panel dub-panel-col">
|
||||
<div className="studio-panel dub-panel-col dub-panel-left">
|
||||
{hasDubbedTrack && (
|
||||
<div
|
||||
className="dub-lang-switch"
|
||||
|
||||
@@ -82,7 +82,7 @@ export default function DubRightColumn({
|
||||
}) {
|
||||
const [pasteOpen, setPasteOpen] = useState(false);
|
||||
return (
|
||||
<div className="studio-panel dub-panel-col">
|
||||
<div className="studio-panel dub-panel-col dub-panel-right">
|
||||
{/* Output options + timing — moved to the top of the right section. */}
|
||||
<div>
|
||||
<div className={OUT_ROW}>
|
||||
@@ -163,7 +163,7 @@ export default function DubRightColumn({
|
||||
},
|
||||
{
|
||||
value: 'strict_slot',
|
||||
label: 'Strict slot',
|
||||
label: 'Lip sync',
|
||||
title:
|
||||
'Legacy: compress audio to fit the original timing. Can sound rushed/chipmunky on high-density target languages.',
|
||||
},
|
||||
|
||||
@@ -1060,7 +1060,7 @@ export default function useDubWorkflow({
|
||||
guidance_scale: cfg,
|
||||
speed,
|
||||
preview,
|
||||
timing_strategy: timingStrategy || 'concise',
|
||||
timing_strategy: timingStrategy || 'strict_slot',
|
||||
// Voice-identity mode for auto-clone bindings (per_line default =
|
||||
// unchanged behaviour; consistent = one reference per speaker).
|
||||
voice_match: voiceMatch || 'per_line',
|
||||
|
||||
+169
-4
@@ -4714,16 +4714,33 @@ button.dub-stepper__action:focus-visible {
|
||||
and letterboxes within the 16:9 box. */
|
||||
flex: 0 0 auto;
|
||||
width: 100%;
|
||||
aspect-ratio: 16 / 9;
|
||||
max-height: 60vh;
|
||||
/* Keep the preview useful without letting it push the waveform/segment lane
|
||||
below the browser viewport. The inner video already uses object-fit:contain,
|
||||
so a bounded viewport is better than a width-driven 16:9 box here. */
|
||||
height: clamp(180px, 36vh, 360px);
|
||||
max-height: 360px;
|
||||
background: #000; border-radius: 4px; overflow: hidden;
|
||||
border: 1px solid transparent; display: flex;
|
||||
}
|
||||
.wfm-wave-wrap {
|
||||
overflow: hidden; flex: 1 1 auto; min-height: 80px; max-height: 160px;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
flex: 0 0 clamp(88px, 12vh, 128px);
|
||||
height: clamp(88px, 12vh, 128px);
|
||||
min-height: 88px;
|
||||
max-height: 128px;
|
||||
z-index: 1;
|
||||
}
|
||||
.wfm-wave-inner {
|
||||
height: 100%; min-height: 80px; border-radius: 4px; width: 100%; overflow: hidden;
|
||||
height: 100%; min-height: 0; border-radius: 4px; width: 100%; overflow: hidden;
|
||||
}
|
||||
.seg-track {
|
||||
flex: 0 0 50px !important;
|
||||
min-height: 50px !important;
|
||||
height: 50px;
|
||||
z-index: 2;
|
||||
overflow: visible;
|
||||
background: var(--chrome-bg);
|
||||
}
|
||||
.wfm-loading {
|
||||
position: absolute; inset: 0; display: flex; align-items: center;
|
||||
@@ -6708,3 +6725,151 @@ button.dub-stepper__action:focus-visible {
|
||||
--chrome-border: transparent; --chrome-border-strong: transparent; --chrome-accent-border: transparent;
|
||||
--glass-border: transparent;
|
||||
}
|
||||
|
||||
|
||||
/* ═══ RESPONSIVE DUB WORKSPACE — local Colab usability patch ═══
|
||||
The original editor treated the two-column workspace as a fixed-height
|
||||
desktop canvas and hid overflow. On laptop/remote-browser viewports that
|
||||
compressed both panes until controls and transcript rows were effectively
|
||||
clipped, forcing users to run the site at ~80% zoom. These rules respond to
|
||||
the ACTUAL dub workspace width (container query), so UI scale/browser zoom
|
||||
no longer makes the breakpoint fire at the wrong size. */
|
||||
.dub-workspace {
|
||||
min-width: 0;
|
||||
min-height: 0;
|
||||
}
|
||||
.dub-editor,
|
||||
.dub-editor-grid,
|
||||
.dub-panel-left,
|
||||
.dub-panel-right {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
@container dub-shell (max-width: 1080px) {
|
||||
.dub-editor {
|
||||
flex: 1 1 auto;
|
||||
overflow-y: auto;
|
||||
overflow-x: hidden;
|
||||
overscroll-behavior: contain;
|
||||
scrollbar-gutter: stable;
|
||||
padding-bottom: 8px;
|
||||
}
|
||||
|
||||
.dub-editor-grid {
|
||||
display: flex;
|
||||
flex: 0 0 auto;
|
||||
flex-direction: column;
|
||||
min-height: 0;
|
||||
overflow: visible;
|
||||
}
|
||||
|
||||
.dub-panel-col {
|
||||
flex: 0 0 auto;
|
||||
overflow: visible;
|
||||
}
|
||||
|
||||
/* Preview / timeline pane: keep enough vertical room to be useful rather
|
||||
than shrinking the waveform to a strip. */
|
||||
.dub-panel-left {
|
||||
min-height: 460px;
|
||||
}
|
||||
|
||||
/* Transcript / output pane needs more room because its segment table is the
|
||||
primary editor. The page scrolls around the pane instead of squeezing it. */
|
||||
.dub-panel-right {
|
||||
min-height: 560px;
|
||||
}
|
||||
|
||||
.dub-panel-right .dub-segment-table__body {
|
||||
min-height: 280px;
|
||||
}
|
||||
|
||||
.dub-command-bar {
|
||||
align-items: flex-start;
|
||||
flex-wrap: wrap;
|
||||
row-gap: 6px;
|
||||
}
|
||||
.dub-command-bar__actions,
|
||||
.dub-command-bar__utilities {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
}
|
||||
|
||||
@container dub-shell (max-width: 720px) {
|
||||
.dub-panel-left,
|
||||
.dub-panel-right {
|
||||
padding-inline: 8px;
|
||||
}
|
||||
.dub-panel-left { min-height: 420px; }
|
||||
.dub-panel-right { min-height: 520px; }
|
||||
|
||||
/* Controls should wrap before their labels/inputs are truncated. */
|
||||
.dub-panel-col .label-row {
|
||||
white-space: normal;
|
||||
overflow: visible;
|
||||
text-overflow: clip;
|
||||
}
|
||||
.dub-lang-switch {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
}
|
||||
|
||||
/* Short screens are common through Colab/Cloudflare because browser chrome
|
||||
consumes a large part of the physical height. Keep the main workspace as the
|
||||
scroll owner instead of allowing fixed-height children to be crushed. */
|
||||
@media (max-height: 760px) {
|
||||
.app-container > .main-content {
|
||||
scrollbar-gutter: stable;
|
||||
}
|
||||
.dub-editor {
|
||||
overscroll-behavior-y: contain;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* Timeline visibility follow-up: keep the transcription lane and transport
|
||||
controls on-screen at normal browser zoom, including Colab/Cloudflare tabs
|
||||
where browser chrome reduces the usable vertical viewport. */
|
||||
@container dub-shell (max-width: 1080px) {
|
||||
.dub-panel-left .wfm-video-preview {
|
||||
height: clamp(180px, 32vh, 300px);
|
||||
max-height: 300px;
|
||||
}
|
||||
.dub-panel-left .wfm-wave-wrap {
|
||||
flex: 0 0 110px;
|
||||
min-height: 100px;
|
||||
max-height: 130px;
|
||||
}
|
||||
.dub-panel-left .seg-track {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
.dub-panel-left .wfm-controls {
|
||||
flex-wrap: wrap;
|
||||
row-gap: 5px;
|
||||
}
|
||||
}
|
||||
|
||||
@container dub-shell (max-width: 720px) {
|
||||
.dub-panel-left .wfm-video-preview {
|
||||
height: clamp(160px, 28vh, 240px);
|
||||
max-height: 240px;
|
||||
}
|
||||
.dub-panel-left .wfm-wave-wrap {
|
||||
flex-basis: 96px;
|
||||
min-height: 90px;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@media (max-height: 820px) {
|
||||
.wfm-video-preview {
|
||||
height: clamp(150px, 28vh, 260px);
|
||||
max-height: 260px;
|
||||
}
|
||||
.wfm-wave-wrap {
|
||||
flex-basis: 96px;
|
||||
height: 96px;
|
||||
min-height: 96px;
|
||||
max-height: 96px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -624,7 +624,7 @@ export default function DubTab(props) {
|
||||
}, [dubJobId, qcRunning, previewMode, dubGenNonce, setDubSegments, t]);
|
||||
|
||||
return (
|
||||
<div className="flex-1 flex flex-col min-h-0">
|
||||
<div className="dub-workspace flex-1 flex flex-col min-h-0 min-w-0 [container-type:inline-size] [container-name:dub-shell]">
|
||||
{/* Pipeline spine — shown once a file/job is in play so the user always
|
||||
knows which stage they're at (Upload → … → Export). In the editor view
|
||||
it's inlined onto the DubHeader row (see below), so only render the
|
||||
@@ -691,7 +691,7 @@ export default function DubTab(props) {
|
||||
|
||||
{/* ── After transcription: side-by-side editor ── */}
|
||||
{dubJobId && (dubStep === 'editing' || dubStep === 'generating' || dubStep === 'done') && (
|
||||
<div className="flex-1 flex flex-col min-h-0">
|
||||
<div className="dub-editor flex-1 flex flex-col min-h-0 min-w-0">
|
||||
<DubHeader
|
||||
t={t}
|
||||
dubFilename={dubFilename}
|
||||
@@ -715,7 +715,7 @@ export default function DubTab(props) {
|
||||
pipelineSteps={pipelineSteps}
|
||||
onPipelineStep={onPipelineStep}
|
||||
/>
|
||||
<div className="grid grid-cols-2 max-[1000px]:grid-cols-1 max-[1000px]:grid-rows-[auto_1fr] gap-[6px] flex-1 min-h-0 overflow-hidden">
|
||||
<div className="dub-editor-grid grid grid-cols-2 gap-[6px] flex-1 min-h-0 min-w-0 overflow-hidden">
|
||||
<DubLeftColumn
|
||||
hasDubbedTrack={hasDubbedTrack}
|
||||
t={t}
|
||||
|
||||
@@ -176,7 +176,7 @@ export const useAppStore = create<AppStore>()(
|
||||
firedMilestones: s.firedMilestones,
|
||||
optedOut: s.optedOut,
|
||||
}),
|
||||
version: 7,
|
||||
version: 8,
|
||||
// Drop old persisted shapes rather than crashing the app. Every field
|
||||
// has a safe default in its slice, so v1/v2/v3 users pick up v4 defaults
|
||||
// for new fields (timingStrategy etc.) and keep any keys we still write
|
||||
@@ -184,6 +184,13 @@ export const useAppStore = create<AppStore>()(
|
||||
migrate: (persisted, version) => {
|
||||
if (!persisted || typeof persisted !== 'object') return {} as Partial<AppStore>; // D1
|
||||
const p = persisted as any;
|
||||
if (version < 8 && p.timingStrategy === 'concise') {
|
||||
// Local dubbing UX migration: the historical default ('concise')
|
||||
// preserves natural speech but can finish before the original mouth
|
||||
// slot. New default is strict_slot for tighter lip sync. Explicit
|
||||
// non-default choices (smart_fit/stretch_video/strict_slot) survive.
|
||||
p.timingStrategy = 'strict_slot';
|
||||
}
|
||||
if (version < 4) {
|
||||
// v1 → v2 added reviewMode; v2 → v3 added mode/sidebar/generate knobs;
|
||||
// v3 → v4 added timingStrategy. All of those have slice defaults, so
|
||||
|
||||
@@ -280,7 +280,7 @@ export const createPrefsSlice: StateCreator<PrefsSlice, [], [], PrefsSlice> = (s
|
||||
glossaryVisible: true,
|
||||
reviewMode: 'on',
|
||||
showHeaderLiveStats: false,
|
||||
timingStrategy: 'concise',
|
||||
timingStrategy: 'strict_slot',
|
||||
fitOptions: null,
|
||||
voiceMatch: 'per_line',
|
||||
whatsNewSeenVersion: null,
|
||||
|
||||
Reference in New Issue
Block a user