upstream-archive/VoiceStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dub): correct long timeline rendering and repeated ASR context

Browse Source
This commit is contained in:
Palash Debnath
2026-09-15 20:02:18 +05:30
parent 36ccec5e61
commit 61d52944f2
11 changed files with 221 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2
View File
@@ -27,6 +27,8 @@ the frozen-backend fallback mirror it for their toolchains.
### Fixed
- Dubbing timelines keep short segments proportional, support zoom, and remove timestamp-confirmed duplicate ASR context (#2129)
- Dubbing translation shares the agent footer with live logs, validated output, cancellation and contextual retries (#2129)
- Agent dubbing translation saves a custom tone and adaptation prompt and preserves it during timing rewrites (#2129)
backend/api/routers/dub_core.py
+4
View File
@@ -1882,6 +1882,8 @@ async def dub_transcribe_stream(
payload["speaker_hint"] = diar_warning["speaker_hint"]
yield _sse_event("warning", payload)
from services.segmentation import deduplicate_chunk_segments
final_segs = deduplicate_chunk_segments(final_segs)
job["segments"] = final_segs
# Auto-speaker-clone: sample each detected speaker's voice from the
@@ -2294,6 +2296,8 @@ async def dub_transcribe(job_id: str, num_speakers: Optional[int] = None):
raise
if job.get("aborted"):
raise HTTPException(status_code=499, detail="Transcription aborted")
from services.segmentation import deduplicate_chunk_segments
segments_result = deduplicate_chunk_segments(segments_result)
job["segments"] = segments_result
source_lang = job.get("source_lang")
_save_job(job_id, job)
backend/services/segmentation.py
+71
View File
@@ -834,3 +834,74 @@ def resplit_segments_by_turns(
and t.get("end") is not None
]
return _resplit_core(segments, words, norm)
def deduplicate_chunk_segments(segments: list[dict]) -> list[dict]:
"""Remove repeated ASR context only when matching words share timestamps.
Preserve different speakers and genuine repeated speech at different times.
Input order retains chunk provenance even when a later chunk starts earlier.
"""
import re
def token(word):
return re.sub(r'[^\w]', '', str(word.get('text', word.get('word', ''))).casefold())
def timed_words(segment):
words = segment.get('words') or []
return words if words and all(isinstance(w, dict) and isinstance(w.get('start'), (int, float))
and isinstance(w.get('end'), (int, float)) for w in words) else []
result = []
for segment in segments:
words = timed_words(segment)
matches = []
if words and segment.get("speaker_id"):
prior_words = [w for previous in result
if previous.get('speaker_id') == segment.get('speaker_id')
for w in timed_words(previous)
if w['end'] >= words[0]['start'] - .35 and w['start'] <= words[-1]['end'] + .35]
for index, word in enumerate(words):
if token(word) and any(token(word) == token(prior)
and abs((word['start'] + word['end']) / 2 - (prior['start'] + prior['end']) / 2) <= .35
for prior in prior_words):
matches.append(index)
# Require a substantial matching prefix; isolated common words cannot
# authorize deleting speech. No timing-only truncation is performed.
if len(matches) >= 3 and len(matches) / (matches[-1] + 1) >= .6:
cutoff = max((w['end'] for w in prior_words), default=0)
remaining = [w for w in words[matches[-1] + 1:] if w['start'] >= cutoff - .05]
if not remaining:
continue
text = _clean(' '.join(str(w.get('text', w.get('word', ''))) for w in remaining))
segment = {**segment, 'start': remaining[0]['start'], 'end': remaining[-1]['end'],
'text': text, 'text_original': text, 'words': remaining}
result.append(segment)
def bounds(row):
words = timed_words(row)
if not words:
return None
if any(a['start'] > b['start'] for a, b in zip(words, words[1:])):
# Older chunk stitching can attach an earlier word to a later
# line. Never invert an interval or move it backwards over speech.
inside = [w for w in words if row['start'] <= w['start'] <= w['end'] <= row['end']]
if len(inside) < .6 * len(words):
return None
words = inside
start, end = min(w['start'] for w in words), max(w['end'] for w in words)
return (start, end) if end > start else None
# Repair stale camera-cut bounds only when timed words prove the two
# spoken intervals are disjoint. Genuine overlapping speech stays intact.
adjust = set()
ordered = sorted(enumerate(result), key=lambda item: item[1]['start'])
for position, (left_index, left) in enumerate(ordered):
for right_index, right in ordered[position + 1:]:
if right['start'] >= left['end']:
break
a, b = bounds(left), bounds(right)
if a and b and (a[1] <= b[0] or b[1] <= a[0]):
adjust.update((left_index, right_index))
result = [{**row, 'start': bounds(row)[0], 'end': bounds(row)[1]}
if index in adjust else row for index, row in enumerate(result)]
return result
docs/electron-dubbing.md
+16
View File
@@ -272,3 +272,19 @@ arrives. No fabricated percentage is shown. Errors remain visible, with Retry fo
failed translation work in the same project. API retries use failed segments when
available; an incomplete CLI response requires retrying that language. Cancel stops
the active translation and prevents late results from applying.
### Long timelines and duplicate ASR context
The timeline draws segments at their actual duration. Use Zoom in/out and Fit all
above it to inspect short lines in long recordings; the zoomed view scrolls
horizontally. Tiny overview bars cannot be accidentally dragged or resized. Click
an overlap warning to zoom to the first affected segment. Nested overlaps are
included in detection; simultaneous speakers are not automatically shifted apart.
After transcription, repeated chunk context is removed only when at least three
matching words form a substantial prefix at matching timestamps for the same
speaker. New words beyond that context remain. Stale segment boundaries are aligned
to their own word timestamps only when those prove that the speech is disjoint.
Existing translations/renders do not become correct merely by editing source text:
changed lines must be translated and regenerated. Preserve the prior project when
repairing an older transcript.
electron/src/renderer/src/features/dub/dub-timeline.test.tsx
+22
View File
@@ -0,0 +1,22 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { cleanup, fireEvent, render, screen } from '@testing-library/react';
import { DubTimeline } from './dub-timeline';
vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) }));
vi.mock('./dub-session', () => ({ deleteDubSegment: vi.fn(), moveResizeDubSegment: vi.fn() }));
vi.mock('@/lib/audio/playback-clock', () => ({ usePlaybackClock: () => ({ duration: 1980, time: 0 }), requestPlaybackRange: vi.fn(), requestPlaybackSeek: vi.fn() }));
beforeEach(() => {
vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null);
vi.stubGlobal('ResizeObserver', class { observe() {} disconnect() {} });
});
afterEach(() => { cleanup(); vi.restoreAllMocks(); vi.unstubAllGlobals(); });
it('keeps short segments proportional on long recordings and offers zoom', () => {
render(<DubTimeline segments={[{id:'a',start:0,end:1,text:'a'},{id:'b',start:2,end:3,text:'b'}]}
disabled={false} mediaDuration={1980} selectedId={null} onSelect={vi.fn()} />);
const options = screen.getAllByRole('option');
expect(parseFloat(options[0].style.width)).toBeCloseTo(100 / 1980, 4);
expect(parseFloat(options[0].style.width)).toBeLessThan(parseFloat(options[1].style.left));
fireEvent.click(screen.getByRole('button', { name: 'trimmer.zoom_in' }));
expect(screen.getByRole('listbox').style.width).toBe('200%');
fireEvent.click(screen.getByRole('button', { name: 'trimmer.fit_all' }));
expect(screen.getByRole('listbox').style.width).toBe('100%');
});
electron/src/renderer/src/features/dub/dub-timeline.tsx
+33 -14
View File
@@ -1,4 +1,4 @@
import { HeadphonesIcon, LoaderCircleIcon, PlayIcon, TriangleAlertIcon } from 'lucide-react';
import { HeadphonesIcon, LoaderCircleIcon, PlayIcon, TriangleAlertIcon, ZoomInIcon, ZoomOutIcon, MaximizeIcon } from 'lucide-react';
import { useEffect, useMemo, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
import {
@@ -8,6 +8,7 @@ import {
snapCandidates,
snapTime,
} from '../../../../../../frontend/src/utils/timeline';
import { Button } from '@/components/ui/button';
import { cn } from '@/lib/utils';
import {
requestPlaybackRange,
@@ -58,6 +59,9 @@ export function DubTimeline({
const { t } = useTranslation();
const playback = usePlaybackClock(playbackSource);
const host = useRef<HTMLDivElement>(null);
const viewport = useRef<HTMLDivElement>(null);
const [zoom, setZoom] = useState(1);
const [timelineWidth, setTimelineWidth] = useState(1000);
const onsetCanvas = useRef<HTMLCanvasElement>(null);
const segmentRefs = useRef(new Map<string, HTMLDivElement>());
const gesture = useRef<Gesture | null>(null);
@@ -94,8 +98,9 @@ export function DubTimeline({
const draw = () => {
const width = container.clientWidth;
const height = container.clientHeight;
setTimelineWidth(width);
const dpr = window.devicePixelRatio || 1;
canvas.width = Math.max(1, Math.round(width * dpr));
canvas.width = Math.min(8192, Math.max(1, Math.round(width * dpr)));
canvas.height = Math.max(1, Math.round(height * dpr));
canvas.style.width = `${width}px`;
canvas.style.height = `${height}px`;
@@ -124,7 +129,7 @@ export function DubTimeline({
context.beginPath();
for (const onset of onsets) {
if (onset < 0 || onset > duration) continue;
const x = Math.round((onset / duration) * width * dpr) + 0.5;
const x = Math.round((onset / duration) * canvas.width) + 0.5;
context.moveTo(x, canvas.height * 0.62);
context.lineTo(x, canvas.height);
}
@@ -146,6 +151,7 @@ export function DubTimeline({
const begin = (event: React.PointerEvent<HTMLDivElement>, index: number) => {
if (disabled || event.button !== 0) return;
if (((effective[index].end - effective[index].start) / duration) * timelineWidth < 16) return;
const segment = effective[index];
const mode =
(event.target as HTMLElement).dataset.edge === 'start'
@@ -304,8 +310,15 @@ export function DubTimeline({
aria-label={t('segmentEditing.timeline')}
className="rounded-xl border border-border/60 bg-card/35 p-3 shadow-sm"
>
<div className="mb-2 flex justify-end gap-1">
<Button size="icon-sm" variant="ghost" aria-label={t('trimmer.zoom_out')} disabled={zoom <= 1} onClick={() => setZoom((z) => Math.max(1, z / 2))}><ZoomOutIcon /></Button>
<Button size="icon-sm" variant="ghost" aria-label={t('trimmer.zoom_in')} disabled={zoom >= 16} onClick={() => setZoom((z) => Math.min(16, z * 2))}><ZoomInIcon /></Button>
<Button size="icon-sm" variant="ghost" aria-label={t('trimmer.fit_all')} onClick={() => setZoom(1)}><MaximizeIcon /></Button>
</div>
<div ref={viewport} className="overflow-x-auto rounded-lg [scrollbar-width:thin]">
<div
ref={host}
style={{ width: `${zoom * 100}%` }}
role="listbox"
aria-orientation="horizontal"
onClick={(event) => {
@@ -356,7 +369,7 @@ export function DubTimeline({
onPointerUp={finish}
onPointerCancel={(event) => finish(event, false)}
className={cn(
'absolute top-2 flex h-10 min-w-2 cursor-grab items-center overflow-hidden rounded-md border border-primary/30 bg-primary/20 px-2 text-[10px] font-medium text-foreground outline-none transition-[box-shadow,background-color] active:cursor-grabbing focus-visible:ring-2 focus-visible:ring-ring',
'absolute top-2 flex h-10 min-w-0 cursor-grab items-center overflow-hidden rounded-sm bg-primary/30 px-0 text-[10px] font-medium text-foreground outline-none transition-[box-shadow,background-color] active:cursor-grabbing focus-visible:ring-2 focus-visible:ring-ring',
selectedId === segment.id && 'border-primary/70 bg-primary/35 shadow-sm',
focusId === segment.id &&
editMode &&
@@ -365,16 +378,16 @@ export function DubTimeline({
)}
style={{
left: `${(segment.start / duration) * 100}%`,
width: `${Math.max(0.6, ((segment.end - segment.start) / duration) * 100)}%`,
width: `${((segment.end - segment.start) / duration) * 100}%`,
}}
>
<span
{((segment.end - segment.start) / duration) * timelineWidth >= 24 && <span
data-edge="start"
aria-hidden="true"
className="absolute inset-y-0 left-0 w-1.5 cursor-ew-resize bg-foreground/15"
/>
<span className="pointer-events-none truncate">{index + 1}</span>
{selectedId === segment.id && ((segment.end - segment.start) / duration) * 100 > 6 && (
/>}
{((segment.end - segment.start) / duration) * timelineWidth >= 18 && <span className="pointer-events-none truncate px-1">{index + 1}</span>}
{selectedId === segment.id && ((segment.end - segment.start) / duration) * timelineWidth > 60 && (
<span className="ml-auto flex shrink-0 gap-0.5">
<button
type="button"
@@ -389,7 +402,7 @@ export function DubTimeline({
>
<PlayIcon className="size-3 fill-current" />
</button>
{onPreviewSegment && ((segment.end - segment.start) / duration) * 100 > 10 && (
{onPreviewSegment && ((segment.end - segment.start) / duration) * timelineWidth > 100 && (
<button
type="button"
aria-label={t('dub.live_preview')}
@@ -411,21 +424,27 @@ export function DubTimeline({
)}
</span>
)}
<span
{((segment.end - segment.start) / duration) * timelineWidth >= 24 && <span
data-edge="end"
aria-hidden="true"
className="absolute inset-y-0 right-0 w-1.5 cursor-ew-resize bg-foreground/15"
/>
/>}
</div>
))}
</div>
</div>
<div className="mt-1 flex items-center justify-between font-mono text-[10px] text-muted-foreground tabular-nums">
<span>0:00.0</span>
{overlaps.size > 0 && (
<span role="status" className="flex items-center gap-1 text-destructive">
<button type="button" className="flex items-center gap-1 text-destructive text-left" onClick={() => {
const id = [...overlaps][0];
setZoom(16);
selectAndFocus(id);
requestAnimationFrame(() => segmentRefs.current.get(id)?.scrollIntoView({ block: 'nearest', inline: 'center' }));
}}>
<TriangleAlertIcon className="size-3" />
{t('segmentEditing.overlap')}
</span>
</button>
)}
<span>{formatTime(duration)}</span>
</div>
electron/src/renderer/src/features/projects/project-format.test.ts
+3
View File
@@ -27,6 +27,7 @@ it('opens legacy projects and preserves unexposed options when saving edits', ()
dubFilename: 'clip.mp4',
dubLang: 'French',
translateQuality: 'cinematic',
translationInstructions: 'Preserve humor.',
fitOptions: { allow_video_retime: false, audio_rate_cap: 1.3 },
dubStep: 'generating',
dubSegments: [{ id: 7, start: 0, end: 2, text: 'Bonjour', translations: { fr: 'Bonjour' } }],
@@ -37,12 +38,14 @@ it('opens legacy projects and preserves unexposed options when saving edits', ()
};
const session = projectSession(project, defaults);
expect(session.quality).toBe('cinematic');
expect(session.translationInstructions).toBe('Preserve humor.');
expect(session.exportOptions).toMatchObject({ preserveBg: false, excluded: ['original'] });
expect(session.fitOptions).toEqual({ allow_video_retime: false, audio_rate_cap: 1.3 });
expect(session.phase).toBe('editing');
expect(session.taskId).toBeNull();
expect(session.segments[0]).toMatchObject({ id: '7', text_original: 'Bonjour' });
const payload = projectPayload({ ...session, target: 'Spanish' }, ' Renamed ');
expect(payload.state.translationInstructions).toBe('Preserve humor.');
expect(payload).toMatchObject({
name: 'Renamed',
audio_path: '/audio.wav',
electron/src/renderer/src/features/projects/project-format.ts
+2
View File
@@ -33,6 +33,7 @@ export function projectSession(project: DubProject, defaults: DubSession): DubSe
reflectPass: s.reflectPass,
condenseSuggest: s.condenseSuggest,
dialect: s.dubDialect,
translationInstructions: s.translationInstructions,
exportOptions: {
...(typeof s.exportOptions === 'object' && s.exportOptions ? s.exportOptions : {}),
preserveBg: s.preserveBg,
@@ -86,6 +87,7 @@ export function projectPayload(session: DubSession, name: string) {
reflectPass: session.reflectPass,
condenseSuggest: session.condenseSuggest,
dubDialect: session.dialect,
translationInstructions: session.translationInstructions,
exportOptions: session.exportOptions,
...(session.exportOptions
? {
frontend/src/utils/timeline.js
+6 -5
View File
@@ -282,13 +282,14 @@ export function detectOverlaps(segments, epsilon = 1e-6) {
const flagged = new Set();
if (segments.length < 2) return flagged;
const sorted = [...segments].sort((x, y) => x.start - y.start || x.end - y.end);
for (let i = 1; i < sorted.length; i++) {
const prev = sorted[i - 1];
const cur = sorted[i];
if (cur.start < prev.end - epsilon) {
flagged.add(String(prev.id));
let active = [];
for (const cur of sorted) {
active = active.filter((previous) => previous.end > cur.start + epsilon);
for (const previous of active) {
flagged.add(String(previous.id));
flagged.add(String(cur.id));
}
active.push(cur);
}
return flagged;
}
frontend/src/utils/timeline.test.js
+8
View File
@@ -318,3 +318,11 @@ describe('REGION_COLORS — opaque JS-pre-blended paint guard (#373, #963)', ()
expect(blendRegionColor([255, 255, 255], [0, 0, 0])).toBe('rgb(115, 115, 115)'); // 0.45·255 = 114.75
});
});
it('flags every nested overlap, not just consecutive intervals', () => {
expect([...detectOverlaps([
{ id: 'long', start: 0, end: 10 },
{ id: 'a', start: 1, end: 2 },
{ id: 'b', start: 3, end: 4 },
])].sort()).toEqual(['a', 'b', 'long']);
});
tests/test_dub_chunk_duplicates.py
+54
View File
@@ -0,0 +1,54 @@
from services.segmentation import deduplicate_chunk_segments
def segment(id, words, speaker='Speaker 1'):
return {'id':id, 'start':words[0]['start'], 'end':words[-1]['end'], 'words':words,
'text':' '.join(w['text'] for w in words), 'speaker_id':speaker}
def words(text, start):
return [{'text':w,'start':start+i*.4,'end':start+(i+1)*.4} for i,w in enumerate(text.split())]
def test_repeated_chunk_context_is_removed_without_losing_the_new_tail():
a=segment('a',words('Thank you very much',10))
b=segment('b',words('Thank you very much Welcome everyone',10))
result=deduplicate_chunk_segments([a,b])
assert result[1]['text']=='Welcome everyone'
assert result[1]['start']==11.6
assert a['text']=='Thank you very much'
def test_exact_duplicate_removed_but_other_speaker_preserved():
a=segment('a',words('Thank you very much',10))
b={**a,'id':'b'}
assert len(deduplicate_chunk_segments([a,b]))==1
assert len(deduplicate_chunk_segments([a,{**b,'speaker_id':'Speaker 2'}]))==2
def test_repeated_phrase_at_different_time_is_not_a_duplicate():
a=segment('a',words('Thank you very much',10))
b=segment('b',words('Thank you very much',12))
assert deduplicate_chunk_segments([a,b])==[a,b]
def test_fix_bounds_only_when_words_prove_speech_is_disjoint():
a=segment('a',words('Good morning everyone',10))
b=segment('b',words('Nice to meet you',12))
a['end']=13
result=deduplicate_chunk_segments([a,b])
assert result[0]['end']==11.2
assert a['end']==13
c=segment('c',words('Other simultaneous words',10.5),'Speaker 2')
assert deduplicate_chunk_segments([a,c])==[a,c]
def test_out_of_order_stitched_word_cannot_invert_or_delete_a_line():
a=segment('a',words('You are so young',10))
a['end']=12.2
a['words'].append({'text':'Earlier?','start':8,'end':9})
b=segment('b',words('How old are you',12))
result=deduplicate_chunk_segments([a,b])
assert len(result)==2
assert result[0]['start']==10
assert result[0]['end']==11.6