upstream-archive/VoiceStudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dub): preserve complete speech and reject silent partial output

Browse Source
This commit is contained in:
Palash Debnath
2026-09-15 15:44:57 +05:30
parent c524ea3235
commit 0a6ea976cb
56 changed files with 599 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -27,6 +27,7 @@ the frozen-backend fallback mirror it for their toolchains.
### Fixed
- Dubbing repairs missing speech caches, rejects incomplete output, avoids oversized speaker references, and fits full speech without early clipping (#2129)
- Workspace sidebars have a working right-edge resize handle, allow 40% more width, remember their size, and keep video controls inside the preview (#2129)
- Pressing Play while a video is loading starts playback when it is ready instead of reporting playback unavailable (#2129)
- Video previews show their thumbnail before playback, including the source video in Dub (#2129)
backend/api/routers/dub_generate.py
+89 -131
View File
@@ -30,7 +30,7 @@ from services.ffmpeg_utils import (
)
from services.rvc import apply_rvc, is_enabled as rvc_is_enabled
from services.incremental import segment_fingerprint, fit_fingerprint
from services.fit_planner import UNDERRUN_TOLERANCE, FitParams, plan_fit
from services.fit_planner import FitParams, plan_fit
from services.watermark import mark_synthetic
from services.speaker_clone import auto_profile_id
from services.segment_bundle import extract_segment_wavs
@@ -39,16 +39,6 @@ from omnivoice.utils.voice_design import heal_design_instruct
logger = logging.getLogger("omnivoice.dub")
# Maximum compression ratio we'll attempt with pitch-preserving stretch
# before declaring "no way to fit cleanly" and falling back. atempo
# remains intelligible up to ~1.5× then introduces audible WSOLA
# artefacts; above ~1.8× speech becomes a fast garbled stream that no
# DSP can rescue. The contributing-factor pipeline (CPS-aware slot-fit
# in services/speech_rate.py, gap absorption below) keeps us under this
# in practice — this is only a guard rail.
MAX_STRETCH_RATIO = 1.8
class _RemoteDubBackend:
"""Sample-rate carrier while Dubbing runs without local TTS weights."""
@@ -378,6 +368,12 @@ def forget_missing_ref_warnings(job_id: str) -> None:
_MISSING_REF_WARNED.pop(str(job_id), None)
def _ref_within_limit(info) -> bool:
from services.speaker_clone import MAX_REF_DURATION_S
return bool(info) and float(info.get("duration") or 0.0) <= MAX_REF_DURATION_S
def resolve_consistent_ref(job: dict, speaker_key: str, memo: dict | None = None):
"""ONE clone reference for every segment of `speaker_key`.
@@ -386,8 +382,9 @@ def resolve_consistent_ref(job: dict, speaker_key: str, memo: dict | None = None
the per-line path uses as its fallback;
2. no speaker clone (heuristic diarization skips extraction entirely —
the key case): a deterministic pick among that speaker's per-segment
clips: longest clip ≥3 s, tie-break lowest segment id. Clips all
shorter than 3 s degrade to "longest overall", same tie-break.
clips: longest usable clip ≥3 s within the shared reference limit,
tie-break lowest segment id. Short clips fall back to longest usable.
Oversized references must never strand every short line for a speaker.
Returns the clone info dict ({"ref_audio", "ref_text", ...}) or None.
Pure function of the job dict; `memo` (keyed by speaker_key) just avoids
@@ -396,7 +393,11 @@ def resolve_consistent_ref(job: dict, speaker_key: str, memo: dict | None = None
if memo is not None and speaker_key in memo:
return memo[speaker_key]
from services.speaker_clone import MAX_REF_DURATION_S
ref = _find_speaker_clone(job.get("speaker_clones") or {}, speaker_key)
if ref and float(ref.get("duration") or 0.0) > MAX_REF_DURATION_S:
ref = None
if ref is None:
seg_clones = job.get("segment_clones") or {}
candidates = []
@@ -408,7 +409,8 @@ def resolve_consistent_ref(job: dict, speaker_key: str, memo: dict | None = None
continue
sid = str(row.get("id", ""))
info = seg_clones.get(sid)
if info and info.get("ref_audio"):
if (info and info.get("ref_audio")
and float(info.get("duration") or 0.0) <= MAX_REF_DURATION_S):
candidates.append((sid, info))
if candidates:
usable = [
@@ -452,6 +454,9 @@ def _remote_voice(job: dict, profile_id: str | None, seg_id, voice_match: str,
info = ((job.get("segment_clones") or {}).get(str(seg_id))
or _find_speaker_clone(job.get("speaker_clones") or {}, key))
single_use = str(seg_id) in (job.get("segment_clones") or {})
if not _ref_within_limit(info):
info = resolve_consistent_ref(job, key, memo)
single_use = False
if info:
ref_audio, ref_text = info.get("ref_audio"), info.get("ref_text")
elif profile_id:
@@ -701,8 +706,29 @@ async def dub_generate(job_id: str, req: DubRequest):
_wav_kind = (
_kind_map.get(lang_code) if isinstance(_kind_map, dict) else job.get("seg_wav_kind")
)
if strategy != "strict_slot" and regen_only is not None and _wav_kind != "natural":
if regen_only is not None and _wav_kind != "natural":
regen_only = None
# A partial rerun must repair absent or corrupt caches, including before
# remote batching decides which lines need synthesis.
if regen_only is not None:
for index, segment in enumerate(req.segments):
sid = seg_ids[index] if index < len(seg_ids) else f"seg_{index}"
if sid in regen_only or not segment.text.strip():
continue
cache = _seg_lang_path(sid)
if not os.path.exists(cache) and _legacy_seg_cache_ok(job, lang_code):
for key in (sid, index):
legacy = dub_seg_path(job_id, key)
if os.path.exists(legacy):
cache = legacy
break
try:
info = torchaudio.info(cache)
intact = info.num_frames > 0 and _cached_payload_intact(cache, info)
except Exception:
intact = False
if not intact:
regen_only.add(sid)
# Manifest: stable segment id per current index. Per-segment WAVs are
# named by stable id (dub_seg_path) so regen reuses the right audio after
# reorder; index-keyed readers (preview/export) resolve via this manifest.
@@ -955,7 +981,7 @@ async def dub_generate(job_id: str, req: DubRequest):
all_segment_wavs.append(
(seg.start, seg.end, seg_wav_path, backend.sample_rate)
)
sync_scores.append(getattr(seg, 'sync_ratio', None) or 1.0)
sync_scores.append(round(cached_info.num_frames / cached_info.sample_rate / max(seg_duration, 0.01), 3))
_t_cache += time.perf_counter() - _t_cache_0
continue
@@ -963,39 +989,20 @@ async def dub_generate(job_id: str, req: DubRequest):
if cached_sr != backend.sample_rate:
import torchaudio.functional as AF
cached_wav = AF.resample(cached_wav, cached_sr, backend.sample_rate)
# strict_slot persists slot-sized buffers. Every other
# strategy consumes natural-rate audio and lets the mix
# loop fit it to the current timeline.
if strategy == "strict_slot":
target_samples = int(seg_duration * backend.sample_rate)
current_samples = cached_wav.shape[-1]
if target_samples > current_samples:
cached_wav = torch.nn.functional.pad(cached_wav, (0, target_samples - current_samples))
elif current_samples > target_samples:
cached_wav = cached_wav[..., :target_samples]
cached_ratio = round(cached_wav.shape[-1] / backend.sample_rate / max(seg_duration, 0.01), 3)
all_segment_wavs.append(_store_mix_wav(seg.start, seg.end, cached_wav, backend.sample_rate, f"mix_{seg_id}"))
try:
del cached_wav
except Exception:
pass
_release_audio_tensors()
sync_scores.append(getattr(seg, 'sync_ratio', None) or 1.0)
sync_scores.append(cached_ratio)
_t_cache += time.perf_counter() - _t_cache_0
continue
except Exception as e:
# Fall through to a silent placeholder if the cached WAV
# is broken — cleaner than aborting the whole mix.
yield f"data: {json.dumps({'type': 'warning', 'segment': i, 'message': f'cached seg lost, padding silence: {str(e)[:120]}'})}\n\n"
sr = backend.sample_rate
silence = torch.zeros(1, max(0, int(seg_duration * sr)))
all_segment_wavs.append(_store_mix_wav(seg.start, seg.end, silence, sr, f"mix_{seg_id}"))
try:
del silence
except Exception:
pass
_release_audio_tensors()
sync_scores.append(1.0)
continue
except Exception:
logger.exception("Dub cached segment could not be read: %s", seg_id)
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'segment_id': seg_id, 'error_code': 'dub_speech_missing', 'error': 'Cached speech could not be read. Regenerate this segment before exporting.'})}\n\n"
return
def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id, effect_preset,
*, execution_target="local", prepare_only=False, current_seg_id=None):
@@ -1092,7 +1099,7 @@ async def dub_generate(job_id: str, req: DubRequest):
if selected_is_segment_speaker
else None
)
if seg_ref:
if _ref_within_limit(seg_ref):
ref_audio = seg_ref.get("ref_audio")
ref_text = seg_ref.get("ref_text")
ref_single_use = True
@@ -1100,7 +1107,7 @@ async def dub_generate(job_id: str, req: DubRequest):
auto = _find_speaker_clone(
job.get("speaker_clones") or {}, key
)
if auto is None:
if not _ref_within_limit(auto):
# Short lines may have no line-specific clip.
# Reuse this speaker's best source instead of
# silently reverting to the engine default.
@@ -1458,27 +1465,17 @@ async def dub_generate(job_id: str, req: DubRequest):
yield f"data: {json.dumps({'type': 'cancelled', 'segments_processed': i + 1})}\n\n"
return
target_samples = int(seg_duration * backend.sample_rate)
current_samples = audio_tensor.shape[-1]
# Capture the real spoken duration before strict-slot padding
# or trimming. This is the evidence used by Agent timing and
# Capture the real spoken duration before assembly fitting.
# This is the evidence used by Agent timing and
# keeps sync badges truthful for every timing strategy.
natural_generated_dur = current_samples / backend.sample_rate
if strategy == "strict_slot":
# Legacy: pad short audio + trim long audio so the mix
# loop receives slot-sized buffers. The atempo squeeze
# in the mix loop never fires here because we already
# forced size = target_samples.
if target_samples > current_samples:
pad_amount = target_samples - current_samples
audio_tensor = torch.nn.functional.pad(audio_tensor, (0, pad_amount))
elif current_samples > target_samples:
audio_tensor = audio_tensor[..., :target_samples]
# concise / stretch_video / smart_fit: keep audio at its
# natural length. The mix loop decides per-mode whether to
# trim, slip, stretch the video, or split audio/video
# retiming (smart_fit) to accommodate it.
# Keep the complete waveform in every cache. Fitting happens
# once during assembly; pre-trimming here destroyed words before
# the pitch-preserving stretcher could see them.
if current_samples == 0 or not torch.isfinite(audio_tensor).all() or not torch.any(audio_tensor.abs() > 1e-6):
raise ValueError("The speech engine returned empty or silent audio")
generated_dur = natural_generated_dur
sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3)
@@ -1488,7 +1485,7 @@ async def dub_generate(job_id: str, req: DubRequest):
# Duration-planner calibration sample: this text length spoke
# for this long at natural rate. Keyed by stable seg id and
# merged into the per-language job map after the loop.
if strategy != "strict_slot" and seg.text.strip() and generated_dur > 0:
if seg.text.strip() and generated_dur > 0:
_natural_dur_records[str(seg_id)] = {
"chars": len(seg.text.strip()),
"dur": round(generated_dur, 4),
@@ -1526,15 +1523,6 @@ async def dub_generate(job_id: str, req: DubRequest):
if rvc_sr == backend.sample_rate:
audio_tensor = rvc_wav
if strategy == "strict_slot":
target_samples = int(seg_duration * backend.sample_rate)
current_samples = audio_tensor.shape[-1]
if target_samples > current_samples:
audio_tensor = torch.nn.functional.pad(
audio_tensor, (0, target_samples - current_samples)
)
elif current_samples > target_samples:
audio_tensor = audio_tensor[..., :target_samples]
except Exception as e:
yield f"data: {json.dumps({'type': 'warning', 'segment': i, 'message': f'RVC skipped: {str(e)[:120]}'})}\n\n"
@@ -1581,10 +1569,9 @@ async def dub_generate(job_id: str, req: DubRequest):
from core.public_errors import stream_generation_failure
error_detail = stream_generation_failure(e)["detail"]
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error': error_detail})}\n\n"
sr = backend.sample_rate
all_segment_wavs.append(_store_mix_wav(seg.start, seg.end, torch.zeros(1, max(0, int(seg_duration * sr))), sr, f"mix_{seg_id}"))
sync_scores.append(1.0)
logger.exception("Dub generation failed for segment %s", seg_id)
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'segment_id': seg_id, 'error': error_detail})}\n\n"
return
_t_loop_end = time.perf_counter()
@@ -1731,19 +1718,16 @@ async def dub_generate(job_id: str, req: DubRequest):
seg_gain = max(0.0, min(2.0, seg_gain))
try:
wav = _load_entry_wav((start, end, wav_path, sr), sr)
except Exception as e:
# A WAV header can be readable while its payload is
# truncated. Direct cache reuse deliberately defers the
# decode to assembly, so preserve the old recovery contract
# here: warn and fill this slot with silence instead of
# aborting the entire dub.
warning = {
"type": "warning",
"segment": i,
"message": f"cached seg lost, padding silence: {str(e)[:120]}",
}
yield f"data: {json.dumps(warning)}\n\n"
wav = torch.zeros(1, max(0, int((end - start) * sr)))
except Exception:
logger.exception("Dub assembly could not read segment %d", i)
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error_code': 'dub_speech_missing', 'error': 'A speech segment could not be read. Regenerate it before exporting.'})}\n\n"
return
from services.audio_dsp import trim_speech_padding
if seg_ref is not None and seg_ref.text.strip():
if wav.numel() == 0 or not torch.isfinite(wav).all() or not torch.any(wav.abs() > 1e-6):
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error_code': 'dub_speech_missing', 'error': 'A speech segment is empty or silent. Regenerate it before exporting.'})}\n\n"
return
wav = trim_speech_padding(wav, sr)
adjusted = wav * seg_gain
if adjusted.ndim == 2 and adjusted.shape[0] > 1:
adjusted = adjusted.mean(dim=0, keepdim=True)
@@ -1791,12 +1775,12 @@ async def dub_generate(job_id: str, req: DubRequest):
align_corners=False,
).squeeze(0)
wl = adjusted.shape[-1]
# Residual overflow → hard-trim to the segment's new video
# slot (fade below keeps the cut pop-free).
# Never publish a complete track with speech discarded by
# the fit caps. The user can shorten text or relax the caps.
new_slot_samples = int(max(0.0, sf.new_end - sf.new_start) * sr)
if new_slot_samples > 0 and wl > new_slot_samples:
adjusted = adjusted[..., :new_slot_samples]
wl = adjusted.shape[-1]
if new_slot_samples > 0 and wl > new_slot_samples + int(sr * 0.02):
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error_code': 'dub_timing_overflow', 'error': 'Speech exceeds the fitting limits. Shorten the translation or choose Strict Slot or Stretch Video before exporting.'})}\n\n"
return
# Truthful per-segment verdict for the UI badge.
entry = {"status": sf.status}
if abs(sf.audio_rate - 1.0) > 1e-6:
@@ -1816,8 +1800,8 @@ async def dub_generate(job_id: str, req: DubRequest):
elif strategy == "concise":
# Mode A: never compress. Allow the audio to extend into the
# silent gap before the next seg (existing heuristic) plus
# any extra `overflow_budget_s`. Beyond that, hard-trim with
# a short fade so we never overlap the next speaker.
# any extra `overflow_budget_s`. Beyond that, require a
# timing/text adjustment instead of discarding speech.
place_at = start
effective_end = end
if i + 1 < len(all_segment_wavs):
@@ -1831,47 +1815,25 @@ async def dub_generate(job_id: str, req: DubRequest):
slot_samples_eff = int(max(0.0, (effective_end - start)) * sr)
if slot_samples_eff > 0 and wl > slot_samples_eff:
overflow_s = (wl - slot_samples_eff) / sr
adjusted = adjusted[..., :slot_samples_eff]
wl = adjusted.shape[-1]
fit_status.append({
"status": "overflows",
"overflow_s": round(overflow_s, 3),
})
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'segment_id': job['seg_order'][i], 'error_code': 'dub_timing_overflow', 'error': 'Speech exceeds its time slot. Shorten the translation or choose Strict Slot or Stretch Video before exporting.', 'overflow_s': round(overflow_s, 3)})}\n\n"
return
else:
fit_status.append({"status": "fits"})
else:
# strict_slot (legacy): preserve the previous atempo / trim /
# off semantics so existing callers and back-compat tests
# keep passing.
# Strict Slot fits the complete speech to the original
# slot. Explicit legacy trim/off choices remain available.
place_at = start
effective_end = end
slowed_rate = None
if i + 1 < len(all_segment_wavs):
next_start = all_segment_wavs[i + 1][0]
gap = next_start - end
if gap > GAP_OVERFLOW_BUFFER_S:
effective_end = end + min(
gap - GAP_OVERFLOW_BUFFER_S, GAP_OVERFLOW_MAX_S,
)
slot_samples = int(max(0.0, (effective_end - start)) * sr)
if slot_fit != "off" and slot_samples > 0 and wl > slot_samples:
if slot_fit == "time_stretch":
ratio = wl / slot_samples
capped_ratio = min(ratio, MAX_STRETCH_RATIO)
capped_target = int(wl / capped_ratio)
try:
adjusted = await _pitch_preserving_stretch(
adjusted, capped_target, sr,
adjusted, slot_samples, sr,
)
if adjusted.shape[-1] > slot_samples:
adjusted = adjusted[..., :slot_samples]
if ratio > MAX_STRETCH_RATIO:
logger.info(
"seg %d compression %.2f× exceeded cap; "
"stretched to %.2f×, tail trimmed",
i, ratio, capped_ratio,
)
except Exception as e:
logger.warning(
"atempo stretch failed for seg %d (%.2f×), "
@@ -1891,15 +1853,14 @@ async def dub_generate(job_id: str, req: DubRequest):
slot_fit == "time_stretch"
and slot_samples > 0
and wl > 0
and wl < slot_samples * UNDERRUN_TOLERANCE
and _underrun_min_rate() < 1.0 - 1e-6
and wl < slot_samples
):
# Underrun fill (mirror of the compression above): the
# dub finished early, leaving the on-screen mouth moving
# over the thin under-speech bed residue — perceived as
# dead air. Slow toward the slot, never below the floor.
rate = max(wl / slot_samples, _underrun_min_rate())
target = min(slot_samples, int(round(wl / rate)))
rate = wl / slot_samples
target = slot_samples
try:
adjusted = await _pitch_preserving_stretch(
adjusted, target, sr,
@@ -2043,12 +2004,9 @@ async def dub_generate(job_id: str, req: DubRequest):
"fit_fp": fit_fp,
}
job["dubbed_tracks"][lang_code]["fit_fp"] = fit_fp
# Record what kind of per-segment WAVs are on disk so a later
# smart_fit run knows whether partial regen / fit-only re-mix can
# reuse them ("natural") or must regen once ("slotted"). Per-track
# (P1.3) — each language renders under its own strategy; the flat
# field stays in lock-step for older readers.
_kind = "slotted" if strategy == "strict_slot" else "natural"
# Every new cache preserves natural speech. Old slotted caches must
# be regenerated once because their missing tails cannot be recovered.
_kind = "natural"
job.setdefault("seg_wav_kind_by_lang", {})[lang_code] = _kind
job["seg_wav_kind"] = _kind
_save_job(job_id, job)
backend/core/tasks.py
+32 -1
View File
@@ -10,6 +10,27 @@ from core import run_sentinel
logger = logging.getLogger("omnivoice.tasks")
def _stream_failure(update):
"""Recognize terminal SSE failures, including generators that do not raise."""
if isinstance(update, bytes):
update = update.decode("utf-8", errors="replace")
if not isinstance(update, str):
return None
lines = update.splitlines()
try:
payload = json.loads("\n".join(line[5:].strip() for line in lines if line.startswith("data:")))
except (ValueError, TypeError):
return None
if not isinstance(payload, dict):
return None
if payload.get("type") != "error" and not any(line.strip() == "event: error" for line in lines):
return None
detail = payload.get("reason") or payload.get("error") or payload.get("detail")
if isinstance(detail, dict):
detail = detail.get("message") or detail.get("reason")
return detail if isinstance(detail, str) and detail else "Task failed"
class TaskManager:
"""In-memory task dispatcher with SQLite-backed metadata.
@@ -125,9 +146,19 @@ class TaskManager:
except Exception: logger.exception("job_store.mark_cancelled failed")
break
await self._push_event(task_id, update)
stream_error = _stream_failure(update)
if stream_error is not None:
t["status"] = "failed"
t["error"] = stream_error
try:
job_store.mark_failed(task_id, stream_error)
except Exception:
logger.exception("job_store.mark_failed failed")
await res.aclose()
break
elif inspect.iscoroutine(res):
await res
if t["status"] != "cancelled":
if t["status"] not in {"cancelled", "failed"}:
t["status"] = "done"
try: job_store.mark_done(task_id)
except Exception: logger.exception("job_store.mark_done failed")
backend/schemas/requests.py
+8 -8
View File
@@ -89,9 +89,9 @@ class DubRequest(BaseModel):
# (Bengali, Hindi, Arabic…). Three modes:
# "concise" — never compress TTS audio. Trim text up-front via
# speech_rate so it fits naturally; if it still
# overflows, hard-trim at slot with a short fade and
# surface fit_status="overflows" so the UI can prompt
# the user to shorten the segment. DEFAULT.
# overflows, fail without replacing the current track;
# the user must shorten it or choose another fit mode.
# DEFAULT.
# "stretch_video" — never compress TTS audio. Re-lay the timeline so
# each segment's video portion is stretched (via
# ffmpeg setpts) to fit the natural-rate dub audio.
@@ -100,13 +100,13 @@ class DubRequest(BaseModel):
# mild pitch-preserving audio speed-up (≤1.2× alone,
# ≤1.5× in hybrid) and a mild per-segment video
# slow-down (≤2.0×), per services/fit_planner.py.
# Residual overflow is trimmed and surfaced.
# "strict_slot" — legacy: keep `slot_fit` semantics (atempo squeeze
# when audio > slot). Kept for back-compat.
# Residual overflow fails without discarding words.
# "strict_slot" — pitch-preserving fit of the complete speech to
# the original start/end; may sound faster or slower.
timing_strategy: Optional[Literal["concise", "stretch_video", "strict_slot", "smart_fit"]] = "concise"
# Per-job slip budget for "concise" mode. Hard-trim only kicks in once
# gap absorption + this much extra time has been consumed.
# Per-job slip budget for "concise" mode. Overflow fails once gap
# absorption + this much extra time has been consumed.
overflow_budget_s: Optional[float] = 0.0
# Knob overrides for `smart_fit` (ignored by other strategies). Omitted
backend/services/audio_dsp.py
+20
View File
@@ -186,6 +186,26 @@ def trim_trailing_silence(
return audio_tensor[..., :end]
def trim_speech_padding(audio_tensor: torch.Tensor, sample_rate: int) -> torch.Tensor:
"""Remove generated edge silence before timing, retaining 50 ms of context.
Never compress silence into the spoken slot or delete internal pauses.
Silent/invalid outputs remain intact for the generation integrity guard.
"""
if audio_tensor.numel() == 0 or sample_rate <= 0:
return audio_tensor
envelope = audio_tensor.abs()
if envelope.ndim > 1:
envelope = envelope.amax(dim=tuple(range(envelope.ndim - 1)))
voiced = torch.nonzero(envelope > 10 ** (-50 / 20))
if voiced.numel() == 0:
return audio_tensor
margin = int(sample_rate * 0.05)
start = max(0, int(voiced[0].item()) - margin)
end = min(audio_tensor.shape[-1], int(voiced[-1].item()) + 1 + margin)
return audio_tensor[..., start:end]
def apply_effects_chain(audio_tensor, sample_rate: int, chain: list[dict]) -> torch.Tensor:
"""Apply a chain of named effects to an audio tensor.
backend/services/segmentation.py
+29
View File
@@ -491,6 +491,35 @@ def _apply_scene_cuts(segments: List[Segment], scene_cuts: Iterable[float]) -> L
dur_total = remaining.duration
if dur_total <= 0:
break
words = remaining.extra.get("words")
if isinstance(words, list) and words:
# A camera cut is not a word boundary. Character-proportional
# splitting assigned words seconds away from their real speech.
candidates = [
i for i in range(1, len(words))
if float(words[i - 1]["end"]) <= float(words[i]["start"])
and abs(float(words[i]["start"]) - cut) <= 0.25
]
if not candidates:
continue
split = min(candidates, key=lambda i: abs(float(words[i]["start"]) - cut))
left, right = words[:split], words[split:]
left_text = _clean(" ".join(str(w.get("text", w.get("word", ""))) for w in left))
right_text = _clean(" ".join(str(w.get("text", w.get("word", ""))) for w in right))
left_end, right_start = float(left[-1]["end"]), float(right[0]["start"])
if (len(left_text) < MIN_CHARS or len(right_text) < MIN_CHARS
or left_end - remaining.start < MIN_DUR
or remaining.end - right_start < MIN_DUR):
continue
out.append(Segment(
start=remaining.start, end=left_end, text=left_text,
speaker_id=remaining.speaker_id, extra={**remaining.extra, "words": left},
))
remaining = Segment(
start=right_start, end=remaining.end, text=right_text,
speaker_id=remaining.speaker_id, extra={**remaining.extra, "words": right},
)
continue
ratio = (cut - remaining.start) / dur_total
tentative_split = int(len(remaining.text) * ratio)
pos = _best_boundary(remaining.text, tentative_split)
docs/electron-dubbing.md
+24
View File
@@ -199,3 +199,27 @@ Below 40rem of workspace width, Dubbing stacks its controls above the editor wit
Global speed/quality changes preserve explicit Dubbing production steps, including settings restored from projects. Unset steps continue to follow the backend's current preset.
NLLB resolves explicit FLORES language/script codes, unambiguous ISO-639-3 codes and common short aliases, including Traditional Chinese. Unsupported or script-ambiguous source, target or per-segment languages are rejected before model loading instead of silently translating to English.
### Speech integrity and timing
A failed, empty or unreadable speech segment stops generation before a new track
replaces the previous output. Partial regeneration repairs missing or corrupt
segment caches, including missing clips outside the requested changed-line list;
it does not substitute silence and report completion. Auto speaker references
exclude oversized clips when selecting a shared fallback, so short lines reuse a
usable reference from their own speaker.
New segment caches retain the full generated speech in every timing mode. Strict
Slot removes edge silence and fits the complete clip to its original start/end
with pitch-preserving speed adjustment. Very long or short translations can still
sound unnaturally fast or slow; shorten or expand the translation for natural
pacing. Legacy clipped caches require regeneration once, because fitting cannot
recover discarded words. Explicit legacy Trim and Off options retain their
respective clipping and overlap behavior.
Concise and Smart Fit stop on unresolved overflow rather than publishing cut-off
words. Shorten the translation, choose Strict Slot, or allow Stretch Video before
retrying. Camera-cut segmentation uses nearby timed word boundaries when available
and skips cuts inside speech that cannot be assigned safely. This improves phrase
timing; it does not promise phoneme-level lip sync or correct inaccurate source
transcripts automatically.
electron/src/renderer/src/components/pipeline-failure.test.tsx
+19
View File
@@ -1,3 +1,4 @@
import i18next from 'i18next';
import { cleanup, render, screen } from '@testing-library/react';
import { afterEach, expect, it, vi } from 'vitest';
import { publicFailureFromEvent } from '@/lib/api/failure';
@@ -39,3 +40,21 @@ it('rejects a non-web documentation URL and uses the shared safe fallback', () =
'https://github.com/debpalash/VoiceStudio/blob/main/docs/install/troubleshooting.md',
);
});
it.each([
['dub_speech_missing', 'dubIntegrity.missingSpeech'],
['dub_timing_overflow', 'dubIntegrity.timingOverflow'],
])('localizes %s while retaining diagnostics', (errorCode, key) => {
const translate = vi.spyOn(i18next, 't').mockReturnValue('Localized recovery instructions');
try {
const failure = publicFailureFromEvent(
{ error_code: errorCode, reason: 'Raw engine error', diagnostic: 'segment a' },
'Fallback',
);
expect(failure.reason).toBe('Localized recovery instructions');
expect(failure.diagnostic).toBe('segment a');
expect(translate).toHaveBeenCalledWith(key);
} finally {
translate.mockRestore();
}
});
electron/src/renderer/src/i18n/locales/ar.json
+4
View File
@@ -2578,5 +2578,9 @@
"captured": "تم التقاط {{count}} من أسطر سجل المشكلات",
"contextNotice": "يتلقى الوكيل المحدد هذا البلاغ والشاشة الحالية وسجلات التطبيق والخلفية الحديثة وتشخيصات النظام.",
"complete": "مكتمل"
},
"dubIntegrity": {
"missingSpeech": "الكلام مفقود أو غير قابل للقراءة. أعد توليد المقطع المتأثر قبل التصدير.",
"timingOverflow": "يتجاوز الكلام الوقت المخصص له. اختصر الترجمة أو اختر التوقيت الصارم أو تمديد الفيديو."
}
}
electron/src/renderer/src/i18n/locales/de.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "{{count}} problematische Protokollzeilen erfasst",
"contextNotice": "Der ausgewählte Agent erhält diesen Bericht, die aktuelle Ansicht, aktuelle App- und Backend-Protokolle sowie Systemdiagnosen.",
"complete": "Abgeschlossen"
},
"dubIntegrity": {
"missingSpeech": "Sprache fehlt oder ist nicht lesbar. Erzeuge das betroffene Segment vor dem Export erneut.",
"timingOverflow": "Die Sprache überschreitet ihr Zeitfenster. Kürze die Übersetzung oder wähle ein festes Zeitfenster oder die Videostreckung."
}
}
electron/src/renderer/src/i18n/locales/en.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "Captured {{count}} problem log lines",
"contextNotice": "The selected agent receives this report, the current screen, recent app and backend logs, and system diagnostics.",
"complete": "Complete"
},
"dubIntegrity": {
"missingSpeech": "Speech is missing or unreadable. Regenerate the affected segment before exporting.",
"timingOverflow": "Speech exceeds its time slot. Shorten the translation or choose Strict Slot or Stretch Video."
}
}
electron/src/renderer/src/i18n/locales/es.json
+4
View File
@@ -2572,5 +2572,9 @@
"captured": "Se capturaron {{count}} líneas de registro con problemas",
"contextNotice": "El agente seleccionado recibe este informe, la pantalla actual, los registros recientes de la aplicación y del backend, y los diagnósticos del sistema.",
"complete": "Completado"
},
"dubIntegrity": {
"missingSpeech": "Falta audio de voz o no se puede leer. Regenera el segmento afectado antes de exportar.",
"timingOverflow": "La voz supera su intervalo de tiempo. Acorta la traducción o elige un intervalo estricto o alargar el vídeo."
}
}
electron/src/renderer/src/i18n/locales/fr.json
+4
View File
@@ -2572,5 +2572,9 @@
"captured": "{{count}} lignes de journal problématiques capturées",
"contextNotice": "L’agent sélectionné reçoit ce rapport, l’écran actuel, les journaux récents de l’application et du backend, ainsi que les diagnostics système.",
"complete": "Complet"
},
"dubIntegrity": {
"missingSpeech": "La parole est absente ou illisible. Régénérez le segment concerné avant l’exportation.",
"timingOverflow": "La parole dépasse son créneau. Raccourcissez la traduction ou choisissez un créneau strict ou l’étirement vidéo."
}
}
electron/src/renderer/src/i18n/locales/hi.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "समस्या वाली {{count}} लॉग पंक्तियाँ कैप्चर की गईं",
"contextNotice": "चुने गए एजेंट को यह रिपोर्ट, वर्तमान स्क्रीन, हाल के ऐप और बैकएंड लॉग तथा सिस्टम निदान मिलते हैं।",
"complete": "पूरा"
},
"dubIntegrity": {
"missingSpeech": "बोली गायब है या पढ़ी नहीं जा सकती। निर्यात से पहले प्रभावित खंड दोबारा बनाएँ।",
"timingOverflow": "बोली अपने समय खंड से लंबी है। अनुवाद छोटा करें या सख्त समय खंड अथवा वीडियो खिंचाव चुनें।"
}
}
electron/src/renderer/src/i18n/locales/id.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "{{count}} baris log bermasalah ditangkap",
"contextNotice": "Agen yang dipilih menerima laporan ini, layar saat ini, log aplikasi dan backend terbaru, serta diagnostik sistem.",
"complete": "Selesai"
},
"dubIntegrity": {
"missingSpeech": "Audio ucapan hilang atau tidak dapat dibaca. Buat ulang segmen terkait sebelum mengekspor.",
"timingOverflow": "Ucapan melebihi jatah waktunya. Persingkat terjemahan atau pilih slot ketat atau rentangkan video."
}
}
electron/src/renderer/src/i18n/locales/it.json
+4
View File
@@ -2572,5 +2572,9 @@
"captured": "Acquisite {{count}} righe di log problematiche",
"contextNotice": "L’agente selezionato riceve questa segnalazione, la schermata corrente, i log recenti dell’app e del backend e la diagnostica di sistema.",
"complete": "Completato"
},
"dubIntegrity": {
"missingSpeech": "Il parlato manca o non è leggibile. Rigenera il segmento interessato prima di esportare.",
"timingOverflow": "Il parlato supera il tempo disponibile. Accorcia la traduzione oppure scegli uno slot rigoroso o estendi il video."
}
}
electron/src/renderer/src/i18n/locales/ja.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "問題のあるログを{{count}}行取得しました",
"contextNotice": "選択したエージェントには、この報告、現在の画面、最近のアプリとバックエンドのログ、システム診断が渡されます。",
"complete": "完了"
},
"dubIntegrity": {
"missingSpeech": "音声がないか読み取れません。書き出す前に該当セグメントを再生成してください。",
"timingOverflow": "音声が割り当て時間を超えています。翻訳を短くするか、厳密な時間枠または動画の引き伸ばしを選んでください。"
}
}
electron/src/renderer/src/i18n/locales/ko.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "문제 로그 {{count}}줄을 수집했습니다",
"contextNotice": "선택한 에이전트는 이 보고서, 현재 화면, 최근 앱 및 백엔드 로그와 시스템 진단 정보를 받습니다.",
"complete": "완료"
},
"dubIntegrity": {
"missingSpeech": "음성이 없거나 읽을 수 없습니다. 내보내기 전에 해당 구간을 다시 생성하세요.",
"timingOverflow": "음성이 할당된 시간을 초과합니다. 번역을 줄이거나 엄격한 시간 구간 또는 비디오 늘이기를 선택하세요."
}
}
electron/src/renderer/src/i18n/locales/nl.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "{{count}} problematische logregels vastgelegd",
"contextNotice": "De geselecteerde agent ontvangt dit rapport, het huidige scherm, recente app- en backendlogboeken en systeemdiagnostiek.",
"complete": "Voltooid"
},
"dubIntegrity": {
"missingSpeech": "Spraak ontbreekt of is onleesbaar. Genereer het betreffende segment opnieuw voordat je exporteert.",
"timingOverflow": "De spraak overschrijdt het tijdvak. Verkort de vertaling of kies een strikt tijdvak of het uitrekken van de video."
}
}
electron/src/renderer/src/i18n/locales/pl.json
+4
View File
@@ -2574,5 +2574,9 @@
"captured": "Przechwycono {{count}} problematycznych wierszy dziennika",
"contextNotice": "Wybrany agent otrzyma to zgłoszenie, bieżący ekran, ostatnie dzienniki aplikacji i backendu oraz diagnostykę systemu.",
"complete": "Zakończono"
},
"dubIntegrity": {
"missingSpeech": "Brak mowy lub nie można jej odczytać. Przed eksportem wygeneruj ponownie dany segment.",
"timingOverflow": "Mowa przekracza przydzielony czas. Skróć tłumaczenie albo wybierz ścisły przedział czasu lub wydłużenie filmu."
}
}
electron/src/renderer/src/i18n/locales/pt.json
+4
View File
@@ -2572,5 +2572,9 @@
"captured": "{{count}} linhas de log com problemas capturadas",
"contextNotice": "O agente selecionado recebe este relatório, a tela atual, os logs recentes do aplicativo e do backend e os diagnósticos do sistema.",
"complete": "Completo"
},
"dubIntegrity": {
"missingSpeech": "A fala está ausente ou ilegível. Gere novamente o segmento afetado antes de exportar.",
"timingOverflow": "A fala excede o intervalo de tempo. Encurte a tradução ou escolha um intervalo estrito ou estique o vídeo."
}
}
electron/src/renderer/src/i18n/locales/ru.json
+4
View File
@@ -2574,5 +2574,9 @@
"captured": "Собрано строк журнала с проблемами: {{count}}",
"contextNotice": "Выбранный агент получит этот отчёт, текущий экран, последние журналы приложения и бэкенда, а также диагностику системы.",
"complete": "Завершено"
},
"dubIntegrity": {
"missingSpeech": "Речь отсутствует или не читается. Перед экспортом заново сгенерируйте этот сегмент.",
"timingOverflow": "Речь выходит за отведённое время. Сократите перевод или выберите строгий интервал либо растяжение видео."
}
}
electron/src/renderer/src/i18n/locales/sv.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "{{count}} problemrader i loggen samlades in",
"contextNotice": "Den valda agenten får den här rapporten, den aktuella vyn, aktuella app- och backendloggar samt systemdiagnostik.",
"complete": "Klart"
},
"dubIntegrity": {
"missingSpeech": "Tal saknas eller kan inte läsas. Generera det berörda segmentet på nytt före export.",
"timingOverflow": "Talet överskrider sin tidslucka. Korta översättningen eller välj strikt tidslucka eller sträck ut videon."
}
}
electron/src/renderer/src/i18n/locales/th.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "บันทึกบรรทัดปัญหาจากล็อกแล้ว {{count}} บรรทัด",
"contextNotice": "เอเจนต์ที่เลือกจะได้รับรายงานนี้ หน้าจอปัจจุบัน บันทึกล่าสุดของแอปและแบ็กเอนด์ และข้อมูลวินิจฉัยระบบ",
"complete": "เสร็จสิ้น"
},
"dubIntegrity": {
"missingSpeech": "เสียงพูดหายไปหรืออ่านไม่ได้ สร้างช่วงที่มีปัญหาใหม่ก่อนส่งออก",
"timingOverflow": "เสียงพูดยาวเกินช่วงเวลาที่กำหนด ย่อคำแปลหรือเลือกช่วงเวลาแบบเคร่งครัดหรือยืดวิดีโอ"
}
}
electron/src/renderer/src/i18n/locales/tr.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "{{count}} sorunlu günlük satırı yakalandı",
"contextNotice": "Seçilen aracı bu raporu, mevcut ekranı, son uygulama ve arka uç günlüklerini ve sistem tanılamasını alır.",
"complete": "Tamamlandı"
},
"dubIntegrity": {
"missingSpeech": "Konuşma eksik veya okunamıyor. Dışa aktarmadan önce ilgili bölümü yeniden oluşturun.",
"timingOverflow": "Konuşma ayrılan süreyi aşıyor. Çeviriyi kısaltın veya kesin zaman aralığını ya da videoyu uzatmayı seçin."
}
}
electron/src/renderer/src/i18n/locales/uk.json
+4
View File
@@ -2574,5 +2574,9 @@
"captured": "Зібрано рядків журналу з проблемами: {{count}}",
"contextNotice": "Вибраний агент отримає цей звіт, поточний екран, останні журнали застосунку й бекенду та діагностику системи.",
"complete": "Завершено"
},
"dubIntegrity": {
"missingSpeech": "Мовлення відсутнє або не читається. Перед експортом повторно згенеруйте цей сегмент.",
"timingOverflow": "Мовлення перевищує відведений час. Скоротіть переклад або виберіть строгий інтервал чи розтягнення відео."
}
}
electron/src/renderer/src/i18n/locales/vi.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "Đã thu thập {{count}} dòng nhật ký có vấn đề",
"contextNotice": "Tác nhân đã chọn nhận báo cáo này, màn hình hiện tại, nhật ký ứng dụng và backend gần đây cùng thông tin chẩn đoán hệ thống.",
"complete": "Hoàn thành"
},
"dubIntegrity": {
"missingSpeech": "Lời nói bị thiếu hoặc không đọc được. Hãy tạo lại đoạn bị ảnh hưởng trước khi xuất.",
"timingOverflow": "Lời nói vượt quá thời gian được phân bổ. Hãy rút ngắn bản dịch hoặc chọn khung thời gian nghiêm ngặt hay kéo dài video."
}
}
electron/src/renderer/src/i18n/locales/zh-CN.json
+4
View File
@@ -2574,5 +2574,9 @@
"captured": "已捕获 {{count}} 行问题日志",
"contextNotice": "所选代理将收到此报告、当前界面、最近的应用与后端日志以及系统诊断信息。",
"complete": "已完成"
},
"dubIntegrity": {
"missingSpeech": "语音缺失或无法读取。请在导出前重新生成受影响的片段。",
"timingOverflow": "语音超出分配的时长。请缩短译文，或选择严格时间段或拉伸视频。"
}
}
electron/src/renderer/src/i18n/locales/zh-TW.json
+4
View File
@@ -2570,5 +2570,9 @@
"captured": "已擷取 {{count}} 行問題日誌",
"contextNotice": "所選代理將收到此報告、目前畫面、最近的應用程式與後端記錄以及系統診斷資訊。",
"complete": "已完成"
},
"dubIntegrity": {
"missingSpeech": "語音缺失或無法讀取。請在匯出前重新產生受影響的片段。",
"timingOverflow": "語音超出分配的時長。請縮短譯文，或選擇嚴格時間區段或延展影片。"
}
}
electron/src/renderer/src/lib/api/failure.ts
+3
View File
@@ -1,3 +1,4 @@
import i18next from 'i18next';
export interface PublicFailure {
reason: string;
errorClass?: string;
@@ -16,6 +17,8 @@ export function publicFailureFromEvent(
): PublicFailure {
return {
reason:
(event.error_code === 'dub_speech_missing' ? i18next.t('dubIntegrity.missingSpeech') :
event.error_code === 'dub_timing_overflow' ? i18next.t('dubIntegrity.timingOverflow') : undefined) ||
text(event.reason) ||
text(event.detail) ||
text(event.error) ||
frontend/src/hooks/useDubWorkflow.js
+10 -3
View File
@@ -1185,6 +1185,7 @@ export default function useDubWorkflow({
const decoder = new TextDecoder();
let buffer = '';
let wasCancelled = false;
let generationError = null;
let sawDone = false;
while (true) {
const { done, value } = await reader.read();
@@ -1261,8 +1262,14 @@ export default function useDubWorkflow({
setDubStep('editing');
setDubError(t('dub_workflow.generation_aborted'));
toast(t('dub_workflow.dubbing_aborted'), { icon: '⏹' });
} else if (evt.type === 'error')
setDubError((p) => p + `\nSeg ${evt.segment}: ${evt.error}`);
} else if (evt.type === 'error') {
generationError = evt.error_code === 'dub_speech_missing'
? t('dubIntegrity.missingSpeech')
: evt.error_code === 'dub_timing_overflow'
? t('dubIntegrity.timingOverflow')
: evt.reason || evt.error || t('dub_workflow.generation_stream_ended');
setDubError(generationError);
}
} catch (err) {
console.warn('Dub generate SSE handler failed:', err);
}
@@ -1271,7 +1278,7 @@ export default function useDubWorkflow({
}
setDubTaskId(null);
if (!wasCancelled) {
if (!sawDone) throw new Error(t('dub_workflow.generation_stream_ended'));
if (!sawDone || generationError) throw new Error(generationError || t('dub_workflow.generation_stream_ended'));
if (dubStep !== 'done') setDubStep('done');
loadDubHistory();
loadProjects();
frontend/src/i18n/locales/ar.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "إيقاف مؤقت",
"play": "تشغيل"
},
"dubIntegrity": {
"missingSpeech": "الكلام مفقود أو غير قابل للقراءة. أعد توليد المقطع المتأثر قبل التصدير.",
"timingOverflow": "يتجاوز الكلام الوقت المخصص له. اختصر الترجمة أو اختر التوقيت الصارم أو تمديد الفيديو."
}
}
frontend/src/i18n/locales/de.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Pausieren",
"play": "Abspielen"
},
"dubIntegrity": {
"missingSpeech": "Sprache fehlt oder ist nicht lesbar. Erzeuge das betroffene Segment vor dem Export erneut.",
"timingOverflow": "Die Sprache überschreitet ihr Zeitfenster. Kürze die Übersetzung oder wähle ein festes Zeitfenster oder die Videostreckung."
}
}
frontend/src/i18n/locales/en.json
+4
View File
@@ -3313,5 +3313,9 @@
"transcript": "What it heard",
"result_kicker": "Converted take",
"need_source_and_voice": "Add a source clip and pick a target voice first."
},
"dubIntegrity": {
"missingSpeech": "Speech is missing or unreadable. Regenerate the affected segment before exporting.",
"timingOverflow": "Speech exceeds its time slot. Shorten the translation or choose Strict Slot or Stretch Video."
}
}
frontend/src/i18n/locales/es.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Pausar",
"play": "Reproducir"
},
"dubIntegrity": {
"missingSpeech": "Falta audio de voz o no se puede leer. Regenera el segmento afectado antes de exportar.",
"timingOverflow": "La voz supera su intervalo de tiempo. Acorta la traducción o elige un intervalo estricto o alargar el vídeo."
}
}
frontend/src/i18n/locales/fr.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Mettre en pause",
"play": "Lire"
},
"dubIntegrity": {
"missingSpeech": "La parole est absente ou illisible. Régénérez le segment concerné avant l’exportation.",
"timingOverflow": "La parole dépasse son créneau. Raccourcissez la traduction ou choisissez un créneau strict ou l’étirement vidéo."
}
}
frontend/src/i18n/locales/hi.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "रोकें",
"play": "चलाएँ"
},
"dubIntegrity": {
"missingSpeech": "बोली गायब है या पढ़ी नहीं जा सकती। निर्यात से पहले प्रभावित खंड दोबारा बनाएँ।",
"timingOverflow": "बोली अपने समय खंड से लंबी है। अनुवाद छोटा करें या सख्त समय खंड अथवा वीडियो खिंचाव चुनें।"
}
}
frontend/src/i18n/locales/id.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Jeda",
"play": "Putar"
},
"dubIntegrity": {
"missingSpeech": "Audio ucapan hilang atau tidak dapat dibaca. Buat ulang segmen terkait sebelum mengekspor.",
"timingOverflow": "Ucapan melebihi jatah waktunya. Persingkat terjemahan atau pilih slot ketat atau rentangkan video."
}
}
frontend/src/i18n/locales/it.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Pausa",
"play": "Riproduci"
},
"dubIntegrity": {
"missingSpeech": "Il parlato manca o non è leggibile. Rigenera il segmento interessato prima di esportare.",
"timingOverflow": "Il parlato supera il tempo disponibile. Accorcia la traduzione oppure scegli uno slot rigoroso o estendi il video."
}
}
frontend/src/i18n/locales/ja.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "一時停止",
"play": "再生"
},
"dubIntegrity": {
"missingSpeech": "音声がないか読み取れません。書き出す前に該当セグメントを再生成してください。",
"timingOverflow": "音声が割り当て時間を超えています。翻訳を短くするか、厳密な時間枠または動画の引き伸ばしを選んでください。"
}
}
frontend/src/i18n/locales/ko.json
+4
View File
@@ -3313,5 +3313,9 @@
"stop": "재생 중지",
"streaming_preview": "스트리밍 미리듣기…",
"untitled": "오디오"
},
"dubIntegrity": {
"missingSpeech": "음성이 없거나 읽을 수 없습니다. 내보내기 전에 해당 구간을 다시 생성하세요.",
"timingOverflow": "음성이 할당된 시간을 초과합니다. 번역을 줄이거나 엄격한 시간 구간 또는 비디오 늘이기를 선택하세요."
}
}
frontend/src/i18n/locales/nl.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Pauzeren",
"play": "Afspelen"
},
"dubIntegrity": {
"missingSpeech": "Spraak ontbreekt of is onleesbaar. Genereer het betreffende segment opnieuw voordat je exporteert.",
"timingOverflow": "De spraak overschrijdt het tijdvak. Verkort de vertaling of kies een strikt tijdvak of het uitrekken van de video."
}
}
frontend/src/i18n/locales/pl.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Wstrzymaj",
"play": "Odtwórz"
},
"dubIntegrity": {
"missingSpeech": "Brak mowy lub nie można jej odczytać. Przed eksportem wygeneruj ponownie dany segment.",
"timingOverflow": "Mowa przekracza przydzielony czas. Skróć tłumaczenie albo wybierz ścisły przedział czasu lub wydłużenie filmu."
}
}
frontend/src/i18n/locales/pt.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Pausar",
"play": "Reproduzir"
},
"dubIntegrity": {
"missingSpeech": "A fala está ausente ou ilegível. Gere novamente o segmento afetado antes de exportar.",
"timingOverflow": "A fala excede o intervalo de tempo. Encurte a tradução ou escolha um intervalo estrito ou estique o vídeo."
}
}
frontend/src/i18n/locales/ru.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Пауза",
"play": "Воспроизвести"
},
"dubIntegrity": {
"missingSpeech": "Речь отсутствует или не читается. Перед экспортом заново сгенерируйте этот сегмент.",
"timingOverflow": "Речь выходит за отведённое время. Сократите перевод или выберите строгий интервал либо растяжение видео."
}
}
frontend/src/i18n/locales/sv.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Pausa",
"play": "Spela upp"
},
"dubIntegrity": {
"missingSpeech": "Tal saknas eller kan inte läsas. Generera det berörda segmentet på nytt före export.",
"timingOverflow": "Talet överskrider sin tidslucka. Korta översättningen eller välj strikt tidslucka eller sträck ut videon."
}
}
frontend/src/i18n/locales/th.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "หยุดชั่วคราว",
"play": "เล่น"
},
"dubIntegrity": {
"missingSpeech": "เสียงพูดหายไปหรืออ่านไม่ได้ สร้างช่วงที่มีปัญหาใหม่ก่อนส่งออก",
"timingOverflow": "เสียงพูดยาวเกินช่วงเวลาที่กำหนด ย่อคำแปลหรือเลือกช่วงเวลาแบบเคร่งครัดหรือยืดวิดีโอ"
}
}
frontend/src/i18n/locales/tr.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Duraklat",
"play": "Oynat"
},
"dubIntegrity": {
"missingSpeech": "Konuşma eksik veya okunamıyor. Dışa aktarmadan önce ilgili bölümü yeniden oluşturun.",
"timingOverflow": "Konuşma ayrılan süreyi aşıyor. Çeviriyi kısaltın veya kesin zaman aralığını ya da videoyu uzatmayı seçin."
}
}
frontend/src/i18n/locales/uk.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Пауза",
"play": "Відтворити"
},
"dubIntegrity": {
"missingSpeech": "Мовлення відсутнє або не читається. Перед експортом повторно згенеруйте цей сегмент.",
"timingOverflow": "Мовлення перевищує відведений час. Скоротіть переклад або виберіть строгий інтервал чи розтягнення відео."
}
}
frontend/src/i18n/locales/vi.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "Tạm dừng",
"play": "Phát"
},
"dubIntegrity": {
"missingSpeech": "Lời nói bị thiếu hoặc không đọc được. Hãy tạo lại đoạn bị ảnh hưởng trước khi xuất.",
"timingOverflow": "Lời nói vượt quá thời gian được phân bổ. Hãy rút ngắn bản dịch hoặc chọn khung thời gian nghiêm ngặt hay kéo dài video."
}
}
frontend/src/i18n/locales/zh-CN.json
+4
View File
@@ -3313,5 +3313,9 @@
"transcript": "识别到的内容",
"result_kicker": "转换后的音频",
"need_source_and_voice": "请先添加源音频并选择目标声音。"
},
"dubIntegrity": {
"missingSpeech": "语音缺失或无法读取。请在导出前重新生成受影响的片段。",
"timingOverflow": "语音超出分配的时长。请缩短译文，或选择严格时间段或拉伸视频。"
}
}
frontend/src/i18n/locales/zh-TW.json
+4
View File
@@ -2832,5 +2832,9 @@
"player": {
"pause": "暫停",
"play": "播放"
},
"dubIntegrity": {
"missingSpeech": "語音缺失或無法讀取。請在匯出前重新產生受影響的片段。",
"timingOverflow": "語音超出分配的時長。請縮短譯文，或選擇嚴格時間區段或延展影片。"
}
}
tests/test_dub_complete_audio.py
+142
View File
@@ -0,0 +1,142 @@
"""A published dub must contain every requested spoken segment, without early clipping."""
import asyncio
import json
from types import SimpleNamespace
import pytest
import torch
import soundfile as sf
from schemas.requests import DubRequest
@pytest.fixture
def render_dub(monkeypatch, tmp_path):
import api.routers.dub_generate as dg
job = {'duration': 4.0, 'dubbed_tracks': {}, 'segments': [], 'seg_wav_kind_by_lang': {'en': 'natural'}}
path = tmp_path / 'job'
path.mkdir()
events = []
generated = []
async def resolve():
return 'omnivoice', SimpleNamespace(target='local', remote=False), backend
def generate(**kwargs):
generated.append(kwargs)
return output[0]()
backend = SimpleNamespace(sample_rate=24000, generate=generate, applies_own_mastering=True)
output = [lambda: torch.ones(1, 24000) * .1]
class Tasks:
def is_cancelled(self, *_): return False
async def add_task(self, tid, kind, func, *args):
async for event in func(*args):
if event.startswith('data: '): events.append(json.loads(event[6:]))
monkeypatch.setattr(dg, '_resolve_dub_execution', resolve)
monkeypatch.setattr(dg, '_get_job', lambda _: job)
monkeypatch.setattr(dg, '_save_job', lambda *_: None)
monkeypatch.setattr(dg, 'DUB_DIR', str(tmp_path))
monkeypatch.setattr(dg, 'dub_seg_path', lambda _, sid: str(path / f'seg_{sid}.wav'))
monkeypatch.setattr(dg, 'task_manager', Tasks())
monkeypatch.setattr(dg, 'rvc_is_enabled', lambda: False)
monkeypatch.setattr(dg, 'mark_synthetic', lambda a, *args, **kw: a)
monkeypatch.setattr(dg, 'get_effect_chain', lambda _: None)
monkeypatch.setattr(dg, 'normalize_audio', lambda a, **kw: a)
def run(**kwargs):
body = dict(segments=[dict(start=0, end=1, text='hello')], segment_ids=['a'], language_code='en', num_step=4)
body.update(kwargs)
asyncio.run(dg.dub_generate('job', DubRequest(**body)))
return events
return SimpleNamespace(run=run, output=output, job=job, path=path, generated=generated)
def test_failed_segment_does_not_publish_complete_track(render_dub):
def fail(): raise RuntimeError('engine failed')
render_dub.output[0] = fail
events = render_dub.run()
assert any(e['type'] == 'error' for e in events)
assert not any(e['type'] == 'done' for e in events)
assert not render_dub.job['dubbed_tracks']
def test_missing_partial_cache_is_regenerated(render_dub):
events = render_dub.run(regen_only=[])
assert render_dub.generated
assert (render_dub.path / 'seg_en_a.wav').exists()
assert any(e['type'] == 'done' for e in events)
@pytest.mark.parametrize("rvc_enabled", [False, True])
def test_strict_slot_keeps_full_cache_and_fits_the_tail(render_dub, monkeypatch, rvc_enabled):
import api.routers.dub_generate as dg
render_dub.output[0] = lambda: torch.cat((torch.ones(1, 24000)*.1, torch.ones(1, 24000)*.3), dim=-1)
monkeypatch.setattr(dg, "rvc_is_enabled", lambda: rvc_enabled)
monkeypatch.setattr(dg, "apply_rvc", lambda _: None)
calls=[]
async def stretch(wav, target, sr):
calls.append((wav.shape[-1], target, float(wav[0,-1])))
return torch.nn.functional.interpolate(wav.unsqueeze(0),size=target,mode='linear').squeeze(0)
monkeypatch.setattr(dg, '_pitch_preserving_stretch', stretch)
events=render_dub.run(timing_strategy='strict_slot')
assert any(e['type']=='done' for e in events)
assert sf.info(render_dub.path/'seg_en_a.wav').duration == 2
assert calls and calls[0][0] == 48000 and calls[0][1] == 24000
assert calls[0][2] == pytest.approx(.3, abs=1e-4)
def test_concise_overrun_does_not_publish_truncated_track(render_dub):
render_dub.output[0] = lambda: torch.ones(1, 48000)*.1
events=render_dub.run()
assert any(e['type']=='error' for e in events)
assert not any(e['type']=='done' for e in events)
assert not render_dub.job['dubbed_tracks']
def test_silent_engine_output_is_not_a_successful_segment(render_dub):
render_dub.output[0] = lambda: torch.zeros(1, 24000)
events = render_dub.run()
assert any(e['type'] == 'error' for e in events)
assert not render_dub.job['dubbed_tracks']
def test_camera_cut_uses_word_times_instead_of_character_ratio():
from services.segmentation import Segment, _apply_scene_cuts
words = [
{'text': 'A very long early phrase', 'start': 0, 'end': 1.8},
{'text': 'Short later phrase', 'start': 4.0, 'end': 6.0},
]
segment = Segment(start=0, end=6, text='A very long early phrase Short later phrase', extra={'words': words})
pieces = _apply_scene_cuts([segment], [4.0])
assert len(pieces) == 2
assert pieces[0].end == 1.8
assert pieces[1].start == 4
assert pieces[0].extra['words'] == words[:1]
assert pieces[1].text == 'Short later phrase'
def test_camera_cut_inside_a_word_does_not_reassign_speech():
from services.segmentation import Segment, _apply_scene_cuts
words = [
{'text': 'A lengthy opening phrase', 'start': 0, 'end': 4},
{'text': 'And the closing phrase', 'start': 4.5, 'end': 7},
]
segment = Segment(start=0, end=7, text='A lengthy opening phrase And the closing phrase', extra={'words': words})
assert _apply_scene_cuts([segment], [3]) == [segment]
def test_failed_regeneration_preserves_previous_track(render_dub):
previous = render_dub.path / 'dubbed_en.wav'
previous.write_bytes(b'previous successful output')
render_dub.job['dubbed_tracks']['en'] = {'path': str(previous)}
def fail(): raise RuntimeError('failure')
render_dub.output[0] = fail
render_dub.run()
assert previous.read_bytes() == b'previous successful output'
assert render_dub.job['dubbed_tracks']['en']['path'] == str(previous)
def test_timing_trims_edge_silence_but_keeps_internal_pauses():
from services.audio_dsp import trim_speech_padding
wav = torch.cat((torch.zeros(1, 1000), torch.ones(1, 200)*.1,
torch.zeros(1, 200), torch.ones(1, 200)*.1, torch.zeros(1,1000)), dim=-1)
result = trim_speech_padding(wav, 1000)
assert result.shape[-1] == 700
assert torch.equal(result[..., 250:450], torch.zeros(1,200))
tests/test_dub_voice_match.py
+10
View File
@@ -449,3 +449,13 @@ def test_consistent_explicit_cross_auto_seg_binding_is_honoured(patched_generate
model = patched_generate(_HEURISTIC_JOB, body)
assert model.refs[0] == ("/v/seg3.wav", "seg3 ref", False) # explicit wins
assert [r[0] for r in model.refs[1:]] == ["/v/seg1.wav"] * 3 # rest unified
def test_consistent_pick_rejects_oversized_fallback_reference():
from api.routers.dub_generate import resolve_consistent_ref
job = _job(
[{'id': 'too-long', 'speaker_id': 'Speaker 1'}, {'id': 'usable', 'speaker_id': 'Speaker 1'}],
{'too-long': {'ref_audio': '/v/long.wav', 'ref_text': 'long', 'duration': 20.92},
'usable': {'ref_audio': '/v/usable.wav', 'ref_text': 'usable', 'duration': 9.0}},
)
assert resolve_consistent_ref(job, 'speaker_1')['ref_audio'] == '/v/usable.wav'
tests/test_redub_incremental.py
+3 -3
View File
@@ -405,9 +405,9 @@ def test_legacy_cache_ignored_once_job_has_another_language(patched_generate):
model.calls.clear()
run(_body(_ES_SEGS, regen_only=[]))
assert model.calls == [] # not in the regen list → still no TTS…
# …but the ambiguous legacy audio was NOT spliced in: the slot is silence.
assert abs(_track_sample(job_dir, "es", 1.25)) < 0.005
# Missing unambiguous speech must be regenerated, never padded with silence.
assert model.calls == ["Buenos dias", "Hasta luego"]
assert abs(_track_sample(job_dir, "es", 1.25) - _amp_for("Hasta luego")) < 0.01
def test_seg_hashes_stored_per_language_with_flat_mirror(patched_generate):
tests/test_task_stream_failure.py
+41
View File
@@ -0,0 +1,41 @@
import asyncio
import pytest
from core.tasks import TaskManager, _stream_failure
@pytest.mark.parametrize('event', [
'data: {"type":"error","error":"segment failed"}\n\n',
'event: error\ndata: {"detail":"segment failed"}\n\n',
])
def test_error_stream_is_terminal(event, monkeypatch):
from core import job_store
from core import run_sentinel
states=[]
for name in ['create','mark_running','append_event']:
monkeypatch.setattr(job_store,name,lambda *a,**kw: None)
monkeypatch.setattr(job_store,'mark_failed',lambda *a: states.append('failed'))
monkeypatch.setattr(job_store,'mark_done',lambda *a: states.append('done'))
monkeypatch.setattr(run_sentinel,'touch_activity',lambda *a: None)
closed=[]
async def stream():
try:
yield event
yield 'data: {"type":"done"}\n\n'
finally: closed.append(True)
async def run():
manager=TaskManager()
await manager.add_task('test','dub_generate',stream)
worker=asyncio.create_task(manager.worker())
try:
await asyncio.wait_for(manager.queue.join(),2)
assert manager.active_tasks['test']['status']=='failed'
assert len(manager.active_tasks['test']['history'])==1
finally:
worker.cancel()
try: await worker
except asyncio.CancelledError: pass
asyncio.run(run())
assert states==['failed']
assert closed==[True]
def test_warnings_remain_non_terminal():
assert _stream_failure('data: {"type":"warning","error":"retrying"}\n\n') is None