feat: implement streaming TTS, A/B voice comparison, and background task processing with SSE updates

This commit is contained in:
debpalash
2026-04-14 04:00:38 +05:30
parent 389b83fb42
commit 3a8adf5dd4
18 changed files with 3392 additions and 2688 deletions
+2 -1
View File
@@ -42,4 +42,5 @@ cloudflared.tgz
# Data directories and sqlite db
omnivoice_data/
*.db
*.db
demo_recording.webp
+9 -9
View File
@@ -13,9 +13,9 @@
<br/>
<div align="center">
<img src="pics/image.png" alt="OmniVoice Studio Design Interface" width="100%"/>
<img src="preview.png" alt="OmniVoice Studio Interface Demo" width="100%"/>
<br/>
<i>High-density Voice Design & Cloning workspace.</i>
<i>The timeline-based cinematic dubbing and workspace UI.</i>
</div>
---
@@ -90,14 +90,14 @@ The studio is highly functional today, but we are aggressively expanding. Watch
- [x] Polished glassmorphism design system with micro-animations, focus rings, and custom scrollbars.
### 🔨 Upcoming Features
- [ ] **Real Speaker Diarization** — ML-based diarization via pyannote.audio for true multi-speaker identification.
- [ ] **Streaming TTS** — Real-time audio streaming during generation instead of blocking.
- [ ] **A/B Voice Comparison** — Side-by-side voice audition for casting decisions.
- [ ] **Scene-Aware Dubbing** — FFmpeg scene detection to auto-split segments at visual cuts.
- [ ] **Lip-Sync Scoring**Analyze dubbed audio duration against original speaker timing.
- [ ] **Batch Processing** — Queue folders full of media to be processed seamlessly overnight.
- [x] **Real Speaker Diarization** — ML-based diarization via pyannote.audio for true multi-speaker identification.
- [x] **A/B Voice Comparison** — Side-by-side voice audition for casting decisions.
- [x] **Scene-Aware Dubbing** — FFmpeg scene detection to auto-split segments at visual cuts.
- [x] **Lip-Sync Scoring** — Analyze dubbed audio duration against original speaker timing with color-coded badges.
- [x] **Batch Processing**Centralized async task queue ensuring sequential GPU execution with reconnectable SSE streams.
- [x] **Advanced Export Suite** — VTT subtitles, per-segment WAV ZIP, compressed MP3, and stem export (vocals + background separate).
- [x] **Streaming TTS** — Chunked WAV streaming with progressive download and auto-playback.
- [ ] **Native Desktop Applications** — Dedicated client apps for macOS, Windows, and Linux.
- [ ] **EBU R128 Loudness Normalization** — Broadcast-standard loudness matching for exported audio.
- [ ] **One-Click Deployment** — Docker image packages engineered for zero-config GPU passthrough.
---
+470 -42
View File
@@ -58,6 +58,72 @@ _cpu_pool = ThreadPoolExecutor(max_workers=os.cpu_count() or 4)
_dub_jobs = {}
# ═══════════════════════════════════════════════════════════════════════
# ASYNC BATCH TASK MANAGER
# ═══════════════════════════════════════════════════════════════════════
class TaskManager:
def __init__(self):
self.queue = None
self.active_tasks = {}
def _init_queue(self):
if self.queue is None:
self.queue = asyncio.Queue()
async def add_task(self, task_id, task_type, func, *args, **kwargs):
self._init_queue()
task_obj = {
"status": "pending",
"type": task_type,
"created_at": time.time(),
"history": [],
"listeners": [],
"error": None
}
self.active_tasks[task_id] = task_obj
await self.queue.put((task_id, func, args, kwargs))
async def _push_event(self, task_id, event_str):
if task_id not in self.active_tasks: return
t = self.active_tasks[task_id]
if event_str is not None:
t["history"].append(event_str)
for q in t["listeners"]:
await q.put(event_str)
async def worker(self):
self._init_queue()
while True:
task_id, func, args, kwargs = await self.queue.get()
t = self.active_tasks.get(task_id)
if not t:
self.queue.task_done()
continue
t["status"] = "running"
try:
import inspect
res = func(*args, **kwargs)
if inspect.isasyncgen(res):
async for update in res:
await self._push_event(task_id, update)
elif inspect.iscoroutine(res):
await res
t["status"] = "done"
except Exception as e:
t["status"] = "failed"
t["error"] = str(e)
try:
await self._push_event(task_id, f"data: {json.dumps({'type': 'error', 'error': str(e)})}\n\n")
except: pass
finally:
await self._push_event(task_id, None) # EOF
self.queue.task_done()
task_manager = TaskManager()
# ═══════════════════════════════════════════════════════════════════════
# SQLITE DATABASE
# ═══════════════════════════════════════════════════════════════════════
@@ -198,8 +264,10 @@ async def _idle_worker():
async def lifespan(app: FastAPI):
_init_db()
idle_task = asyncio.create_task(_idle_worker())
worker_task = asyncio.create_task(task_manager.worker())
yield
idle_task.cancel()
worker_task.cancel()
from fastapi.middleware.cors import CORSMiddleware
@@ -666,17 +734,27 @@ async def generate_speech(
conn.commit()
conn.close()
# Also return the WAV bytes for immediate playback
# Stream WAV bytes in chunks for progressive playback
buffer = io.BytesIO()
torchaudio.save(buffer, audio_tensor, model.sampling_rate, format="wav")
buffer.seek(0)
return Response(
content=buffer.read(), media_type="audio/wav",
wav_bytes = buffer.read()
async def _stream_wav():
chunk_size = 16384 # 16KB chunks for smooth streaming
for i in range(0, len(wav_bytes), chunk_size):
yield wav_bytes[i:i + chunk_size]
return StreamingResponse(
_stream_wav(),
media_type="audio/wav",
headers={
"X-Audio-Id": audio_id,
"X-Gen-Time": str(gen_time),
"X-Audio-Path": audio_filename,
"X-Seed": str(used_seed) if used_seed is not None else "",
"X-Audio-Duration": str(audio_dur),
"Content-Length": str(len(wav_bytes)),
}
)
except Exception as e:
@@ -690,6 +768,28 @@ async def generate_speech(
# VIDEO DUBBING PIPELINE
# ═══════════════════════════════════════════════════════════════════════
_diar_pipeline = None
def _get_diarization_pipeline():
global _diar_pipeline
hf_token = os.environ.get("HF_TOKEN")
if not hf_token:
return None
if _diar_pipeline is not None:
return _diar_pipeline
try:
import torch
from pyannote.audio import Pipeline
logger.info("Loading Pyannote Diarization Pipeline...")
_diar_pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", use_auth_token=hf_token)
if torch.cuda.is_available():
_diar_pipeline.to(torch.device("cuda"))
logger.info("Pyannote Diarization Pipeline loaded successfully.")
return _diar_pipeline
except Exception as e:
logger.error(f"Failed to load Pyannote pipeline: {e}")
return None
def _find_ffmpeg():
for path in ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "ffmpeg"]:
if shutil.which(path):
@@ -743,32 +843,50 @@ async def dub_upload(video: UploadFile = File(...)):
except Exception:
dur = 0.0
# Run demucs to isolate vocals vs background music
vocals_path = os.path.join(job_dir, "vocals.wav")
no_vocals_path = os.path.join(job_dir, "no_vocals.wav")
try:
# Run demucs CLI asynchronously to strictly output 2 stems
proc = await asyncio.create_subprocess_exec(
"uv", "run", "demucs", "--two-stems", "vocals", "-n", "htdemucs", "-d", get_best_device(),
audio_path, "-o", job_dir,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
_, stderr = await proc.communicate()
if proc.returncode != 0:
raise Exception(stderr.decode())
# Demucs creates an output structure: htdemucs/audio/vocals.wav
demucs_out = os.path.join(job_dir, "htdemucs", "audio")
if os.path.exists(os.path.join(demucs_out, "vocals.wav")):
import shutil
shutil.move(os.path.join(demucs_out, "vocals.wav"), vocals_path)
shutil.move(os.path.join(demucs_out, "no_vocals.wav"), no_vocals_path)
# Remove demucs temp dir
shutil.rmtree(os.path.join(job_dir, "htdemucs"))
except Exception as e:
logger.warning(f"Demucs failed, falling back to mixed audio. {e}")
vocals_path = audio_path
no_vocals_path = None
scene_cuts = []
async def run_demucs():
nonlocal vocals_path, no_vocals_path
try:
# Run demucs CLI asynchronously to strictly output 2 stems
proc = await asyncio.create_subprocess_exec(
"uv", "run", "demucs", "--two-stems", "vocals", "-n", "htdemucs", "-d", get_best_device(),
audio_path, "-o", job_dir,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
_, stderr = await proc.communicate()
if proc.returncode != 0:
raise Exception(stderr.decode())
# Demucs creates an output structure: htdemucs/audio/vocals.wav
demucs_out = os.path.join(job_dir, "htdemucs", "audio")
if os.path.exists(os.path.join(demucs_out, "vocals.wav")):
import shutil
shutil.move(os.path.join(demucs_out, "vocals.wav"), vocals_path)
shutil.move(os.path.join(demucs_out, "no_vocals.wav"), no_vocals_path)
shutil.rmtree(os.path.join(job_dir, "htdemucs"))
except Exception as e:
logger.warning(f"Demucs failed, falling back to mixed audio. {e}")
vocals_path = audio_path
no_vocals_path = None
async def run_scene_detection():
nonlocal scene_cuts
try:
scene_proc = await asyncio.create_subprocess_exec(
ffmpeg, "-i", video_path, "-filter:v", "select='gt(scene,0.3)',showinfo", "-f", "null", "-",
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
_, stderr_scene = await scene_proc.communicate()
import re
matches = re.finditer(r"pts_time:([\d\.]+)", stderr_scene.decode())
scene_cuts = [float(m.group(1)) for m in matches]
except Exception as e:
logger.warning(f"Scene detection failed: {e}")
await asyncio.gather(run_demucs(), run_scene_detection())
_dub_jobs[job_id] = {
"video_path": video_path,
@@ -777,6 +895,7 @@ async def dub_upload(video: UploadFile = File(...)):
"no_vocals_path": no_vocals_path,
"duration": dur, "filename": video.filename,
"segments": None, "dubbed_tracks": {},
"scene_cuts": scene_cuts,
}
return {"job_id": job_id, "duration": round(dur, 2), "filename": video.filename}
@@ -862,19 +981,88 @@ async def dub_transcribe(job_id: str):
t += sent_dur
else:
segments.append({"start": 0.0, "end": job["duration"], "text": result.get("text", "").strip()})
# Apply basic Diarization (Heuristic fallback / pyannote skeleton)
# If segments have gaps > 1.2s, assume natural speaker alternation or same speaker.
# For a true production deployment, plug pyannote.audio pipeline here guarded by HF_TOKEN.
current_speaker_idx = 1
last_end = 0.0
# Apply Diarization (pyannote.audio if HF_TOKEN is set, else Heuristic fallback)
diar_pipe = _get_diarization_pipeline()
for i, s in enumerate(segments):
if i > 0 and (s["start"] - last_end) > 1.2:
# Toggle speaker on significant silence gaps for multi-speaker simulation
current_speaker_idx = 2 if current_speaker_idx == 1 else 1
s["speaker_id"] = f"Speaker {current_speaker_idx}"
s["id"] = str(uuid.uuid4())[:8] # assign fresh ID
last_end = s["end"]
if diar_pipe:
try:
asr_audio_target = job.get("vocals_path", job.get("audio_path"))
diarization = diar_pipe(asr_audio_target)
for s in segments:
seg_mid = (s["start"] + s["end"]) / 2.0
assigned_speaker = "Speaker 1"
for turn, _, speaker in diarization.itertracks(yield_label=True):
if turn.start <= seg_mid <= turn.end:
# Map pyannote generic "SPEAKER_00" to "Speaker 1"
speaker_idx = int(speaker.split("_")[-1]) + 1
assigned_speaker = f"Speaker {speaker_idx}"
break
s["speaker_id"] = assigned_speaker
s["id"] = str(uuid.uuid4())[:8] # assign fresh ID
except Exception as e:
logger.error(f"Pyannote diarization failed during inference: {e}. Falling back to heuristic.")
diar_pipe = None
if not diar_pipe:
# Fallback heuristic
current_speaker_idx = 1
last_end = 0.0
for i, s in enumerate(segments):
if i > 0 and (s["start"] - last_end) > 1.2:
current_speaker_idx = 2 if current_speaker_idx == 1 else 1
s["speaker_id"] = f"Speaker {current_speaker_idx}"
s["id"] = str(uuid.uuid4())[:8] # assign fresh ID
last_end = s["end"]
# --- SCENE-AWARE DUBBING ---
scene_cuts = job.get("scene_cuts", [])
if scene_cuts:
sorted_cuts = sorted(scene_cuts)
new_segments = []
for s in segments:
s_start = s["start"]
s_end = s["end"]
valid_cuts = [c for c in sorted_cuts if c > s_start + 0.2 and c < s_end - 0.2]
if not valid_cuts:
new_segments.append(s)
else:
curr_start = s_start
curr_text = s["text"]
total_dur = s_end - s_start
for cut in valid_cuts:
ratio = (cut - curr_start) / max(total_dur, 0.01)
split_idx = int(len(curr_text) * ratio)
# Avoid splitting words exactly in half if possible
space_idx = curr_text.rfind(' ', 0, split_idx + 5)
if space_idx != -1 and space_idx > split_idx - 10:
split_idx = space_idx
part_text = curr_text[:split_idx].strip()
curr_text = curr_text[split_idx:].strip()
if part_text:
new_seg = dict(s)
new_seg["start"] = round(curr_start, 2)
new_seg["end"] = round(cut, 2)
new_seg["text"] = part_text
new_seg["id"] = str(uuid.uuid4())[:8]
new_segments.append(new_seg)
curr_start = cut
total_dur = s_end - curr_start
if curr_text:
new_seg = dict(s)
new_seg["start"] = round(curr_start, 2)
new_seg["end"] = round(s_end, 2)
new_seg["text"] = curr_text
new_seg["id"] = str(uuid.uuid4())[:8]
new_segments.append(new_seg)
segments = new_segments
# Store full transcript
job["full_transcript"] = " ".join(s["text"] for s in segments)
@@ -923,7 +1111,7 @@ class DubRequest(BaseModel):
@app.post("/dub/generate/{job_id}")
async def dub_generate(job_id: str, req: DubRequest):
"""Generate TTS per segment. Returns SSE progress stream."""
"""Adds a dub generation job to the async batch task pool."""
job = _get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
@@ -933,6 +1121,7 @@ async def dub_generate(job_id: str, req: DubRequest):
async def _stream():
total = len(req.segments)
all_segment_wavs = []
sync_scores = []
for i, seg in enumerate(req.segments):
yield f"data: {json.dumps({'type': 'progress', 'current': i, 'total': total, 'text': seg.text[:50]})}\n\n"
@@ -942,6 +1131,7 @@ async def dub_generate(job_id: str, req: DubRequest):
sr = _model.sampling_rate
silence = torch.zeros(1, int(seg_duration * sr))
all_segment_wavs.append((seg.start, seg.end, silence, sr))
sync_scores.append(1.0)
continue
def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id=None):
@@ -999,6 +1189,12 @@ async def dub_generate(job_id: str, req: DubRequest):
seg.text, req.language, seg_instruct, seg_duration,
req.num_step, req.guidance_scale, seg_speed, seg_profile,
)
# Calculate lip-sync score natively from tensor
generated_dur = audio_tensor.shape[-1] / _model.sampling_rate
sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3)
sync_scores.append(sync_ratio)
# Save individual segment WAV for preview
seg_wav_path = os.path.join(DUB_DIR, job_id, f"seg_{i}.wav")
torchaudio.save(seg_wav_path, audio_tensor, _model.sampling_rate)
@@ -1007,6 +1203,7 @@ async def dub_generate(job_id: str, req: DubRequest):
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error': str(e)})}\n\n"
sr = _model.sampling_rate
all_segment_wavs.append((seg.start, seg.end, torch.zeros(1, int(seg_duration * sr)), sr))
sync_scores.append(1.0)
yield f"data: {json.dumps({'type': 'assembling'})}\n\n"
@@ -1048,9 +1245,39 @@ async def dub_generate(job_id: str, req: DubRequest):
except Exception as e:
logger.error(f"Failed to save dub history: {e}")
yield f"data: {json.dumps({'type': 'done', 'segments_processed': total, 'language_code': lang_code, 'tracks': list(job['dubbed_tracks'].keys())})}\n\n"
yield f"data: {json.dumps({'type': 'done', 'segments_processed': total, 'language_code': lang_code, 'tracks': list(job['dubbed_tracks'].keys()), 'sync_scores': sync_scores})}\n\n"
return StreamingResponse(_stream(), media_type="text/event-stream")
task_id = f"dub_{job_id}_{int(time.time())}"
await task_manager.add_task(task_id, "dub_generate", _stream)
return {"task_id": task_id}
@app.get("/tasks/stream/{task_id}")
async def stream_task(task_id: str):
"""Universal Server-Sent Event stream for background tasks."""
if task_id not in task_manager.active_tasks:
raise HTTPException(status_code=404, detail="Task not found")
async def _reader():
t = task_manager.active_tasks[task_id]
q = asyncio.Queue()
t["listeners"].append(q)
try:
for evt in t["history"]:
yield evt
if t["status"] in ("done", "failed"):
return
while True:
evt = await q.get()
if evt is None:
break
yield evt
finally:
t["listeners"].remove(q)
return StreamingResponse(_reader(), media_type="text/event-stream")
@app.get("/dub/tracks/{job_id}")
@@ -1380,6 +1607,207 @@ async def dub_export_srt(job_id: str):
)
# ═══════════════════════════════════════════════════════════════════════
# VTT SUBTITLE EXPORT
# ═══════════════════════════════════════════════════════════════════════
def _format_vtt_time(seconds):
"""Format seconds as VTT timestamp: HH:MM:SS.mmm"""
h = int(seconds // 3600)
m = int((seconds % 3600) // 60)
s = int(seconds % 60)
ms = int((seconds % 1) * 1000)
return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}"
@app.get("/dub/vtt/{job_id}")
@app.get("/dub/vtt/{job_id}/{filename}")
async def dub_export_vtt(job_id: str):
"""Export transcript segments as a WebVTT subtitle file."""
job = _get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
segments = job.get("segments", [])
if not segments:
raise HTTPException(status_code=400, detail="No transcript segments available")
vtt_lines = ["WEBVTT", ""]
for i, seg in enumerate(segments):
start_ts = _format_vtt_time(seg["start"])
end_ts = _format_vtt_time(seg["end"])
vtt_lines.append(str(i + 1))
vtt_lines.append(f"{start_ts} --> {end_ts}")
vtt_lines.append(seg["text"])
vtt_lines.append("")
vtt_content = "\n".join(vtt_lines)
base_name = os.path.splitext(job.get('filename', 'video'))[0]
return Response(
content=vtt_content,
media_type="text/vtt",
headers={
"Content-Disposition": f'attachment; filename="subtitles_{base_name}.vtt"',
},
)
# ═══════════════════════════════════════════════════════════════════════
# PER-SEGMENT WAV ZIP EXPORT
# ═══════════════════════════════════════════════════════════════════════
@app.get("/dub/export-segments/{job_id}")
async def dub_export_segments_zip(job_id: str):
"""Export individually named WAV files for each dubbed segment as a ZIP archive."""
import zipfile
job = _get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
segments = job.get("segments", [])
if not segments:
raise HTTPException(status_code=400, detail="No segments available")
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
for i, seg in enumerate(segments):
seg_path = os.path.join(DUB_DIR, job_id, f"seg_{i}.wav")
if os.path.exists(seg_path):
speaker = seg.get("speaker_id", "Speaker1").replace(" ", "")
start_str = f"{seg['start']:.2f}"
end_str = f"{seg['end']:.2f}"
arc_name = f"{i+1:03d}_{start_str}-{end_str}_{speaker}.wav"
zf.write(seg_path, arc_name)
zip_buffer.seek(0)
base_name = os.path.splitext(job.get('filename', 'video'))[0]
safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'segments'
return Response(
content=zip_buffer.read(),
media_type="application/zip",
headers={
"Content-Disposition": f'attachment; filename="segments_{safe_name}.zip"',
},
)
# ═══════════════════════════════════════════════════════════════════════
# MP3 AUDIO EXPORT
# ═══════════════════════════════════════════════════════════════════════
@app.get("/dub/download-mp3/{job_id}")
@app.get("/dub/download-mp3/{job_id}/{filename}")
async def dub_download_mp3(job_id: str, lang: str = Query(None), preserve_bg: bool = Query(True)):
"""Export dubbed audio as compressed MP3 (192kbps). ~10x smaller than WAV."""
job = _get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
tracks = job.get("dubbed_tracks", {})
if lang and lang in tracks:
wav_path = tracks[lang]["path"]
elif tracks:
wav_path = list(tracks.values())[0]["path"]
else:
raise HTTPException(status_code=400, detail="No dubbed audio track generated yet")
if not os.path.exists(wav_path):
raise HTTPException(status_code=404, detail="Audio file not found")
lang_label = lang or list(tracks.keys())[0]
ffmpeg = _find_ffmpeg()
# Optionally mix with background audio first
source_path = wav_path
bg_audio = job.get("no_vocals_path") if preserve_bg else None
if bg_audio and os.path.exists(bg_audio):
mixed_path = os.path.join(DUB_DIR, job_id, f"mixed_mp3_{lang_label}.wav")
cmd_mix = [
ffmpeg, "-i", bg_audio, "-i", wav_path,
"-filter_complex", "[0:a][1:a]amix=inputs=2:duration=longest:dropout_transition=2:weights=0.8 1.2[aout]",
"-map", "[aout]", "-c:a", "pcm_s16le", "-y", mixed_path
]
try:
proc = await asyncio.create_subprocess_exec(
*cmd_mix, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
_, stderr = await proc.communicate()
if proc.returncode == 0:
source_path = mixed_path
except Exception as e:
logger.error(f"Failed to mix audio for MP3: {e}")
# Convert to MP3
mp3_path = os.path.join(DUB_DIR, job_id, f"dubbed_{lang_label}.mp3")
cmd = [ffmpeg, "-i", source_path, "-codec:a", "libmp3lame", "-b:a", "192k", "-y", mp3_path]
try:
proc = await asyncio.create_subprocess_exec(
*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
_, stderr = await proc.communicate()
if proc.returncode != 0:
raise Exception(stderr.decode())
except Exception as e:
raise HTTPException(status_code=500, detail=f"MP3 encoding failed: {str(e)}")
base_name = os.path.splitext(job.get('filename', 'audio'))[0]
safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'audio'
dl_name = f"dubbed_{lang_label}_{safe_name}.mp3"
return FileResponse(
mp3_path, media_type="audio/mpeg",
headers={"Content-Disposition": f'attachment; filename="{dl_name}"'},
)
# ═══════════════════════════════════════════════════════════════════════
# STEM EXPORT (Vocals + Background Separate)
# ═══════════════════════════════════════════════════════════════════════
@app.get("/dub/export-stems/{job_id}")
async def dub_export_stems(job_id: str, lang: str = Query(None)):
"""Export dubbed vocals and original background as separate WAV files in a ZIP."""
import zipfile
job = _get_job(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
tracks = job.get("dubbed_tracks", {})
if not tracks:
raise HTTPException(status_code=400, detail="No dubbed tracks generated yet")
if lang and lang in tracks:
vocals_path = tracks[lang]["path"]
lang_label = lang
elif tracks:
first_key = list(tracks.keys())[0]
vocals_path = tracks[first_key]["path"]
lang_label = first_key
else:
raise HTTPException(status_code=400, detail="No dubbed audio track")
bg_path = job.get("no_vocals_path")
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
if os.path.exists(vocals_path):
zf.write(vocals_path, f"vocals_dubbed_{lang_label}.wav")
if bg_path and os.path.exists(bg_path):
zf.write(bg_path, "background_original.wav")
zip_buffer.seek(0)
base_name = os.path.splitext(job.get('filename', 'video'))[0]
safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'stems'
return Response(
content=zip_buffer.read(),
media_type="application/zip",
headers={
"Content-Disposition": f'attachment; filename="stems_{safe_name}.zip"',
},
)
# ═══════════════════════════════════════════════════════════════════════
# STUDIO PROJECTS — Save / Load / List / Delete
# ═══════════════════════════════════════════════════════════════════════
+236 -10
View File
@@ -9,7 +9,8 @@ import {
Settings2, ChevronDown, ChevronUp, Play, Search, Film, Trash2,
FileText, Loader, Check, AlertCircle, Plus, User, Save, Languages, Headphones,
FolderOpen, FolderPlus, Pencil, Clock, Lock, Unlock, Mic, MicOff, Square,
CheckCircle, Circle, ChevronRight, Target, PanelLeftClose, PanelLeftOpen
CheckCircle, Circle, ChevronRight, Target, PanelLeftClose, PanelLeftOpen, Scale,
Layers, Music, Package
} from 'lucide-react';
const TAGS = [
@@ -138,6 +139,16 @@ function App() {
const [selectedProfile, setSelectedProfile] = useState(null);
const [showSaveProfile, setShowSaveProfile] = useState(false);
const [profileName, setProfileName] = useState('');
// A/B Voice Comparison State
const [isCompareModalOpen, setIsCompareModalOpen] = useState(false);
const [compareVoiceA, setCompareVoiceA] = useState("");
const [compareVoiceB, setCompareVoiceB] = useState("");
const [compareText, setCompareText] = useState("The quick brown fox jumps over the lazy dog, proving that this voice sounds much better.");
const [compareResultA, setCompareResultA] = useState(null);
const [compareResultB, setCompareResultB] = useState(null);
const [isComparing, setIsComparing] = useState(false);
const [compareProgress, setCompareProgress] = useState("");
const [showAllProjects, setShowAllProjects] = useState(false);
const [previewLoading, setPreviewLoading] = useState(null);
const [segmentPreviewLoading, setSegmentPreviewLoading] = useState(null);
@@ -396,6 +407,38 @@ function App() {
const response = await fetch(`${API}/generate`, { method: "POST", body: formData });
if (!response.ok) throw new Error(await response.text());
// Streaming TTS: read audio bytes as they arrive
const reader = response.body.getReader();
const chunks = [];
let receivedLength = 0;
const contentLength = parseInt(response.headers.get('Content-Length') || '0', 10);
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
receivedLength += value.length;
// Update generation time to show streaming progress
if (contentLength > 0) {
const pct = Math.round((receivedLength / contentLength) * 100);
setGenerationTime(prev => `${prev.toString().split(' ')[0]} (${pct}%)`);
}
}
// Construct final blob and create instant playback URL
const blob = new Blob(chunks, { type: 'audio/wav' });
const audioUrl = URL.createObjectURL(blob);
// Auto-play the streamed result immediately
try {
const audio = new Audio(audioUrl);
audio.play().catch(() => {});
// Cleanup after playback
audio.onended = () => URL.revokeObjectURL(audioUrl);
} catch (e) {}
// Refresh history from server and explicitly switch to history tab automatically so user can see it
await loadHistory();
setSidebarTab('history');
@@ -740,7 +783,12 @@ function App() {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
const reader = res.body.getReader();
const data = await res.json();
if (!res.ok) throw new Error(data.detail || "Failed to start generation");
// Connect to background task SSE stream
const streamRes = await fetch(`${API}/tasks/stream/${data.task_id}`);
const reader = streamRes.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
while (true) {
@@ -753,7 +801,13 @@ function App() {
try {
const evt = JSON.parse(line.slice(6));
if (evt.type === 'progress') setDubProgress({ current: evt.current + 1, total: evt.total, text: evt.text });
else if (evt.type === 'done') { setDubStep('done'); setDubTracks(evt.tracks || []); }
else if (evt.type === 'done') {
setDubStep('done');
setDubTracks(evt.tracks || []);
if (evt.sync_scores) {
setDubSegments(prev => prev.map((s, idx) => ({ ...s, sync_ratio: evt.sync_scores[idx] })));
}
}
else if (evt.type === 'error') setDubError(p => p + `\nSeg ${evt.segment}: ${evt.error}`);
} catch (e) {}
}
@@ -996,8 +1050,15 @@ function App() {
{mode === 'launchpad' ? (
<div className="glass-panel" style={{flex:1, display:'flex', flexDirection:'column', overflowY:'auto'}}>
<div style={{padding:'20px 30px', borderBottom:'1px solid rgba(255,255,255,0.08)'}}>
<h2 style={{margin:0, fontSize:'1.4rem', display:'flex', alignItems:'center', gap:'10px'}}><Globe color="#ebdbb2"/> Unified Workspace</h2>
<p style={{margin:'4px 0 0 0', color:'#a89984', fontSize:'0.85rem'}}>Select a cloned voice, design preset, or dubbing project to load into the studio.</p>
<div style={{display:'flex', justifyContent:'space-between', alignItems:'flex-start'}}>
<div>
<h2 style={{margin:0, fontSize:'1.4rem', display:'flex', alignItems:'center', gap:'10px'}}><Globe color="#ebdbb2"/> Unified Workspace</h2>
<p style={{margin:'4px 0 0 0', color:'#a89984', fontSize:'0.85rem'}}>Select a cloned voice, design preset, or dubbing project to load into the studio.</p>
</div>
<button className="btn-primary" onClick={() => setIsCompareModalOpen(true)} style={{display:'flex', alignItems:'center', gap:6, padding:'6px 14px', fontSize:'0.85rem', width:'auto', marginTop:0}}>
<Scale size={16}/> A/B Voice Comparison
</button>
</div>
{/* Quick Start Guide */}
<div style={{marginTop:'20px', padding:'15px', background:'rgba(255,255,255,0.02)', borderRadius:'8px', border:'1px solid rgba(255,255,255,0.05)'}}>
@@ -1468,11 +1529,28 @@ function App() {
</div>
{dubSegments.map((seg, idx) => (
<div key={seg.id} className={`segment-row ${dubStep==='generating'&&dubProgress.current===idx+1?'segment-active':''} ${dubStep==='generating'&&dubProgress.current>idx+1?'segment-done':''}`}>
<span className="segment-time" style={{width:55}}>
{formatTime(seg.start)}{formatTime(seg.end)}
{seg.speed && seg.speed !== 1.0 && (
<span style={{fontSize:'0.55rem', color: seg.speed > 1 ? '#d3869b' : '#8ec07c', marginLeft:2}}>
{seg.speed.toFixed(2)}x
<span className="segment-time" style={{width:55, display:'flex', flexDirection:'column'}}>
<span>
{formatTime(seg.start)}{formatTime(seg.end)}
{seg.speed && seg.speed !== 1.0 && (
<span style={{fontSize:'0.55rem', color: seg.speed > 1 ? '#d3869b' : '#8ec07c', marginLeft:2}}>
{seg.speed.toFixed(2)}x
</span>
)}
</span>
{seg.sync_ratio !== undefined && (
<span style={{
fontSize: '0.5rem',
marginTop: 2,
display: 'inline-flex',
alignItems: 'center',
gap: 2,
color: seg.sync_ratio >= 0.95 && seg.sync_ratio <= 1.05 ? '#b8bb26' :
seg.sync_ratio > 1.25 ? '#fb4934' : '#fabd2f'
}} title={`Generated audio is ${Math.round(seg.sync_ratio * 100)}% the duration of original`}>
{seg.sync_ratio >= 0.95 && seg.sync_ratio <= 1.05 ? <CheckCircle size={8}/> :
seg.sync_ratio > 1.25 ? <AlertCircle size={8}/> : <Circle size={8}/>}
Sync: {Math.round(seg.sync_ratio * 100)}%
</span>
)}
</span>
@@ -1574,6 +1652,25 @@ function App() {
<FileText size={11}/> SRT
</button>
</div>
{/* Advanced Export Row */}
<div style={{display:'flex', gap:4, marginTop:4}}>
<button className="btn-primary" style={{marginTop:0, flex:1, padding:'4px 7px', fontSize:'0.62rem', background:dubSegments.length?'linear-gradient(135deg,#b8bb26,#98971a)':undefined}}
onClick={() => triggerDownload(`${API}/dub/vtt/${dubJobId}/subtitles.vtt`, 'subtitles.vtt')} disabled={!dubSegments.length}>
<FileText size={10}/> VTT
</button>
<button className="btn-primary" style={{marginTop:0, flex:1, padding:'4px 7px', fontSize:'0.62rem', background:dubStep==='done'?'linear-gradient(135deg,#fabd2f,#d79921)':undefined}}
onClick={() => triggerDownload(`${API}/dub/download-mp3/${dubJobId}/audio.mp3?preserve_bg=${preserveBg}`, 'dubbed_audio.mp3')} disabled={dubStep!=='done'}>
<Music size={10}/> MP3
</button>
<button className="btn-primary" style={{marginTop:0, flex:1, padding:'4px 7px', fontSize:'0.62rem', background:dubStep==='done'?'linear-gradient(135deg,#fe8019,#d65d0e)':undefined}}
onClick={() => triggerDownload(`${API}/dub/export-segments/${dubJobId}`, 'segments.zip')} disabled={dubStep!=='done'}>
<Package size={10}/> Clips
</button>
<button className="btn-primary" style={{marginTop:0, flex:1, padding:'4px 7px', fontSize:'0.62rem', background:dubStep==='done'?'linear-gradient(135deg,#d3869b,#b16286)':undefined}}
onClick={() => triggerDownload(`${API}/dub/export-stems/${dubJobId}`, 'stems.zip')} disabled={dubStep!=='done'}>
<Layers size={10}/> Stems
</button>
</div>
</div>
</div>
)}
@@ -2002,6 +2099,135 @@ function App() {
)}
</div>
)}
{/* ═══ A/B VOICE COMPARISON MODAL ═══ */}
{isCompareModalOpen && (
<div style={{
position: 'fixed', top: 0, left: 0, right: 0, bottom: 0,
background: 'rgba(0,0,0,0.85)', backdropFilter: 'blur(8px)',
display: 'flex', alignItems: 'center', justifyContent: 'center', zIndex: 9999
}}>
<div className="glass-panel" style={{ width: 600, maxWidth: '90vw', padding: '24px', display: 'flex', flexDirection: 'column', gap: '16px', position: 'relative' }}>
<h2 style={{ margin: 0, color: '#ebdbb2', display: 'flex', alignItems: 'center', gap: '8px', fontSize: '1.2rem' }}>
<Scale /> A/B Voice Comparison
</h2>
<p style={{ margin: 0, fontSize: '0.85rem', color: '#a89984' }}>Compare two voices side by side to make casting decisions.</p>
<div style={{ display: 'flex', flexDirection: 'column', gap: 6 }}>
<label style={{ fontSize: '0.75rem', color: '#a89984', fontWeight: 600 }}>Test Phrase</label>
<textarea
className="input-base"
value={compareText}
onChange={e => setCompareText(e.target.value)}
rows={2}
style={{ resize: 'none' }}
/>
</div>
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: '20px' }}>
{/* Voice A */}
<div style={{ display: 'flex', flexDirection: 'column', gap: 8, padding: 12, border: '1px solid rgba(255,255,255,0.05)', borderRadius: 8, background: 'rgba(255,255,255,0.01)' }}>
<h3 style={{ margin: 0, color: '#d3869b', fontSize: '0.9rem', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6 }}><Fingerprint size={14}/> Voice A</h3>
<select className="input-base" value={compareVoiceA} onChange={e => setCompareVoiceA(e.target.value)}>
<option value="">-- Select Voice --</option>
{profiles.map(p => <option key={p.id} value={p.id}>{p.name}</option>)}
{PRESETS.map(p => <option key={p.id} value={`preset:${p.id}`}>{p.name} (Preset)</option>)}
</select>
{compareResultA ? (
<audio src={compareResultA} controls style={{ width: '100%', height: 32, outline: 'none' }} />
) : (
<div style={{ height: 32, display: 'flex', alignItems: 'center', justifyContent: 'center', color: '#665c54', fontSize: '0.75rem', background: 'rgba(0,0,0,0.2)', borderRadius: 4 }}>No Audio</div>
)}
</div>
{/* Voice B */}
<div style={{ display: 'flex', flexDirection: 'column', gap: 8, padding: 12, border: '1px solid rgba(255,255,255,0.05)', borderRadius: 8, background: 'rgba(255,255,255,0.01)' }}>
<h3 style={{ margin: 0, color: '#8ec07c', fontSize: '0.9rem', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6 }}><Fingerprint size={14}/> Voice B</h3>
<select className="input-base" value={compareVoiceB} onChange={e => setCompareVoiceB(e.target.value)}>
<option value="">-- Select Voice --</option>
{profiles.map(p => <option key={p.id} value={p.id}>{p.name}</option>)}
{PRESETS.map(p => <option key={p.id} value={`preset:${p.id}`}>{p.name} (Preset)</option>)}
</select>
{compareResultB ? (
<audio src={compareResultB} controls style={{ width: '100%', height: 32, outline: 'none' }} />
) : (
<div style={{ height: 32, display: 'flex', alignItems: 'center', justifyContent: 'center', color: '#665c54', fontSize: '0.75rem', background: 'rgba(0,0,0,0.2)', borderRadius: 4 }}>No Audio</div>
)}
</div>
</div>
<div style={{ display: 'flex', justifyContent: 'flex-end', gap: '10px', marginTop: '10px' }}>
<button className="btn-primary" style={{ background: 'transparent', color: '#a89984', padding: '6px 14px' }} onClick={() => setIsCompareModalOpen(false)}>
Close
</button>
<button
className="btn-primary"
disabled={isComparing || !compareVoiceA || !compareVoiceB || !compareText.trim()}
onClick={async () => {
setIsComparing(true);
setCompareResultA(null);
setCompareResultB(null);
const generateVoice = async (voiceId, setProgress) => {
setProgress(`Preparing voice...`);
const formData = new FormData();
formData.append("text", compareText);
let fin_prof = voiceId;
let fin_inst = "";
if (fin_prof.startsWith('preset:')) {
const pr = PRESETS.find(p => p.id === fin_prof.replace('preset:', ''));
if (pr) {
const parts = Object.values(pr.attrs).filter(v => v !== 'Auto');
fin_inst = parts.join(', ');
}
fin_prof = '';
} else if (profiles.find(p => p.id === fin_prof)?.instruct) {
fin_inst = profiles.find(p => p.id === fin_prof).instruct;
}
if (fin_prof) formData.append("profile_id", fin_prof);
if (fin_inst) formData.append("instruct", fin_inst);
formData.append("num_step", steps);
formData.append("guidance_scale", cfg);
formData.append("speed", speed);
formData.append("denoise", denoise);
formData.append("postprocess_output", postprocess);
const res = await fetch(`${API}/generate`, { method: "POST", body: formData });
if (!res.ok) throw new Error(await res.text());
return URL.createObjectURL(await res.blob());
};
try {
setCompareProgress("Generating Voice A...");
const audioA = await generateVoice(compareVoiceA, setCompareProgress);
setCompareResultA(audioA);
setCompareProgress("Generating Voice B...");
const audioB = await generateVoice(compareVoiceB, setCompareProgress);
setCompareResultB(audioB);
setCompareProgress("");
toast.success("Comparison complete!");
loadHistory();
} catch (err) {
toast.error("Play failed: " + err.message);
setCompareProgress("");
} finally {
setIsComparing(false);
}
}}
style={{ padding: '6px 14px', width: 200, display: 'flex', justifyContent: 'center', alignItems: 'center' }}
>
{isComparing ? <><Loader className="spinner" size={14}/> {compareProgress}</> : <><Play size={14}/> Compare</>}
</button>
</div>
</div>
</div>
)}
</div>
);
}
-5
View File
@@ -1,5 +0,0 @@
import json
from omnivoice.utils.lang_map import LANG_NAMES, lang_display_name
languages = ["Auto"] + sorted(lang_display_name(n) for n in LANG_NAMES)
with open("frontend/src/languages.json", "w") as f:
json.dump(languages, f)
View File
-2575
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -7,7 +7,7 @@
"dev": "turbo run dev //#dev:api",
"build": "turbo run build",
"start": "turbo run start",
"dev:api": "uv run uvicorn api:app --host 0.0.0.0 --port 8000 --reload"
"dev:api": "uv run uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload"
},
"workspaces": [
"frontend"
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.2 MiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 962 KiB

+8
View File
@@ -41,6 +41,7 @@ dependencies = [
"numpy",
"soundfile",
"psutil>=7.2.2",
"pyannote-audio>=4.0.4",
]
[project.optional-dependencies]
@@ -105,3 +106,10 @@ include = ["omnivoice"]
[tool.hatch.build.targets.wheel]
packages = ["omnivoice"]
[dependency-groups]
dev = [
"httpx>=0.28.1",
"pytest>=9.0.3",
"pytest-asyncio>=1.3.0",
]
+11
View File
@@ -0,0 +1,11 @@
{
"name": "omnivoice-promo-recorder",
"version": "1.0.0",
"type": "module",
"scripts": {
"record": "node record_promo.js"
},
"dependencies": {
"playwright": "^1.43.0"
}
}
+177
View File
@@ -0,0 +1,177 @@
import { chromium } from 'playwright';
import { execSync } from 'child_process';
import path from 'path';
import fs from 'fs';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const URL = 'http://localhost:5173';
const OUTPUT_DIR = path.join(__dirname, '..', 'pics');
const RAW_VIDEO = path.join(OUTPUT_DIR, 'raw_record.webm');
const FINAL_MP4 = path.join(OUTPUT_DIR, 'promo.mp4');
const FINAL_GIF = path.join(OUTPUT_DIR, 'promo.gif');
// Ensure output directory exists
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
}
async function run() {
console.log('🎬 Starting Cinematic Promo Recording...');
const browser = await chromium.launch({
headless: false,
args: [
'--window-size=1920,1080',
'--force-device-scale-factor=2', // retina
'--disable-font-subpixel-positioning',
'--disable-smooth-scrolling'
]
});
const context = await browser.newContext({
viewport: { width: 1920, height: 1080 },
deviceScaleFactor: 2,
recordVideo: {
dir: OUTPUT_DIR,
size: { width: 1920, height: 1080 }
}
});
const page = await context.newPage();
let recordingOffsetSeconds = 0;
const scriptStartTime = Date.now();
// Helper for cinematic pausing
const wait = (ms) => new Promise(r => setTimeout(r, ms));
// Helper for smooth zoom injections
const cinematicZoom = async (selector, scale, xOffset = 0, yOffset = 0) => {
await page.evaluate(({ selector, scale, xOffset, yOffset }) => {
const el = document.querySelector('body');
el.style.transition = 'transform 2s cubic-bezier(0.25, 1, 0.5, 1)';
el.style.transformOrigin = `${xOffset}% ${yOffset}%`;
el.style.transform = `scale(${scale})`;
}, { selector, scale, xOffset, yOffset });
};
const resetZoom = async () => {
await page.evaluate(() => {
const el = document.querySelector('body');
el.style.transform = 'scale(1)';
});
};
try {
console.log('📍 Navigating to OmniVoice Studio Launchpad...');
await page.goto(URL, { waitUntil: 'load' });
// Give it enough time to stabilize the layout
await wait(2500);
console.log('📍 Switching to History tab in sidebar...');
await page.locator('button:has-text("History")').first().click();
await wait(1500);
console.log('📍 Loading Project from History...');
// Click the "Load" button on the first history item
await page.locator('button:has-text("Load")').first().click();
// Wait for the workspace to fully load, waveforms to draw, etc
console.log('⏳ Waiting for dubbing workspace to render...');
await wait(3500);
// 🎥 Cinematic Zoom on the Video Player
console.log('📍 Zooming on Video Playback...');
// Assuming video is roughly in the top left/center of the main view
await cinematicZoom('.video-player-container', 1.4, 20, 30);
await wait(2000);
// Play the video
console.log('📍 Playing timeline...');
// Click play button (WaveSurfer play or native)
// Finding the play button by its lucide icon class or explicit text is best
// Using a broad selector for the play button in the waveform bar
const playBtn = page.locator('button:has(.lucide-play), .waveform-controls button').first();
if (await playBtn.isVisible()) {
await playBtn.click();
}
// Let it play for a bit while zoomed
await wait(6000);
// Reset zoom and zoom into the segment table
console.log('📍 Highlighting Segment Controls...');
await resetZoom();
await wait(1500);
await cinematicZoom('.segment-table', 1.3, 80, 50);
await wait(3000);
// Highlight Output Options
console.log('📍 Showing Export Options...');
await resetZoom();
await wait(1000);
await cinematicZoom('text=Output Options', 1.5, 50, 95);
await wait(3000);
// Reset zoom
await resetZoom();
await wait(1500);
// Show Clone Tab
console.log('📍 Navigating Tabs (Clone, Design)...');
await page.click('text=Clone');
await wait(2500);
// Show Design Tab
console.log('📍 Showing Voice Design...');
await page.click('text=Design');
await wait(2500);
// Back to Dub
await page.click('text=Dub');
await wait(2000);
console.log('✅ Browser automation complete.');
} catch (error) {
console.error('❌ Error during automation:', error);
} finally {
// Stop recording and close
const videoPath = await page.video().path();
await context.close();
await browser.close();
// Rename the raw recording
if (fs.existsSync(videoPath)) {
fs.renameSync(videoPath, RAW_VIDEO);
console.log(`\n💾 Raw recording saved to: ${RAW_VIDEO}`);
console.log('✂️ Processing with FFmpeg (Trimming, Speeding, GIF Generation)...');
try {
console.log(`✂️ Processing final MP4 without trimming...`);
// Generate MP4 Promo (Lossless quality)
// preset=veryslow, crf=16 is virtually lossless
// Speed up the entire video slightly (0.85*PTS is about 15% faster) since UI interactions can feel slow on playback
execSync(`ffmpeg -y -i "${RAW_VIDEO}" -filter_complex "[0:v]setpts=0.85*PTS[v]" -map "[v]" -c:v libx264 -preset veryslow -crf 16 -pix_fmt yuv420p "${FINAL_MP4}"`, { stdio: 'inherit' });
console.log(`\n✨ HIGH QUALITY MP4 generated: ${FINAL_MP4}`);
// Generate GIF Promo (24fps, 1080p width, high quality palette)
execSync(`ffmpeg -y -i "${FINAL_MP4}" -vf "fps=24,scale=1080:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=255:stats_mode=diff[p];[s1][p]paletteuse=dither=bayer:bayer_scale=5" -loop 0 "${FINAL_GIF}"`, { stdio: 'inherit' });
console.log(`✨ HIGH QUALITY GIF generated: ${FINAL_GIF}`);
// Clean up raw
fs.unlinkSync(RAW_VIDEO);
} catch (e) {
console.error('❌ FFmpeg processing failed. Please ensure ffmpeg is installed natively.');
console.error(e.message);
}
}
}
}
run();
-21
View File
@@ -1,21 +0,0 @@
import asyncio
from playwright.async_api import async_playwright
async def main():
async with async_playwright() as p:
browser = await p.chromium.launch()
page = await browser.new_page(viewport={"width": 1280, "height": 800}, device_scale_factor=2)
await page.goto("http://localhost:5173", wait_until="networkidle")
await asyncio.sleep(2) # Wait for initial load
# Take screenshot of design layout
await page.screenshot(path="pics/omnivoice_studio_1.png")
# Click dub tab
await page.click("button:has-text('Dub')")
await asyncio.sleep(1)
await page.screenshot(path="pics/omnivoice_studio_2.png")
await browser.close()
asyncio.run(main())
-13
View File
@@ -1,13 +0,0 @@
const fs = require('fs');
const acorn = require('acorn');
const jsx = require('acorn-jsx');
const code = fs.readFileSync('frontend/src/App.jsx', 'utf8');
try {
acorn.Parser.extend(jsx()).parse(code, { sourceType: 'module', ecmaVersion: 2020 });
console.log("SUCCESS NO PARSE ERRORS");
} catch (e) {
console.log("PARSE ERROR AT:", e.loc);
console.log("Message:", e.message);
}
+526
View File
@@ -0,0 +1,526 @@
"""
OmniVoice Studio API — Unit Test Suite
Tests all roadmap features: TaskManager, scene detection, lip-sync scoring,
export endpoints (VTT, SRT, MP3, segments ZIP, stems ZIP), streaming TTS.
Uses FastAPI's TestClient (synchronous httpx) to avoid needing a running server.
GPU/model inference is mocked so tests run on any machine in seconds.
"""
import io
import os
import json
import uuid
import wave
import struct
import time
import pytest
import asyncio
# Patch environment before importing api
os.environ.setdefault("OMNIVOICE_MODEL", "test")
from unittest.mock import patch, MagicMock, AsyncMock
import torch
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def make_wav_bytes(duration_s=1.0, sample_rate=24000, channels=1) -> bytes:
"""Create a valid WAV file in memory for testing."""
n_samples = int(duration_s * sample_rate)
buf = io.BytesIO()
with wave.open(buf, "wb") as wf:
wf.setnchannels(channels)
wf.setsampwidth(2)
wf.setframerate(sample_rate)
wf.writeframes(struct.pack(f"<{n_samples}h", *([0] * n_samples)))
buf.seek(0)
return buf.read()
def make_audio_tensor(duration_s=1.0, sample_rate=24000) -> torch.Tensor:
"""Create a torch audio tensor of the given duration."""
n_samples = int(duration_s * sample_rate)
return torch.zeros(1, n_samples)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="session", autouse=True)
def _mock_model():
"""Prevent real model loading across the entire test session."""
mock = MagicMock()
mock.sampling_rate = 24000
mock.generate.return_value = [make_audio_tensor(1.0)]
import backend.main as api_mod
api_mod.model = mock
api_mod._init_db()
yield mock
@pytest.fixture()
def client():
"""Create a TestClient for the FastAPI app (no server needed)."""
from fastapi.testclient import TestClient
from backend.main import app
return TestClient(app)
@pytest.fixture()
def seeded_job(client):
"""Create a fake dub job with segments, tracks, and WAV files on disk."""
import backend.main as api_mod
job_id = str(uuid.uuid4())[:8]
job_dir = os.path.join(api_mod.DUB_DIR, job_id)
os.makedirs(job_dir, exist_ok=True)
# Write fake segment WAVs
for i in range(3):
seg_path = os.path.join(job_dir, f"seg_{i}.wav")
with open(seg_path, "wb") as f:
f.write(make_wav_bytes(0.5))
# Write a fake dubbed track
track_path = os.path.join(job_dir, "dubbed_en.wav")
with open(track_path, "wb") as f:
f.write(make_wav_bytes(2.0))
# Write a fake background audio
bg_path = os.path.join(job_dir, "no_vocals.wav")
with open(bg_path, "wb") as f:
f.write(make_wav_bytes(2.0))
# Write a fake video
video_path = os.path.join(job_dir, "original.mp4")
with open(video_path, "wb") as f:
f.write(b"\x00" * 100)
job = {
"video_path": video_path,
"audio_path": os.path.join(job_dir, "audio.wav"),
"vocals_path": os.path.join(job_dir, "vocals.wav"),
"no_vocals_path": bg_path,
"duration": 3.0,
"filename": "test_video.mp4",
"segments": [
{"id": "a1", "start": 0.0, "end": 1.0, "text": "Hello world", "speaker_id": "Speaker 1"},
{"id": "a2", "start": 1.0, "end": 2.0, "text": "How are you", "speaker_id": "Speaker 1"},
{"id": "a3", "start": 2.0, "end": 3.0, "text": "Goodbye", "speaker_id": "Speaker 2"},
],
"dubbed_tracks": {
"en": {"path": track_path, "language": "English", "language_code": "en"},
},
"scene_cuts": [1.5],
}
api_mod._dub_jobs[job_id] = job
yield job_id, job
# Cleanup
api_mod._dub_jobs.pop(job_id, None)
# ═══════════════════════════════════════════════════════════════════════
# TASK MANAGER TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestTaskManager:
"""Tests for the centralized async batch task queue."""
def test_task_manager_init(self):
from backend.main import TaskManager
tm = TaskManager()
assert tm.active_tasks == {}
assert tm.queue is None
@pytest.mark.asyncio
async def test_add_task_creates_entry(self):
from backend.main import TaskManager
tm = TaskManager()
tm._init_queue()
async def dummy():
pass
await tm.add_task("t1", "test", dummy)
assert "t1" in tm.active_tasks
assert tm.active_tasks["t1"]["status"] == "pending"
assert tm.active_tasks["t1"]["type"] == "test"
@pytest.mark.asyncio
async def test_worker_processes_task(self):
from backend.main import TaskManager
tm = TaskManager()
results = []
async def work():
results.append("done")
await tm.add_task("t2", "test", work)
# Run worker for a brief period
worker = asyncio.create_task(tm.worker())
await asyncio.sleep(0.2)
worker.cancel()
assert "done" in results
assert tm.active_tasks["t2"]["status"] == "done"
@pytest.mark.asyncio
async def test_worker_handles_failure(self):
from backend.main import TaskManager
tm = TaskManager()
async def fail():
raise ValueError("boom")
await tm.add_task("t3", "test", fail)
worker = asyncio.create_task(tm.worker())
await asyncio.sleep(0.2)
worker.cancel()
assert tm.active_tasks["t3"]["status"] == "failed"
assert "boom" in tm.active_tasks["t3"]["error"]
# ═══════════════════════════════════════════════════════════════════════
# SRT EXPORT TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestSRTExport:
def test_srt_export(self, client, seeded_job):
job_id, _ = seeded_job
res = client.get(f"/dub/srt/{job_id}")
assert res.status_code == 200
content = res.text
assert "1\n" in content
assert "Hello world" in content
assert "-->" in content
def test_srt_404_missing_job(self, client):
res = client.get("/dub/srt/nonexistent")
assert res.status_code == 404
# ═══════════════════════════════════════════════════════════════════════
# VTT EXPORT TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestVTTExport:
def test_vtt_export(self, client, seeded_job):
job_id, _ = seeded_job
res = client.get(f"/dub/vtt/{job_id}")
assert res.status_code == 200
content = res.text
assert content.startswith("WEBVTT")
assert "Hello world" in content
assert "-->" in content
# VTT uses periods not commas
assert "." in content.split("-->")[0]
def test_vtt_format_correct(self, client, seeded_job):
job_id, _ = seeded_job
res = client.get(f"/dub/vtt/{job_id}")
lines = res.text.strip().split("\n")
assert lines[0] == "WEBVTT"
# Find a timestamp line
ts_lines = [l for l in lines if "-->" in l]
assert len(ts_lines) == 3
# Verify format: HH:MM:SS.mmm
for ts in ts_lines:
start, end = ts.split("-->")
assert "." in start.strip()
assert "." in end.strip()
def test_vtt_404_missing_job(self, client):
res = client.get("/dub/vtt/nonexistent")
assert res.status_code == 404
# ═══════════════════════════════════════════════════════════════════════
# PER-SEGMENT ZIP EXPORT TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestSegmentZipExport:
def test_segments_zip_export(self, client, seeded_job):
job_id, _ = seeded_job
res = client.get(f"/dub/export-segments/{job_id}")
assert res.status_code == 200
assert res.headers["content-type"] == "application/zip"
import zipfile
zf = zipfile.ZipFile(io.BytesIO(res.content))
names = zf.namelist()
assert len(names) == 3
# Verify naming convention: 001_0.00-1.00_Speaker1.wav
assert names[0].startswith("001_")
assert names[0].endswith(".wav")
assert "Speaker" in names[0]
def test_segments_zip_404(self, client):
res = client.get("/dub/export-segments/nonexistent")
assert res.status_code == 404
# ═══════════════════════════════════════════════════════════════════════
# STEM EXPORT TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestStemExport:
def test_stems_zip_export(self, client, seeded_job):
job_id, _ = seeded_job
res = client.get(f"/dub/export-stems/{job_id}")
assert res.status_code == 200
assert res.headers["content-type"] == "application/zip"
import zipfile
zf = zipfile.ZipFile(io.BytesIO(res.content))
names = zf.namelist()
assert any("vocals" in n for n in names)
assert any("background" in n for n in names)
def test_stems_404_no_tracks(self, client):
import backend.main as api_mod
job_id = "stems_test"
api_mod._dub_jobs[job_id] = {
"segments": [], "dubbed_tracks": {}, "filename": "t.mp4",
"video_path": "", "duration": 0,
}
res = client.get(f"/dub/export-stems/{job_id}")
assert res.status_code == 400
api_mod._dub_jobs.pop(job_id, None)
# ═══════════════════════════════════════════════════════════════════════
# SCENE-AWARE DUBBING TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestSceneAwareDubbing:
def test_scene_cuts_stored(self, seeded_job):
_, job = seeded_job
assert "scene_cuts" in job
assert isinstance(job["scene_cuts"], list)
def test_scene_split_algorithm(self):
"""Test the segment splitting logic directly."""
segments = [
{"id": "s1", "start": 0.0, "end": 3.0, "text": "Hello world this is a test sentence", "speaker_id": "Speaker 1"},
]
scene_cuts = [1.5]
# Run the algorithm inline (mirrors api.py logic)
sorted_cuts = sorted(scene_cuts)
new_segments = []
for s in segments:
s_start = s["start"]
s_end = s["end"]
valid_cuts = [c for c in sorted_cuts if c > s_start + 0.2 and c < s_end - 0.2]
if not valid_cuts:
new_segments.append(s)
else:
curr_start = s_start
curr_text = s["text"]
total_dur = s_end - s_start
for cut in valid_cuts:
ratio = (cut - curr_start) / max(total_dur, 0.01)
split_idx = int(len(curr_text) * ratio)
space_idx = curr_text.rfind(' ', 0, split_idx + 5)
if space_idx != -1 and space_idx > split_idx - 10:
split_idx = space_idx
part_text = curr_text[:split_idx].strip()
curr_text = curr_text[split_idx:].strip()
if part_text:
new_seg = dict(s)
new_seg["start"] = round(curr_start, 2)
new_seg["end"] = round(cut, 2)
new_seg["text"] = part_text
new_seg["id"] = "new1"
new_segments.append(new_seg)
curr_start = cut
total_dur = s_end - curr_start
if curr_text:
new_seg = dict(s)
new_seg["start"] = round(curr_start, 2)
new_seg["end"] = round(s_end, 2)
new_seg["text"] = curr_text
new_seg["id"] = "new2"
new_segments.append(new_seg)
assert len(new_segments) == 2
assert new_segments[0]["end"] == 1.5
assert new_segments[1]["start"] == 1.5
# Text should be split
combined = new_segments[0]["text"] + " " + new_segments[1]["text"]
assert combined == "Hello world this is a test sentence"
def test_no_split_when_cut_too_close_to_edge(self):
"""Cuts within 0.2s of segment edges should NOT split."""
segments = [{"id": "s1", "start": 0.0, "end": 1.0, "text": "Short", "speaker_id": "Speaker 1"}]
scene_cuts = [0.1, 0.9] # Both within 0.2s padding
sorted_cuts = sorted(scene_cuts)
new_segments = []
for s in segments:
valid_cuts = [c for c in sorted_cuts if c > s["start"] + 0.2 and c < s["end"] - 0.2]
if not valid_cuts:
new_segments.append(s)
assert len(new_segments) == 1 # No split occurred
# ═══════════════════════════════════════════════════════════════════════
# LIP-SYNC SCORING TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestLipSyncScoring:
def test_sync_ratio_calculation(self):
"""Test the sync ratio math directly."""
seg_duration = 2.0 # original segment is 2 seconds
sample_rate = 24000
# Generated audio is exactly 2 seconds → ratio = 1.0
audio_tensor = make_audio_tensor(2.0, sample_rate)
generated_dur = audio_tensor.shape[-1] / sample_rate
sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3)
assert sync_ratio == 1.0
def test_sync_ratio_fast(self):
"""Generated audio shorter than original → ratio < 1."""
seg_duration = 2.0
audio_tensor = make_audio_tensor(1.5, 24000)
generated_dur = audio_tensor.shape[-1] / 24000
sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3)
assert sync_ratio == 0.75
def test_sync_ratio_slow(self):
"""Generated audio longer than original → ratio > 1."""
seg_duration = 2.0
audio_tensor = make_audio_tensor(3.0, 24000)
generated_dur = audio_tensor.shape[-1] / 24000
sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3)
assert sync_ratio == 1.5
def test_sync_ratio_thresholds(self):
"""Verify color-coded classification logic."""
def classify(ratio):
if 0.95 <= ratio <= 1.05:
return "green"
elif ratio > 1.25:
return "red"
else:
return "yellow"
assert classify(1.0) == "green"
assert classify(0.95) == "green"
assert classify(1.05) == "green"
assert classify(0.8) == "yellow"
assert classify(1.2) == "yellow"
assert classify(1.3) == "red"
assert classify(1.5) == "red"
# ═══════════════════════════════════════════════════════════════════════
# SRT/VTT TIMESTAMP FORMATTING TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestTimestampFormatting:
def test_srt_time_format(self):
from backend.main import _format_srt_time
assert _format_srt_time(0.0) == "00:00:00,000"
assert _format_srt_time(61.5) == "00:01:01,500"
assert _format_srt_time(3661.123) == "01:01:01,123"
def test_vtt_time_format(self):
from backend.main import _format_vtt_time
assert _format_vtt_time(0.0) == "00:00:00.000"
assert _format_vtt_time(61.5) == "00:01:01.500"
# SRT uses comma, VTT uses period
assert "." in _format_vtt_time(1.0)
# ═══════════════════════════════════════════════════════════════════════
# API ENDPOINT VALIDATION TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestAPIEndpoints:
def test_model_status(self, client):
res = client.get("/model/status")
assert res.status_code == 200
data = res.json()
assert "loaded" in data
assert "status" in data
def test_sysinfo(self, client):
res = client.get("/sysinfo")
assert res.status_code == 200
data = res.json()
assert "cpu" in data
assert "ram" in data
def test_dub_tracks(self, client, seeded_job):
job_id, _ = seeded_job
res = client.get(f"/dub/tracks/{job_id}")
assert res.status_code == 200
data = res.json()
assert "tracks" in data
assert "en" in data["tracks"]
def test_tasks_stream_404(self, client):
res = client.get("/tasks/stream/nonexistent")
assert res.status_code == 404
def test_dub_download_404(self, client):
res = client.get("/dub/download/nonexistent")
assert res.status_code == 404
# ═══════════════════════════════════════════════════════════════════════
# STREAMING TTS TESTS
# ═══════════════════════════════════════════════════════════════════════
class TestStreamingTTS:
def test_generate_returns_streaming_response(self, client):
"""POST /generate should return streamed WAV with metadata headers."""
with patch("backend.main.get_model") as mock_get:
mock_model = MagicMock()
mock_model.sampling_rate = 24000
mock_model.generate.return_value = [make_audio_tensor(1.0)]
async def _get():
return mock_model
mock_get.return_value = _get()
import backend.main as api_mod
api_mod.model = mock_model
res = client.post("/generate", data={
"text": "Hello world",
"num_step": "4",
"guidance_scale": "2.0",
"speed": "1.0",
"denoise": "true",
"t_shift": "0.1",
"position_temperature": "5.0",
"class_temperature": "0.0",
"layer_penalty_factor": "5.0",
"postprocess_output": "true",
})
assert res.status_code == 200
assert res.headers.get("content-type") == "audio/wav"
assert res.headers.get("x-audio-id") is not None
assert res.headers.get("x-gen-time") is not None
assert res.headers.get("x-audio-duration") is not None
# Verify it's valid WAV
assert len(res.content) > 44 # WAV header is 44 bytes minimum
Generated
+1952 -11
View File
File diff suppressed because it is too large Load Diff