diff --git a/.gitignore b/.gitignore index b1a14b7c..96b16638 100644 --- a/.gitignore +++ b/.gitignore @@ -37,10 +37,22 @@ bun.lockb # Generated logs and binaries *.log +crash_log.txt cloudflared cloudflared.tgz +# Root-level ad-hoc test scripts (should live in /tests) +/test_crash.py +/test_mock.py +/test_pyannote.py +/test_server.py +/test_whisper.py + # Data directories and sqlite db omnivoice_data/ *.db demo_recording.webp + +# Local archives / reference clones +omnivoice.zip +/research/ diff --git a/backend/api/__init__.py b/backend/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/routers/__init__.py b/backend/api/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/routers/dub_core.py b/backend/api/routers/dub_core.py new file mode 100644 index 00000000..8d45465c --- /dev/null +++ b/backend/api/routers/dub_core.py @@ -0,0 +1,464 @@ +import os +import io +import sys +import uuid +import json +import time +import asyncio +import logging +import shutil +import subprocess +import threading +import soundfile as sf +import torch +import torchaudio +from typing import Optional, List +from fastapi import APIRouter, File, Form, UploadFile, HTTPException, Query +from fastapi.responses import FileResponse, Response, StreamingResponse, JSONResponse + +from core.db import get_db, db_conn +from core.config import DATA_DIR, DUB_DIR, PREVIEW_DIR, VOICES_DIR +from core.tasks import task_manager +from schemas.requests import DubRequest, TranslateRequest +from services.model_manager import get_model, _gpu_pool, _cpu_pool, get_best_device, get_diarization_pipeline +from services.audio_dsp import apply_mastering, normalize_audio +from services.ffmpeg_utils import find_ffmpeg +from services.segmentation import ( + segment_transcript, + assign_speakers_from_diarization, + assign_speakers_heuristic, + clean_up_segments, +) + +router = APIRouter() +logger = logging.getLogger("omnivoice.api") + +_dub_jobs = {} +# Tracks live subprocesses per job so POST /dub/abort/{job_id} can terminate them. +_active_procs: dict[str, list] = {} +_active_procs_lock = threading.Lock() + +_DUB_DIR_REAL = os.path.realpath(DUB_DIR) + + +def _safe_job_dir(job_id: str) -> Optional[str]: + """Resolve a job directory under DUB_DIR, rejecting traversal.""" + if not job_id or "/" in job_id or "\\" in job_id or job_id in (".", ".."): + return None + candidate = os.path.realpath(os.path.join(DUB_DIR, job_id)) + if not candidate.startswith(_DUB_DIR_REAL + os.sep): + return None + return candidate + + +def _register_proc(job_id: str, proc): + with _active_procs_lock: + _active_procs.setdefault(job_id, []).append(proc) + + +def _unregister_proc(job_id: str, proc): + with _active_procs_lock: + lst = _active_procs.get(job_id) + if lst and proc in lst: + lst.remove(proc) + if lst is not None and not lst: + _active_procs.pop(job_id, None) + + +def _kill_job_procs(job_id: str): + with _active_procs_lock: + procs = list(_active_procs.get(job_id, [])) + for proc in procs: + try: + if proc.returncode is None: + proc.kill() + except ProcessLookupError: + pass + except Exception as e: + logger.warning("Failed to kill subprocess for %s: %s", job_id, e) + with _active_procs_lock: + _active_procs.pop(job_id, None) + +def _get_job(job_id: str): + if job_id in _dub_jobs: + return _dub_jobs[job_id] + conn = get_db() + try: + row = conn.execute("SELECT job_data FROM dub_history WHERE id=?", (job_id,)).fetchone() + finally: + conn.close() + if row and row["job_data"]: + try: + job = json.loads(row["job_data"]) + _dub_jobs[job_id] = job + return job + except json.JSONDecodeError as e: + logger.error("Failed to decode dub_history.job_data for %s: %s", job_id, e) + return None + +def _save_job(job_id: str, job: dict, filename: str = "", duration: float = 0.0): + """Persist dub job state to SQLite so it survives restarts.""" + try: + segments = job.get("segments") or [] + tracks = list((job.get("dubbed_tracks") or {}).keys()) + with db_conn() as conn: + conn.execute( + """INSERT INTO dub_history + (id, filename, duration, segments_count, language, language_code, tracks, job_data, created_at) + VALUES (?,?,?,?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET + filename=excluded.filename, + duration=excluded.duration, + segments_count=excluded.segments_count, + tracks=excluded.tracks, + job_data=excluded.job_data""", + (job_id, filename or job.get("filename", ""), + duration or job.get("duration", 0.0), + len(segments), job.get("language", ""), job.get("language_code", ""), + json.dumps(tracks), json.dumps(job, default=str), time.time()), + ) + except Exception as e: + logger.error("Failed to persist dub job %s: %s", job_id, e) + +@router.post("/dub/cleanup-segments/{job_id}") +def dub_cleanup_segments(job_id: str): + """Re-run merge/stitch passes on a job's existing segments to drop fragments.""" + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + segments = job.get("segments") or [] + cleaned = clean_up_segments(segments) + job["segments"] = cleaned + _save_job(job_id, job) + return {"segments": cleaned, "before": len(segments), "after": len(cleaned)} + + +@router.post("/dub/abort/{job_id}") +def dub_abort(job_id: str): + """Cancel in-flight upload/transcribe subprocesses for a job.""" + with _active_procs_lock: + had_procs = bool(_active_procs.get(job_id)) + _kill_job_procs(job_id) + job = _dub_jobs.get(job_id) + if job is not None: + job["aborted"] = True + try: + task_manager.cancel_task(job_id) + except Exception: + pass + return {"aborted": True, "had_active_procs": had_procs} + + +@router.get("/dub/history") +def list_dub_history(): + conn = get_db() + try: + rows = conn.execute("SELECT * FROM dub_history ORDER BY created_at DESC LIMIT 30").fetchall() + finally: + conn.close() + return [dict(r) for r in rows] + +@router.delete("/dub/history") +def clear_dub_history(): + """Delete persisted dub rows and their on-disk dirs (scoped to known IDs).""" + conn = get_db() + try: + ids = [r["id"] for r in conn.execute("SELECT id FROM dub_history").fetchall()] + conn.execute("DELETE FROM dub_history") + conn.commit() + finally: + conn.close() + for jid in ids: + safe = _safe_job_dir(jid) + if safe and os.path.isdir(safe): + shutil.rmtree(safe, ignore_errors=True) + return {"cleared": True, "count": len(ids)} + +@router.delete("/dub/history/{history_id}") +def delete_single_dub_history(history_id: str): + with db_conn() as conn: + conn.execute("DELETE FROM dub_history WHERE id=?", (history_id,)) + safe = _safe_job_dir(history_id) + if safe and os.path.isdir(safe): + shutil.rmtree(safe, ignore_errors=True) + _dub_jobs.pop(history_id, None) + return {"deleted": True} + +@router.post("/preview/upload") +async def preview_upload(video: UploadFile = File(...)): + ext = os.path.splitext(video.filename or "video.mp4")[1].lower() + safe_name = f"{uuid.uuid4().hex[:12]}" + vid_path = os.path.join(PREVIEW_DIR, f"{safe_name}{ext}") + wav_path = os.path.join(PREVIEW_DIR, f"{safe_name}.wav") + + with open(vid_path, "wb") as f: + f.write(await video.read()) + + has_audio = False + if ext not in [".wav", ".mp3", ".m4a", ".aac"]: + try: + ffmpeg_cmd = [ + find_ffmpeg(), "-y", "-i", vid_path, + "-vn", "-acodec", "pcm_s16le", "-ar", "22050", "-ac", "1", + wav_path + ] + subprocess.run( + ffmpeg_cmd, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=300, + ) + has_audio = True + except Exception as e: + logger.warning(f"FFmpeg extraction failed: {e}") + pass + + return { + "url": f"/preview/{safe_name}{ext}", + "audioUrl": f"/preview/{safe_name}.wav" if has_audio else f"/preview/{safe_name}{ext}", + "filename": video.filename, + } + +@router.get("/preview/{filename}") +async def preview_serve(filename: str): + if not filename or "/" in filename or "\\" in filename or filename.startswith("."): + raise HTTPException(400, "Invalid preview filename") + preview_real = os.path.realpath(PREVIEW_DIR) + path = os.path.realpath(os.path.join(PREVIEW_DIR, filename)) + if not path.startswith(preview_real + os.sep): + raise HTTPException(400, "Invalid preview filename") + if not os.path.isfile(path): + raise HTTPException(404, "Preview not found") + ext = os.path.splitext(filename)[1].lower() + media_types = { + ".mp4": "video/mp4", ".mov": "video/quicktime", + ".mkv": "video/x-matroska", ".webm": "video/webm", + ".avi": "video/x-msvideo", ".wav": "audio/wav", + ".mp3": "audio/mpeg" + } + return FileResponse(path, media_type=media_types.get(ext, "application/octet-stream")) + +@router.post("/dub/upload") +async def dub_upload(video: UploadFile = File(...), job_id: Optional[str] = Form(None)): + job_id = job_id or str(uuid.uuid4())[:8] + job_dir = _safe_job_dir(job_id) + if job_dir is None: + raise HTTPException(status_code=400, detail="invalid job_id") + os.makedirs(job_dir, exist_ok=True) + + ext = os.path.splitext(video.filename or "video.mp4")[1] + video_path = os.path.join(job_dir, f"original{ext}") + with open(video_path, "wb") as f: + f.write(await video.read()) + + audio_path = os.path.join(job_dir, "audio.wav") + ffmpeg = find_ffmpeg() + + async def _run_proc(cmd, timeout: float = 900.0): + p = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + ) + _register_proc(job_id, p) + try: + try: + stdout, stderr = await asyncio.wait_for(p.communicate(), timeout=timeout) + except asyncio.TimeoutError: + try: + p.kill() + except ProcessLookupError: + pass + raise HTTPException(status_code=504, detail=f"subprocess timed out after {timeout}s") + return p, stdout, stderr + finally: + _unregister_proc(job_id, p) + + try: + try: + p, _, stderr = await _run_proc([ + ffmpeg, "-i", video_path, "-vn", "-acodec", "pcm_s16le", + "-ar", "16000", "-ac", "1", audio_path, "-y", + ]) + if p.returncode != 0: + raise Exception(stderr.decode()) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500, detail=f"ffmpeg failed: {str(e)}") + + try: + dur = float(sf.info(audio_path).frames) / float(sf.info(audio_path).samplerate) + except Exception: + dur = 0.0 + + vocals_path = os.path.join(job_dir, "vocals.wav") + no_vocals_path = os.path.join(job_dir, "no_vocals.wav") + scene_cuts = [] + + async def run_demucs(): + nonlocal vocals_path, no_vocals_path + try: + demucs_cmd = [sys.executable, "-m", "demucs.separate", + "--two-stems", "vocals", "-n", "htdemucs", "-d", get_best_device(), + audio_path, "-o", job_dir] + p, _, stderr = await _run_proc(demucs_cmd, timeout=1800.0) + if p.returncode != 0: + raise Exception(stderr.decode()) + + demucs_out = os.path.join(job_dir, "htdemucs", "audio") + if os.path.exists(os.path.join(demucs_out, "vocals.wav")): + shutil.move(os.path.join(demucs_out, "vocals.wav"), vocals_path) + shutil.move(os.path.join(demucs_out, "no_vocals.wav"), no_vocals_path) + shutil.rmtree(os.path.join(job_dir, "htdemucs"), ignore_errors=True) + except asyncio.CancelledError: + raise + except Exception as e: + logger.warning(f"Demucs failed, falling back to mixed audio. {e}") + vocals_path = audio_path + no_vocals_path = None + + async def run_scene_detection(): + nonlocal scene_cuts + try: + p, _, stderr_scene = await _run_proc([ + ffmpeg, "-i", video_path, "-filter:v", + "select='gt(scene,0.3)',showinfo", "-f", "null", "-", + ], timeout=600.0) + import re + matches = re.finditer(r"pts_time:([\d\.]+)", stderr_scene.decode()) + scene_cuts = [float(m.group(1)) for m in matches] + except asyncio.CancelledError: + raise + except Exception as e: + logger.warning(f"Scene detection failed: {e}") + + await asyncio.gather(run_demucs(), run_scene_detection()) + + _dub_jobs[job_id] = { + "video_path": video_path, + "audio_path": audio_path, + "vocals_path": vocals_path, + "no_vocals_path": no_vocals_path, + "duration": dur, "filename": video.filename, + "segments": None, "dubbed_tracks": {}, + "scene_cuts": scene_cuts, + } + _save_job(job_id, _dub_jobs[job_id], video.filename, dur) + return {"job_id": job_id, "duration": round(dur, 2), "filename": video.filename} + + except asyncio.CancelledError: + logger.info("Dub upload cancelled for job %s; killing subprocesses and cleaning up", job_id) + _kill_job_procs(job_id) + try: + shutil.rmtree(job_dir, ignore_errors=True) + finally: + _dub_jobs.pop(job_id, None) + raise + finally: + with _active_procs_lock: + _active_procs.pop(job_id, None) + + +@router.post("/dub/transcribe/{job_id}") +async def dub_transcribe(job_id: str): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + _model = await get_model() + if _model._asr_pipe is None: + raise HTTPException(status_code=503, detail="ASR not loaded") + + def _transcribe(): + import re + import traceback + + asr_audio_target = job.get("vocals_path") + if not asr_audio_target or not os.path.exists(asr_audio_target): + asr_audio_target = job.get("audio_path") + + import torch + + detected_lang = None + + if torch.backends.mps.is_available(): + try: + import mlx_whisper + asr_model = os.environ.get("ASR_MODEL", "mlx-community/whisper-large-v3-mlx") + logger.info(f"Transcribing via MLX CoreML Engine ({asr_model})...") + result = mlx_whisper.transcribe( + asr_audio_target, + path_or_hf_repo=asr_model, + word_timestamps=True + ) + detected_lang = result.get("language") + + if "segments" in result: + result["chunks"] = [] + for seg in result["segments"]: + result["chunks"].append({ + "text": seg["text"], + "timestamp": (seg["start"], seg["end"]) + }) + except Exception as e: + logger.error(f"MLX Whisper failed, falling back to PyTorch: {e}") + audio_np, sr = sf.read(asr_audio_target, dtype="float32") + if audio_np.ndim > 1: audio_np = audio_np.mean(axis=1) + bs = 16 if torch.cuda.is_available() else 2 + result = _model._asr_pipe({"array": audio_np, "sampling_rate": sr}, return_timestamps=True, chunk_length_s=15, batch_size=bs) + detected_lang = (result.get("language") if isinstance(result, dict) else None) + else: + audio_np, sr = sf.read(asr_audio_target, dtype="float32") + if audio_np.ndim > 1: audio_np = audio_np.mean(axis=1) + bs = 16 if torch.cuda.is_available() else 1 + result = _model._asr_pipe({"array": audio_np, "sampling_rate": sr}, return_timestamps=True, chunk_length_s=15, batch_size=bs) + detected_lang = (result.get("language") if isinstance(result, dict) else None) + + job["source_lang"] = (detected_lang or "en").split("_")[0][:2].lower() + + scene_cuts = job.get("scene_cuts") or [] + segments = segment_transcript(result, duration=job.get("duration", 0.0), scene_cuts=scene_cuts) + + diar_pipe = get_diarization_pipeline() + if diar_pipe: + try: + diar_target = job.get("vocals_path") or job.get("audio_path") + diarization = diar_pipe(diar_target) + segments = assign_speakers_from_diarization(segments, diarization) + except Exception as e: + logger.error(f"Pyannote diarization failed during inference: {e}. Falling back to heuristic.") + segments = assign_speakers_heuristic(segments) + else: + segments = assign_speakers_heuristic(segments) + + job["full_transcript"] = " ".join(s["text"] for s in segments) + + if torch.backends.mps.is_available(): + torch.mps.empty_cache() + + return segments + + try: + loop = asyncio.get_event_loop() + try: + segments_result = await loop.run_in_executor(_gpu_pool, _transcribe) + except asyncio.CancelledError: + job["aborted"] = True + raise + if job.get("aborted"): + raise HTTPException(status_code=499, detail="Transcription aborted") + job["segments"] = segments_result + source_lang = job.get("source_lang") + _save_job(job_id, job) + return { + "job_id": job_id, + "segments": segments_result, + "full_transcript": job.get("full_transcript", ""), + "source_lang": source_lang, + } + except HTTPException: + raise + except asyncio.CancelledError: + raise + except Exception as e: + import traceback + traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/api/routers/dub_export.py b/backend/api/routers/dub_export.py new file mode 100644 index 00000000..4006f0d6 --- /dev/null +++ b/backend/api/routers/dub_export.py @@ -0,0 +1,513 @@ +import os +import io +import time +import uuid +import asyncio +import logging +from fastapi import APIRouter, HTTPException, Query, Response +from fastapi.responses import FileResponse, StreamingResponse + +from core.db import get_db +from core.config import DUB_DIR +from core.tasks import task_manager +from api.routers.dub_core import _get_job +from services.ffmpeg_utils import find_ffmpeg, run_ffmpeg + +router = APIRouter() +logger = logging.getLogger("omnivoice.api") + + +def _unique_stamp() -> str: + """Return a short unique suffix like '20260415T142301-ab12cd34' for export files.""" + return f"{time.strftime('%Y%m%dT%H%M%S')}-{uuid.uuid4().hex[:8]}" + + +def _native_save(source: str, destination: str, display_name: str, media_type: str): + """Copy a generated export file to a user-chosen destination and return JSON.""" + import shutil + dest = os.path.expanduser(destination) + # Reject traversal against the user's home dir — Tauri save dialog returns abs path. + if not os.path.isabs(dest): + raise HTTPException(status_code=400, detail="save_path must be absolute") + try: + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + shutil.copy2(source, dest) + except PermissionError as e: + raise HTTPException(status_code=403, detail=f"Permission denied: {e}") + except OSError as e: + raise HTTPException(status_code=500, detail=f"Copy failed: {e}") + if not os.path.exists(dest) or os.path.getsize(dest) == 0: + raise HTTPException(status_code=500, detail="Copy produced empty file at destination") + logger.info("Native save wrote %s (%d bytes)", dest, os.path.getsize(dest)) + return { + "saved": True, + "path": dest, + "size": os.path.getsize(dest), + "media_type": media_type, + "display_name": display_name, + } + +@router.get("/tasks/stream/{task_id}") +async def stream_task(task_id: str): + """Universal Server-Sent Event stream for background tasks.""" + if task_id not in task_manager.active_tasks: + raise HTTPException(status_code=404, detail="Task not found") + + async def _reader(): + t = task_manager.active_tasks.get(task_id) + if t is None: + return + q = asyncio.Queue() + await task_manager.add_listener(task_id, q) + + try: + for evt in t["history"]: + yield evt + + if t["status"] in ("done", "failed"): + return + + while True: + evt = await q.get() + if evt is None: + break + yield evt + finally: + await task_manager.remove_listener(task_id, q) + + return StreamingResponse(_reader(), media_type="text/event-stream") + +@router.post("/tasks/cancel/{task_id}") +async def cancel_task(task_id: str): + """Cancel a running background task (e.g. dub generation).""" + ok = task_manager.cancel_task(task_id) + if not ok: + raise HTTPException(status_code=404, detail="Task not found") + return {"cancelled": True, "task_id": task_id} + + +@router.get("/dub/tracks/{job_id}") +async def dub_list_tracks(job_id: str): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + return {"tracks": job.get("dubbed_tracks", {})} + + +@router.get("/dub/download/{job_id}") +@router.get("/dub/download/{job_id}/{filename}") +async def dub_download( + job_id: str, + preserve_bg: bool = Query(True, description="Mix background noise into dubbed tracks"), + default_track: str = Query("original"), + include_tracks: str = Query("", description="Comma-separated list of tracks to include (e.g. 'original,de,es'). Empty = include all."), + save_path: str = Query("", description="Absolute destination path. If set, mux output is copied there and JSON returned instead of FileResponse."), +): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + tracks = job.get("dubbed_tracks", {}) + if not tracks: + raise HTTPException(status_code=400, detail="No dubbed tracks generated yet") + + include_set = set(t.strip() for t in include_tracks.split(",") if t.strip()) if include_tracks else None + include_original = include_set is None or "original" in include_set + + if include_set: + filtered_tracks = {k: v for k, v in tracks.items() if k in include_set} + else: + filtered_tracks = dict(tracks) + + if not filtered_tracks and not include_original: + raise HTTPException(status_code=400, detail="No tracks selected for export") + + video_path = job["video_path"] + stamp = _unique_stamp() + exports_dir = os.path.join(DUB_DIR, job_id, "exports") + os.makedirs(exports_dir, exist_ok=True) + output_path = os.path.join(exports_dir, f"dubbed_video_{stamp}.mp4") + ffmpeg = find_ffmpeg() + + cmd = [ffmpeg, "-i", video_path] + input_idx = 1 + + bg_audio = job.get("no_vocals_path") if preserve_bg else None + bg_idx = None + if bg_audio and os.path.exists(bg_audio) and filtered_tracks: + cmd += ["-i", bg_audio] + bg_idx = input_idx + input_idx += 1 + + tracks_to_process = [] + for lang_code, track_info in filtered_tracks.items(): + cmd += ["-i", track_info["path"]] + tracks_to_process.append({"lang_code": lang_code, "idx": input_idx, "info": track_info}) + input_idx += 1 + + cmd += ["-map", "0:v:0"] + if include_original: + cmd += ["-map", "0:a:0"] + + if bg_idx is not None: + filters = [] + for i, t in enumerate(tracks_to_process): + out_label = f"[aout{i}]" + filters.append(f"[{bg_idx}:a][{t['idx']}:a]amix=inputs=2:duration=longest:dropout_transition=2:weights=0.8 1.2{out_label}") + t["out_label"] = out_label + cmd += ["-filter_complex", ";".join(filters)] + for t in tracks_to_process: + cmd += ["-map", t["out_label"]] + else: + for t in tracks_to_process: + cmd += ["-map", f"{t['idx']}:a:0"] + + cmd += ["-c:v", "copy", "-c:a", "aac", "-b:a", "192k"] + + audio_stream_idx = 0 + if include_original: + cmd += [f"-metadata:s:a:{audio_stream_idx}", "language=und", f"-metadata:s:a:{audio_stream_idx}", "title=Original"] + audio_stream_idx += 1 + + for t in tracks_to_process: + cmd += [ + f"-metadata:s:a:{audio_stream_idx}", f"language={t['lang_code']}", + f"-metadata:s:a:{audio_stream_idx}", f"title={t['info']['language']}" + ] + t["stream_idx"] = audio_stream_idx + audio_stream_idx += 1 + + total_audio = (1 if include_original else 0) + len(tracks_to_process) + for i in range(total_audio): + cmd += [f"-disposition:a:{i}", "0"] + + if default_track == "original" and include_original: + cmd += ["-disposition:a:0", "default"] + else: + target_idx = 0 + for t in tracks_to_process: + if t['lang_code'] == default_track: + target_idx = t["stream_idx"] + break + cmd += [f"-disposition:a:{target_idx}", "default"] + + cmd += ["-shortest", output_path, "-y"] + + try: + proc = await asyncio.create_subprocess_exec( + *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + _, stderr = await proc.communicate() + if proc.returncode != 0: + raise Exception(stderr.decode()) + except Exception as e: + raise HTTPException(status_code=500, detail=f"ffmpeg mux failed: {str(e)}") + + if not os.path.exists(output_path) or os.path.getsize(output_path) == 0: + raise HTTPException(status_code=500, detail="ffmpeg mux produced no output file") + logger.info("Dub mux wrote %s (%d bytes)", output_path, os.path.getsize(output_path)) + + base_name = os.path.splitext(job.get('filename', 'output'))[0] + safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'output' + dl_name = f"dubbed_{safe_name}_{stamp}.mp4" + + if save_path: + return _native_save(output_path, save_path, dl_name, media_type="video/mp4") + + return FileResponse( + output_path, media_type="video/mp4", + headers={"Content-Disposition": f'attachment; filename="{dl_name}"'}, + ) + + +@router.get("/dub/media/{job_id}") +async def dub_get_media(job_id: str): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + if not os.path.exists(job["video_path"]): + raise HTTPException(status_code=404, detail="Media file not found") + return FileResponse(job["video_path"]) + +@router.get("/dub/audio/{job_id}") +async def dub_get_audio(job_id: str): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + audio = job.get("audio_path") + if not audio or not os.path.exists(audio): + raise HTTPException(status_code=404, detail="Audio file not found") + return FileResponse(audio, media_type="audio/wav") + +@router.get("/dub/preview/{job_id}/{segment_index}") +async def dub_preview_segment(job_id: str, segment_index: int): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + seg_path = os.path.join(DUB_DIR, job_id, f"seg_{segment_index}.wav") + if not os.path.exists(seg_path): + raise HTTPException(status_code=404, detail="Segment not generated yet") + return FileResponse(seg_path, media_type="audio/wav") + + +@router.get("/dub/download-audio/{job_id}") +@router.get("/dub/download-audio/{job_id}/{filename}") +async def dub_download_audio(job_id: str, lang: str = Query(None), preserve_bg: bool = Query(True), save_path: str = Query("")): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + tracks = job.get("dubbed_tracks", {}) + if lang and lang in tracks: + wav_path = tracks[lang]["path"] + elif tracks: + wav_path = list(tracks.values())[0]["path"] + else: + raise HTTPException(status_code=400, detail="No dubbed audio track generated yet") + + if not os.path.exists(wav_path): + raise HTTPException(status_code=404, detail="Audio file not found") + + lang_label = lang or list(tracks.keys())[0] + stamp = _unique_stamp() + exports_dir = os.path.join(DUB_DIR, job_id, "exports") + os.makedirs(exports_dir, exist_ok=True) + + bg_audio = job.get("no_vocals_path") if preserve_bg else None + if bg_audio and os.path.exists(bg_audio): + ffmpeg = find_ffmpeg() + final_audio_path = os.path.join(exports_dir, f"mixed_dub_{lang_label}_{stamp}.wav") + cmd = [ + ffmpeg, "-i", bg_audio, "-i", wav_path, + "-filter_complex", "[0:a][1:a]amix=inputs=2:duration=longest:dropout_transition=2:weights=0.8 1.2[aout]", + "-map", "[aout]", "-c:a", "pcm_s16le", "-y", final_audio_path + ] + try: + rc, _, stderr = await run_ffmpeg(cmd, timeout=900.0) + if rc != 0: + raise Exception(stderr.decode(errors="replace") if stderr else "ffmpeg mix non-zero") + if not os.path.exists(final_audio_path) or os.path.getsize(final_audio_path) == 0: + raise Exception("ffmpeg mix produced no output file") + wav_path = final_audio_path + logger.info("Dub audio mix wrote %s (%d bytes)", final_audio_path, os.path.getsize(final_audio_path)) + except Exception as e: + logger.error(f"Failed to mix audio: {str(e)}") + + base_name = os.path.splitext(job.get('filename', 'audio'))[0] + safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'audio' + dl_name = f"dubbed_audio_{lang_label}_{safe_name}_{stamp}.wav" + if save_path: + return _native_save(wav_path, save_path, dl_name, media_type="audio/wav") + return FileResponse( + wav_path, media_type="audio/wav", + headers={"Content-Disposition": f'attachment; filename="{dl_name}"'}, + ) + + +def _format_srt_time(seconds): + h = int(seconds // 3600) + m = int((seconds % 3600) // 60) + s = int(seconds % 60) + ms = int((seconds % 1) * 1000) + return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" + +@router.get("/dub/srt/{job_id}") +@router.get("/dub/srt/{job_id}/{filename}") +async def dub_export_srt(job_id: str): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + segments = job.get("segments", []) + if not segments: + raise HTTPException(status_code=400, detail="No transcript segments available") + + srt_lines = [] + for i, seg in enumerate(segments): + start_ts = _format_srt_time(seg["start"]) + end_ts = _format_srt_time(seg["end"]) + srt_lines.append(f"{i + 1}") + srt_lines.append(f"{start_ts} --> {end_ts}") + srt_lines.append(seg["text"]) + srt_lines.append("") + + srt_content = "\n".join(srt_lines) + base_name = os.path.splitext(job.get('filename', 'video'))[0] + return Response( + content=srt_content, + media_type="text/plain", + headers={"Content-Disposition": f'attachment; filename="subtitles_{base_name}.srt"'}, + ) + +def _format_vtt_time(seconds): + h = int(seconds // 3600) + m = int((seconds % 3600) // 60) + s = int(seconds % 60) + ms = int((seconds % 1) * 1000) + return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + +@router.get("/dub/vtt/{job_id}") +@router.get("/dub/vtt/{job_id}/{filename}") +async def dub_export_vtt(job_id: str): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + segments = job.get("segments", []) + if not segments: + raise HTTPException(status_code=400, detail="No transcript segments available") + + vtt_lines = ["WEBVTT", ""] + for i, seg in enumerate(segments): + start_ts = _format_vtt_time(seg["start"]) + end_ts = _format_vtt_time(seg["end"]) + vtt_lines.append(str(i + 1)) + vtt_lines.append(f"{start_ts} --> {end_ts}") + vtt_lines.append(seg["text"]) + vtt_lines.append("") + + vtt_content = "\n".join(vtt_lines) + base_name = os.path.splitext(job.get('filename', 'video'))[0] + return Response( + content=vtt_content, + media_type="text/vtt", + headers={"Content-Disposition": f'attachment; filename="subtitles_{base_name}.vtt"'}, + ) + + +@router.get("/dub/export-segments/{job_id}") +async def dub_export_segments_zip(job_id: str): + import zipfile + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + segments = job.get("segments", []) + if not segments: + raise HTTPException(status_code=400, detail="No segments available") + + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: + for i, seg in enumerate(segments): + seg_path = os.path.join(DUB_DIR, job_id, f"seg_{i}.wav") + if os.path.exists(seg_path): + speaker = seg.get("speaker_id", "Speaker1").replace(" ", "") + start_str = f"{seg['start']:.2f}" + end_str = f"{seg['end']:.2f}" + arc_name = f"{i+1:03d}_{start_str}-{end_str}_{speaker}.wav" + zf.write(seg_path, arc_name) + + zip_buffer.seek(0) + base_name = os.path.splitext(job.get('filename', 'video'))[0] + safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'segments' + return Response( + content=zip_buffer.read(), + media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="segments_{safe_name}.zip"'}, + ) + +@router.get("/dub/download-mp3/{job_id}") +@router.get("/dub/download-mp3/{job_id}/{filename}") +async def dub_download_mp3(job_id: str, lang: str = Query(None), preserve_bg: bool = Query(True), save_path: str = Query("")): + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + tracks = job.get("dubbed_tracks", {}) + if lang and lang in tracks: + wav_path = tracks[lang]["path"] + elif tracks: + wav_path = list(tracks.values())[0]["path"] + else: + raise HTTPException(status_code=400, detail="No dubbed audio track generated yet") + + if not os.path.exists(wav_path): + raise HTTPException(status_code=404, detail="Audio file not found") + + lang_label = lang or list(tracks.keys())[0] + ffmpeg = find_ffmpeg() + stamp = _unique_stamp() + exports_dir = os.path.join(DUB_DIR, job_id, "exports") + os.makedirs(exports_dir, exist_ok=True) + + source_path = wav_path + bg_audio = job.get("no_vocals_path") if preserve_bg else None + if bg_audio and os.path.exists(bg_audio): + mixed_path = os.path.join(exports_dir, f"mixed_mp3_{lang_label}_{stamp}.wav") + cmd_mix = [ + ffmpeg, "-i", bg_audio, "-i", wav_path, + "-filter_complex", "[0:a][1:a]amix=inputs=2:duration=longest:dropout_transition=2:weights=0.8 1.2[aout]", + "-map", "[aout]", "-c:a", "pcm_s16le", "-y", mixed_path + ] + try: + rc, _, _ = await run_ffmpeg(cmd_mix, timeout=900.0) + if rc == 0 and os.path.exists(mixed_path) and os.path.getsize(mixed_path) > 0: + source_path = mixed_path + except Exception as e: + logger.error(f"Failed to mix audio for MP3: {e}") + + mp3_path = os.path.join(exports_dir, f"dubbed_{lang_label}_{stamp}.mp3") + cmd = [ffmpeg, "-i", source_path, "-codec:a", "libmp3lame", "-b:a", "192k", "-y", mp3_path] + try: + rc, _, stderr = await run_ffmpeg(cmd, timeout=600.0) + if rc != 0: + raise Exception(stderr.decode(errors="replace") if stderr else "MP3 encode non-zero") + except asyncio.TimeoutError: + raise HTTPException(status_code=504, detail="MP3 encoding timed out") + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500, detail=f"MP3 encoding failed: {str(e)}") + + if not os.path.exists(mp3_path) or os.path.getsize(mp3_path) == 0: + raise HTTPException(status_code=500, detail="MP3 encoding produced no output file") + logger.info("Dub MP3 encoded %s (%d bytes)", mp3_path, os.path.getsize(mp3_path)) + + base_name = os.path.splitext(job.get('filename', 'audio'))[0] + safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'audio' + dl_name = f"dubbed_{lang_label}_{safe_name}_{stamp}.mp3" + if save_path: + return _native_save(mp3_path, save_path, dl_name, media_type="audio/mpeg") + return FileResponse( + mp3_path, media_type="audio/mpeg", + headers={"Content-Disposition": f'attachment; filename="{dl_name}"'}, + ) + +@router.get("/dub/export-stems/{job_id}") +async def dub_export_stems(job_id: str, lang: str = Query(None)): + import zipfile + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + tracks = job.get("dubbed_tracks", {}) + if not tracks: + raise HTTPException(status_code=400, detail="No dubbed tracks generated yet") + + if lang and lang in tracks: + vocals_path = tracks[lang]["path"] + lang_label = lang + elif tracks: + first_key = list(tracks.keys())[0] + vocals_path = tracks[first_key]["path"] + lang_label = first_key + else: + raise HTTPException(status_code=400, detail="No dubbed audio track") + + bg_path = job.get("no_vocals_path") + + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: + if os.path.exists(vocals_path): + zf.write(vocals_path, f"vocals_dubbed_{lang_label}.wav") + if bg_path and os.path.exists(bg_path): + zf.write(bg_path, "background_original.wav") + + zip_buffer.seek(0) + base_name = os.path.splitext(job.get('filename', 'video'))[0] + safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'stems' + return Response( + content=zip_buffer.read(), + media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="stems_{safe_name}.zip"'}, + ) diff --git a/backend/api/routers/dub_generate.py b/backend/api/routers/dub_generate.py new file mode 100644 index 00000000..523c8e17 --- /dev/null +++ b/backend/api/routers/dub_generate.py @@ -0,0 +1,182 @@ +import os +import json +import time +import asyncio +import torch +import torchaudio +from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse + +from core.db import get_db +from core.config import DUB_DIR, VOICES_DIR +from core.tasks import task_manager +from schemas.requests import DubRequest +from services.model_manager import get_model, _gpu_pool +from services.audio_dsp import apply_mastering, normalize_audio +from api.routers.dub_core import _get_job, _save_job + +router = APIRouter() + +@router.post("/dub/generate/{job_id}") +async def dub_generate(job_id: str, req: DubRequest): + """Adds a dub generation job to the async batch task pool.""" + job = _get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + _model = await get_model() + + async def _stream(task_id): + total = len(req.segments) + all_segment_wavs = [] + sync_scores = [] + + for i, seg in enumerate(req.segments): + # Check abort flag before each segment + if task_manager.is_cancelled(task_id): + yield f"data: {json.dumps({'type': 'cancelled', 'segments_processed': i})}\n\n" + return + + yield f"data: {json.dumps({'type': 'progress', 'current': i, 'total': total, 'text': seg.text[:50]})}\n\n" + + seg_duration = seg.end - seg.start + if seg_duration <= 0.05 or not seg.text.strip(): + sr = _model.sampling_rate + silence = torch.zeros(1, int(seg_duration * sr)) + all_segment_wavs.append((seg.start, seg.end, silence, sr)) + sync_scores.append(1.0) + continue + + def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id=None): + ref_audio = None + ref_text = None + used_seed = None + + if profile_id: + conn = get_db() + try: + row = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() + finally: + conn.close() + if row: + if row["is_locked"] and row["locked_audio_path"]: + ref_audio = os.path.join(VOICES_DIR, row["locked_audio_path"]) + ref_text = row["ref_text"] + used_seed = row["seed"] + elif row["instruct"] and not row["is_locked"]: + used_seed = row["seed"] + else: + ref_audio = os.path.join(VOICES_DIR, row["ref_audio_path"]) + ref_text = row["ref_text"] + used_seed = row["seed"] + + if not instruct_str: + instruct_str = row["instruct"] + + if used_seed is not None: + torch.manual_seed(used_seed) + + try: + audios = _model.generate( + text=text, language=lang if lang != "Auto" else None, + ref_audio=ref_audio, ref_text=ref_text, + instruct=instruct_str if instruct_str else None, + duration=dur_s, num_step=nstep, guidance_scale=cfg, + speed=spd, denoise=True, postprocess_output=True, + ) + audio_out = audios[0] + mastered_audio = apply_mastering(audio_out, sample_rate=_model.sampling_rate if hasattr(_model, 'sampling_rate') else 24000) + return normalize_audio(mastered_audio, target_dBFS=-2.0) + except Exception as e: + import gc + gc.collect() + if torch.backends.mps.is_available(): + torch.mps.empty_cache() + elif torch.cuda.is_available(): + torch.cuda.empty_cache() + raise RuntimeError(f"Engine VRAM crash on segment: {str(e)}") + + seg_instruct = seg.instruct or req.instruct + seg_profile = seg.profile_id or None + seg_speed = seg.speed if hasattr(seg, 'speed') and seg.speed is not None else req.speed + seg_lang = seg.target_lang if getattr(seg, 'target_lang', None) else req.language + + loop = asyncio.get_event_loop() + try: + audio_tensor = await loop.run_in_executor( + _gpu_pool, _gen, + seg.text, seg_lang, seg_instruct, seg_duration, + req.num_step, req.guidance_scale, seg_speed, seg_profile, + ) + + # Check abort immediately after GPU work completes + if task_manager.is_cancelled(task_id): + yield f"data: {json.dumps({'type': 'cancelled', 'segments_processed': i + 1})}\n\n" + return + + target_samples = int(seg_duration * _model.sampling_rate) + current_samples = audio_tensor.shape[-1] + + if target_samples > current_samples: + pad_amount = target_samples - current_samples + audio_tensor = torch.nn.functional.pad(audio_tensor, (0, pad_amount)) + elif current_samples > target_samples: + audio_tensor = audio_tensor[..., :target_samples] + + generated_dur = audio_tensor.shape[-1] / _model.sampling_rate + sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3) + + sync_scores.append(sync_ratio) + + seg_wav_path = os.path.join(DUB_DIR, job_id, f"seg_{i}.wav") + torchaudio.save(seg_wav_path, audio_tensor, _model.sampling_rate) + all_segment_wavs.append((seg.start, seg.end, audio_tensor, _model.sampling_rate)) + except Exception as e: + yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error': str(e)})}\n\n" + sr = _model.sampling_rate + all_segment_wavs.append((seg.start, seg.end, torch.zeros(1, int(seg_duration * sr)), sr)) + sync_scores.append(1.0) + + yield f"data: {json.dumps({'type': 'assembling'})}\n\n" + + sr = _model.sampling_rate + total_samples = int(job["duration"] * sr) + full_audio = torch.zeros(1, total_samples) + + for i, (start, end, wav, _) in enumerate(all_segment_wavs): + s = int(start * sr) + seg_ref = req.segments[i] if i < len(req.segments) else None + seg_gain = getattr(seg_ref, "gain", None) if seg_ref is not None else None + seg_gain = seg_gain if seg_gain is not None else 1.0 + seg_gain = max(0.0, min(2.0, seg_gain)) + adjusted = wav * seg_gain + wl = adjusted.shape[-1] + fade_ms = 15 + fade_samples = int((fade_ms / 1000.0) * sr) + if wl > fade_samples * 2: + ramp_up = torch.linspace(0, 1, fade_samples, device=adjusted.device) + ramp_down = torch.linspace(1, 0, fade_samples, device=adjusted.device) + adjusted[0, :fade_samples] *= ramp_up + adjusted[0, -fade_samples:] *= ramp_down + + e = min(s + wl, total_samples) + full_audio[:, s:e] += adjusted[:, :e - s] + + lang_code = req.language_code or "und" + track_path = os.path.join(DUB_DIR, job_id, f"dubbed_{lang_code}.wav") + torchaudio.save(track_path, full_audio, sr) + job["dubbed_tracks"][lang_code] = { + "path": track_path, + "language": req.language, + "language_code": lang_code, + } + + job["language"] = req.language + job["language_code"] = lang_code + _save_job(job_id, job) + + yield f"data: {json.dumps({'type': 'done', 'segments_processed': total, 'language_code': lang_code, 'tracks': list(job['dubbed_tracks'].keys()), 'sync_scores': sync_scores})}\n\n" + + task_id = f"dub_{job_id}_{int(time.time())}" + await task_manager.add_task(task_id, "dub_generate", _stream, task_id) + return {"task_id": task_id} diff --git a/backend/api/routers/dub_translate.py b/backend/api/routers/dub_translate.py new file mode 100644 index 00000000..fbd8a19b --- /dev/null +++ b/backend/api/routers/dub_translate.py @@ -0,0 +1,253 @@ +import os +import asyncio +import logging +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +from schemas.requests import TranslateRequest +from services.model_manager import _cpu_pool, _gpu_pool +from api.routers.dub_core import _get_job + +router = APIRouter() +logger = logging.getLogger("omnivoice.api") + +TRANSLATE_CODES = { + "en": "en", "es": "es", "fr": "fr", "de": "de", "it": "it", "pt": "pt", + "ru": "ru", "ja": "ja", "ko": "ko", "zh": "zh-CN", "ar": "ar", "hi": "hi", + "tr": "tr", "pl": "pl", "nl": "nl", "sv": "sv", "th": "th", "vi": "vi", + "id": "id", "uk": "uk", +} + +FLORES_CODES = { + "en": "eng_Latn", "es": "spa_Latn", "fr": "fra_Latn", "de": "deu_Latn", + "it": "ita_Latn", "pt": "por_Latn", "ru": "rus_Cyrl", "ja": "jpn_Jpan", + "ko": "kor_Hang", "zh": "zho_Hans", "zh-CN": "zho_Hans", "ar": "arb_Arab", + "hi": "hin_Deva", "tr": "tur_Latn", "pl": "pol_Latn", "nl": "nld_Latn", + "sv": "swe_Latn", "th": "tha_Thai", "vi": "vie_Latn", "id": "ind_Latn", + "uk": "ukr_Cyrl", +} + +_nllb_model = None +_nllb_tokenizer = None +_nllb_device = None + + +def _resolve_source_lang(req: TranslateRequest) -> str: + """Pick source language: explicit request > job.source_lang > 'en' fallback.""" + if getattr(req, "source_lang", None): + return req.source_lang + if getattr(req, "job_id", None): + job = _get_job(req.job_id) + if job and job.get("source_lang"): + return job["source_lang"] + return "en" + + +def _unload_nllb(): + """Release NLLB VRAM so TTS model can reload.""" + global _nllb_model, _nllb_tokenizer + import gc + _nllb_model = None + _nllb_tokenizer = None + gc.collect() + try: + import torch + if torch.cuda.is_available(): + torch.cuda.empty_cache() + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + torch.mps.empty_cache() + except Exception: + pass + + +@router.post("/dub/translate") +async def dub_translate(req: TranslateRequest): + try: + provider = (req.provider if req.provider else os.environ.get("TRANSLATE_PROVIDER", "google")).lower() + lang_code = TRANSLATE_CODES.get(req.target_lang, req.target_lang) + api_key = os.environ.get("TRANSLATE_API_KEY", "") + loop = asyncio.get_event_loop() + src_lang = _resolve_source_lang(req) + + # Offline NLLB Transformer Translation + if provider == "nllb": + flores_tgt = FLORES_CODES.get(req.target_lang, "eng_Latn") + flores_src = FLORES_CODES.get(src_lang, "eng_Latn") + + def _translate_nllb(): + global _nllb_model, _nllb_tokenizer, _nllb_device + import torch + from transformers import AutoTokenizer, AutoModelForSeq2SeqLM + + if torch.cuda.is_available(): + target_device = "cuda" + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + target_device = "mps" + else: + target_device = "cpu" + + try: + if _nllb_tokenizer is None: + _nllb_tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M") + if _nllb_model is None: + _nllb_model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M") + if target_device != "cpu": + try: + _nllb_model = _nllb_model.to(target_device) + _nllb_device = target_device + except Exception as e: + logger.warning("NLLB %s placement failed, falling back to CPU: %s", target_device, e) + _nllb_device = "cpu" + else: + _nllb_device = "cpu" + except Exception as e: + logger.exception("NLLB model load failed") + return [{"id": seg.id, "text": seg.text, "error": f"Model load error: {str(e)}"} for seg in req.segments] + + results = [] + for seg in req.segments: + try: + if not seg.text or not seg.text.strip(): + results.append({"id": seg.id, "text": seg.text}) + continue + + tgt = FLORES_CODES.get(seg.target_lang, flores_tgt) if seg.target_lang else flores_tgt + + _nllb_tokenizer.src_lang = flores_src + inputs = _nllb_tokenizer(seg.text, return_tensors="pt") + if _nllb_device and _nllb_device != "cpu": + inputs = {k: v.to(_nllb_device) for k, v in inputs.items()} + + forced_bos_token_id = _nllb_tokenizer.convert_tokens_to_ids(tgt) + try: + translated_tokens = _nllb_model.generate( + **inputs, forced_bos_token_id=forced_bos_token_id, max_length=400 + ) + except (RuntimeError, NotImplementedError) as e: + if _nllb_device == "mps": + logger.warning("MPS generate failed, retrying on CPU: %s", e) + _nllb_model.to("cpu") + _nllb_device = "cpu" + inputs = {k: v.to("cpu") for k, v in inputs.items()} + translated_tokens = _nllb_model.generate( + **inputs, forced_bos_token_id=forced_bos_token_id, max_length=400 + ) + else: + raise + translated_text = _nllb_tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0] + results.append({"id": seg.id, "text": translated_text}) + except Exception as e: + results.append({"id": seg.id, "text": seg.text, "error": str(e)}) + return results + + translated = await loop.run_in_executor(_gpu_pool, _translate_nllb) + if os.environ.get("OMNIVOICE_UNLOAD_NLLB", "1") == "1": + _unload_nllb() + return {"translated": translated, "target_lang": req.target_lang, "source_lang": src_lang} + + # OpenAI / Ollama Local LLM Translation + if provider == "openai": + base_url = os.environ.get("TRANSLATE_BASE_URL") + model_name = os.environ.get("TRANSLATE_MODEL", "gpt-3.5-turbo") + from openai import OpenAI + client = OpenAI(base_url=base_url, api_key=api_key or "local") + + def _translate_llm(seg): + try: + if not seg.text or not seg.text.strip(): + return {"id": seg.id, "text": seg.text} + tgt = seg.target_lang if seg.target_lang else req.target_lang + res = client.chat.completions.create( + model=model_name, + messages=[ + {"role": "system", "content": f"You are a professional dubbing translator. Translate the user's text from {src_lang} into {tgt}. Reply ONLY with the translated text, do not add any quotes, notes, or explanations."}, + {"role": "user", "content": seg.text} + ] + ) + out_text = res.choices[0].message.content.strip() + return {"id": seg.id, "text": out_text} + except Exception as e: + return {"id": seg.id, "text": seg.text, "error": str(e)} + + tasks = [loop.run_in_executor(_cpu_pool, _translate_llm, seg) for seg in req.segments] + translated = await asyncio.gather(*tasks) + translated.sort(key=lambda x: str(x["id"])) + return {"translated": translated, "target_lang": req.target_lang, "source_lang": src_lang} + + # Offline Argos Translate + if provider == "argos" or provider == "libretranslate": + def _translate_argos(): + cache_dir = os.environ.get("OMNIVOICE_CACHE_DIR") + if cache_dir: + argos_cache = os.path.join(cache_dir, "argos-translate") + os.makedirs(argos_cache, exist_ok=True) + os.environ.setdefault("ARGOS_PACKAGES_DIR", argos_cache) + os.environ.setdefault("ARGOS_DATA_DIR", argos_cache) + import argostranslate.package + import argostranslate.translate + + from_code = src_lang + available_packages = argostranslate.package.get_installed_packages() + + results = [] + for seg in req.segments: + try: + if not seg.text or not seg.text.strip(): + results.append({"id": seg.id, "text": seg.text}) + continue + to_code = seg.target_lang if seg.target_lang else req.target_lang + installed_pkg = next(filter(lambda x: x.from_code == from_code and x.to_code == to_code, available_packages), None) + + if installed_pkg is None: + argostranslate.package.update_package_index() + all_packages = argostranslate.package.get_available_packages() + package_to_install = next(filter(lambda x: x.from_code == from_code and x.to_code == to_code, all_packages), None) + if package_to_install: + argostranslate.package.install_from_path(package_to_install.download()) + available_packages = argostranslate.package.get_installed_packages() + else: + raise Exception(f"No Argos package available for {from_code} -> {to_code}") + + translated_text = argostranslate.translate.translate(seg.text, from_code, to_code) + results.append({"id": seg.id, "text": translated_text}) + except Exception as e: + results.append({"id": seg.id, "text": seg.text, "error": str(e)}) + return results + + translated = await loop.run_in_executor(_cpu_pool, _translate_argos) + return {"translated": translated, "target_lang": req.target_lang, "source_lang": src_lang} + + # Legacy / API Deep_Translator logic + src_arg = TRANSLATE_CODES.get(src_lang, src_lang) or "auto" + + def _translate_single(seg): + try: + if not seg.text or not seg.text.strip(): + return {"id": seg.id, "text": seg.text} + seg_lc = TRANSLATE_CODES.get(seg.target_lang, seg.target_lang) if seg.target_lang else lang_code + if provider == "deepl": + from deep_translator import DeepL + translator = DeepL(api_key=api_key, source=src_arg, target=seg_lc) + elif provider == "mymemory": + from deep_translator import MyMemoryTranslator + translator = MyMemoryTranslator(source=src_arg, target=seg_lc) + elif provider == "microsoft": + from deep_translator import MicrosoftTranslator + translator = MicrosoftTranslator(api_key=api_key, source=src_arg, target=seg_lc) + else: + from deep_translator import GoogleTranslator + translator = GoogleTranslator(source=src_arg, target=seg_lc) + + translated = translator.translate(seg.text) + return {"id": seg.id, "text": translated or seg.text} + except Exception as e: + return {"id": seg.id, "text": seg.text, "error": str(e)} + + tasks = [loop.run_in_executor(_cpu_pool, _translate_single, seg) for seg in req.segments] + translated = await asyncio.gather(*tasks) + translated.sort(key=lambda x: str(x["id"])) + + return {"translated": translated, "target_lang": req.target_lang, "source_lang": src_lang} + except Exception as e: + import traceback; traceback.print_exc() + return JSONResponse(status_code=500, content={"error": str(e)}) diff --git a/backend/api/routers/exports.py b/backend/api/routers/exports.py new file mode 100644 index 00000000..53014a6c --- /dev/null +++ b/backend/api/routers/exports.py @@ -0,0 +1,115 @@ +import os +import uuid +import time +import shutil +import subprocess +import platform +from fastapi import APIRouter, HTTPException + +from core.db import get_db +from core.config import OUTPUTS_DIR +from schemas.requests import ExportRequest, ExportRecordRequest, RevealRequest + +router = APIRouter() + + +def _safe_destination(raw: str) -> str: + """Resolve + validate an export destination. Rejects relative/empty paths.""" + if not raw or not raw.strip(): + raise HTTPException(status_code=400, detail="destination_path required") + dest = os.path.realpath(os.path.expanduser(raw)) + if not os.path.isabs(dest): + raise HTTPException(status_code=400, detail="destination_path must be absolute") + parent = os.path.dirname(dest) + if not parent or not os.path.isdir(parent): + raise HTTPException(status_code=400, detail="destination directory does not exist") + return dest + + +def _safe_source(filename: str) -> str: + """Resolve a source filename against OUTPUTS_DIR / dub outputs, blocking traversal.""" + base = os.path.basename(filename or "") + if not base or base != filename: + raise HTTPException(status_code=400, detail="invalid source_filename") + for root in (OUTPUTS_DIR, os.path.join("dub", "outputs")): + candidate = os.path.realpath(os.path.join(root, base)) + root_real = os.path.realpath(root) + if candidate.startswith(root_real + os.sep) and os.path.exists(candidate): + return candidate + raise HTTPException(status_code=404, detail="Source file not found") + + +@router.post("/export") +def export_file(req: ExportRequest): + src = _safe_source(req.source_filename) + dest = _safe_destination(req.destination_path) + try: + shutil.copy2(src, dest) + except OSError as e: + raise HTTPException(status_code=500, detail=str(e)) + + export_id = str(uuid.uuid4())[:8] + conn = get_db() + try: + conn.execute( + "INSERT INTO export_history (id, filename, destination_path, mode, created_at) VALUES (?, ?, ?, ?, ?)", + (export_id, req.source_filename, dest, req.mode, time.time()), + ) + conn.commit() + finally: + conn.close() + return {"success": True, "id": export_id} + + +@router.post("/export/record") +def record_export(req: ExportRecordRequest): + export_id = str(uuid.uuid4())[:8] + conn = get_db() + try: + conn.execute( + "INSERT INTO export_history (id, filename, destination_path, mode, created_at) VALUES (?, ?, ?, ?, ?)", + (export_id, req.filename, req.destination_path, req.mode, time.time()), + ) + conn.commit() + finally: + conn.close() + return {"success": True, "id": export_id} + + +@router.get("/export/history") +def get_export_history(): + conn = get_db() + try: + rows = conn.execute("SELECT * FROM export_history ORDER BY created_at DESC LIMIT 50").fetchall() + finally: + conn.close() + return [dict(r) for r in rows] + + +@router.post("/export/reveal") +def reveal_in_folder(req: RevealRequest): + # Tauri/native dialog-provided path; subprocess uses list args (no shell interpolation). + if not req.path or not req.path.strip(): + raise HTTPException(status_code=400, detail="path required") + target = os.path.realpath(os.path.expanduser(req.path)) + if not os.path.exists(target): + raise HTTPException(status_code=404, detail="path not found") + + folder = target if os.path.isdir(target) else os.path.dirname(target) + system = platform.system() + try: + if system == "Darwin": + if os.path.isfile(target): + subprocess.Popen(["open", "-R", target]) + else: + subprocess.Popen(["open", folder]) + elif system == "Windows": + if os.path.isfile(target): + subprocess.Popen(["explorer", "/select,", target.replace("/", "\\")]) + else: + subprocess.Popen(["explorer", folder.replace("/", "\\")]) + else: + subprocess.Popen(["xdg-open", folder]) + return {"success": True} + except OSError as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/api/routers/generation.py b/backend/api/routers/generation.py new file mode 100644 index 00000000..17084f3c --- /dev/null +++ b/backend/api/routers/generation.py @@ -0,0 +1,224 @@ +import os +import io +import uuid +import time +import asyncio +import tempfile +import contextlib +import torch +import torchaudio +from typing import Optional +from fastapi import APIRouter, File, Form, UploadFile, HTTPException +from fastapi.responses import StreamingResponse + +from core.db import get_db, db_conn +from core.config import OUTPUTS_DIR, VOICES_DIR +from services.model_manager import get_model, _gpu_pool +from services.audio_dsp import apply_mastering, normalize_audio + +router = APIRouter() + +def _run_inference( + model, text, language, ref_audio_path, ref_text, instruct, duration, + num_step, guidance_scale, speed, t_shift, denoise, + postprocess_output, layer_penalty_factor, position_temperature, + class_temperature, used_seed, +): + try: + if used_seed is not None: + torch.manual_seed(used_seed) + + kwargs = {} + if t_shift is not None: kwargs["t_shift"] = t_shift + if layer_penalty_factor is not None: kwargs["layer_penalty_factor"] = layer_penalty_factor + if position_temperature is not None: kwargs["position_temperature"] = position_temperature + if class_temperature is not None: kwargs["class_temperature"] = class_temperature + + audios = model.generate( + text=text, language=language, ref_audio=ref_audio_path, + ref_text=ref_text, instruct=instruct, duration=duration, + num_step=num_step, guidance_scale=guidance_scale, speed=speed, + denoise=denoise, postprocess_output=postprocess_output, + **kwargs + ) + audio_out = audios[0] + + mastered_audio = apply_mastering(audio_out, sample_rate=model.sampling_rate if hasattr(model, 'sampling_rate') else 24000) + return normalize_audio(mastered_audio, target_dBFS=-2.0) + + except Exception as e: + import gc + gc.collect() + if torch.backends.mps.is_available(): + torch.mps.empty_cache() + elif torch.cuda.is_available(): + torch.cuda.empty_cache() + raise RuntimeError(f"F5-TTS Engine crashed: {str(e)}") + + +@router.post("/generate") +async def generate_speech( + text: str = Form(...), + language: Optional[str] = Form(None), + ref_audio: Optional[UploadFile] = File(None), + ref_text: Optional[str] = Form(None), + instruct: Optional[str] = Form(None), + duration: Optional[float] = Form(None), + num_step: int = Form(16), + guidance_scale: float = Form(2.0), + speed: float = Form(1.0), + t_shift: Optional[float] = Form(None), + denoise: bool = Form(True), + postprocess_output: bool = Form(True), + layer_penalty_factor: Optional[float] = Form(None), + position_temperature: Optional[float] = Form(None), + class_temperature: Optional[float] = Form(None), + profile_id: Optional[str] = Form(None), + seed: Optional[int] = Form(None), +): + _model = await get_model() + + ref_audio_path = None + cleanup_ref = False + used_seed = seed + + if profile_id: + conn = get_db() + try: + row = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() + finally: + conn.close() + if row: + if row["is_locked"] and row["locked_audio_path"]: + ref_audio_path = os.path.join(VOICES_DIR, row["locked_audio_path"]) + if not ref_text: + ref_text = row["ref_text"] + if not instruct: + instruct = row["instruct"] + if used_seed is None and row["seed"] is not None: + used_seed = row["seed"] + elif row["instruct"] and not row["is_locked"]: + if not instruct: + instruct = row["instruct"] + if used_seed is None and row["seed"] is not None: + used_seed = row["seed"] + else: + ref_audio_path = os.path.join(VOICES_DIR, row["ref_audio_path"]) if row["ref_audio_path"] else None + if not ref_text and row["ref_text"]: + ref_text = row["ref_text"] + if not instruct and row["instruct"]: + instruct = row["instruct"] + if used_seed is None and row["seed"] is not None: + used_seed = row["seed"] + if language == "Auto": + language = None + elif ref_audio is not None: + try: + with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f: + f.write(await ref_audio.read()) + ref_audio_path = f.name + cleanup_ref = True + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + start_time = time.time() + try: + loop = asyncio.get_event_loop() + audio_tensor = await loop.run_in_executor( + _gpu_pool, _run_inference, + _model, text, language, ref_audio_path, ref_text, instruct, duration, + num_step, guidance_scale, speed, t_shift, denoise, + postprocess_output, layer_penalty_factor, position_temperature, + class_temperature, used_seed, + ) + gen_time = round(time.time() - start_time, 2) + + audio_id = str(uuid.uuid4())[:8] + audio_filename = f"{audio_id}.wav" + audio_path = os.path.join(OUTPUTS_DIR, audio_filename) + torchaudio.save(audio_path, audio_tensor, _model.sampling_rate) + + audio_dur = round(audio_tensor.shape[-1] / _model.sampling_rate, 2) + + with db_conn() as conn: + conn.execute( + "INSERT INTO generation_history (id, text, mode, language, instruct, profile_id, audio_path, duration_seconds, generation_time, seed, created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (audio_id, text[:200], "clone" if ref_audio_path else "design", + language or "Auto", instruct or "", profile_id or "", + audio_filename, audio_dur, gen_time, used_seed, time.time()) + ) + + buffer = io.BytesIO() + torchaudio.save(buffer, audio_tensor, _model.sampling_rate, format="wav") + buffer.seek(0) + wav_bytes = buffer.read() + + async def _stream_wav(): + chunk_size = 16384 + for i in range(0, len(wav_bytes), chunk_size): + yield wav_bytes[i:i + chunk_size] + + return StreamingResponse( + _stream_wav(), + media_type="audio/wav", + headers={ + "X-Audio-Id": audio_id, + "X-Gen-Time": str(gen_time), + "X-Audio-Path": audio_filename, + "X-Seed": str(used_seed) if used_seed is not None else "", + "X-Audio-Duration": str(audio_dur), + "Content-Length": str(len(wav_bytes)), + } + ) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Inference failed: {str(e)}") + finally: + if cleanup_ref and ref_audio_path: + with contextlib.suppress(OSError): + os.remove(ref_audio_path) + +def _safe_output_path(name): + if not name: + return None + base = os.path.basename(name) + if base != name: + return None + outputs_real = os.path.realpath(OUTPUTS_DIR) + candidate = os.path.realpath(os.path.join(OUTPUTS_DIR, base)) + if not candidate.startswith(outputs_real + os.sep): + return None + return candidate + + +@router.get("/history") +def list_history(): + conn = get_db() + try: + rows = conn.execute("SELECT * FROM generation_history ORDER BY created_at DESC LIMIT 50").fetchall() + finally: + conn.close() + return [dict(r) for r in rows] + +@router.delete("/history") +def clear_history(): + with db_conn() as conn: + rows = conn.execute("SELECT audio_path FROM generation_history").fetchall() + for r in rows: + p = _safe_output_path(r["audio_path"]) + if p and os.path.exists(p): + with contextlib.suppress(OSError): + os.remove(p) + conn.execute("DELETE FROM generation_history") + return {"cleared": True} + +@router.delete("/history/{history_id}") +def delete_single_history(history_id: str): + with db_conn() as conn: + row = conn.execute("SELECT audio_path FROM generation_history WHERE id=?", (history_id,)).fetchone() + if row and row["audio_path"]: + p = _safe_output_path(row["audio_path"]) + if p and os.path.exists(p): + with contextlib.suppress(OSError): + os.remove(p) + conn.execute("DELETE FROM generation_history WHERE id=?", (history_id,)) + return {"deleted": True} diff --git a/backend/api/routers/profiles.py b/backend/api/routers/profiles.py new file mode 100644 index 00000000..87853bfd --- /dev/null +++ b/backend/api/routers/profiles.py @@ -0,0 +1,132 @@ +import os +import uuid +import time +import shutil +from typing import Optional +from fastapi import APIRouter, File, Form, UploadFile, HTTPException +from fastapi.responses import FileResponse, Response + +from core.db import get_db +from core.config import VOICES_DIR, OUTPUTS_DIR + +router = APIRouter() + +@router.get("/profiles") +def list_profiles(): + conn = get_db() + rows = conn.execute("SELECT * FROM voice_profiles ORDER BY created_at DESC").fetchall() + conn.close() + return [dict(r) for r in rows] + +@router.post("/profiles") +async def create_profile( + name: str = Form(...), + ref_audio: UploadFile = File(...), + ref_text: str = Form(""), + instruct: str = Form(""), + language: str = Form("Auto"), + seed: Optional[int] = Form(None), +): + profile_id = str(uuid.uuid4())[:8] + ext = os.path.splitext(ref_audio.filename or ".wav")[1] + audio_filename = f"{profile_id}{ext}" + audio_path = os.path.join(VOICES_DIR, audio_filename) + + with open(audio_path, "wb") as f: + f.write(await ref_audio.read()) + + conn = get_db() + conn.execute( + "INSERT INTO voice_profiles (id, name, ref_audio_path, ref_text, instruct, language, seed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (profile_id, name, audio_filename, ref_text, instruct, language, seed, time.time()) + ) + conn.commit() + conn.close() + return {"id": profile_id, "name": name} + +@router.get("/profiles/{profile_id}/audio") +def get_profile_audio(profile_id: str): + conn = get_db() + row = conn.execute("SELECT ref_audio_path, locked_audio_path FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() + conn.close() + if not row: + return Response("Profile not found", status_code=404) + audio_file = row["locked_audio_path"] or row["ref_audio_path"] + if not audio_file: + return Response("No audio available", status_code=404) + audio_path = os.path.join(VOICES_DIR, audio_file) + if not os.path.exists(audio_path): + return Response("Audio file missing", status_code=404) + return FileResponse(audio_path, media_type="audio/wav") + +@router.post("/profiles/{profile_id}/lock") +async def lock_profile( + profile_id: str, + history_id: str = Form(...), + seed: Optional[int] = Form(None), +): + conn = get_db() + profile = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() + if not profile: + conn.close() + raise HTTPException(status_code=404, detail="Profile not found") + + history = conn.execute("SELECT * FROM generation_history WHERE id=?", (history_id,)).fetchone() + if not history or not history["audio_path"]: + conn.close() + raise HTTPException(status_code=404, detail="History item not found or has no audio") + + src_path = os.path.join(OUTPUTS_DIR, history["audio_path"]) + if not os.path.exists(src_path): + conn.close() + raise HTTPException(status_code=404, detail="Audio file not found on disk") + + locked_filename = f"{profile_id}_locked.wav" + locked_path = os.path.join(VOICES_DIR, locked_filename) + shutil.copy2(src_path, locked_path) + + ref_text = history["text"][:100] if history["text"] else "" + + conn.execute( + "UPDATE voice_profiles SET locked_audio_path=?, seed=?, is_locked=1, ref_text=? WHERE id=?", + (locked_filename, seed, ref_text, profile_id) + ) + conn.commit() + conn.close() + return {"locked": True, "profile_id": profile_id, "locked_audio_path": locked_filename} + +@router.post("/profiles/{profile_id}/unlock") +async def unlock_profile(profile_id: str): + conn = get_db() + profile = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() + if not profile: + conn.close() + raise HTTPException(status_code=404, detail="Profile not found") + + if profile["locked_audio_path"]: + locked_path = os.path.join(VOICES_DIR, profile["locked_audio_path"]) + if os.path.exists(locked_path): + os.remove(locked_path) + + conn.execute( + "UPDATE voice_profiles SET locked_audio_path='', seed=NULL, is_locked=0 WHERE id=?", + (profile_id,) + ) + conn.commit() + conn.close() + return {"unlocked": True, "profile_id": profile_id} + +@router.delete("/profiles/{profile_id}") +def delete_profile(profile_id: str): + conn = get_db() + row = conn.execute("SELECT ref_audio_path, locked_audio_path FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() + if row: + for col in ["ref_audio_path", "locked_audio_path"]: + if row[col]: + path = os.path.join(VOICES_DIR, row[col]) + if os.path.exists(path): + os.remove(path) + conn.execute("DELETE FROM voice_profiles WHERE id=?", (profile_id,)) + conn.commit() + conn.close() + return {"deleted": profile_id} diff --git a/backend/api/routers/projects.py b/backend/api/routers/projects.py new file mode 100644 index 00000000..c6468e86 --- /dev/null +++ b/backend/api/routers/projects.py @@ -0,0 +1,72 @@ +import uuid +import time +import json +from fastapi import APIRouter, HTTPException + +from core.db import get_db +from schemas.requests import ProjectSaveRequest + +router = APIRouter() + +@router.get("/projects") +async def list_projects(): + conn = get_db() + rows = conn.execute( + "SELECT id, name, video_path, duration, created_at, updated_at FROM studio_projects ORDER BY updated_at DESC" + ).fetchall() + conn.close() + return [dict(r) for r in rows] + +@router.get("/projects/{project_id}") +async def get_project(project_id: str): + conn = get_db() + row = conn.execute("SELECT * FROM studio_projects WHERE id=?", (project_id,)).fetchone() + conn.close() + if not row: + raise HTTPException(status_code=404, detail="Project not found") + result = dict(row) + if result.get("state_json"): + try: + result["state"] = json.loads(result["state_json"]) + except Exception: + result["state"] = {} + else: + result["state"] = {} + return result + +@router.post("/projects") +async def create_project(req: ProjectSaveRequest): + project_id = str(uuid.uuid4())[:8] + now = time.time() + conn = get_db() + conn.execute( + "INSERT INTO studio_projects (id, name, video_path, audio_path, duration, state_json, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?)", + (project_id, req.name, req.video_path, req.audio_path, req.duration, json.dumps(req.state), now, now), + ) + conn.commit() + conn.close() + return {"id": project_id, "name": req.name, "created_at": now} + +@router.put("/projects/{project_id}") +async def update_project(project_id: str, req: ProjectSaveRequest): + conn = get_db() + row = conn.execute("SELECT id FROM studio_projects WHERE id=?", (project_id,)).fetchone() + if not row: + conn.close() + raise HTTPException(status_code=404, detail="Project not found") + now = time.time() + conn.execute( + "UPDATE studio_projects SET name=?, video_path=?, audio_path=?, duration=?, state_json=?, updated_at=? WHERE id=?", + (req.name, req.video_path, req.audio_path, req.duration, json.dumps(req.state), now, project_id), + ) + conn.commit() + conn.close() + return {"id": project_id, "name": req.name, "updated_at": now} + +@router.delete("/projects/{project_id}") +async def delete_project(project_id: str): + conn = get_db() + conn.execute("DELETE FROM studio_projects WHERE id=?", (project_id,)) + conn.commit() + conn.close() + return {"deleted": project_id} diff --git a/backend/api/routers/system.py b/backend/api/routers/system.py new file mode 100644 index 00000000..ac2a47de --- /dev/null +++ b/backend/api/routers/system.py @@ -0,0 +1,132 @@ +import os +import sys +import uuid +import psutil +import asyncio +import logging +from fastapi import APIRouter, File, UploadFile, HTTPException +from fastapi.responses import FileResponse +import torch +import shutil + +from core.config import OUTPUTS_DIR +from services.model_manager import get_model_status, get_best_device +from services.ffmpeg_utils import find_ffmpeg + +router = APIRouter() +logger = logging.getLogger("omnivoice.api") + +@router.get("/model/status") +def model_status(): + """Report model loading state for frontend warm-up indicators.""" + return get_model_status() + +@router.get("/sysinfo") +def get_sys_info(): + vram = 0.0 + gpu_active = False + + is_mac = hasattr(torch.backends, "mps") and torch.backends.mps.is_available() + is_cuda = torch.cuda.is_available() + + try: + if is_mac: + alloc = getattr(torch.mps, "current_allocated_memory", None) + driver = getattr(torch.mps, "driver_allocated_memory", None) + if driver: + vram = driver() / (1024**3) + elif alloc: + vram = alloc() / (1024**3) + elif is_cuda: + vram = torch.cuda.memory_allocated() / (1024**3) + except Exception: + pass + + if vram > 0.01: + gpu_active = True + + return { + "cpu": psutil.cpu_percent(interval=0.1), + "ram": psutil.virtual_memory().used / (1024**3), + "total_ram": psutil.virtual_memory().total / (1024**3), + "vram": round(vram, 2), + "gpu_active": gpu_active + } + +@router.post("/clean-audio") +async def clean_audio(audio: UploadFile = File(...)): + """Accept a raw mic recording, run demucs vocal isolation, return clean WAV.""" + clean_id = str(uuid.uuid4())[:8] + tmp_dir = os.path.join(OUTPUTS_DIR, f"_clean_{clean_id}") + os.makedirs(tmp_dir, exist_ok=True) + try: + return await _do_clean_audio(audio, tmp_dir, clean_id) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +async def _do_clean_audio(audio, tmp_dir, clean_id): + raw_path = os.path.join(tmp_dir, "raw.wav") + with open(raw_path, "wb") as f: + f.write(await audio.read()) + + converted_path = os.path.join(tmp_dir, "converted.wav") + ffmpeg = find_ffmpeg() + proc = await asyncio.create_subprocess_exec( + ffmpeg, "-y", "-i", raw_path, "-ar", "24000", "-ac", "1", converted_path, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + ) + try: + _, stderr = await asyncio.wait_for(proc.communicate(), timeout=120.0) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + converted_path = raw_path + else: + if proc.returncode != 0: + converted_path = raw_path + + clean_path = converted_path + try: + proc = await asyncio.create_subprocess_exec( + sys.executable, "-m", "demucs.separate", "--two-stems", "vocals", "-n", "htdemucs", + "-d", get_best_device(), converted_path, "-o", tmp_dir, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + ) + try: + _, stderr = await asyncio.wait_for(proc.communicate(), timeout=900.0) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + raise Exception("demucs timed out") + if proc.returncode == 0: + demucs_out = os.path.join(tmp_dir, "htdemucs", "converted") + vocals_file = os.path.join(demucs_out, "vocals.wav") + if os.path.exists(vocals_file): + clean_path = vocals_file + except Exception as e: + logger.warning(f"Demucs failed for mic audio, using raw: {e}") + + clean_filename = f"mic_{clean_id}.wav" + final_path = os.path.join(OUTPUTS_DIR, clean_filename) + + proc = await asyncio.create_subprocess_exec( + ffmpeg, "-y", "-i", clean_path, "-ar", "24000", "-ac", "1", final_path, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + ) + try: + await asyncio.wait_for(proc.communicate(), timeout=120.0) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + if not os.path.exists(final_path): + shutil.copy2(clean_path, final_path) + + return FileResponse(final_path, media_type="audio/wav", filename=clean_filename, + headers={"X-Clean-Filename": clean_filename}) diff --git a/backend/core/__init__.py b/backend/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/core/config.py b/backend/core/config.py new file mode 100644 index 00000000..329db991 --- /dev/null +++ b/backend/core/config.py @@ -0,0 +1,37 @@ +import os +import sys + +def get_app_data_dir(): + custom_dir = os.environ.get("OMNIVOICE_DATA_DIR") + if custom_dir: + return custom_dir + + if sys.platform == "darwin": + return os.path.expanduser("~/Library/Application Support/OmniVoice") + elif sys.platform == "win32": + return os.path.join(os.environ.get("APPDATA", ""), "OmniVoice") + else: + return os.path.expanduser("~/.omnivoice") + +DATA_DIR = get_app_data_dir() +VOICES_DIR = os.path.join(DATA_DIR, "voices") # Reference audio for profiles +OUTPUTS_DIR = os.path.join(DATA_DIR, "outputs") # Generated audio files +DUB_DIR = os.path.join(DATA_DIR, "dub_jobs") +DB_PATH = os.path.join(DATA_DIR, "omnivoice.db") +PREVIEW_DIR = os.path.join(DATA_DIR, "preview") +CRASH_LOG_PATH = os.path.join(DATA_DIR, "crash_log.txt") + +IDLE_TIMEOUT_SECONDS = int(os.environ.get("OMNIVOICE_IDLE_TIMEOUT", "300")) +CPU_POOL_WORKERS = int(os.environ.get("OMNIVOICE_CPU_POOL", "0")) or min(8, (os.cpu_count() or 4)) + +def ensure_dirs(): + for d in [DATA_DIR, VOICES_DIR, OUTPUTS_DIR, DUB_DIR, PREVIEW_DIR]: + os.makedirs(d, exist_ok=True) + +ensure_dirs() + +# Ensure ffmpeg is on PATH for Whisper and other subprocesses (mostly relevant for Mac/Linux) +if sys.platform != "win32": + for _fpath in ["/opt/homebrew/bin", "/usr/local/bin"]: + if _fpath not in os.environ.get("PATH", "") and os.path.exists(_fpath): + os.environ["PATH"] = _fpath + os.pathsep + os.environ.get("PATH", "") diff --git a/backend/core/db.py b/backend/core/db.py new file mode 100644 index 00000000..df201212 --- /dev/null +++ b/backend/core/db.py @@ -0,0 +1,141 @@ +import re +import sqlite3 +import logging +from contextlib import contextmanager +from core.config import DB_PATH + +logger = logging.getLogger("omnivoice.db") + +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_TYPE_RE = re.compile(r"^[A-Za-z0-9_ '\"\(\)\-\.]+$") + + +def get_db(): + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + return conn + + +@contextmanager +def db_conn(): + """Context-managed SQLite connection that commits on clean exit and always closes.""" + conn = get_db() + try: + yield conn + conn.commit() + except Exception: + try: + conn.rollback() + except Exception: + pass + raise + finally: + conn.close() + + +_BASE_SCHEMA = """ + CREATE TABLE IF NOT EXISTS voice_profiles ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + ref_audio_path TEXT, + ref_text TEXT DEFAULT '', + instruct TEXT DEFAULT '', + language TEXT DEFAULT 'Auto', + locked_audio_path TEXT DEFAULT '', + seed INTEGER DEFAULT NULL, + is_locked INTEGER DEFAULT 0, + created_at REAL + ); + CREATE TABLE IF NOT EXISTS generation_history ( + id TEXT PRIMARY KEY, + text TEXT, + mode TEXT, + language TEXT, + instruct TEXT, + profile_id TEXT, + audio_path TEXT, + duration_seconds REAL, + generation_time REAL, + seed INTEGER DEFAULT NULL, + created_at REAL, + FOREIGN KEY (profile_id) REFERENCES voice_profiles(id) + ); + CREATE TABLE IF NOT EXISTS dub_history ( + id TEXT PRIMARY KEY, + filename TEXT, + duration REAL, + segments_count INTEGER, + language TEXT, + language_code TEXT, + tracks TEXT DEFAULT '[]', + job_data TEXT, + created_at REAL + ); + CREATE TABLE IF NOT EXISTS studio_projects ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + video_path TEXT, + audio_path TEXT, + duration REAL, + state_json TEXT, + created_at REAL, + updated_at REAL + ); + CREATE TABLE IF NOT EXISTS export_history ( + id TEXT PRIMARY KEY, + filename TEXT, + destination_path TEXT, + mode TEXT, + created_at REAL + ); +""" + +# Only tables/columns this module is allowed to ALTER. Prevents SQL injection via +# the f-string ALTER below if these helpers ever get exposed to user input. +_ALLOWED_MIGRATIONS = { + ("voice_profiles", "locked_audio_path"), + ("voice_profiles", "seed"), + ("voice_profiles", "is_locked"), + ("generation_history", "seed"), +} + + +def _add_column_if_missing(conn, table: str, column: str, typedef: str): + if (table, column) not in _ALLOWED_MIGRATIONS: + raise ValueError(f"Migration not allowed: {table}.{column}") + if not _IDENT_RE.match(table) or not _IDENT_RE.match(column): + raise ValueError(f"Invalid identifier: {table}.{column}") + if not _TYPE_RE.match(typedef): + raise ValueError(f"Invalid typedef: {typedef!r}") + try: + conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {typedef}") + except sqlite3.OperationalError as e: + if "duplicate column" not in str(e).lower(): + logger.warning("ALTER %s.%s failed: %s", table, column, e) + + +def _migrate(conn, current: int) -> int: + """Apply migrations sequentially. Return new version.""" + if current < 1: + _add_column_if_missing(conn, "voice_profiles", "locked_audio_path", "TEXT DEFAULT ''") + _add_column_if_missing(conn, "voice_profiles", "seed", "INTEGER DEFAULT NULL") + _add_column_if_missing(conn, "voice_profiles", "is_locked", "INTEGER DEFAULT 0") + _add_column_if_missing(conn, "generation_history", "seed", "INTEGER DEFAULT NULL") + current = 1 + # Future migrations: if current < 2: ...; current = 2 + return current + + +def init_db(): + conn = get_db() + try: + conn.executescript(_BASE_SCHEMA) + version = conn.execute("PRAGMA user_version").fetchone()[0] + new_version = _migrate(conn, version) + if new_version != version: + conn.execute(f"PRAGMA user_version = {new_version}") + conn.commit() + finally: + conn.close() diff --git a/backend/core/tasks.py b/backend/core/tasks.py new file mode 100644 index 00000000..54815c00 --- /dev/null +++ b/backend/core/tasks.py @@ -0,0 +1,103 @@ +import asyncio +import time +import json + +class TaskManager: + def __init__(self): + self.queue = None + self.active_tasks = {} + + def _init_queue(self): + if self.queue is None: + self.queue = asyncio.Queue() + + async def add_task(self, task_id, task_type, func, *args, **kwargs): + self._init_queue() + task_obj = { + "status": "pending", + "type": task_type, + "created_at": time.time(), + "history": [], + "listeners": [], + "listeners_lock": asyncio.Lock(), + "error": None, + "cancelled": False, + } + self.active_tasks[task_id] = task_obj + await self.queue.put((task_id, func, args, kwargs)) + + def cancel_task(self, task_id): + if task_id in self.active_tasks: + self.active_tasks[task_id]["cancelled"] = True + return True + return False + + def is_cancelled(self, task_id): + t = self.active_tasks.get(task_id) + return t["cancelled"] if t else False + + async def add_listener(self, task_id, q): + t = self.active_tasks.get(task_id) + if not t: + return False + async with t["listeners_lock"]: + t["listeners"].append(q) + return True + + async def remove_listener(self, task_id, q): + t = self.active_tasks.get(task_id) + if not t: + return + async with t["listeners_lock"]: + if q in t["listeners"]: + t["listeners"].remove(q) + + async def _push_event(self, task_id, event_str): + t = self.active_tasks.get(task_id) + if t is None: + return + if event_str is not None: + t["history"].append(event_str) + # Snapshot listeners under lock so concurrent add/remove can't mutate mid-iteration. + async with t["listeners_lock"]: + listeners = list(t["listeners"]) + for q in listeners: + await q.put(event_str) + + async def worker(self): + self._init_queue() + while True: + task_id, func, args, kwargs = await self.queue.get() + t = self.active_tasks.get(task_id) + if not t: + self.queue.task_done() + continue + + t["status"] = "running" + try: + import inspect + res = func(*args, **kwargs) + if inspect.isasyncgen(res): + async for update in res: + if t.get("cancelled"): + await self._push_event(task_id, f"data: {json.dumps({'type': 'cancelled'})}\n\n") + t["status"] = "cancelled" + break + await self._push_event(task_id, update) + elif inspect.iscoroutine(res): + await res + t["status"] = "done" + except Exception as e: + import logging + logging.getLogger("omnivoice.tasks").exception("Task %s failed", task_id) + t["status"] = "failed" + t["error"] = str(e) + try: + await self._push_event(task_id, f"data: {json.dumps({'type': 'error', 'error': str(e)})}\n\n") + except Exception as push_err: + logging.getLogger("omnivoice.tasks").warning("Failed to push error event for %s: %s", task_id, push_err) + finally: + await self._push_event(task_id, None) # EOF + self.queue.task_done() + +task_manager = TaskManager() diff --git a/backend/main.py b/backend/main.py index bf7d238f..3f755f27 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,2106 +1,109 @@ -import io -import os -import uuid -import json -import shutil -import sqlite3 -import tempfile -import asyncio -import subprocess -import logging -import time -import psutil -from contextlib import asynccontextmanager -from typing import Optional, List -from concurrent.futures import ThreadPoolExecutor - import os import sys -import site -import shutil -# [CRITICAL PRODUCTION PATCH] -# Systematically purge 'torchcodec' from the environment before PyTorch loads. -# This violently neutralizes PyTorch's broken dynamic linkage sequence against -# fragile Homebrew FFmpeg (.dylibs) on macOS, mandating a safe fallback to `soundfile`. -for sp in [sys.prefix] + site.getsitepackages(): - if "site-packages" not in sp: sp = os.path.join(sp, "lib", f"python3.{sys.version_info.minor}", "site-packages") - tc_path = os.path.join(sp, "torchcodec") - if os.path.exists(tc_path): - try: shutil.rmtree(tc_path); print("Sanitized broken torchcodec module.") - except: pass +try: + import dotenv + dotenv.load_dotenv() +except ImportError: + pass + +# Route HF/Torch caches to a single external directory when requested. +_cache_dir = os.environ.get("OMNIVOICE_CACHE_DIR") +if _cache_dir: + os.makedirs(_cache_dir, exist_ok=True) + os.environ["HF_HOME"] = _cache_dir + os.environ["HF_HUB_CACHE"] = _cache_dir + os.environ["TORCH_HOME"] = _cache_dir + +# Prevent torchaudio from lazy-importing torchcodec (broken on some installs). +# Proper fix = exclude torchcodec in pyproject.toml; this is a belt-and-braces guard. +os.environ.setdefault("TORCHAUDIO_USE_TORCHCODEC", "0") +sys.modules.setdefault("torchcodec", None) import soundfile as sf import torch import torchaudio - -# Enforce fully static soundfile backend import warnings +import logging + warnings.filterwarnings("ignore", category=UserWarning) torchaudio.set_audio_backend("soundfile") -from fastapi import FastAPI, File, Form, UploadFile, HTTPException, Query -from fastapi.responses import FileResponse, Response, StreamingResponse, JSONResponse, RedirectResponse -from fastapi.staticfiles import StaticFiles -from pydantic import BaseModel - -from omnivoice.models.omnivoice import OmniVoice - +logging.basicConfig( + level=os.environ.get("OMNIVOICE_LOG_LEVEL", "INFO"), + format="%(asctime)s %(levelname)s [%(name)s] %(message)s", +) logger = logging.getLogger("omnivoice.api") -# ═══════════════════════════════════════════════════════════════════════ -import platform -import sys -def get_app_data_dir(): - if sys.platform == "darwin": - return os.path.expanduser("~/Library/Application Support/OmniVoice") - elif sys.platform == "win32": - return os.path.join(os.environ.get("APPDATA", ""), "OmniVoice") - else: - return os.path.expanduser("~/.omnivoice") +import asyncio +import time +import threading +from contextlib import asynccontextmanager +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles +from fastapi.middleware.cors import CORSMiddleware +import traceback -DATA_DIR = get_app_data_dir() -VOICES_DIR = os.path.join(DATA_DIR, "voices") # Reference audio for profiles -OUTPUTS_DIR = os.path.join(DATA_DIR, "outputs") # Generated audio files -DUB_DIR = os.path.join(DATA_DIR, "dub_jobs") -DB_PATH = os.path.join(DATA_DIR, "omnivoice.db") +_crash_log_lock = threading.Lock() -for d in [DATA_DIR, VOICES_DIR, OUTPUTS_DIR, DUB_DIR]: - os.makedirs(d, exist_ok=True) +from core.db import init_db +from core.config import OUTPUTS_DIR, VOICES_DIR, CRASH_LOG_PATH +from core.tasks import task_manager +from services.model_manager import idle_worker - - -# Ensure ffmpeg is on PATH for Whisper and other subprocesses (mostly relevant for Mac/Linux) -if sys.platform != "win32": - for _fpath in ["/opt/homebrew/bin", "/usr/local/bin"]: - if _fpath not in os.environ.get("PATH", "") and os.path.exists(_fpath): - os.environ["PATH"] = _fpath + os.pathsep + os.environ.get("PATH", "") - -model: Optional[OmniVoice] = None -_model_lock = asyncio.Lock() -_last_used = time.time() -_IDLE_TIMEOUT_SECONDS = 300 # 5 minutes - -_gpu_pool = ThreadPoolExecutor(max_workers=1) -_cpu_pool = ThreadPoolExecutor(max_workers=os.cpu_count() or 4) -_dub_jobs = {} - - -# ═══════════════════════════════════════════════════════════════════════ -# ASYNC BATCH TASK MANAGER -# ═══════════════════════════════════════════════════════════════════════ - -class TaskManager: - def __init__(self): - self.queue = None - self.active_tasks = {} - - def _init_queue(self): - if self.queue is None: - self.queue = asyncio.Queue() - - async def add_task(self, task_id, task_type, func, *args, **kwargs): - self._init_queue() - task_obj = { - "status": "pending", - "type": task_type, - "created_at": time.time(), - "history": [], - "listeners": [], - "error": None - } - self.active_tasks[task_id] = task_obj - await self.queue.put((task_id, func, args, kwargs)) - - async def _push_event(self, task_id, event_str): - if task_id not in self.active_tasks: return - t = self.active_tasks[task_id] - if event_str is not None: - t["history"].append(event_str) - for q in t["listeners"]: - await q.put(event_str) - - async def worker(self): - self._init_queue() - while True: - task_id, func, args, kwargs = await self.queue.get() - t = self.active_tasks.get(task_id) - if not t: - self.queue.task_done() - continue - - t["status"] = "running" - try: - import inspect - res = func(*args, **kwargs) - if inspect.isasyncgen(res): - async for update in res: - await self._push_event(task_id, update) - elif inspect.iscoroutine(res): - await res - t["status"] = "done" - except Exception as e: - t["status"] = "failed" - t["error"] = str(e) - try: - await self._push_event(task_id, f"data: {json.dumps({'type': 'error', 'error': str(e)})}\n\n") - except: pass - finally: - await self._push_event(task_id, None) # EOF - self.queue.task_done() - -task_manager = TaskManager() - - -# ═══════════════════════════════════════════════════════════════════════ -# SQLITE DATABASE -# ═══════════════════════════════════════════════════════════════════════ - -def _get_db(): - conn = sqlite3.connect(DB_PATH) - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA journal_mode=WAL") - return conn - - -def _init_db(): - conn = _get_db() - conn.executescript(""" - CREATE TABLE IF NOT EXISTS voice_profiles ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - ref_audio_path TEXT, - ref_text TEXT DEFAULT '', - instruct TEXT DEFAULT '', - language TEXT DEFAULT 'Auto', - locked_audio_path TEXT DEFAULT '', - seed INTEGER DEFAULT NULL, - is_locked INTEGER DEFAULT 0, - created_at REAL - ); - CREATE TABLE IF NOT EXISTS generation_history ( - id TEXT PRIMARY KEY, - text TEXT, - mode TEXT, - language TEXT, - instruct TEXT, - profile_id TEXT, - audio_path TEXT, - duration_seconds REAL, - generation_time REAL, - seed INTEGER DEFAULT NULL, - created_at REAL, - FOREIGN KEY (profile_id) REFERENCES voice_profiles(id) - ); - CREATE TABLE IF NOT EXISTS dub_history ( - id TEXT PRIMARY KEY, - filename TEXT, - duration REAL, - segments_count INTEGER, - language TEXT, - language_code TEXT, - tracks TEXT DEFAULT '[]', - job_data TEXT, - created_at REAL - ); - CREATE TABLE IF NOT EXISTS studio_projects ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - video_path TEXT, - audio_path TEXT, - duration REAL, - state_json TEXT, - created_at REAL, - updated_at REAL - ); - CREATE TABLE IF NOT EXISTS export_history ( - id TEXT PRIMARY KEY, - filename TEXT, - destination_path TEXT, - mode TEXT, - created_at REAL - ); - """) - # Safe migrations for existing databases - for col, typedef in [ - ("locked_audio_path", "TEXT DEFAULT ''"), - ("seed", "INTEGER DEFAULT NULL"), - ("is_locked", "INTEGER DEFAULT 0"), - ]: - try: - conn.execute(f"ALTER TABLE voice_profiles ADD COLUMN {col} {typedef}") - except Exception: - pass # Column already exists - try: - conn.execute("ALTER TABLE generation_history ADD COLUMN seed INTEGER DEFAULT NULL") - except Exception: - pass - conn.commit() - conn.close() - - -# ═══════════════════════════════════════════════════════════════════════ -# APP LIFECYCLE -# ═══════════════════════════════════════════════════════════════════════ - -def get_best_device(): - if torch.cuda.is_available(): - return "cuda" - if torch.backends.mps.is_available(): - return "mps" - return "cpu" - - -def _load_model_sync(): - global model - device = get_best_device() - print(f"Loading OmniVoice model lazily on device: {device}...") - checkpoint = os.environ.get("OMNIVOICE_MODEL", "k2-fsa/OmniVoice") - _model = OmniVoice.from_pretrained( - checkpoint, device_map=device, dtype=torch.float16, load_asr=True, - ) - try: - if device == "cuda": - _model.llm = torch.compile(_model.llm, mode="reduce-overhead") - print("torch.compile applied.") - except Exception as e: - print(f"torch.compile skipped: {e}") - print("OmniVoice model loaded successfully.") - return _model - -async def get_model() -> OmniVoice: - global model, _last_used - _last_used = time.time() - if model is not None: - return model - - async with _model_lock: - if model is None: - loop = asyncio.get_running_loop() - model = await loop.run_in_executor(_gpu_pool, _load_model_sync) - return model - -async def _idle_worker(): - global model - while True: - await asyncio.sleep(30) - async with _model_lock: - if model is not None and time.time() - _last_used > _IDLE_TIMEOUT_SECONDS: - print("Idle timeout reached. Unloading OmniVoice model to free VRAM...") - model = None - import gc - gc.collect() - if torch.backends.mps.is_available(): - torch.mps.empty_cache() - elif torch.cuda.is_available(): - torch.cuda.empty_cache() +from api.routers import system, profiles, exports, generation, dub_core, dub_generate, dub_export, dub_translate, projects @asynccontextmanager async def lifespan(app: FastAPI): - _init_db() - idle_task = asyncio.create_task(_idle_worker()) + init_db() + idle_task = asyncio.create_task(idle_worker()) worker_task = asyncio.create_task(task_manager.worker()) yield idle_task.cancel() worker_task.cancel() - -from fastapi.middleware.cors import CORSMiddleware - app = FastAPI(title="OmniVoice Studio API", version="0.4.0", lifespan=lifespan) -from fastapi import Request -import traceback @app.exception_handler(Exception) async def global_exception_handler(request: Request, exc: Exception): - with open("crash_log.txt", "w") as f: - f.write(f"Request: {request.url}\n") - f.write(traceback.format_exc()) + try: + # Serialize writes so concurrent unhandled exceptions don't interleave frames. + with _crash_log_lock, open(CRASH_LOG_PATH, "a") as f: + f.write(f"\n--- {time.strftime('%Y-%m-%dT%H:%M:%S')} ---\n") + f.write(f"Request: {request.url}\n") + f.write(traceback.format_exc()) + except Exception: + logger.exception("Failed to write crash log") + logger.exception("Unhandled exception for %s", request.url) return JSONResponse({"detail": str(exc)}, status_code=500) +_allowed = os.environ.get( + "OMNIVOICE_ALLOWED_ORIGINS", + "http://localhost:5173,http://127.0.0.1:5173,tauri://localhost,http://tauri.localhost", +).split(",") + app.add_middleware( CORSMiddleware, - allow_origins=["*"], allow_credentials=True, + allow_origins=[o.strip() for o in _allowed if o.strip()], + allow_credentials=True, allow_methods=["*"], allow_headers=["*"], expose_headers=["Content-Disposition"], ) -# Serve generated audio files statically app.mount("/audio", StaticFiles(directory=OUTPUTS_DIR), name="audio") app.mount("/voice_audio", StaticFiles(directory=VOICES_DIR), name="voice_audio") -# ═══════════════════════════════════════════════════════════════════════ -# MODEL STATUS -# ═══════════════════════════════════════════════════════════════════════ -@app.get("/model/status") -def model_status(): - """Report model loading state for frontend warm-up indicators.""" - is_loaded = model is not None - is_loading = _model_lock.locked() if hasattr(_model_lock, 'locked') else False - return { - "loaded": is_loaded, - "loading": is_loading, - "status": "loading" if is_loading else ("ready" if is_loaded else "idle"), - } +app.include_router(system.router) +app.include_router(profiles.router) +app.include_router(exports.router) +app.include_router(generation.router) +app.include_router(dub_core.router) +app.include_router(dub_generate.router) +app.include_router(dub_export.router) +app.include_router(dub_translate.router) +app.include_router(projects.router) -# ═══════════════════════════════════════════════════════════════════════ -# SYSTEM STATS -# ═══════════════════════════════════════════════════════════════════════ - -@app.get("/sysinfo") -def get_sys_info(): - vram = 0.0 - gpu_active = False - - # Safely handle cross-platform (Mac Apple Silicon, Windows/Linux NVIDIA, CPU-only) - is_mac = hasattr(torch.backends, "mps") and torch.backends.mps.is_available() - is_cuda = torch.cuda.is_available() - - try: - if is_mac: - # PyTorch MPS uses current_allocated_memory / driver_allocated_memory - alloc = getattr(torch.mps, "current_allocated_memory", None) - driver = getattr(torch.mps, "driver_allocated_memory", None) - if driver: - vram = driver() / (1024**3) - elif alloc: - vram = alloc() / (1024**3) - elif is_cuda: - vram = torch.cuda.memory_allocated() / (1024**3) - except Exception: - pass - - if vram > 0.01: - gpu_active = True - - return { - "cpu": psutil.cpu_percent(interval=0.1), - "ram": psutil.virtual_memory().used / (1024**3), - "total_ram": psutil.virtual_memory().total / (1024**3), - "vram": round(vram, 2), - "gpu_active": gpu_active - } - -# ═══════════════════════════════════════════════════════════════════════ -# VOICE PROFILES (SQLite + disk) -# ═══════════════════════════════════════════════════════════════════════ - -@app.get("/profiles") -def list_profiles(): - conn = _get_db() - rows = conn.execute("SELECT * FROM voice_profiles ORDER BY created_at DESC").fetchall() - conn.close() - return [dict(r) for r in rows] - - -@app.post("/profiles") -async def create_profile( - name: str = Form(...), - ref_audio: UploadFile = File(...), - ref_text: str = Form(""), - instruct: str = Form(""), - language: str = Form("Auto"), - seed: Optional[int] = Form(None), -): - profile_id = str(uuid.uuid4())[:8] - ext = os.path.splitext(ref_audio.filename or ".wav")[1] - audio_filename = f"{profile_id}{ext}" - audio_path = os.path.join(VOICES_DIR, audio_filename) - - with open(audio_path, "wb") as f: - f.write(await ref_audio.read()) - - conn = _get_db() - conn.execute( - "INSERT INTO voice_profiles (id, name, ref_audio_path, ref_text, instruct, language, seed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - (profile_id, name, audio_filename, ref_text, instruct, language, seed, time.time()) - ) - conn.commit() - conn.close() - - return {"id": profile_id, "name": name} - - -@app.get("/profiles/{profile_id}/audio") -def get_profile_audio(profile_id: str): - """Serve the reference audio for a voice profile (for preview/playback).""" - conn = _get_db() - row = conn.execute("SELECT ref_audio_path, locked_audio_path FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() - conn.close() - if not row: - return Response("Profile not found", status_code=404) - # Prefer locked audio, fall back to ref audio - audio_file = row["locked_audio_path"] or row["ref_audio_path"] - if not audio_file: - return Response("No audio available", status_code=404) - audio_path = os.path.join(VOICES_DIR, audio_file) - if not os.path.exists(audio_path): - return Response("Audio file missing", status_code=404) - return FileResponse(audio_path, media_type="audio/wav") - -@app.post("/profiles/{profile_id}/lock") -async def lock_profile( - profile_id: str, - history_id: str = Form(...), - seed: Optional[int] = Form(None), -): - """Lock a voice profile by anchoring it to a specific generation's audio. - This converts a stochastic Design voice into a deterministic Clone-like voice.""" - conn = _get_db() - profile = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() - if not profile: - conn.close() - raise HTTPException(status_code=404, detail="Profile not found") - - history = conn.execute("SELECT * FROM generation_history WHERE id=?", (history_id,)).fetchone() - if not history or not history["audio_path"]: - conn.close() - raise HTTPException(status_code=404, detail="History item not found or has no audio") - - # Copy the generation's audio into the voices directory as the locked reference - src_path = os.path.join(OUTPUTS_DIR, history["audio_path"]) - if not os.path.exists(src_path): - conn.close() - raise HTTPException(status_code=404, detail="Audio file not found on disk") - - locked_filename = f"{profile_id}_locked.wav" - locked_path = os.path.join(VOICES_DIR, locked_filename) - import shutil - shutil.copy2(src_path, locked_path) - - # Also store the ref_text from the history item for better clone quality - ref_text = history["text"][:100] if history["text"] else "" - - conn.execute( - "UPDATE voice_profiles SET locked_audio_path=?, seed=?, is_locked=1, ref_text=? WHERE id=?", - (locked_filename, seed, ref_text, profile_id) - ) - conn.commit() - conn.close() - return {"locked": True, "profile_id": profile_id, "locked_audio_path": locked_filename} - - -@app.post("/profiles/{profile_id}/unlock") -async def unlock_profile(profile_id: str): - """Unlock a voice profile, reverting it to stochastic Design mode.""" - conn = _get_db() - profile = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() - if not profile: - conn.close() - raise HTTPException(status_code=404, detail="Profile not found") - - # Remove the locked audio file from disk - if profile["locked_audio_path"]: - locked_path = os.path.join(VOICES_DIR, profile["locked_audio_path"]) - if os.path.exists(locked_path): - os.remove(locked_path) - - conn.execute( - "UPDATE voice_profiles SET locked_audio_path='', seed=NULL, is_locked=0 WHERE id=?", - (profile_id,) - ) - conn.commit() - conn.close() - return {"unlocked": True, "profile_id": profile_id} - - -@app.delete("/profiles/{profile_id}") -def delete_profile(profile_id: str): - conn = _get_db() - row = conn.execute("SELECT ref_audio_path, locked_audio_path FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() - if row: - for col in ["ref_audio_path", "locked_audio_path"]: - if row[col]: - path = os.path.join(VOICES_DIR, row[col]) - if os.path.exists(path): - os.remove(path) - conn.execute("DELETE FROM voice_profiles WHERE id=?", (profile_id,)) - conn.commit() - conn.close() - return {"deleted": profile_id} - - -# ═══════════════════════════════════════════════════════════════════════ -# AUDIO CLEANING (Denoise mic recordings for cloning) -# ═══════════════════════════════════════════════════════════════════════ - -@app.post("/clean-audio") -async def clean_audio(audio: UploadFile = File(...)): - """Accept a raw mic recording, run demucs vocal isolation, return clean WAV.""" - clean_id = str(uuid.uuid4())[:8] - tmp_dir = os.path.join(OUTPUTS_DIR, f"_clean_{clean_id}") - os.makedirs(tmp_dir, exist_ok=True) - - # Save uploaded audio to a temp WAV - raw_path = os.path.join(tmp_dir, "raw.wav") - with open(raw_path, "wb") as f: - f.write(await audio.read()) - - # Convert to proper WAV format with ffmpeg (in case browser sends webm/ogg) - converted_path = os.path.join(tmp_dir, "converted.wav") - ffmpeg = _find_ffmpeg() - proc = await asyncio.create_subprocess_exec( - ffmpeg, "-y", "-i", raw_path, "-ar", "24000", "-ac", "1", converted_path, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - # Fallback: use raw file directly - converted_path = raw_path - - # Run demucs to isolate vocals - clean_path = converted_path # Fallback if demucs fails - try: - proc = await asyncio.create_subprocess_exec( - "uv", "run", "demucs", "--two-stems", "vocals", "-n", "htdemucs", - "-d", get_best_device(), converted_path, "-o", tmp_dir, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode == 0: - demucs_out = os.path.join(tmp_dir, "htdemucs", "converted") - vocals_file = os.path.join(demucs_out, "vocals.wav") - if os.path.exists(vocals_file): - clean_path = vocals_file - except Exception as e: - logger.warning(f"Demucs failed for mic audio, using raw: {e}") - - # Save final cleaned audio to outputs dir - clean_filename = f"mic_{clean_id}.wav" - final_path = os.path.join(OUTPUTS_DIR, clean_filename) - - # Re-encode to ensure proper WAV format - proc = await asyncio.create_subprocess_exec( - ffmpeg, "-y", "-i", clean_path, "-ar", "24000", "-ac", "1", final_path, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - ) - await proc.communicate() - if not os.path.exists(final_path): - import shutil - shutil.copy2(clean_path, final_path) - - # Clean up temp dir - import shutil - shutil.rmtree(tmp_dir, ignore_errors=True) - - # Return the cleaned WAV - return FileResponse(final_path, media_type="audio/wav", filename=clean_filename, - headers={"X-Clean-Filename": clean_filename}) - - -# ═══════════════════════════════════════════════════════════════════════ -# GENERATION HISTORY & EXPORTS (SQLite + disk) -# ═══════════════════════════════════════════════════════════════════════ - -class ExportRequest(BaseModel): - source_filename: str - destination_path: str - mode: str = "history" - -@app.post("/export") -def export_file(req: ExportRequest): - src_paths = [ - os.path.join(OUTPUTS_DIR, req.source_filename), - os.path.join("dub/outputs", req.source_filename) - ] - - found = False - for sp in src_paths: - if os.path.exists(sp): - try: - shutil.copy2(sp, req.destination_path) - found = True - break - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - if not found: - raise HTTPException(status_code=404, detail="Source file not found") - - export_id = str(uuid.uuid4())[:8] - conn = _get_db() - conn.execute( - "INSERT INTO export_history (id, filename, destination_path, mode, created_at) VALUES (?, ?, ?, ?, ?)", - (export_id, req.source_filename, req.destination_path, req.mode, time.time()) - ) - conn.commit() - conn.close() - return {"success": True, "id": export_id} - -class ExportRecordRequest(BaseModel): - filename: str - destination_path: str = "~/Downloads" - mode: str = "file" - -@app.post("/export/record") -def record_export(req: ExportRecordRequest): - """Record a blob-based download in export history (no file copy).""" - export_id = str(uuid.uuid4())[:8] - conn = _get_db() - conn.execute( - "INSERT INTO export_history (id, filename, destination_path, mode, created_at) VALUES (?, ?, ?, ?, ?)", - (export_id, req.filename, req.destination_path, req.mode, time.time()) - ) - conn.commit() - conn.close() - return {"success": True, "id": export_id} - -@app.get("/export/history") -def get_export_history(): - conn = _get_db() - rows = conn.execute("SELECT * FROM export_history ORDER BY created_at DESC LIMIT 50").fetchall() - conn.close() - return [dict(r) for r in rows] - -class RevealRequest(BaseModel): - path: str - -@app.post("/export/reveal") -def reveal_in_folder(req: RevealRequest): - """Open the containing folder of a file in the native OS file manager.""" - import platform - target = os.path.expanduser(req.path) - - # If the path is a file, reveal it selected; if dir, just open it - folder = target if os.path.isdir(target) else os.path.dirname(target) - - system = platform.system() - try: - if system == "Darwin": - # macOS: open Finder with file selected - if os.path.isfile(target): - subprocess.Popen(["open", "-R", target]) - else: - subprocess.Popen(["open", folder]) - elif system == "Windows": - # Windows: Explorer with file selected - if os.path.isfile(target): - subprocess.Popen(["explorer", "/select,", target.replace("/", "\\")]) - else: - subprocess.Popen(["explorer", folder.replace("/", "\\")]) - else: - # Linux: xdg-open the containing folder - subprocess.Popen(["xdg-open", folder]) - return {"success": True} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - -@app.get("/history") -def list_history(): - conn = _get_db() - rows = conn.execute("SELECT * FROM generation_history ORDER BY created_at DESC LIMIT 50").fetchall() - conn.close() - return [dict(r) for r in rows] - - -@app.delete("/history") -def clear_history(): - conn = _get_db() - rows = conn.execute("SELECT audio_path FROM generation_history").fetchall() - for r in rows: - if r["audio_path"]: - p = os.path.join(OUTPUTS_DIR, r["audio_path"]) - if os.path.exists(p): - os.remove(p) - conn.execute("DELETE FROM generation_history") - conn.commit() - conn.close() - return {"cleared": True} - -@app.delete("/history/{history_id}") -def delete_single_history(history_id: int): - conn = _get_db() - row = conn.execute("SELECT audio_path FROM generation_history WHERE id=?", (history_id,)).fetchone() - if row and row["audio_path"]: - p = os.path.join(OUTPUTS_DIR, row["audio_path"]) - if os.path.exists(p): - os.remove(p) - conn.execute("DELETE FROM generation_history WHERE id=?", (history_id,)) - conn.commit() - conn.close() - return {"deleted": True} - - -@app.get("/dub/history") -def list_dub_history(): - conn = _get_db() - rows = conn.execute("SELECT * FROM dub_history ORDER BY created_at DESC LIMIT 30").fetchall() - conn.close() - return [dict(r) for r in rows] - -@app.delete("/dub/history") -def clear_dub_history(): - conn = _get_db() - conn.execute("DELETE FROM dub_history") - conn.commit() - conn.close() - for item in os.listdir(DUB_DIR): - p = os.path.join(DUB_DIR, item) - if os.path.isdir(p): - import shutil - shutil.rmtree(p) - return {"cleared": True} - -@app.delete("/dub/history/{history_id}") -def delete_single_dub_history(history_id: int): - conn = _get_db() - conn.execute("DELETE FROM dub_history WHERE id=?", (history_id,)) - conn.commit() - conn.close() - return {"deleted": True} - - -# ═══════════════════════════════════════════════════════════════════════ -# TTS GENERATION -# ═══════════════════════════════════════════════════════════════════════ - -def _run_inference( - text, language, ref_audio_path, ref_text, instruct, duration, - num_step, guidance_scale, speed, t_shift, denoise, - postprocess_output, layer_penalty_factor, position_temperature, - class_temperature, used_seed, -): - if used_seed is not None: - torch.manual_seed(used_seed) - - kwargs = {} - if t_shift is not None: kwargs["t_shift"] = t_shift - if layer_penalty_factor is not None: kwargs["layer_penalty_factor"] = layer_penalty_factor - if position_temperature is not None: kwargs["position_temperature"] = position_temperature - if class_temperature is not None: kwargs["class_temperature"] = class_temperature - - audios = model.generate( - text=text, language=language, ref_audio=ref_audio_path, - ref_text=ref_text, instruct=instruct, duration=duration, - num_step=num_step, guidance_scale=guidance_scale, speed=speed, - denoise=denoise, postprocess_output=postprocess_output, - **kwargs - ) - return audios[0] # shape (1, T) - - -@app.post("/generate") -async def generate_speech( - text: str = Form(...), - language: Optional[str] = Form(None), - ref_audio: Optional[UploadFile] = File(None), - ref_text: Optional[str] = Form(None), - instruct: Optional[str] = Form(None), - duration: Optional[float] = Form(None), - num_step: int = Form(16), - guidance_scale: float = Form(2.0), - speed: float = Form(1.0), - t_shift: Optional[float] = Form(None), - denoise: bool = Form(True), - postprocess_output: bool = Form(True), - layer_penalty_factor: Optional[float] = Form(None), - position_temperature: Optional[float] = Form(None), - class_temperature: Optional[float] = Form(None), - profile_id: Optional[str] = Form(None), - seed: Optional[int] = Form(None), -): - _model = await get_model() - - ref_audio_path = None - cleanup_ref = False - used_seed = seed - - # Load from voice profile if specified - if profile_id: - conn = _get_db() - row = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() - conn.close() - if row: - # If the profile is LOCKED, use the locked audio as ref_audio (clone path) - # This is the key to voice consistency — the locked audio anchors the identity - if row["is_locked"] and row["locked_audio_path"]: - ref_audio_path = os.path.join(VOICES_DIR, row["locked_audio_path"]) - if not ref_text: - ref_text = row["ref_text"] - # Still pass instruct for style control on top of the locked voice - if not instruct: - instruct = row["instruct"] - # Use the profile's saved seed for maximum determinism - if used_seed is None and row["seed"] is not None: - used_seed = row["seed"] - elif row["instruct"] and not row["is_locked"]: - # UNLOCKED Design voice (personality): - # DO NOT pass ref_audio/ref_text (the test words from creation), - # because short dub segments cause F5-TTS to leak the test words into output. - # Instead, we just pass the personality instruct and the seed (if any) - if not instruct: - instruct = row["instruct"] - if used_seed is None and row["seed"] is not None: - used_seed = row["seed"] - else: - # Pure Clone voice (no instruct) - ref_audio_path = os.path.join(VOICES_DIR, row["ref_audio_path"]) if row["ref_audio_path"] else None - if not ref_text and row["ref_text"]: - ref_text = row["ref_text"] - if not instruct and row["instruct"]: - instruct = row["instruct"] - if used_seed is None and row["seed"] is not None: - used_seed = row["seed"] - if language == "Auto": - language = None - elif ref_audio is not None: - try: - with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f: - f.write(await ref_audio.read()) - ref_audio_path = f.name - cleanup_ref = True - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - start_time = time.time() - try: - loop = asyncio.get_event_loop() - audio_tensor = await loop.run_in_executor( - _gpu_pool, _run_inference, - text, language, ref_audio_path, ref_text, instruct, duration, - num_step, guidance_scale, speed, t_shift, denoise, - postprocess_output, layer_penalty_factor, position_temperature, - class_temperature, used_seed, - ) - gen_time = round(time.time() - start_time, 2) - - # Save to disk + DB - audio_id = str(uuid.uuid4())[:8] - audio_filename = f"{audio_id}.wav" - audio_path = os.path.join(OUTPUTS_DIR, audio_filename) - torchaudio.save(audio_path, audio_tensor, model.sampling_rate) - - audio_dur = round(audio_tensor.shape[-1] / model.sampling_rate, 2) - - conn = _get_db() - conn.execute( - "INSERT INTO generation_history (id, text, mode, language, instruct, profile_id, audio_path, duration_seconds, generation_time, seed, created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)", - (audio_id, text[:200], "clone" if ref_audio_path else "design", - language or "Auto", instruct or "", profile_id or "", - audio_filename, audio_dur, gen_time, used_seed, time.time()) - ) - conn.commit() - conn.close() - - # Stream WAV bytes in chunks for progressive playback - buffer = io.BytesIO() - torchaudio.save(buffer, audio_tensor, model.sampling_rate, format="wav") - buffer.seek(0) - wav_bytes = buffer.read() - - async def _stream_wav(): - chunk_size = 16384 # 16KB chunks for smooth streaming - for i in range(0, len(wav_bytes), chunk_size): - yield wav_bytes[i:i + chunk_size] - - return StreamingResponse( - _stream_wav(), - media_type="audio/wav", - headers={ - "X-Audio-Id": audio_id, - "X-Gen-Time": str(gen_time), - "X-Audio-Path": audio_filename, - "X-Seed": str(used_seed) if used_seed is not None else "", - "X-Audio-Duration": str(audio_dur), - "Content-Length": str(len(wav_bytes)), - } - ) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Inference failed: {str(e)}") - finally: - if cleanup_ref and ref_audio_path and os.path.exists(ref_audio_path): - os.remove(ref_audio_path) - - -# ═══════════════════════════════════════════════════════════════════════ -# VIDEO DUBBING PIPELINE -# ═══════════════════════════════════════════════════════════════════════ - -_diar_pipeline = None - -def _get_diarization_pipeline(): - global _diar_pipeline - hf_token = os.environ.get("HF_TOKEN") - if not hf_token: - return None - if _diar_pipeline is not None: - return _diar_pipeline - try: - import torch - from pyannote.audio import Pipeline - logger.info("Loading Pyannote Diarization Pipeline...") - _diar_pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", use_auth_token=hf_token) - if torch.cuda.is_available(): - _diar_pipeline.to(torch.device("cuda")) - logger.info("Pyannote Diarization Pipeline loaded successfully.") - return _diar_pipeline - except Exception as e: - logger.error(f"Failed to load Pyannote pipeline: {e}") - return None - -def _find_ffmpeg(): - try: - import imageio_ffmpeg - # This will natively extract and return an architecture-specific static FFmpeg binary! - return imageio_ffmpeg.get_ffmpeg_exe() - except Exception as e: - logger.warning(f"imageio_ffmpeg failed to provide static binary: {e}. Falling back to default system path.") - for path in ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "ffmpeg"]: - if shutil.which(path): - return path - raise RuntimeError("ffmpeg not found in bundle or system path") - - - -# ── Preview file proxy (avoids blob: URLs which fail in Tauri's WebKit) ──────── -PREVIEW_DIR = os.path.join(DATA_DIR, "preview") -os.makedirs(PREVIEW_DIR, exist_ok=True) - -@app.post("/preview/upload") -async def preview_upload(video: UploadFile = File(...)): - """Save a video or audio to a temp location and return HTTP URLs for playback. - Automatically extracts an audio-only WAV to bypass WebKit MediaElement decoding bugs on macOS. - """ - ext = os.path.splitext(video.filename or "video.mp4")[1].lower() - safe_name = f"{uuid.uuid4().hex[:12]}" - vid_path = os.path.join(PREVIEW_DIR, f"{safe_name}{ext}") - wav_path = os.path.join(PREVIEW_DIR, f"{safe_name}.wav") - - with open(vid_path, "wb") as f: - f.write(await video.read()) - - has_audio = False - if ext not in [".wav", ".mp3", ".m4a", ".aac"]: - try: - ffmpeg_cmd = [ - _find_ffmpeg(), "-y", "-i", vid_path, - "-vn", "-acodec", "pcm_s16le", "-ar", "22050", "-ac", "1", - wav_path - ] - subprocess.run(ffmpeg_cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - has_audio = True - except Exception as e: - logger.warning(f"FFmpeg extraction failed: {e}") - pass - - return { - "url": f"/preview/{safe_name}{ext}", - "audioUrl": f"/preview/{safe_name}.wav" if has_audio else f"/preview/{safe_name}{ext}", - "filename": video.filename, "TESTING": "YES" - } - -@app.get("/preview/{filename}") -async def preview_serve(filename: str): - path = os.path.join(PREVIEW_DIR, filename) - if not os.path.isfile(path): - raise HTTPException(404, "Preview not found") - ext = os.path.splitext(filename)[1].lower() - media_types = { - ".mp4": "video/mp4", ".mov": "video/quicktime", - ".mkv": "video/x-matroska", ".webm": "video/webm", - ".avi": "video/x-msvideo", ".wav": "audio/wav", - ".mp3": "audio/mpeg" - } - return FileResponse(path, media_type=media_types.get(ext, "application/octet-stream")) - -@app.post("/dub/upload") -async def dub_upload(video: UploadFile = File(...)): - job_id = str(uuid.uuid4())[:8] - job_dir = os.path.join(DUB_DIR, job_id) - os.makedirs(job_dir, exist_ok=True) - - ext = os.path.splitext(video.filename or "video.mp4")[1] - video_path = os.path.join(job_dir, f"original{ext}") - with open(video_path, "wb") as f: - f.write(await video.read()) - - audio_path = os.path.join(job_dir, "audio.wav") - ffmpeg = _find_ffmpeg() - try: - proc = await asyncio.create_subprocess_exec( - ffmpeg, "-i", video_path, "-vn", "-acodec", "pcm_s16le", - "-ar", "16000", "-ac", "1", audio_path, "-y", - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - raise Exception(stderr.decode()) - except Exception as e: - raise HTTPException(status_code=500, detail=f"ffmpeg failed: {str(e)}") - - try: - dur = float(sf.info(audio_path).frames) / float(sf.info(audio_path).samplerate) - except Exception: - dur = 0.0 - - vocals_path = os.path.join(job_dir, "vocals.wav") - no_vocals_path = os.path.join(job_dir, "no_vocals.wav") - scene_cuts = [] - - async def run_demucs(): - nonlocal vocals_path, no_vocals_path - try: - # Run demucs CLI asynchronously to strictly output 2 stems - proc = await asyncio.create_subprocess_exec( - "uv", "run", "demucs", "--two-stems", "vocals", "-n", "htdemucs", "-d", get_best_device(), - audio_path, "-o", job_dir, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - raise Exception(stderr.decode()) - - # Demucs creates an output structure: htdemucs/audio/vocals.wav - demucs_out = os.path.join(job_dir, "htdemucs", "audio") - if os.path.exists(os.path.join(demucs_out, "vocals.wav")): - import shutil - shutil.move(os.path.join(demucs_out, "vocals.wav"), vocals_path) - shutil.move(os.path.join(demucs_out, "no_vocals.wav"), no_vocals_path) - shutil.rmtree(os.path.join(job_dir, "htdemucs")) - except Exception as e: - logger.warning(f"Demucs failed, falling back to mixed audio. {e}") - vocals_path = audio_path - no_vocals_path = None - - async def run_scene_detection(): - nonlocal scene_cuts - try: - scene_proc = await asyncio.create_subprocess_exec( - ffmpeg, "-i", video_path, "-filter:v", "select='gt(scene,0.3)',showinfo", "-f", "null", "-", - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _, stderr_scene = await scene_proc.communicate() - import re - matches = re.finditer(r"pts_time:([\d\.]+)", stderr_scene.decode()) - scene_cuts = [float(m.group(1)) for m in matches] - except Exception as e: - logger.warning(f"Scene detection failed: {e}") - - await asyncio.gather(run_demucs(), run_scene_detection()) - - _dub_jobs[job_id] = { - "video_path": video_path, - "audio_path": audio_path, - "vocals_path": vocals_path, - "no_vocals_path": no_vocals_path, - "duration": dur, "filename": video.filename, "TESTING": "YES", - "segments": None, "dubbed_tracks": {}, - "scene_cuts": scene_cuts, - } - return {"job_id": job_id, "duration": round(dur, 2), "filename": video.filename, "TESTING": "YES"} - - -def _get_job(job_id: str): - if job_id in _dub_jobs: - return _dub_jobs[job_id] - conn = _get_db() - row = conn.execute("SELECT job_data FROM dub_history WHERE id=?", (job_id,)).fetchone() - conn.close() - if row and row["job_data"]: - try: - job = json.loads(row["job_data"]) - _dub_jobs[job_id] = job - return job - except: - pass - return None - -@app.post("/dub/transcribe/{job_id}") -async def dub_transcribe(job_id: str): - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - _model = await get_model() - if _model._asr_pipe is None: - raise HTTPException(status_code=503, detail="ASR not loaded") - - def _transcribe(): - import re - import traceback - - # Safe fallback check if user omitted Demucs or the pipeline completely crashed during source vocal separation! - asr_audio_target = job.get("vocals_path") - if not asr_audio_target or not os.path.exists(asr_audio_target): - asr_audio_target = job.get("audio_path") - - audio_np, sr = sf.read(asr_audio_target, dtype="float32") - if audio_np.ndim > 1: - audio_np = audio_np.mean(axis=1) - audio_input = {"array": audio_np, "sampling_rate": sr} - - bs = 16 if torch.cuda.is_available() else (2 if torch.backends.mps.is_available() else 1) - # Use chunk-level timestamps - result = _model._asr_pipe( - audio_input, return_timestamps=True, - chunk_length_s=15, batch_size=bs, - ) - - # Split chunks into sentences using punctuation - sentence_enders = re.compile(r'(?<=[.!?。？！])\s+') - segments = [] - - if "chunks" in result: - for chunk in result["chunks"]: - ts = chunk.get("timestamp", (0, 0)) - chunk_start = ts[0] if ts[0] is not None else 0.0 - chunk_end = ts[1] if ts[1] is not None else chunk_start + 1.0 - chunk_text = chunk.get("text", "").strip() - - if not chunk_text: - continue - - # Split this chunk into sentences - sentences = sentence_enders.split(chunk_text) - sentences = [s.strip() for s in sentences if s.strip()] - - if len(sentences) <= 1: - segments.append({ - "start": round(chunk_start, 2), - "end": round(chunk_end, 2), - "text": chunk_text, - }) - else: - # Distribute time proportionally across sentences - total_chars = sum(len(s) for s in sentences) - chunk_dur = chunk_end - chunk_start - t = chunk_start - for sent in sentences: - ratio = len(sent) / max(total_chars, 1) - sent_dur = chunk_dur * ratio - segments.append({ - "start": round(t, 2), - "end": round(t + sent_dur, 2), - "text": sent, - }) - t += sent_dur - else: - segments.append({"start": 0.0, "end": job["duration"], "text": result.get("text", "").strip()}) - # Apply Diarization (pyannote.audio if HF_TOKEN is set, else Heuristic fallback) - diar_pipe = _get_diarization_pipeline() - - if diar_pipe: - try: - asr_audio_target = job.get("vocals_path", job.get("audio_path")) - diarization = diar_pipe(asr_audio_target) - - for s in segments: - seg_mid = (s["start"] + s["end"]) / 2.0 - assigned_speaker = "Speaker 1" - for turn, _, speaker in diarization.itertracks(yield_label=True): - if turn.start <= seg_mid <= turn.end: - # Map pyannote generic "SPEAKER_00" to "Speaker 1" - speaker_idx = int(speaker.split("_")[-1]) + 1 - assigned_speaker = f"Speaker {speaker_idx}" - break - s["speaker_id"] = assigned_speaker - s["id"] = str(uuid.uuid4())[:8] # assign fresh ID - except Exception as e: - logger.error(f"Pyannote diarization failed during inference: {e}. Falling back to heuristic.") - diar_pipe = None - - if not diar_pipe: - # Fallback heuristic - current_speaker_idx = 1 - last_end = 0.0 - - for i, s in enumerate(segments): - if i > 0 and (s["start"] - last_end) > 1.2: - current_speaker_idx = 2 if current_speaker_idx == 1 else 1 - s["speaker_id"] = f"Speaker {current_speaker_idx}" - s["id"] = str(uuid.uuid4())[:8] # assign fresh ID - last_end = s["end"] - - # --- SCENE-AWARE DUBBING --- - scene_cuts = job.get("scene_cuts", []) - if scene_cuts: - sorted_cuts = sorted(scene_cuts) - new_segments = [] - for s in segments: - s_start = s["start"] - s_end = s["end"] - valid_cuts = [c for c in sorted_cuts if c > s_start + 0.2 and c < s_end - 0.2] - - if not valid_cuts: - new_segments.append(s) - else: - curr_start = s_start - curr_text = s["text"] - total_dur = s_end - s_start - - for cut in valid_cuts: - ratio = (cut - curr_start) / max(total_dur, 0.01) - split_idx = int(len(curr_text) * ratio) - # Avoid splitting words exactly in half if possible - space_idx = curr_text.rfind(' ', 0, split_idx + 5) - if space_idx != -1 and space_idx > split_idx - 10: - split_idx = space_idx - - part_text = curr_text[:split_idx].strip() - curr_text = curr_text[split_idx:].strip() - - if part_text: - new_seg = dict(s) - new_seg["start"] = round(curr_start, 2) - new_seg["end"] = round(cut, 2) - new_seg["text"] = part_text - new_seg["id"] = str(uuid.uuid4())[:8] - new_segments.append(new_seg) - - curr_start = cut - total_dur = s_end - curr_start - - if curr_text: - new_seg = dict(s) - new_seg["start"] = round(curr_start, 2) - new_seg["end"] = round(s_end, 2) - new_seg["text"] = curr_text - new_seg["id"] = str(uuid.uuid4())[:8] - new_segments.append(new_seg) - segments = new_segments - - # Store full transcript - job["full_transcript"] = " ".join(s["text"] for s in segments) - - # Free MPS memory - if torch.backends.mps.is_available(): - torch.mps.empty_cache() - - return segments - - try: - loop = asyncio.get_event_loop() - segments_result = await loop.run_in_executor(_gpu_pool, _transcribe) - job["segments"] = segments_result - return { - "job_id": job_id, - "segments": segments_result, - "full_transcript": job.get("full_transcript", ""), - } - except Exception as e: - import traceback - traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) - - -class DubSegment(BaseModel): - start: float - end: float - text: str - instruct: str = "" # Per-segment voice override - profile_id: str = "" # Per-segment voice profile - speed: Optional[float] = None - gain: Optional[float] = None # Per-segment volume (0.0 - 2.0, default 1.0) - - - -class DubRequest(BaseModel): - segments: List[DubSegment] - language: str = "Auto" - language_code: str = "und" # ISO 639-1 for ffmpeg metadata (e.g. "es", "fr", "de") - instruct: str = "" - num_step: int = 16 - guidance_scale: float = 2.0 - speed: float = 1.0 - - -@app.post("/dub/generate/{job_id}") -async def dub_generate(job_id: str, req: DubRequest): - """Adds a dub generation job to the async batch task pool.""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - _model = await get_model() - - async def _stream(): - total = len(req.segments) - all_segment_wavs = [] - sync_scores = [] - - for i, seg in enumerate(req.segments): - yield f"data: {json.dumps({'type': 'progress', 'current': i, 'total': total, 'text': seg.text[:50]})}\n\n" - - seg_duration = seg.end - seg.start - if seg_duration <= 0.05 or not seg.text.strip(): - sr = _model.sampling_rate - silence = torch.zeros(1, int(seg_duration * sr)) - all_segment_wavs.append((seg.start, seg.end, silence, sr)) - sync_scores.append(1.0) - continue - - def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id=None): - ref_audio = None - ref_text = None - used_seed = None - - # Load per-segment voice profile if specified - if profile_id: - conn = _get_db() - row = conn.execute("SELECT * FROM voice_profiles WHERE id=?", (profile_id,)).fetchone() - conn.close() - if row: - if row["is_locked"] and row["locked_audio_path"]: - # Locked voice: Use anchor audio & transcript - ref_audio = os.path.join(VOICES_DIR, row["locked_audio_path"]) - ref_text = row["ref_text"] - used_seed = row["seed"] - elif row["instruct"] and not row["is_locked"]: - # UNLOCKED Design voice (personality): - # DO NOT pass ref_audio/ref_text (the test words from creation), - # because short dub segments cause F5-TTS to leak the test words into output. - # Instead, we just pass the personality instruct and the seed (if any) - # to get a seamless zero-shot personality locking. - used_seed = row["seed"] - else: - # Pure Clone voice (no instruct) - ref_audio = os.path.join(VOICES_DIR, row["ref_audio_path"]) - ref_text = row["ref_text"] - used_seed = row["seed"] - - if not instruct_str: - instruct_str = row["instruct"] - - if used_seed is not None: - torch.manual_seed(used_seed) - - return _model.generate( - text=text, language=lang if lang != "Auto" else None, - ref_audio=ref_audio, ref_text=ref_text, - instruct=instruct_str if instruct_str else None, - duration=dur_s, num_step=nstep, guidance_scale=cfg, - speed=spd, denoise=True, postprocess_output=True, - )[0] - - # Use per-segment parameters if set, otherwise fall back to request-level - seg_instruct = seg.instruct or req.instruct - seg_profile = seg.profile_id or None - seg_speed = seg.speed if hasattr(seg, 'speed') and seg.speed is not None else req.speed - - loop = asyncio.get_event_loop() - try: - audio_tensor = await loop.run_in_executor( - _gpu_pool, _gen, - seg.text, req.language, seg_instruct, seg_duration, - req.num_step, req.guidance_scale, seg_speed, seg_profile, - ) - - # Calculate lip-sync score natively from tensor - generated_dur = audio_tensor.shape[-1] / _model.sampling_rate - sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3) - - # Auto time-stretch to fit segment window if off by >5% - if sync_ratio > 1.05 or sync_ratio < 0.95: - target_samples = int(seg_duration * _model.sampling_rate) - current_samples = audio_tensor.shape[-1] - if target_samples > 0 and current_samples > 0: - # Resample to effectively time-stretch: change "perceived" sample rate - # then resample back to actual rate - stretch_ratio = current_samples / target_samples - # Use interpolation for clean time-stretching - audio_tensor = torch.nn.functional.interpolate( - audio_tensor.unsqueeze(0), # add batch dim - size=target_samples, - mode='linear', - align_corners=False, - ).squeeze(0) # remove batch dim - generated_dur = audio_tensor.shape[-1] / _model.sampling_rate - sync_ratio = round(generated_dur / max(seg_duration, 0.01), 3) - - sync_scores.append(sync_ratio) - - # Save individual segment WAV for preview - seg_wav_path = os.path.join(DUB_DIR, job_id, f"seg_{i}.wav") - torchaudio.save(seg_wav_path, audio_tensor, _model.sampling_rate) - all_segment_wavs.append((seg.start, seg.end, audio_tensor, _model.sampling_rate)) - except Exception as e: - yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error': str(e)})}\n\n" - sr = _model.sampling_rate - all_segment_wavs.append((seg.start, seg.end, torch.zeros(1, int(seg_duration * sr)), sr)) - sync_scores.append(1.0) - - yield f"data: {json.dumps({'type': 'assembling'})}\n\n" - - sr = _model.sampling_rate - total_samples = int(job["duration"] * sr) - full_audio = torch.zeros(1, total_samples) - - for i, (start, end, wav, _) in enumerate(all_segment_wavs): - s = int(start * sr) - # Apply per-segment gain - seg_gain = req.segments[i].gain if req.segments[i].gain is not None else 1.0 - seg_gain = max(0.0, min(2.0, seg_gain)) # Clamp 0-2x - adjusted = wav * seg_gain - wl = adjusted.shape[-1] - e = min(s + wl, total_samples) - full_audio[:, s:e] = adjusted[:, :e - s] - - # Save this dubbed track with the language code - lang_code = req.language_code or "und" - track_path = os.path.join(DUB_DIR, job_id, f"dubbed_{lang_code}.wav") - torchaudio.save(track_path, full_audio, sr) - job["dubbed_tracks"][lang_code] = { - "path": track_path, - "language": req.language, - "language_code": lang_code, - } - - # Save to dub_history - try: - conn = _get_db() - conn.execute( - "INSERT OR REPLACE INTO dub_history (id, filename, duration, segments_count, language, language_code, tracks, job_data, created_at) VALUES (?,?,?,?,?,?,?,?,?)", - (job_id, job.get("filename", ""), job.get("duration", 0), total, - req.language, lang_code, json.dumps(list(job["dubbed_tracks"].keys())), - json.dumps(job, default=str), time.time()) - ) - conn.commit() - conn.close() - except Exception as e: - logger.error(f"Failed to save dub history: {e}") - - yield f"data: {json.dumps({'type': 'done', 'segments_processed': total, 'language_code': lang_code, 'tracks': list(job['dubbed_tracks'].keys()), 'sync_scores': sync_scores})}\n\n" - - task_id = f"dub_{job_id}_{int(time.time())}" - await task_manager.add_task(task_id, "dub_generate", _stream) - return {"task_id": task_id} - -@app.get("/tasks/stream/{task_id}") -async def stream_task(task_id: str): - """Universal Server-Sent Event stream for background tasks.""" - if task_id not in task_manager.active_tasks: - raise HTTPException(status_code=404, detail="Task not found") - - async def _reader(): - t = task_manager.active_tasks[task_id] - q = asyncio.Queue() - t["listeners"].append(q) - - try: - for evt in t["history"]: - yield evt - - if t["status"] in ("done", "failed"): - return - - while True: - evt = await q.get() - if evt is None: - break - yield evt - finally: - t["listeners"].remove(q) - - return StreamingResponse(_reader(), media_type="text/event-stream") - - -@app.get("/dub/tracks/{job_id}") -async def dub_list_tracks(job_id: str): - """List all dubbed language tracks for a job.""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - return {"tracks": job.get("dubbed_tracks", {})} - - -@app.get("/dub/download/{job_id}") -@app.get("/dub/download/{job_id}/{filename}") -async def dub_download(job_id: str, preserve_bg: bool = Query(True, description="Mix background noise into dubbed tracks"), default_track: str = Query("original"), include_tracks: str = Query("", description="Comma-separated list of tracks to include (e.g. 'original,de,es'). Empty = include all.")): - """Mux selected dubbed language tracks into the video. - If preserve_bg=true, mixes isolated background noise seamlessly into each dubbed string. - If default_track is 'original', sets the original audio as default track. - If default_track is a language code, sets that dubbed track as default. - If include_tracks is provided, only include those specific tracks.""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - tracks = job.get("dubbed_tracks", {}) - if not tracks: - raise HTTPException(status_code=400, detail="No dubbed tracks generated yet") - - # Parse include_tracks filter - include_set = set(t.strip() for t in include_tracks.split(",") if t.strip()) if include_tracks else None - include_original = include_set is None or "original" in include_set - - # Filter dubbed tracks if include_set is specified - if include_set: - filtered_tracks = {k: v for k, v in tracks.items() if k in include_set} - else: - filtered_tracks = dict(tracks) - - if not filtered_tracks and not include_original: - raise HTTPException(status_code=400, detail="No tracks selected for export") - - video_path = job["video_path"] - output_path = os.path.join(DUB_DIR, job_id, "dubbed_video_final.mp4") - ffmpeg = _find_ffmpeg() - - cmd = [ffmpeg, "-i", video_path] - input_idx = 1 - - bg_audio = job.get("no_vocals_path") if preserve_bg else None - bg_idx = None - if bg_audio and os.path.exists(bg_audio) and filtered_tracks: - cmd += ["-i", bg_audio] - bg_idx = input_idx - input_idx += 1 - - tracks_to_process = [] - for lang_code, track_info in filtered_tracks.items(): - cmd += ["-i", track_info["path"]] - tracks_to_process.append({"lang_code": lang_code, "idx": input_idx, "info": track_info}) - input_idx += 1 - - # Map original video - cmd += ["-map", "0:v:0"] - - # Only include original audio if selected - if include_original: - cmd += ["-map", "0:a:0"] - - if bg_idx is not None: - filters = [] - for i, t in enumerate(tracks_to_process): - out_label = f"[aout{i}]" - # Normalize mixing so neither drops off unexpectedly - filters.append(f"[{bg_idx}:a][{t['idx']}:a]amix=inputs=2:duration=longest:dropout_transition=2:weights=0.8 1.2{out_label}") - t["out_label"] = out_label - cmd += ["-filter_complex", ";".join(filters)] - for t in tracks_to_process: - cmd += ["-map", t["out_label"]] - else: - for t in tracks_to_process: - cmd += ["-map", f"{t['idx']}:a:0"] - - cmd += ["-c:v", "copy", "-c:a", "aac", "-b:a", "192k"] - - # Build audio stream metadata with correct indices - audio_stream_idx = 0 - - if include_original: - cmd += [f"-metadata:s:a:{audio_stream_idx}", "language=und", f"-metadata:s:a:{audio_stream_idx}", "title=Original"] - audio_stream_idx += 1 - - for t in tracks_to_process: - cmd += [ - f"-metadata:s:a:{audio_stream_idx}", f"language={t['lang_code']}", - f"-metadata:s:a:{audio_stream_idx}", f"title={t['info']['language']}" - ] - t["stream_idx"] = audio_stream_idx - audio_stream_idx += 1 - - # Set all dispositions to 0 first, then set the default - total_audio = (1 if include_original else 0) + len(tracks_to_process) - for i in range(total_audio): - cmd += [f"-disposition:a:{i}", "0"] - - if default_track == "original" and include_original: - cmd += ["-disposition:a:0", "default"] - else: - # Find the stream index for the default track - target_idx = 0 - for t in tracks_to_process: - if t['lang_code'] == default_track: - target_idx = t["stream_idx"] - break - cmd += [f"-disposition:a:{target_idx}", "default"] - - cmd += ["-shortest", output_path, "-y"] - - try: - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - raise Exception(stderr.decode()) - except Exception as e: - raise HTTPException(status_code=500, detail=f"ffmpeg mux failed: {str(e)}") - - base_name = os.path.splitext(job.get('filename', 'output'))[0] - safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'output' - dl_name = f"dubbed_{safe_name}.mp4" - return FileResponse( - output_path, media_type="video/mp4", - headers={"Content-Disposition": f'attachment; filename="{dl_name}"'}, - ) - - -# ═══════════════════════════════════════════════════════════════════════ -# TRANSLATION -# ═══════════════════════════════════════════════════════════════════════ - -# Google Translate language codes for common dub targets -TRANSLATE_CODES = { - "en": "en", "es": "es", "fr": "fr", "de": "de", "it": "it", "pt": "pt", - "ru": "ru", "ja": "ja", "ko": "ko", "zh": "zh-CN", "ar": "ar", "hi": "hi", - "tr": "tr", "pl": "pl", "nl": "nl", "sv": "sv", "th": "th", "vi": "vi", - "id": "id", "uk": "uk", -} - - -class TranslateRequest(BaseModel): - segments: List[dict] # [{"id": 0, "text": "..."}] - target_lang: str # ISO 639-1 code like "es", "fr" - - -@app.post("/dub/translate") -async def dub_translate(req: TranslateRequest): - """Translate all segment texts to the target language using Google Translate.""" - try: - from deep_translator import GoogleTranslator - - lang_code = TRANSLATE_CODES.get(req.target_lang, req.target_lang) - loop = asyncio.get_event_loop() - - def _translate_single(seg): - try: - translator = GoogleTranslator(source="auto", target=lang_code) - translated = translator.translate(seg["text"]) - return {"id": seg["id"], "text": translated or seg["text"]} - except Exception as e: - return {"id": seg["id"], "text": seg["text"], "error": str(e)} - - tasks = [ - loop.run_in_executor(_cpu_pool, _translate_single, seg) - for seg in req.segments - ] - translated = await asyncio.gather(*tasks) - translated.sort(key=lambda x: str(x["id"])) - - return {"translated": translated, "target_lang": req.target_lang} - except Exception as e: - import traceback; traceback.print_exc() - return JSONResponse(status_code=500, content={"error": str(e)}) - - -# ═══════════════════════════════════════════════════════════════════════ -# SEGMENT PREVIEW & MEDIA -# ═══════════════════════════════════════════════════════════════════════ - -@app.get("/dub/media/{job_id}") -async def dub_get_media(job_id: str): - """Return the original video file for timeline preview streaming.""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - if not os.path.exists(job["video_path"]): - raise HTTPException(status_code=404, detail="Media file not found") - return FileResponse(job["video_path"]) - - -@app.get("/dub/audio/{job_id}") -async def dub_get_audio(job_id: str): - """Return extracted audio.wav for waveform rendering (lighter than full video).""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - audio = job.get("audio_path") - if not audio or not os.path.exists(audio): - raise HTTPException(status_code=404, detail="Audio file not found") - return FileResponse(audio, media_type="audio/wav") - - -@app.get("/dub/preview/{job_id}/{segment_index}") -async def dub_preview_segment(job_id: str, segment_index: int): - """Return the WAV for a single dubbed segment (generated during /dub/generate).""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - seg_path = os.path.join(DUB_DIR, job_id, f"seg_{segment_index}.wav") - if not os.path.exists(seg_path): - raise HTTPException(status_code=404, detail="Segment not generated yet") - return FileResponse(seg_path, media_type="audio/wav") - - -# ═══════════════════════════════════════════════════════════════════════ -# AUDIO-ONLY DOWNLOAD (timestamp-synced) -# ═══════════════════════════════════════════════════════════════════════ - -@app.get("/dub/download-audio/{job_id}") -@app.get("/dub/download-audio/{job_id}/{filename}") -async def dub_download_audio(job_id: str, lang: str = Query(None), preserve_bg: bool = Query(True)): - """Download just the dubbed audio track (WAV). Timestamp-synced with original video.""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - - tracks = job.get("dubbed_tracks", {}) - - if lang and lang in tracks: - wav_path = tracks[lang]["path"] - elif tracks: - # Return first available track - wav_path = list(tracks.values())[0]["path"] - else: - raise HTTPException(status_code=400, detail="No dubbed audio track generated yet") - - if not os.path.exists(wav_path): - raise HTTPException(status_code=404, detail="Audio file not found") - - lang_label = lang or list(tracks.keys())[0] - base_name = os.path.splitext(job.get('filename', 'audio'))[0] - - bg_audio = job.get("no_vocals_path") if preserve_bg else None - if bg_audio and os.path.exists(bg_audio): - ffmpeg = _find_ffmpeg() - final_audio_path = os.path.join(DUB_DIR, job_id, f"mixed_dub_{lang_label}.wav") - cmd = [ - ffmpeg, "-i", bg_audio, "-i", wav_path, - "-filter_complex", "[0:a][1:a]amix=inputs=2:duration=longest:dropout_transition=2:weights=0.8 1.2[aout]", - "-map", "[aout]", "-c:a", "pcm_s16le", "-y", final_audio_path - ] - try: - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - raise Exception(stderr.decode()) - wav_path = final_audio_path - except Exception as e: - logger.error(f"Failed to mix audio: {str(e)}") - - base_name = os.path.splitext(job.get('filename', 'audio'))[0] - safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'audio' - dl_name = f"dubbed_audio_{lang_label}_{safe_name}.wav" - return FileResponse( - wav_path, media_type="audio/wav", - headers={"Content-Disposition": f'attachment; filename="{dl_name}"'}, - ) - - -# ═══════════════════════════════════════════════════════════════════════ -# SRT SUBTITLE EXPORT -# ═══════════════════════════════════════════════════════════════════════ - -def _format_srt_time(seconds): - """Format seconds as SRT timestamp: HH:MM:SS,mmm""" - h = int(seconds // 3600) - m = int((seconds % 3600) // 60) - s = int(seconds % 60) - ms = int((seconds % 1) * 1000) - return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" - - -@app.get("/dub/srt/{job_id}") -@app.get("/dub/srt/{job_id}/{filename}") -async def dub_export_srt(job_id: str): - """Export transcript segments as an SRT subtitle file.""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - - segments = job.get("segments", []) - if not segments: - raise HTTPException(status_code=400, detail="No transcript segments available") - - srt_lines = [] - for i, seg in enumerate(segments): - start_ts = _format_srt_time(seg["start"]) - end_ts = _format_srt_time(seg["end"]) - srt_lines.append(f"{i + 1}") - srt_lines.append(f"{start_ts} --> {end_ts}") - srt_lines.append(seg["text"]) - srt_lines.append("") - - srt_content = "\n".join(srt_lines) - - base_name = os.path.splitext(job.get('filename', 'video'))[0] - return Response( - content=srt_content, - media_type="text/plain", - headers={ - "Content-Disposition": f'attachment; filename="subtitles_{base_name}.srt"', - }, - ) - - -# ═══════════════════════════════════════════════════════════════════════ -# VTT SUBTITLE EXPORT -# ═══════════════════════════════════════════════════════════════════════ - -def _format_vtt_time(seconds): - """Format seconds as VTT timestamp: HH:MM:SS.mmm""" - h = int(seconds // 3600) - m = int((seconds % 3600) // 60) - s = int(seconds % 60) - ms = int((seconds % 1) * 1000) - return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" - - -@app.get("/dub/vtt/{job_id}") -@app.get("/dub/vtt/{job_id}/{filename}") -async def dub_export_vtt(job_id: str): - """Export transcript segments as a WebVTT subtitle file.""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - segments = job.get("segments", []) - if not segments: - raise HTTPException(status_code=400, detail="No transcript segments available") - - vtt_lines = ["WEBVTT", ""] - for i, seg in enumerate(segments): - start_ts = _format_vtt_time(seg["start"]) - end_ts = _format_vtt_time(seg["end"]) - vtt_lines.append(str(i + 1)) - vtt_lines.append(f"{start_ts} --> {end_ts}") - vtt_lines.append(seg["text"]) - vtt_lines.append("") - - vtt_content = "\n".join(vtt_lines) - base_name = os.path.splitext(job.get('filename', 'video'))[0] - return Response( - content=vtt_content, - media_type="text/vtt", - headers={ - "Content-Disposition": f'attachment; filename="subtitles_{base_name}.vtt"', - }, - ) - - -# ═══════════════════════════════════════════════════════════════════════ -# PER-SEGMENT WAV ZIP EXPORT -# ═══════════════════════════════════════════════════════════════════════ - -@app.get("/dub/export-segments/{job_id}") -async def dub_export_segments_zip(job_id: str): - """Export individually named WAV files for each dubbed segment as a ZIP archive.""" - import zipfile - - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - segments = job.get("segments", []) - if not segments: - raise HTTPException(status_code=400, detail="No segments available") - - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: - for i, seg in enumerate(segments): - seg_path = os.path.join(DUB_DIR, job_id, f"seg_{i}.wav") - if os.path.exists(seg_path): - speaker = seg.get("speaker_id", "Speaker1").replace(" ", "") - start_str = f"{seg['start']:.2f}" - end_str = f"{seg['end']:.2f}" - arc_name = f"{i+1:03d}_{start_str}-{end_str}_{speaker}.wav" - zf.write(seg_path, arc_name) - - zip_buffer.seek(0) - base_name = os.path.splitext(job.get('filename', 'video'))[0] - safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'segments' - return Response( - content=zip_buffer.read(), - media_type="application/zip", - headers={ - "Content-Disposition": f'attachment; filename="segments_{safe_name}.zip"', - }, - ) - - -# ═══════════════════════════════════════════════════════════════════════ -# MP3 AUDIO EXPORT -# ═══════════════════════════════════════════════════════════════════════ - -@app.get("/dub/download-mp3/{job_id}") -@app.get("/dub/download-mp3/{job_id}/{filename}") -async def dub_download_mp3(job_id: str, lang: str = Query(None), preserve_bg: bool = Query(True)): - """Export dubbed audio as compressed MP3 (192kbps). ~10x smaller than WAV.""" - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - tracks = job.get("dubbed_tracks", {}) - if lang and lang in tracks: - wav_path = tracks[lang]["path"] - elif tracks: - wav_path = list(tracks.values())[0]["path"] - else: - raise HTTPException(status_code=400, detail="No dubbed audio track generated yet") - - if not os.path.exists(wav_path): - raise HTTPException(status_code=404, detail="Audio file not found") - - lang_label = lang or list(tracks.keys())[0] - ffmpeg = _find_ffmpeg() - - # Optionally mix with background audio first - source_path = wav_path - bg_audio = job.get("no_vocals_path") if preserve_bg else None - if bg_audio and os.path.exists(bg_audio): - mixed_path = os.path.join(DUB_DIR, job_id, f"mixed_mp3_{lang_label}.wav") - cmd_mix = [ - ffmpeg, "-i", bg_audio, "-i", wav_path, - "-filter_complex", "[0:a][1:a]amix=inputs=2:duration=longest:dropout_transition=2:weights=0.8 1.2[aout]", - "-map", "[aout]", "-c:a", "pcm_s16le", "-y", mixed_path - ] - try: - proc = await asyncio.create_subprocess_exec( - *cmd_mix, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _, stderr = await proc.communicate() - if proc.returncode == 0: - source_path = mixed_path - except Exception as e: - logger.error(f"Failed to mix audio for MP3: {e}") - - # Convert to MP3 - mp3_path = os.path.join(DUB_DIR, job_id, f"dubbed_{lang_label}.mp3") - cmd = [ffmpeg, "-i", source_path, "-codec:a", "libmp3lame", "-b:a", "192k", "-y", mp3_path] - try: - proc = await asyncio.create_subprocess_exec( - *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - raise Exception(stderr.decode()) - except Exception as e: - raise HTTPException(status_code=500, detail=f"MP3 encoding failed: {str(e)}") - - base_name = os.path.splitext(job.get('filename', 'audio'))[0] - safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'audio' - dl_name = f"dubbed_{lang_label}_{safe_name}.mp3" - return FileResponse( - mp3_path, media_type="audio/mpeg", - headers={"Content-Disposition": f'attachment; filename="{dl_name}"'}, - ) - - -# ═══════════════════════════════════════════════════════════════════════ -# STEM EXPORT (Vocals + Background Separate) -# ═══════════════════════════════════════════════════════════════════════ - -@app.get("/dub/export-stems/{job_id}") -async def dub_export_stems(job_id: str, lang: str = Query(None)): - """Export dubbed vocals and original background as separate WAV files in a ZIP.""" - import zipfile - - job = _get_job(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - tracks = job.get("dubbed_tracks", {}) - if not tracks: - raise HTTPException(status_code=400, detail="No dubbed tracks generated yet") - - if lang and lang in tracks: - vocals_path = tracks[lang]["path"] - lang_label = lang - elif tracks: - first_key = list(tracks.keys())[0] - vocals_path = tracks[first_key]["path"] - lang_label = first_key - else: - raise HTTPException(status_code=400, detail="No dubbed audio track") - - bg_path = job.get("no_vocals_path") - - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: - if os.path.exists(vocals_path): - zf.write(vocals_path, f"vocals_dubbed_{lang_label}.wav") - if bg_path and os.path.exists(bg_path): - zf.write(bg_path, "background_original.wav") - - zip_buffer.seek(0) - base_name = os.path.splitext(job.get('filename', 'video'))[0] - safe_name = ''.join(c for c in base_name if c.isalnum() or c in '-_ ').strip() or 'stems' - return Response( - content=zip_buffer.read(), - media_type="application/zip", - headers={ - "Content-Disposition": f'attachment; filename="stems_{safe_name}.zip"', - }, - ) - - -# ═══════════════════════════════════════════════════════════════════════ -# STUDIO PROJECTS — Save / Load / List / Delete -# ═══════════════════════════════════════════════════════════════════════ - -class ProjectSaveRequest(BaseModel): - name: str - video_path: Optional[str] = None - audio_path: Optional[str] = None - duration: Optional[float] = None - state: dict # Full JSON blob: segments, settings, tracks, etc. - - -@app.get("/projects") -async def list_projects(): - conn = _get_db() - rows = conn.execute( - "SELECT id, name, video_path, duration, created_at, updated_at FROM studio_projects ORDER BY updated_at DESC" - ).fetchall() - conn.close() - return [dict(r) for r in rows] - - -@app.get("/projects/{project_id}") -async def get_project(project_id: str): - conn = _get_db() - row = conn.execute("SELECT * FROM studio_projects WHERE id=?", (project_id,)).fetchone() - conn.close() - if not row: - raise HTTPException(status_code=404, detail="Project not found") - result = dict(row) - if result.get("state_json"): - try: - result["state"] = json.loads(result["state_json"]) - except Exception: - result["state"] = {} - else: - result["state"] = {} - return result - - -@app.post("/projects") -async def create_project(req: ProjectSaveRequest): - project_id = str(uuid.uuid4())[:8] - now = time.time() - conn = _get_db() - conn.execute( - "INSERT INTO studio_projects (id, name, video_path, audio_path, duration, state_json, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?)", - (project_id, req.name, req.video_path, req.audio_path, req.duration, json.dumps(req.state), now, now), - ) - conn.commit() - conn.close() - return {"id": project_id, "name": req.name, "created_at": now} - - -@app.put("/projects/{project_id}") -async def update_project(project_id: str, req: ProjectSaveRequest): - conn = _get_db() - row = conn.execute("SELECT id FROM studio_projects WHERE id=?", (project_id,)).fetchone() - if not row: - conn.close() - raise HTTPException(status_code=404, detail="Project not found") - now = time.time() - conn.execute( - "UPDATE studio_projects SET name=?, video_path=?, audio_path=?, duration=?, state_json=?, updated_at=? WHERE id=?", - (req.name, req.video_path, req.audio_path, req.duration, json.dumps(req.state), now, project_id), - ) - conn.commit() - conn.close() - return {"id": project_id, "name": req.name, "updated_at": now} - - -@app.delete("/projects/{project_id}") -async def delete_project(project_id: str): - conn = _get_db() - conn.execute("DELETE FROM studio_projects WHERE id=?", (project_id,)) - conn.commit() - conn.close() - return {"deleted": project_id} - -# Mount frontend at root. Placed last so it doesn't shadow API routes. frontend_path = os.path.join(os.path.dirname(__file__), "..", "frontend", "dist") if os.path.exists(frontend_path): app.mount("/", StaticFiles(directory=frontend_path, html=True), name="frontend") @@ -2109,7 +112,6 @@ else: def _dev_fallback(): return RedirectResponse(url="http://localhost:5173") - if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/backend/schemas/__init__.py b/backend/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/schemas/requests.py b/backend/schemas/requests.py new file mode 100644 index 00000000..f46c2687 --- /dev/null +++ b/backend/schemas/requests.py @@ -0,0 +1,53 @@ +from pydantic import BaseModel +from typing import List, Optional + +class ExportRequest(BaseModel): + source_filename: str + destination_path: str + mode: str = "history" + +class ExportRecordRequest(BaseModel): + filename: str + destination_path: str = "~/Downloads" + mode: str = "file" + +class RevealRequest(BaseModel): + path: str + +class DubSegment(BaseModel): + start: float + end: float + text: str + instruct: str = "" # Per-segment voice override + profile_id: str = "" # Per-segment voice profile + speed: Optional[float] = None + gain: Optional[float] = None # Per-segment volume (0.0 - 2.0, default 1.0) + target_lang: Optional[str] = None # Per-segment language override (ISO code) + +class DubRequest(BaseModel): + segments: List[DubSegment] + language: str = "Auto" + language_code: str = "und" # ISO 639-1 for ffmpeg metadata (e.g. "es", "fr", "de") + instruct: str = "" + num_step: int = 16 + guidance_scale: float = 2.0 + speed: float = 1.0 + +class TranslateSegment(BaseModel): + id: str + text: str + target_lang: Optional[str] = None + +class TranslateRequest(BaseModel): + segments: List[TranslateSegment] + target_lang: str # ISO 639-1 code like "es", "fr" + provider: Optional[str] = None + source_lang: Optional[str] = None # ISO 639-1; overrides job detection + job_id: Optional[str] = None # Dub job id, used to resolve detected source_lang + +class ProjectSaveRequest(BaseModel): + name: str + video_path: Optional[str] = None + audio_path: Optional[str] = None + duration: Optional[float] = None + state: dict # Full JSON blob: segments, settings, tracks, etc. diff --git a/backend/services/__init__.py b/backend/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/services/audio_dsp.py b/backend/services/audio_dsp.py new file mode 100644 index 00000000..ceb3ba09 --- /dev/null +++ b/backend/services/audio_dsp.py @@ -0,0 +1,32 @@ +import torch + +def apply_mastering(audio_tensor, sample_rate=24000): + """Applies professional Broadcast-grade DSP (EQ, Compressor, light Reverb) to the clone voice.""" + try: + from pedalboard import Pedalboard, Compressor, Reverb, HighpassFilter + import numpy as np + board = Pedalboard([ + HighpassFilter(cutoff_frequency_hz=60), + Compressor(threshold_db=-15, ratio=1.5, attack_ms=2.0, release_ms=100), + Reverb(room_size=0.10, wet_level=0.08, dry_level=0.95) + ]) + audio_np = audio_tensor.cpu().numpy() + if audio_np.ndim == 1: + audio_np = audio_np[np.newaxis, :] + effected = board(audio_np, sample_rate, reset=False) + return torch.from_numpy(effected).to(audio_tensor.device) + except ImportError: + return audio_tensor # Fail gracefully if pedalboard isn't installed + except Exception as e: + print(f"Mastering DSP Error: {e}") + return audio_tensor + +def normalize_audio(audio_tensor, target_dBFS=-2.0): + """Peak-normalizes the audio to a standard broadcasting level (-2 dB) to fix F5TTS volume fluctuations.""" + if audio_tensor.numel() == 0: + return audio_tensor + max_val = torch.abs(audio_tensor).max() + if max_val > 0: + target_amp = 10 ** (target_dBFS / 20.0) + audio_tensor = audio_tensor * (target_amp / max_val) + return audio_tensor diff --git a/backend/services/ffmpeg_utils.py b/backend/services/ffmpeg_utils.py new file mode 100644 index 00000000..578c07a5 --- /dev/null +++ b/backend/services/ffmpeg_utils.py @@ -0,0 +1,90 @@ +import asyncio +import errno +import logging +import shutil + +logger = logging.getLogger("omnivoice.api") + +# Cap concurrent ffmpeg jobs so macOS posix_spawn can't hit EAGAIN under load. +_FFMPEG_SEMAPHORE: "asyncio.Semaphore | None" = None +_FFMPEG_CONCURRENCY = 2 + + +def _get_semaphore() -> asyncio.Semaphore: + global _FFMPEG_SEMAPHORE + if _FFMPEG_SEMAPHORE is None: + _FFMPEG_SEMAPHORE = asyncio.Semaphore(_FFMPEG_CONCURRENCY) + return _FFMPEG_SEMAPHORE + + +def find_ffmpeg(): + try: + import imageio_ffmpeg + # This will natively extract and return an architecture-specific static FFmpeg binary! + return imageio_ffmpeg.get_ffmpeg_exe() + except Exception as e: + logger.warning(f"imageio_ffmpeg failed to provide static binary: {e}. Falling back to default system path.") + for path in ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "ffmpeg"]: + if shutil.which(path): + return path + raise RuntimeError("ffmpeg not found in bundle or system path") + + +async def _spawn_with_retry(cmd, **kwargs): + """Spawn a subprocess, retrying briefly on EAGAIN (posix_spawn resource pressure).""" + delay = 0.1 + last_err = None + for _ in range(5): + try: + return await asyncio.create_subprocess_exec(*cmd, **kwargs) + except BlockingIOError as e: + last_err = e + if e.errno != errno.EAGAIN: + raise + await asyncio.sleep(delay) + delay *= 2 + except OSError as e: + if e.errno == errno.EAGAIN: + last_err = e + await asyncio.sleep(delay) + delay *= 2 + continue + raise + raise last_err if last_err else RuntimeError("spawn failed") + + +async def run_ffmpeg(cmd, timeout: float = 1800.0, capture: bool = True): + """Run an ffmpeg subprocess with concurrency cap, timeout, and proper cleanup. + + Returns (returncode, stdout_bytes, stderr_bytes). Raises asyncio.TimeoutError + on hard timeout (after killing + reaping the process). + """ + stdout = asyncio.subprocess.PIPE if capture else asyncio.subprocess.DEVNULL + stderr = asyncio.subprocess.PIPE + async with _get_semaphore(): + proc = await _spawn_with_retry(cmd, stdout=stdout, stderr=stderr) + try: + try: + out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + try: + await asyncio.wait_for(proc.wait(), timeout=5.0) + except asyncio.TimeoutError: + pass + raise + return proc.returncode, out, err + finally: + # Guarantee reaping — prevents zombie pileup under timeouts or errors. + if proc.returncode is None: + try: + proc.kill() + except ProcessLookupError: + pass + try: + await asyncio.wait_for(proc.wait(), timeout=5.0) + except asyncio.TimeoutError: + pass diff --git a/backend/services/model_manager.py b/backend/services/model_manager.py new file mode 100644 index 00000000..db6580aa --- /dev/null +++ b/backend/services/model_manager.py @@ -0,0 +1,118 @@ +import os +import time +import asyncio +import logging +import torch +from typing import Optional +from concurrent.futures import ThreadPoolExecutor + +from omnivoice.models.omnivoice import OmniVoice +from core.config import IDLE_TIMEOUT_SECONDS, CPU_POOL_WORKERS + +logger = logging.getLogger("omnivoice.model") + +_gpu_pool = ThreadPoolExecutor(max_workers=1) +_cpu_pool = ThreadPoolExecutor(max_workers=CPU_POOL_WORKERS) + +model: Optional[OmniVoice] = None +_model_lock = asyncio.Lock() +_last_used = time.time() +_IDLE_TIMEOUT_SECONDS = IDLE_TIMEOUT_SECONDS + +def get_best_device(): + if torch.cuda.is_available(): + return "cuda" + if torch.backends.mps.is_available(): + return "mps" + return "cpu" + +def _load_model_sync(): + global model + device = get_best_device() + print(f"Loading OmniVoice model lazily on device: {device}...") + checkpoint = os.environ.get("OMNIVOICE_MODEL", "k2-fsa/OmniVoice") + _model = OmniVoice.from_pretrained( + checkpoint, device_map=device, dtype=torch.float16, load_asr=True, + ) + try: + if device == "cuda": + _model.llm = torch.compile(_model.llm, mode="reduce-overhead") + print("torch.compile applied.") + except Exception as e: + print(f"torch.compile skipped: {e}") + print("OmniVoice model loaded successfully.") + return _model + +async def get_model() -> OmniVoice: + global model, _last_used + _last_used = time.time() + if model is not None: + return model + + async with _model_lock: + if model is None: + loop = asyncio.get_running_loop() + model = await loop.run_in_executor(_gpu_pool, _load_model_sync) + return model + +def get_model_status(): + is_loaded = model is not None + # asyncio.Lock exposes .locked() on all supported Python versions; wrap in try for safety. + try: + is_loading = (not is_loaded) and _model_lock.locked() + except Exception: + is_loading = False + return { + "loaded": is_loaded, + "loading": is_loading, + "status": "loading" if is_loading else ("ready" if is_loaded else "idle"), + } + +async def idle_worker(): + global model + while True: + await asyncio.sleep(30) + async with _model_lock: + if model is not None and time.time() - _last_used > _IDLE_TIMEOUT_SECONDS: + print("Idle timeout reached. Unloading OmniVoice model to free VRAM...") + model = None + import gc + gc.collect() + if torch.backends.mps.is_available(): + torch.mps.empty_cache() + elif torch.cuda.is_available(): + torch.cuda.empty_cache() + +def free_vram(): + import gc + gc.collect() + if torch.backends.mps.is_available(): + torch.mps.empty_cache() + elif torch.cuda.is_available(): + torch.cuda.empty_cache() + +_diar_pipeline = None + +def get_diarization_pipeline(): + global _diar_pipeline + hf_token = os.environ.get("HF_TOKEN") + if not hf_token: + return None + if _diar_pipeline is not None: + return _diar_pipeline + try: + import torch + from pyannote.audio import Pipeline + import logging + logger = logging.getLogger("omnivoice.api") + logger.info("Loading Pyannote Diarization Pipeline...") + _diar_pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", use_auth_token=hf_token) + if torch.cuda.is_available(): + _diar_pipeline.to(torch.device("cuda")) + logger.info("Pyannote Diarization Pipeline loaded successfully.") + return _diar_pipeline + except Exception as e: + import logging + logger = logging.getLogger("omnivoice.api") + logger.error(f"Failed to load Pyannote pipeline: {e}") + return None diff --git a/backend/services/segmentation.py b/backend/services/segmentation.py new file mode 100644 index 00000000..e25260ef --- /dev/null +++ b/backend/services/segmentation.py @@ -0,0 +1,506 @@ +"""Broadcast-grade segmentation for dubbing. + +Rules (in priority order): + 1. Never split mid-word. Whitespace or nothing. + 2. Prefer sentence punctuation > clause punctuation (, ; : —) > word boundaries. + 3. Reject any candidate split that leaves either side below the minimum floor. + 4. Fragments below the floor merge into same-speaker neighbor; gap < MERGE_GAP + prefers previous, else next. + 5. Scene-cut assisted splits apply only when both halves remain viable. + 6. Never merge across a speaker boundary. +""" + +from __future__ import annotations + +import re +import uuid +from dataclasses import dataclass, field, asdict +from typing import Iterable, List, Optional, Sequence + + +MIN_DUR = 1.5 # seconds — below this, a segment must merge +MIN_CHARS = 12 # characters — below this, a segment must merge (Latin-ish) +MIN_WORDS = 3 # words — below this, a segment is considered a fragment +STITCH_DUR = 2.5 # seconds — pair of short neighbors under this combine even when each is legal +STITCH_GAP = 0.9 # seconds — max silence between two stitch candidates +IDEAL_DUR = 4.5 # seconds — target length for splits +MAX_DUR = 9.0 # seconds — above this, force a split +MAX_CHARS = 140 # characters — above this, force a split +MERGE_GAP = 0.6 # seconds — tolerated silence when folding a fragment backward +MERGE_GAP_ULTRA = 2.0 # seconds — wider gap tolerated for ultra-short (< 0.5s or < 3 chars) +ULTRA_SHORT_DUR = 0.5 # seconds — threshold for "always fold" regardless of neighbor match +ULTRA_SHORT_CHARS = 4 # chars — same tier +SPEAKER_GAP = 1.2 # seconds — heuristic speaker-change gap (no pyannote) + +# Sentence-end punctuation across Latin, CJK, Bengali, Arabic, Thai, Armenian, Hindi, etc. +_SENTENCE_END = re.compile( + r'([.!?。！？।؟…؛܀։՝።။၊।]["\')\]]?)(\s+|$)' +) +_CLAUSE_END = re.compile(r'([,;:—、،؍])(\s+|$)') +_WS = re.compile(r'\s+') + + +def _word_count(text: str) -> int: + if not text: + return 0 + # Latin-like scripts use whitespace; CJK scripts count each glyph as a word. + tokens = [t for t in text.split() if t] + if len(tokens) >= MIN_WORDS: + return len(tokens) + # For scripts without spaces (CJK), approximate word count as graphemes / 2. + non_space = sum(1 for ch in text if not ch.isspace()) + approx = max(len(tokens), non_space // 2) + return approx + + +def _is_short(seg) -> bool: + return ( + seg.duration < MIN_DUR + or seg.char_count < MIN_CHARS + or _word_count(seg.text) < MIN_WORDS + ) + + +def _is_ultra_short(seg) -> bool: + return seg.duration < ULTRA_SHORT_DUR or seg.char_count < ULTRA_SHORT_CHARS + + +@dataclass +class Word: + start: float + end: float + text: str + + +@dataclass +class Segment: + start: float + end: float + text: str + speaker_id: str = "Speaker 1" + id: str = field(default_factory=lambda: str(uuid.uuid4())[:8]) + + @property + def duration(self) -> float: + return max(0.0, self.end - self.start) + + @property + def char_count(self) -> int: + return len(self.text) + + def to_dict(self) -> dict: + return { + "id": self.id, + "start": round(self.start, 2), + "end": round(self.end, 2), + "text": self.text, + "speaker_id": self.speaker_id, + } + + +def _clean(text: str) -> str: + return _WS.sub(" ", (text or "").strip()) + + +def _best_boundary(text: str, ideal_pos: int) -> int: + """Return a character offset to split at. Prefer sentence > clause > word. + + Scans the full text for each candidate class and picks the one whose offset + is closest to `ideal_pos`. Sentence endings always beat clause endings, which + always beat bare word boundaries. + """ + if not text: + return 0 + length = len(text) + if length <= 1: + return length + + def _closest(offsets: List[int]) -> Optional[int]: + if not offsets: + return None + return min(offsets, key=lambda o: abs(o - ideal_pos)) + + sentence_offsets = [m.end(1) for m in _SENTENCE_END.finditer(text)] + pick = _closest(sentence_offsets) + if pick is not None: + return pick + + clause_offsets = [m.end(1) for m in _CLAUSE_END.finditer(text)] + pick = _closest(clause_offsets) + if pick is not None: + return pick + + # Bare word boundaries: every space position. + space_offsets = [i for i, ch in enumerate(text) if ch == " "] + pick = _closest(space_offsets) + if pick is not None: + return pick + return length + + +def _words_from_whisper(result: dict) -> List[Word]: + """Extract word-level timing if available, otherwise fall back to chunk-level.""" + words: List[Word] = [] + segs = result.get("segments") if isinstance(result, dict) else None + if segs: + for seg in segs: + for w in seg.get("words", []) or []: + wt = (w.get("word") or w.get("text") or "").strip() + if not wt: + continue + ws = float(w.get("start", seg.get("start", 0.0))) + we = float(w.get("end", seg.get("end", ws + 0.1))) + if we <= ws: + we = ws + 0.05 + words.append(Word(start=ws, end=we, text=wt)) + if words: + return words + + # Fallback: chunk-level timings (no per-word granularity) + for chunk in result.get("chunks", []) or []: + ts = chunk.get("timestamp") or (0.0, 0.0) + s = float(ts[0] or 0.0) + e = float(ts[1] or s + 0.1) + text = _clean(chunk.get("text", "")) + if not text or e <= s: + continue + # Distribute time evenly across the tokens inside the chunk + tokens = text.split(" ") + dur = (e - s) / max(len(tokens), 1) + t = s + for tok in tokens: + words.append(Word(start=t, end=t + dur, text=tok)) + t += dur + return words + + +def _build_segments_from_words(words: Sequence[Word]) -> List[Segment]: + """Greedy grouping of words into IDEAL_DUR sentences, cut at natural boundaries.""" + segments: List[Segment] = [] + if not words: + return segments + + buf: List[Word] = [] + buf_start = words[0].start + + def flush_buf(force: bool = False) -> None: + nonlocal buf, buf_start + if not buf: + return + text = _clean(" ".join(w.text for w in buf)) + if not text: + buf = [] + return + segments.append(Segment(start=buf_start, end=buf[-1].end, text=text)) + buf = [] + if not force: + buf_start = 0.0 + + for i, w in enumerate(words): + if not buf: + buf_start = w.start + buf.append(w) + buf_dur = buf[-1].end - buf_start + buf_chars = sum(len(x.text) + 1 for x in buf) + next_gap = 0.0 + if i + 1 < len(words): + next_gap = max(0.0, words[i + 1].start - w.end) + + ends_sentence = bool(_SENTENCE_END.search(w.text)) + ends_clause = bool(_CLAUSE_END.search(w.text)) + + too_long = buf_dur >= MAX_DUR or buf_chars >= MAX_CHARS + at_ideal = buf_dur >= IDEAL_DUR and buf_chars >= MIN_CHARS + + # Natural-boundary flush at target length. + if at_ideal and ends_sentence: + flush_buf() + elif too_long and (ends_sentence or ends_clause): + flush_buf() + elif too_long and next_gap >= 0.35: + flush_buf() + elif too_long: + # Last-resort split on a word boundary. Choose the word whose + # cumulative position is closest to IDEAL_DUR from buf_start. + best_idx = None + best_score = float("inf") + for k, bw in enumerate(buf[:-1]): # must leave ≥1 word on right + left_dur = bw.end - buf_start + if left_dur < MIN_DUR: + continue + right_dur = buf[-1].end - buf[k + 1].start + if right_dur < MIN_DUR: + continue + # Prefer words ending in sentence / clause punctuation. + boundary_bonus = 0.0 + if _SENTENCE_END.search(bw.text): + boundary_bonus = -2.0 + elif _CLAUSE_END.search(bw.text): + boundary_bonus = -0.8 + score = abs(left_dur - IDEAL_DUR) + boundary_bonus + if score < best_score: + best_score = score + best_idx = k + + if best_idx is not None: + left_buf = buf[: best_idx + 1] + right_buf = buf[best_idx + 1 :] + segments.append(Segment( + start=buf_start, + end=left_buf[-1].end, + text=_clean(" ".join(x.text for x in left_buf)), + )) + buf = list(right_buf) + buf_start = right_buf[0].start + else: + flush_buf() + + flush_buf(force=True) + return segments + + +def _merge_short(segments: List[Segment]) -> List[Segment]: + """Fold fragments below the floor into adjacent same-speaker segment. + + Runs multi-pass until no further merges happen. Ultra-short segments + (< 0.5s or < 4 chars) fold across larger gaps and across speakers when + no same-speaker neighbor is close — stray tokens like "STR" are never + allowed to survive as standalone segments. + """ + if not segments: + return segments + + for _ in range(64): # bounded iterations so misuse can't hang + did_merge = False + i = 0 + while i < len(segments): + s = segments[i] + if not _is_short(s): + i += 1 + continue + + prev = segments[i - 1] if i > 0 else None + nxt = segments[i + 1] if i + 1 < len(segments) else None + gap_tolerance = MERGE_GAP_ULTRA if _is_ultra_short(s) else MERGE_GAP + + prev_same = bool(prev and prev.speaker_id == s.speaker_id) + next_same = bool(nxt and nxt.speaker_id == s.speaker_id) + prev_gap = (s.start - prev.end) if prev else float("inf") + next_gap = (nxt.start - s.end) if nxt else float("inf") + + prev_ok = prev_same and prev_gap <= gap_tolerance + next_ok = next_same and next_gap <= gap_tolerance + + target = None + if prev_ok and next_ok: + target = prev if prev.duration <= nxt.duration else nxt + elif prev_ok: + target = prev + elif next_ok: + target = nxt + elif prev_same: + target = prev + elif next_same: + target = nxt + elif _is_ultra_short(s): + # Stray token — fold into closest neighbor regardless of speaker. + if prev and nxt: + target = prev if prev_gap <= next_gap else nxt + else: + target = prev or nxt + elif prev: + target = prev + elif nxt: + target = nxt + + if target is None: + i += 1 + continue + if target is prev: + prev.text = _clean(prev.text + " " + s.text) + prev.end = max(prev.end, s.end) + segments.pop(i) + did_merge = True + continue + if target is nxt: + nxt.text = _clean(s.text + " " + nxt.text) + nxt.start = min(nxt.start, s.start) + segments.pop(i) + did_merge = True + continue + + i += 1 + if not did_merge: + break + return segments + + +def _stitch_adjacent_shorts(segments: List[Segment]) -> List[Segment]: + """Combine adjacent same-speaker segments when both are short and close. + + Catches the case where each segment individually passes MIN_DUR but a + rapid-fire pair produces a jittery dub. Only stitches when both halves + live under STITCH_DUR and the gap between them is minimal. + """ + if len(segments) < 2: + return segments + + for _ in range(32): + did = False + i = 0 + while i + 1 < len(segments): + a, b = segments[i], segments[i + 1] + same = a.speaker_id == b.speaker_id + gap = b.start - a.end + combined_dur = (b.end - a.start) + if ( + same + and gap <= STITCH_GAP + and a.duration <= STITCH_DUR + and b.duration <= STITCH_DUR + and combined_dur <= MAX_DUR + ): + a.text = _clean(a.text + " " + b.text) + a.end = b.end + segments.pop(i + 1) + did = True + continue + i += 1 + if not did: + break + return segments + + +def clean_up_segments(segments: List[dict]) -> List[dict]: + """Public entry: run merge + stitch passes on already-persisted segments. + + Used by the UI's "Clean up segments" action so users can repair jobs + that were segmented under older, looser rules. + """ + objs: List[Segment] = [] + for s in segments or []: + try: + objs.append(Segment( + start=float(s.get("start", 0.0)), + end=float(s.get("end", 0.0)), + text=_clean(str(s.get("text", ""))), + speaker_id=str(s.get("speaker_id") or "Speaker 1"), + id=str(s.get("id") or uuid.uuid4().hex[:8]), + )) + except (TypeError, ValueError): + continue + objs = [s for s in objs if s.end > s.start and s.text] + objs = _merge_short(objs) + objs = _stitch_adjacent_shorts(objs) + objs = _merge_short(objs) + return [s.to_dict() for s in objs] + + +def _apply_scene_cuts(segments: List[Segment], scene_cuts: Iterable[float]) -> List[Segment]: + """Split segments at scene cuts only if both halves remain viable.""" + cuts = sorted(c for c in scene_cuts if c > 0) + if not cuts: + return segments + + out: List[Segment] = [] + for s in segments: + inner_cuts = [c for c in cuts if s.start + MIN_DUR < c < s.end - MIN_DUR] + if not inner_cuts: + out.append(s) + continue + + remaining = s + for cut in inner_cuts: + dur_total = remaining.duration + if dur_total <= 0: + break + ratio = (cut - remaining.start) / dur_total + tentative_split = int(len(remaining.text) * ratio) + pos = _best_boundary(remaining.text, tentative_split) + left_text = remaining.text[:pos].strip() + right_text = remaining.text[pos:].strip() + # Viability check — refuse the cut if either half would be a fragment. + if ( + not left_text + or not right_text + or len(left_text) < MIN_CHARS + or len(right_text) < MIN_CHARS + or (cut - remaining.start) < MIN_DUR + or (remaining.end - cut) < MIN_DUR + ): + continue + out.append(Segment( + start=remaining.start, end=cut, text=left_text, speaker_id=remaining.speaker_id, + )) + remaining = Segment( + start=cut, end=remaining.end, text=right_text, speaker_id=remaining.speaker_id, + ) + out.append(remaining) + return out + + +def segment_transcript( + whisper_result: dict, + duration: float, + scene_cuts: Optional[Iterable[float]] = None, +) -> List[dict]: + """Public entry point: whisper result → clean dub segments (as dicts).""" + words = _words_from_whisper(whisper_result) + if not words: + text = _clean((whisper_result or {}).get("text", "")) + if text: + return [Segment(start=0.0, end=max(duration, 0.1), text=text).to_dict()] + return [] + + segments = _build_segments_from_words(words) + segments = _merge_short(segments) + if scene_cuts: + segments = _apply_scene_cuts(segments, scene_cuts) + segments = _merge_short(segments) + segments = _stitch_adjacent_shorts(segments) + segments = _merge_short(segments) + return [s.to_dict() for s in segments] + + +def assign_speakers_from_diarization( + segments: List[dict], + diarization, +) -> List[dict]: + """Replace speaker_id based on a pyannote diarization result (overlap-weighted).""" + for s in segments: + start, end = s["start"], s["end"] + mid = (start + end) / 2.0 + overlap: dict[str, float] = {} + for turn, _, speaker in diarization.itertracks(yield_label=True): + left = max(start, turn.start) + right = min(end, turn.end) + if right > left: + overlap[speaker] = overlap.get(speaker, 0.0) + (right - left) + if overlap: + winner = max(overlap.items(), key=lambda kv: kv[1])[0] + else: + # fall back to midpoint membership + winner = None + for turn, _, speaker in diarization.itertracks(yield_label=True): + if turn.start <= mid <= turn.end: + winner = speaker + break + if winner is not None: + try: + idx = int(winner.split("_")[-1]) + 1 + s["speaker_id"] = f"Speaker {idx}" + except ValueError: + s["speaker_id"] = winner + return segments + + +def assign_speakers_heuristic(segments: List[dict]) -> List[dict]: + """Two-speaker alternation based on silence gaps.""" + current = 1 + last_end = 0.0 + for i, s in enumerate(segments): + if i > 0 and (s["start"] - last_end) > SPEAKER_GAP: + current = 2 if current == 1 else 1 + s["speaker_id"] = f"Speaker {current}" + last_end = s["end"] + return segments diff --git a/benchmark.py b/benchmark.py new file mode 100644 index 00000000..12788e27 --- /dev/null +++ b/benchmark.py @@ -0,0 +1,51 @@ +import sys +import os +os.environ["PATH"] += os.pathsep + "/opt/homebrew/bin:/usr/local/bin" + +try: + from imageio_ffmpeg import get_ffmpeg_exe + ffmpeg_path = get_ffmpeg_exe() + os.environ["PATH"] = os.path.dirname(ffmpeg_path) + os.pathsep + os.environ.get("PATH", "") +except Exception as e: + pass + +import mlx_whisper +import time +import subprocess + +audio_file = "/Users/user4/Desktop/voice-design/OmniVoice/data/preview/1c43531cb0ae.mov" + +print("Starting transcription...", flush=True) +start = time.time() +result = mlx_whisper.transcribe(audio_file, path_or_hf_repo="mlx-community/whisper-large-v3-mlx") +transcript = result.get("text", "").strip() +trans_time = time.time() - start +print(f"Transcript ({trans_time:.2f}s):\n{transcript}\n") + +# Baseline Google NMT +try: + from deep_translator import GoogleTranslator + print("Translating with Google NMT...", flush=True) + start = time.time() + google_tgt = GoogleTranslator(source="auto", target="bn") + google_text = google_tgt.translate(transcript) + google_time = time.time() - start + print(f"Google Bengali ({google_time:.2f}s):\n{google_text}\n") +except Exception as e: + print(f"Google Failed: {e}\n") + +# APFEL +print("Translating with Apfel...", flush=True) +start = time.time() +try: + prompt = f"You are a professional dubbing translator. Translate the following text into Bengali. Output ONLY the translated text.\n{transcript}" + # Use login shell to ensure apfel alias/function is loaded + prompt_esc = prompt.replace('"', '\\"') + apfel_res = subprocess.run(["zsh", "-lc", f'apfel "{prompt_esc}"'], capture_output=True, text=True) + apfel_time = time.time() - start + if apfel_res.returncode == 0: + print(f"Apfel Bengali ({apfel_time:.2f}s):\n{apfel_res.stdout.strip()}\n") + else: + print(f"Apfel CLI failed: {apfel_res.stderr}\n") +except Exception as e: + print(f"Apfel Failed: {e}\n") diff --git a/crash_log.txt b/crash_log.txt deleted file mode 100644 index fdfa24da..00000000 --- a/crash_log.txt +++ /dev/null @@ -1,37 +0,0 @@ -Request: http://localhost:8000/generate -Traceback (most recent call last): - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 164, in __call__ - await self.app(scope, receive, _send) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/middleware/cors.py", line 95, in __call__ - await self.simple_response(scope, receive, send, request_headers=headers) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/middleware/cors.py", line 153, in simple_response - await self.app(scope, receive, send) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 63, in __call__ - await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app - raise exc - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app - await app(scope, receive, sender) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__ - await self.app(scope, receive, send) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/routing.py", line 716, in __call__ - await self.middleware_stack(scope, receive, send) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/routing.py", line 736, in app - await route.handle(scope, receive, send) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/routing.py", line 290, in handle - await self.app(scope, receive, send) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/fastapi/routing.py", line 134, in app - await wrap_app_handling_exceptions(app, request)(scope, receive, send) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app - raise exc - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app - await app(scope, receive, sender) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/fastapi/routing.py", line 120, in app - response = await f(request) - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/fastapi/routing.py", line 674, in app - raw_response = await run_endpoint_function( - File "/Users/user4/Desktop/voice-design/OmniVoice/.venv/lib/python3.10/site-packages/fastapi/routing.py", line 328, in run_endpoint_function - return await dependant.call(**values) - File "/Users/user4/Desktop/voice-design/OmniVoice/backend/main.py", line 837, in generate_speech - ref_audio_path = os.path.join(VOICES_DIR, row["ref_audio_path"]) if row.get("ref_audio_path") else None -AttributeError: 'sqlite3.Row' object has no attribute 'get' diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index ef0dbe5e..5771d0e5 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,6 +1,10 @@ import React, { useState, useRef, useEffect, useCallback } from 'react'; import './index.css'; import WaveformTimeline from './components/WaveformTimeline'; +import SearchableSelect from './components/SearchableSelect'; + +const POPULAR_LANGS = ['English','Spanish','French','German','Italian','Portuguese','Russian','Chinese','Japanese','Korean','Arabic','Hindi']; +const POPULAR_ISO = ['en','es','fr','de','it','pt','ru','zh','ja','ko','ar','hi']; import { Toaster, toast } from 'react-hot-toast'; import ALL_LANGUAGES from './languages.json'; import { @@ -10,7 +14,7 @@ import { FileText, Loader, Check, AlertCircle, Plus, User, Save, Languages, Headphones, FolderOpen, FolderPlus, Pencil, Clock, Lock, Unlock, Mic, MicOff, Square, CheckCircle, Circle, ChevronRight, Target, PanelLeftClose, PanelLeftOpen, Scale, - Layers, Music, Package, DownloadCloud + Layers, Music, Package, DownloadCloud, RefreshCw } from 'lucide-react'; // Tauri: pre-import window API to avoid async delays in event handlers @@ -29,7 +33,7 @@ const doubleClickMaximize = () => { * We upload to the backend's /preview endpoint and serve via HTTP instead. * Falls back to createObjectURL for regular browsers. */ -const _PREVIEW_API = 'http://localhost:8000'; +const _PREVIEW_API = import.meta.env.VITE_OMNIVOICE_API || 'http://localhost:8000'; const fileToMediaUrl = async (file, prevUrls) => { // Revoke previous blob URLs if they exist if (prevUrls?.videoUrl?.startsWith('blob:')) URL.revokeObjectURL(prevUrls.videoUrl); @@ -60,17 +64,33 @@ const fileToMediaUrl = async (file, prevUrls) => { const playBlobAudio = async (blob) => { if (isTauri) { const ctx = new (window.AudioContext || window.webkitAudioContext)(); - const buf = await blob.arrayBuffer(); - const decoded = await ctx.decodeAudioData(buf); - const src = ctx.createBufferSource(); - src.buffer = decoded; - src.connect(ctx.destination); - src.start(0); - src.onended = () => ctx.close(); + // WebKit suspends AudioContext by default — must resume before decoding + if (ctx.state === 'suspended') await ctx.resume(); + try { + const buf = await blob.arrayBuffer(); + const decoded = await ctx.decodeAudioData(buf); + const src = ctx.createBufferSource(); + src.buffer = decoded; + src.connect(ctx.destination); + src.start(0); + src.onended = () => ctx.close(); + } catch (e) { + console.error('playBlobAudio decode error:', e); + ctx.close(); + // Fallback: try the standard Audio() path even in Tauri + try { + const url = URL.createObjectURL(blob); + const a = new Audio(url); + await a.play(); + a.onended = () => URL.revokeObjectURL(url); + } catch (e2) { + console.error('playBlobAudio fallback error:', e2); + } + } } else { const url = URL.createObjectURL(blob); const a = new Audio(url); - a.play().catch(() => {}); + a.play().catch((e) => console.error('playBlobAudio play error:', e)); a.onended = () => URL.revokeObjectURL(url); } }; @@ -172,7 +192,6 @@ function App() { const [refText, setRefText] = useState(''); const [instruct, setInstruct] = useState(''); const [language, setLanguage] = useState('Auto'); - const [langSearch, setLangSearch] = useState(''); const [isGenerating, setIsGenerating] = useState(false); const [history, setHistory] = useState([]); const [exportHistory, setExportHistory] = useState([]); @@ -229,8 +248,8 @@ function App() { const [dubStep, setDubStep] = useState('idle'); const [dubSegments, setDubSegments] = useState([]); const [dubLang, setDubLang] = useState('Auto'); - const [dubLangSearch, setDubLangSearch] = useState(''); const [dubLangCode, setDubLangCode] = useState('en'); + const [translateProvider, setTranslateProvider] = useState('argos'); const [dubInstruct, setDubInstruct] = useState(''); const [dubProgress, setDubProgress] = useState({ current: 0, total: 0, text: '' }); const [dubFilename, setDubFilename] = useState(''); @@ -238,6 +257,14 @@ function App() { const [dubError, setDubError] = useState(''); const [dubVideoFile, setDubVideoFile] = useState(null); const [dubLocalBlobUrl, setDubLocalBlobUrl] = useState(null); + const dubBlobUrlRef = useRef(null); + useEffect(() => { dubBlobUrlRef.current = dubLocalBlobUrl; }, [dubLocalBlobUrl]); + useEffect(() => () => { + // Release any outstanding blob URLs on unmount to avoid leaks. + const urls = dubBlobUrlRef.current; + if (urls?.videoUrl?.startsWith('blob:')) URL.revokeObjectURL(urls.videoUrl); + if (urls?.audioUrl?.startsWith('blob:') && urls.audioUrl !== urls.videoUrl) URL.revokeObjectURL(urls.audioUrl); + }, []); const [dubTracks, setDubTracks] = useState([]); const [dubTranscript, setDubTranscript] = useState(''); const [showTranscript, setShowTranscript] = useState(false); @@ -249,6 +276,7 @@ function App() { const [exportTracks, setExportTracks] = useState({original: true}); // {original: true, es: true, de: false, ...} const [transcribeStart, setTranscribeStart] = useState(null); const [transcribeElapsed, setTranscribeElapsed] = useState(0); + const [dubTaskId, setDubTaskId] = useState(null); // ═══ STUDIO PROJECTS ═══ const [studioProjects, setStudioProjects] = useState([]); @@ -456,6 +484,17 @@ function App() { if (saved.dubTracks) setDubTracks(saved.dubTracks); if (saved.dubStep) setDubStep(saved.dubStep); if (saved.dubTranscript) setDubTranscript(saved.dubTranscript); + // Extra UI State + if (saved.exportTracks) setExportTracks(saved.exportTracks); + if (saved.preserveBg !== undefined) setPreserveBg(saved.preserveBg); + if (saved.defaultTrack) setDefaultTrack(saved.defaultTrack); + if (saved.exportHistory) setExportHistory(saved.exportHistory); + // Inference Parameters + if (saved.speed) setSpeed(saved.speed); + if (saved.steps) setSteps(saved.steps); + if (saved.cfg) setCfg(saved.cfg); + if (saved.denoise !== undefined) setDenoise(saved.denoise); + if (saved.showOverrides !== undefined) setShowOverrides(saved.showOverrides); } catch (e) {} return () => { cancelled = true; }; }, []); @@ -465,9 +504,16 @@ function App() { uiScale, text, mode, vdStates, language, isSidebarCollapsed, sidebarTab, dubJobId, dubFilename, dubDuration, dubSegments, - dubLang, dubLangCode, dubTracks, dubStep, dubTranscript + dubLang, dubLangCode, dubTracks, dubStep, dubTranscript, + exportTracks, preserveBg, defaultTrack, exportHistory, + speed, steps, cfg, denoise, showOverrides })); - }, [uiScale, text, mode, vdStates, language, isSidebarCollapsed, sidebarTab, dubJobId, dubFilename, dubDuration, dubSegments, dubLang, dubLangCode, dubTracks, dubStep, dubTranscript]); + }, [ + uiScale, text, mode, vdStates, language, isSidebarCollapsed, sidebarTab, + dubJobId, dubFilename, dubDuration, dubSegments, dubLang, dubLangCode, + dubTracks, dubStep, dubTranscript, exportTracks, preserveBg, defaultTrack, + exportHistory, speed, steps, cfg, denoise, showOverrides + ]); // ── KEYBOARD SHORTCUTS ── useEffect(() => { @@ -710,10 +756,11 @@ function App() { if (fin_prof) formData.append("profile_id", fin_prof); if (fin_inst) formData.append("instruct", fin_inst); + const fin_lang = seg.target_lang || dubLang; + if (fin_lang !== 'Auto') formData.append("language", fin_lang); - if (dubLang !== 'Auto') formData.append("language", dubLang); - - formData.append("num_step", steps || 16); + // Hardcode lightweight inference steps for live preview to drastically boost timeline responsiveness + formData.append("num_step", 8); formData.append("guidance_scale", cfg || 2.0); if (seg.speed && seg.speed !== 1.0) formData.append("speed", seg.speed); @@ -855,26 +902,54 @@ function App() { }; // ═══ DUB WORKFLOW ═══ + const dubAbortCtrlRef = useRef(null); + const dubClientJobIdRef = useRef(null); + const handleDubUpload = async () => { if (!dubVideoFile) return; setDubStep('uploading'); setDubError(''); setDubTracks([]); + const ctrl = new AbortController(); + dubAbortCtrlRef.current = ctrl; + // Generate job_id client-side so we can POST /dub/abort even during upload. + const clientJobId = Math.random().toString(36).slice(2, 10); + dubClientJobIdRef.current = clientJobId; + setDubJobId(clientJobId); try { const fd = new FormData(); fd.append("video", dubVideoFile); - const res = await fetch(`${API}/dub/upload`, { method: "POST", body: fd }); + fd.append("job_id", clientJobId); + const res = await fetch(`${API}/dub/upload`, { method: "POST", body: fd, signal: ctrl.signal }); if (!res.ok) throw new Error(await res.text()); const data = await res.json(); setDubJobId(data.job_id); setDubFilename(data.filename); setDubDuration(data.duration); setDubStep('transcribing'); setTranscribeStart(Date.now()); - const tRes = await fetch(`${API}/dub/transcribe/${data.job_id}`, { method: "POST" }); + const tRes = await fetch(`${API}/dub/transcribe/${data.job_id}`, { method: "POST", signal: ctrl.signal }); if (!tRes.ok) throw new Error(await tRes.text()); const tData = await tRes.json(); setDubSegments(tData.segments.map((s, i) => ({ ...s, id: i }))); setDubTranscript(tData.full_transcript || ''); setTranscribeStart(null); setDubStep('editing'); - } catch (err) { setDubError(err.message); setDubStep('idle'); setTranscribeStart(null); } + } catch (err) { + if (err.name === 'AbortError') { + toast('Upload cancelled'); + setDubStep('idle'); + } else { + setDubError(err.message); setDubStep('idle'); + } + setTranscribeStart(null); + } finally { + dubAbortCtrlRef.current = null; + } + }; + + const handleDubAbort = async () => { + const jobId = dubClientJobIdRef.current || dubJobId; + if (dubAbortCtrlRef.current) dubAbortCtrlRef.current.abort(); + if (jobId) { + try { await fetch(`${API}/dub/abort/${jobId}`, { method: 'POST' }); } catch (_) {} + } }; // Transcription elapsed timer @@ -885,6 +960,21 @@ function App() { }, [transcribeStart]); // ── AUTO-TRANSLATE ── + const handleCleanupSegments = async () => { + if (!dubJobId || !dubSegments.length) return; + const before = dubSegments.length; + try { + const res = await fetch(`${API}/dub/cleanup-segments/${dubJobId}`, { method: 'POST' }); + if (!res.ok) throw new Error(await res.text()); + const data = await res.json(); + setDubSegments(data.segments || []); + const delta = before - (data.after ?? data.segments.length); + toast.success(delta > 0 ? `Cleaned ${delta} fragment${delta === 1 ? '' : 's'}` : 'Segments already clean'); + } catch (err) { + toast.error('Clean up failed: ' + err.message); + } + }; + const handleTranslateAll = async () => { if (!dubSegments.length || !dubLangCode) return; setIsTranslating(true); @@ -893,8 +983,9 @@ function App() { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - segments: dubSegments.map(s => ({ id: s.id, text: s.text })), + segments: dubSegments.map(s => ({ id: s.id, text: s.text, target_lang: s.target_lang })), target_lang: dubLangCode, + provider: translateProvider, }), }); if (!res.ok) throw new Error(await res.text()); @@ -929,6 +1020,7 @@ function App() { instruct: fin_inst, profile_id: fin_prof, speed: s.speed || undefined, gain: s.gain !== undefined && s.gain !== 1.0 ? s.gain : undefined, + target_lang: s.target_lang || undefined, }; }), language: dubLang === 'Auto' ? 'Auto' : dubLang, @@ -943,11 +1035,14 @@ function App() { const data = await res.json(); if (!res.ok) throw new Error(data.detail || "Failed to start generation"); + setDubTaskId(data.task_id); + // Connect to background task SSE stream const streamRes = await fetch(`${API}/tasks/stream/${data.task_id}`); const reader = streamRes.body.getReader(); const decoder = new TextDecoder(); let buffer = ''; + let wasCancelled = false; while (true) { const { done, value } = await reader.read(); if (done) break; @@ -965,15 +1060,34 @@ function App() { setDubSegments(prev => prev.map((s, idx) => ({ ...s, sync_ratio: evt.sync_scores[idx] }))); } } + else if (evt.type === 'cancelled') { + wasCancelled = true; + setDubStep('editing'); + setDubError('Generation aborted.'); + toast('Dubbing aborted', { icon: '⏹' }); + } else if (evt.type === 'error') setDubError(p => p + `\nSeg ${evt.segment}: ${evt.error}`); } catch (e) {} } } } - if (dubStep !== 'done') setDubStep('done'); - loadDubHistory(); - playPing(); - } catch (err) { setDubError(err.message); setDubStep('editing'); } + setDubTaskId(null); + if (!wasCancelled) { + if (dubStep !== 'done') setDubStep('done'); + loadDubHistory(); + playPing(); + } + } catch (err) { setDubError(err.message); setDubStep('editing'); setDubTaskId(null); } + }; + + const handleDubStop = async () => { + if (!dubTaskId) return; + setDubStep('stopping'); + try { + await fetch(`${API}/tasks/cancel/${dubTaskId}`, { method: 'POST' }); + } catch (e) { + toast.error('Failed to stop'); + } }; const handleNativeExport = async (e, sourceIdentifier, fallbackName, mode) => { @@ -1012,28 +1126,73 @@ function App() { toast.error(`Could not open folder: ${err.message}`); } }; + const parseFilenameFromContentDisposition = (header) => { + if (!header) return null; + const utf8 = header.match(/filename\*=(?:UTF-8|utf-8)''([^;]+)/i); + if (utf8) { try { return decodeURIComponent(utf8[1].trim().replace(/^"|"$/g, '')); } catch { /* ignore */ } } + const plain = header.match(/filename="?([^";]+)"?/i); + return plain ? plain[1].trim() : null; + }; + const triggerDownload = async (url, fallbackName) => { + const extGuess = (fallbackName.includes('.') ? fallbackName.split('.').pop() : 'bin').toLowerCase(); + const modeGuess = ['mp4','mov','mkv','webm'].includes(extGuess) + ? 'video' : ['wav','mp3','flac'].includes(extGuess) ? 'audio' : 'file'; + + // In Tauri, WebKit silently drops blob downloads. Use native save dialog + // + server-side copy so the file actually lands on disk at a known path. + if (isTauri) { + try { + const { save } = await import('@tauri-apps/plugin-dialog'); + const destPath = await save({ + defaultPath: fallbackName, + filters: [{ name: modeGuess === 'video' ? 'Video' : 'Audio', extensions: [extGuess] }], + }); + if (!destPath) return; // user cancelled + toast.loading(`Saving ${fallbackName}...`, { id: fallbackName }); + const sep = url.includes('?') ? '&' : '?'; + const res = await fetch(`${url}${sep}save_path=${encodeURIComponent(destPath)}`); + if (!res.ok) { + const err = await res.json().catch(() => ({})); + throw new Error(err.detail || 'Save failed'); + } + const data = await res.json(); + toast.success(`Saved: ${data.path}`, { id: fallbackName }); + try { + await fetch(`${API}/export/record`, { + method: 'POST', headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ filename: data.display_name || fallbackName, destination_path: data.path, mode: modeGuess }) + }); + loadExportHistory(); + } catch (_) {} + } catch (err) { + console.error(err); + toast.error(`Save error: ${err.message}`, { id: fallbackName }); + } + return; + } + + // Browser path: standard blob download. try { toast.loading(`Processing ${fallbackName}...`, { id: fallbackName }); const response = await fetch(url); if (!response.ok) throw new Error("Download failed"); + const serverName = parseFilenameFromContentDisposition(response.headers.get('content-disposition')); + const finalName = serverName || fallbackName || 'download'; const blob = await response.blob(); const localUrl = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = localUrl; - a.download = fallbackName || 'download'; + a.download = finalName; document.body.appendChild(a); a.click(); document.body.removeChild(a); URL.revokeObjectURL(localUrl); - toast.success(`Downloaded ${fallbackName}`, { id: fallbackName }); - // Record to export history + toast.success(`Downloaded ${finalName}`, { id: fallbackName }); try { - const ext = fallbackName.split('.').pop() || ''; - const mode = ['mp4','mov','mkv','webm'].includes(ext) ? 'video' : ['wav','mp3','flac'].includes(ext) ? 'audio' : 'file'; await fetch(`${API}/export/record`, { method: 'POST', headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({ filename: fallbackName, destination_path: `~/Downloads/${fallbackName}`, mode }) + body: JSON.stringify({ filename: finalName, destination_path: `~/Downloads/${finalName}`, mode: modeGuess }) }); loadExportHistory(); } catch (_) {} @@ -1056,7 +1215,11 @@ function App() { setDubDuration(0); setDubError(''); setDubVideoFile(null); setDubTracks([]); setDubProgress({ current: 0, total: 0, text: '' }); setDubTranscript(''); setShowTranscript(false); setPreviewAudios({}); - setDubLocalBlobUrl(prev => { if (prev && prev.startsWith('blob:')) URL.revokeObjectURL(prev); return null; }); + setDubLocalBlobUrl(prev => { + if (prev?.videoUrl?.startsWith('blob:')) URL.revokeObjectURL(prev.videoUrl); + if (prev?.audioUrl?.startsWith('blob:') && prev.audioUrl !== prev.videoUrl) URL.revokeObjectURL(prev.audioUrl); + return null; + }); setActiveProjectId(null); setActiveProjectName(''); }; @@ -1189,8 +1352,6 @@ function App() { } }; - const filteredLangs = langSearch ? ALL_LANGUAGES.filter(l => l.toLowerCase().includes(langSearch.toLowerCase())) : ALL_LANGUAGES; - const filteredDubLangs = dubLangSearch ? ALL_LANGUAGES.filter(l => l.toLowerCase().includes(dubLangSearch.toLowerCase())) : ALL_LANGUAGES; return (
@@ -1199,35 +1360,37 @@ function App() { error: { iconTheme: { primary: '#fb4934', secondary: '#fff' } }, success: { iconTheme: { primary: '#b8bb26', secondary: '#fff' } } }}/> -
- {/* Empty placeholder for traffic light buffer */} -
- - {/* Center cluster: logo + tabs */} -
-
- - - - - - - - OmniVoice -
- -
- +
+ {/* Left cluster: traffic light buffer + tabs + dev panel */} +
+
+ {import.meta.env.DEV && ( + + )} +
+ + {/* Center: logo */} +
+ + + + + + + + OmniVoice
{/* Right cluster: scale + stats */} -
+
@@ -1472,9 +1635,14 @@ function App() { disabled={true} overlayContent={ dubStep === 'uploading' ? ( -
+
Extracting audio… +
) : dubStep === 'transcribing' ? (
@@ -1496,6 +1664,11 @@ function App() { }}/>
)} +
) : null } @@ -1680,21 +1853,25 @@ function App() { videoSrc={`${API}/dub/media/${dubJobId}`} segments={dubSegments} onSegmentsChange={setDubSegments} - disabled={dubStep === 'generating'} - overlayContent={dubStep === 'generating' ? ( + disabled={dubStep === 'generating' || dubStep === 'stopping'} + overlayContent={(dubStep === 'generating' || dubStep === 'stopping') ? (
- - - Dubbing {dubProgress.current}/{dubProgress.total}… + {dubStep === 'stopping' ? : } + + {dubStep === 'stopping' ? 'Stopping…' : `Dubbing ${dubProgress.current}/${dubProgress.total}…`}
-
-
-
- {dubProgress.text && {dubProgress.text}} + {dubStep === 'generating' && ( + <> +
+
+
+ {dubProgress.text && {dubProgress.text}} + + )}
) : null} /> @@ -1738,31 +1915,50 @@ function App() {
Language
- + { + setDubLang(lang); + const match = LANG_CODES.find(lc => lc.label.toLowerCase() === lang.toLowerCase()); + if (match) setDubLangCode(match.code); + }} + />
ISO Code
- + ({ value: lc.code, label: `${lc.code} — ${lc.label}` }))} + popular={POPULAR_ISO} + recentsKey="omnivoice.recents.dubIso" + onChange={setDubLangCode} + />
Style
setDubInstruct(e.target.value)} style={{fontSize:'0.65rem'}}/>
+
+
Engine
+ +
+
{/* Full transcript toggle */} @@ -1817,12 +2013,13 @@ function App() { Time Spkr Text + Lang Voice Vol
{dubSegments.map((seg, idx) => ( -
idx+1?'segment-done':''}`}> +
idx+1?'segment-done':''}`}> {formatTime(seg.start)}–{formatTime(seg.end)} @@ -1851,9 +2048,15 @@ function App() { {seg.speaker_id || ''} editSegments(dubSegments.map(s => s.id===seg.id?{...s,text:e.target.value}:s))} - disabled={dubStep==='generating'}/> + disabled={dubStep==='generating'||dubStep==='stopping'}/> + editSegments(dubSegments.map(s => s.id===seg.id?{...s,gain:Number(e.target.value)/100}:s))} style={{width:30, height:2, padding:0, margin:0, accentColor: (seg.gain ?? 1.0) > 1.2 ? '#fb4934' : (seg.gain ?? 1.0) < 0.5 ? '#83a598' : '#a89984'}} />
- -
@@ -1930,9 +2133,20 @@ function App() {
)}
- + {dubStep==='stopping' ? ( + + ) : dubStep==='generating' ? ( + + ) : ( + + )} -
@@ -2283,32 +2500,41 @@ function App() {
) : ( (mode === 'clone' ? profiles.filter(p => !p.instruct) : profiles.filter(p => !!p.instruct)).map(proj => ( -
-
-
+
+
+ {proj.is_locked ? : (mode === 'clone' ? : )} {proj.is_locked ? 'LOCKED' : (mode === 'clone' ? 'CLONE' : 'DESIGN')} -
+ {proj.is_locked && ( -
Consistent
+
Consistent
)}
-
+
{proj.name}
- {proj.instruct &&
{proj.instruct}
} -
- - {proj.is_locked && ( - )} -
@@ -2357,30 +2583,33 @@ function App() { <> {/* Dub history */} {!isSidebarCollapsed && dubHistory.map(item => ( -
-
-
+
+
+ DUB -
-
{item.segments_count} segs
+ +
{item.segments_count} segs
-
+
{item.filename}
-
- +
+ {item.language} ({item.language_code}) - + {Math.round(item.duration)}s
- -
- -
@@ -2389,35 +2618,49 @@ function App() { {/* Clone/Design history */} {!isSidebarCollapsed && history.map(item => ( -
-
-
- {item.mode === 'clone' ? : } {(item.mode||'').toUpperCase()} +
+
+ + {item.mode === 'clone' ? : } {(item.mode||'synth')} + +
+ {item.language && item.language !== 'Auto' && {item.language}} + {item.generation_time && {item.generation_time}s}
- {item.generation_time &&
{item.generation_time}s
}
- {item.language && item.language !== 'Auto' &&
{item.language}
} - {item.seed &&
seed: {item.seed}
} -
{item.text}
- {item.audio_path &&