diff --git a/backend/api/routers/dub_generate.py b/backend/api/routers/dub_generate.py index 06913c82..cd353e23 100644 --- a/backend/api/routers/dub_generate.py +++ b/backend/api/routers/dub_generate.py @@ -13,7 +13,7 @@ from core.config import DUB_DIR, VOICES_DIR from core.tasks import task_manager from schemas.requests import DubRequest from services.model_manager import get_model, _gpu_pool -from services.audio_dsp import apply_mastering, normalize_audio +from services.audio_dsp import apply_mastering, normalize_audio, apply_effects_chain, get_effect_chain from services.audio_io import atomic_save_wav, _safe_torchaudio_save from services.rvc import apply_rvc, is_enabled as rvc_is_enabled from services.incremental import segment_fingerprint @@ -108,7 +108,7 @@ async def dub_generate(job_id: str, req: DubRequest): sync_scores.append(1.0) continue - def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id=None): + def _gen(text, lang, instruct_str, dur_s, nstep, cfg, spd, profile_id, effect_preset): ref_audio = None ref_text = None used_seed = None @@ -161,7 +161,21 @@ async def dub_generate(job_id: str, req: DubRequest): speed=spd, denoise=True, postprocess_output=True, ) audio_out = audios[0] - mastered_audio = apply_mastering(audio_out, sample_rate=_model.sampling_rate if hasattr(_model, 'sampling_rate') else 24000) + sr = _model.sampling_rate if hasattr(_model, 'sampling_rate') else 24000 + + # Apply per-segment DSP effect preset (default: broadcast) + seg_effect_preset = effect_preset or "broadcast" + if seg_effect_preset == "raw": + return audio_out + + mastered_audio = apply_mastering(audio_out, sample_rate=sr) + effect_chain = get_effect_chain(seg_effect_preset) + if effect_chain: + mastered_audio = apply_effects_chain( + mastered_audio, + sample_rate=sr, + chain=effect_chain, + ) return normalize_audio(mastered_audio, target_dBFS=-2.0) except Exception as e: is_oom = ( @@ -169,7 +183,6 @@ async def dub_generate(job_id: str, req: DubRequest): or "out of memory" in str(e).lower() or "CUDA error" in str(e) ) - # Always try to reclaim VRAM regardless of error type. import gc gc.collect() if torch.cuda.is_available(): @@ -178,9 +191,8 @@ async def dub_generate(job_id: str, req: DubRequest): torch.mps.empty_cache() if not is_oom: - raise # Non-OOM — propagate the real error, don't mask it. + raise - # OOM recovery: retry once with reduced steps (less VRAM). retry_steps = min(nstep, 8) logger.warning( "OOM on segment (nstep=%d), retrying with %d steps after cache flush", @@ -195,7 +207,20 @@ async def dub_generate(job_id: str, req: DubRequest): speed=spd, denoise=True, postprocess_output=True, ) audio_out = audios[0] - mastered_audio = apply_mastering(audio_out, sample_rate=_model.sampling_rate if hasattr(_model, 'sampling_rate') else 24000) + sr = _model.sampling_rate if hasattr(_model, 'sampling_rate') else 24000 + + seg_effect_preset = effect_preset or "broadcast" + if seg_effect_preset == "raw": + return audio_out + + mastered_audio = apply_mastering(audio_out, sample_rate=sr) + effect_chain = get_effect_chain(seg_effect_preset) + if effect_chain: + mastered_audio = apply_effects_chain( + mastered_audio, + sample_rate=sr, + chain=effect_chain, + ) return normalize_audio(mastered_audio, target_dBFS=-2.0) except Exception as retry_err: raise RuntimeError( @@ -204,13 +229,13 @@ async def dub_generate(job_id: str, req: DubRequest): f"Try the Flush button in the header to free VRAM, " f"or switch to CPU in Settings. " f"Underlying error: {retry_err}" - ) + ) from retry_err - seg_instruct = seg.instruct or req.instruct seg_profile = seg.profile_id or None seg_speed = seg.speed if hasattr(seg, 'speed') and seg.speed is not None else req.speed seg_lang = seg.target_lang if getattr(seg, 'target_lang', None) else req.language + seg_instruct = seg.instruct or req.instruct # Phase 4.2 — if the segment carries a free-form direction, parse it # and append the taxonomy instruct (e.g. "urgent, surprised") on top # of whatever instruct was already set. Also apply the director's @@ -240,10 +265,11 @@ async def dub_generate(job_id: str, req: DubRequest): # flag to restore num_step=req.num_step quality. _num_step = 8 if req.preview else req.num_step _t_tts_0 = time.perf_counter() + seg_effect_preset = getattr(seg, "effect_preset", None) or "broadcast" audio_tensor = await loop.run_in_executor( _gpu_pool, _gen, seg.text, seg_lang, seg_instruct, seg_duration, - _num_step, req.guidance_scale, seg_speed, seg_profile, + _num_step, req.guidance_scale, seg_speed, seg_profile, seg_effect_preset, ) _t_tts += time.perf_counter() - _t_tts_0 @@ -277,6 +303,7 @@ async def dub_generate(job_id: str, req: DubRequest): "instruct": getattr(seg, "instruct", None), "speed": getattr(seg, "speed", None), "direction": getattr(seg, "direction", None), + "effect_preset": getattr(seg, "effect_preset", None), }) except Exception as e: logger.debug("seg fingerprint skipped for %s: %s", seg_id, e) diff --git a/backend/api/routers/engines.py b/backend/api/routers/engines.py index bdc08896..8e33589f 100644 --- a/backend/api/routers/engines.py +++ b/backend/api/routers/engines.py @@ -23,6 +23,8 @@ from pydantic import BaseModel from api.dependencies import require_loopback from core import prefs from services import tts_backend, asr_backend, llm_backend, translation_engines +from services.audio_dsp import list_effect_presets +from api.schemas import EffectPresetsResponse router = APIRouter() @@ -66,6 +68,16 @@ def list_llm_backends(): return {"active": llm_backend.active_backend_id(), "backends": llm_backend.list_backends()} +@router.get("/engines/effects/presets", response_model=EffectPresetsResponse) +def list_effects_presets(): + """Return available DSP effect presets for the dub pipeline. + + Each preset is a named chain of audio effects (EQ, compressor, reverb, etc.) + that can be applied to generated TTS audio on a per-segment basis. + """ + return {"presets": list_effect_presets()} + + @router.get("/engines/translation") def list_translation_engines(): """Translation engines with per-engine pip-package availability. diff --git a/backend/api/routers/generation.py b/backend/api/routers/generation.py index 0c85552c..c1a8a6b3 100644 --- a/backend/api/routers/generation.py +++ b/backend/api/routers/generation.py @@ -25,8 +25,9 @@ def _run_inference( model, text, language, ref_audio_path, ref_text, instruct, duration, num_step, guidance_scale, speed, t_shift, denoise, postprocess_output, layer_penalty_factor, position_temperature, - class_temperature, used_seed, + class_temperature, used_seed, effect_preset="broadcast", ): + from services.audio_dsp import apply_mastering, normalize_audio, apply_effects_chain, get_effect_chain import torch try: if used_seed is not None: @@ -47,7 +48,30 @@ def _run_inference( ) audio_out = audios[0] - mastered_audio = apply_mastering(audio_out, sample_rate=model.sampling_rate if hasattr(model, 'sampling_rate') else 24000) + sr = model.sampling_rate if hasattr(model, 'sampling_rate') else 24000 + + # Apply DSP effect preset + _effect_preset = effect_preset or "broadcast" + + # Validate preset ID + from services.audio_dsp import EFFECT_PRESETS + if _effect_preset not in EFFECT_PRESETS: + raise ValueError( + f"Unknown effect preset: {_effect_preset!r}. " + f"Valid: {list(EFFECT_PRESETS.keys())}" + ) + + if _effect_preset == "raw": + # Raw: skip all DSP — return raw model output + return audio_out + + mastered_audio = apply_mastering(audio_out, sample_rate=sr) + _chain = get_effect_chain(_effect_preset) + if _chain: + mastered_audio = apply_effects_chain( + mastered_audio, sample_rate=sr, chain=_chain, + ) + return normalize_audio(mastered_audio, target_dBFS=-2.0) except ValueError as e: @@ -85,6 +109,7 @@ async def generate_speech( class_temperature: Optional[float] = Form(None), profile_id: Optional[str] = Form(None), seed: Optional[int] = Form(None), + effect_preset: str = Form("broadcast"), ): _model = await get_model() @@ -138,7 +163,7 @@ async def generate_speech( _model, text, language, ref_audio_path, ref_text, instruct, duration, num_step, guidance_scale, speed, t_shift, denoise, postprocess_output, layer_penalty_factor, position_temperature, - class_temperature, used_seed, + class_temperature, used_seed, effect_preset, ) gen_time = round(time.time() - start_time, 2) diff --git a/backend/api/schemas.py b/backend/api/schemas.py index 2ded90f7..5379ea2e 100644 --- a/backend/api/schemas.py +++ b/backend/api/schemas.py @@ -147,3 +147,20 @@ class ModelEntry(BaseModel): supported: bool = True size_on_disk: int | None = None nb_files: int | None = None + + +# ── Effect presets ───────────────────────────────────────────────────── + +class EffectPresetEntry(BaseModel): + """One DSP effect preset.""" + model_config = ConfigDict(extra="allow") + + id: str + label: str + icon: str + description: str + + +class EffectPresetsResponse(BaseModel): + """GET /engines/effects/presets""" + presets: list[EffectPresetEntry] = Field(default_factory=list) diff --git a/backend/schemas/requests.py b/backend/schemas/requests.py index 6dc25927..cc319979 100644 --- a/backend/schemas/requests.py +++ b/backend/schemas/requests.py @@ -1,6 +1,8 @@ -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from typing import List, Optional +from services.audio_dsp import EFFECT_PRESETS + class ExportRequest(BaseModel): source_filename: str destination_path: str @@ -23,6 +25,17 @@ class DubSegment(BaseModel): speed: Optional[float] = None gain: Optional[float] = None # Per-segment volume (0.0 - 2.0, default 1.0) target_lang: Optional[str] = None # Per-segment language override (ISO code) + effect_preset: str = "broadcast" # NEW: DSP preset id (default: broadcast) + + @field_validator("effect_preset") + @classmethod + def validate_effect_preset(cls, v: str) -> str: + if v not in EFFECT_PRESETS: + raise ValueError( + f"Unknown effect preset: {v!r}. " + f"Valid: {list(EFFECT_PRESETS.keys())}" + ) + return v class DubRequest(BaseModel): segments: List[DubSegment] diff --git a/backend/services/batched_tts.py b/backend/services/batched_tts.py index 300efaba..04d4c5b1 100644 --- a/backend/services/batched_tts.py +++ b/backend/services/batched_tts.py @@ -42,7 +42,7 @@ class SegmentSpec: __slots__ = ( "index", "text", "language", "instruct", "speed", "duration", "num_step", "guidance_scale", "profile_id", - "ref_audio", "ref_text", "start", "end", + "ref_audio", "ref_text", "start", "end", "effect_preset", ) def __init__(self, **kwargs): @@ -93,7 +93,7 @@ async def generate_segments_batched( List of (segment_index, audio_tensor, sample_rate) tuples, ordered by segment_index. """ - from services.audio_dsp import apply_mastering, normalize_audio + from services.audio_dsp import apply_mastering, normalize_audio, apply_effects_chain, get_effect_chain sr = getattr(model, "sampling_rate", 24000) loop = asyncio.get_running_loop() @@ -144,7 +144,20 @@ async def generate_segments_batched( postprocess_output=True, ) audio_out = audios[0] + + # Apply per-segment DSP effect preset (default: broadcast) + seg_effect_preset = getattr(s, "effect_preset", None) or "broadcast" + if seg_effect_preset == "raw": + # Raw: skip all DSP — return raw model output + return audio_out + mastered = apply_mastering(audio_out, sample_rate=sr) + effect_chain = get_effect_chain(seg_effect_preset) + if effect_chain: + mastered = apply_effects_chain( + mastered, sample_rate=sr, chain=effect_chain + ) + return normalize_audio(mastered, target_dBFS=-2.0) audio = await loop.run_in_executor(gpu_pool, _gen_one) diff --git a/backend/services/incremental.py b/backend/services/incremental.py index a7b19025..99922189 100644 --- a/backend/services/incremental.py +++ b/backend/services/incremental.py @@ -20,7 +20,7 @@ import hashlib import json -_GEN_INPUT_FIELDS = ("text", "target_lang", "profile_id", "instruct", "speed", "direction") +_GEN_INPUT_FIELDS = ("text", "target_lang", "profile_id", "instruct", "speed", "direction", "effect_preset") def segment_fingerprint(seg: dict) -> str: @@ -29,6 +29,9 @@ def segment_fingerprint(seg: dict) -> str: Any change to `_GEN_INPUT_FIELDS` flips the hash and the segment becomes a re-gen candidate. Changes to position / selection state / lip-sync badge don't trigger regen, which is what we want. + + Currently includes: text, target_lang, profile_id, instruct, speed, + direction, effect_preset. """ payload = {k: (seg.get(k) if seg.get(k) is not None else "") for k in _GEN_INPUT_FIELDS} blob = json.dumps(payload, sort_keys=True, ensure_ascii=False) diff --git a/frontend/src/api/engines.ts b/frontend/src/api/engines.ts index 2d6b6f18..89774599 100644 --- a/frontend/src/api/engines.ts +++ b/frontend/src/api/engines.ts @@ -98,3 +98,20 @@ export async function getJob(id: string): Promise { export async function getJobEvents(id: string, afterSeq: number = 0): Promise { return apiJson(`/jobs/${id}/events?after_seq=${afterSeq}`); } + +// ── Effect presets ────────────────────────────────────────────────────── + +export interface EffectPreset { + id: string; + label: string; + icon: string; + description: string; +} + +export interface EffectPresetsResponse { + presets: EffectPreset[]; +} + +export async function fetchEffectPresets(): Promise { + return apiJson('/engines/effects/presets'); +} diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 4c642dd9..4c2965aa 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -153,6 +153,18 @@ export interface DubHistoryResponse { jobs: DubJobMeta[]; } +export interface DubSegment { + start: number; + end: number; + text: string; + instruct?: string; + profile_id?: string; + speed?: number; + gain?: number; + target_lang?: string; + effect_preset?: string; +} + export interface DubTranslateResponse { segments: { id: string; text: string; text_original?: string; rate_ratio?: number; rate_error?: string }[]; } diff --git a/frontend/src/store/dubSlice.ts b/frontend/src/store/dubSlice.ts index cab8c5b9..0f4edbe3 100644 --- a/frontend/src/store/dubSlice.ts +++ b/frontend/src/store/dubSlice.ts @@ -15,6 +15,7 @@ * history restore explicitly rehydrates the relevant fields. */ import type { StateCreator } from 'zustand'; +import type { EffectPreset } from '../api/engines'; export type DubStep = | 'idle' @@ -84,6 +85,12 @@ export interface DubSlice { source_count: number; }>; + // ── Effect Presets ──────────────────────────────────────────────────── + segmentEffectPresets: Record; + setSegmentEffectPreset: (segId: string, presetId: string) => void; + availableEffectPresets: EffectPreset[]; + setAvailableEffectPresets: (presets: EffectPreset[]) => void; + // ── Setters (React-style; accept value or updater fn) ───────────────── setDubJobId: (v: Updater) => void; setDubStep: (v: Updater) => void; @@ -115,7 +122,8 @@ const INITIAL: Omit = { dubJobId: null, dubStep: 'idle', @@ -137,6 +145,8 @@ const INITIAL: Omit = (set, get) => ({ @@ -162,6 +172,11 @@ export const createDubSlice: StateCreator = (set, ge setExportTracks: (v) => set((s) => ({ exportTracks: resolve(v, s.exportTracks) })), setPreviewSegIds:(v) => set((s) => ({ previewSegIds:resolve(v, s.previewSegIds) })), setSpeakerClones:(v) => set((s) => ({ speakerClones: resolve(v, s.speakerClones) })), + setSegmentEffectPreset: (segId, presetId) => + set((s) => ({ + segmentEffectPresets: { ...s.segmentEffectPresets, [segId]: presetId }, + })), + setAvailableEffectPresets: (presets) => set({ availableEffectPresets: presets }), resetDubState: () => { // Touch `get` so strict-mode double-invocation of the initializer doesn't diff --git a/tests/test_effects_chain.py b/tests/test_effects_chain.py new file mode 100644 index 00000000..58e76325 --- /dev/null +++ b/tests/test_effects_chain.py @@ -0,0 +1,139 @@ +""" +Tests for the audio effects chain DSP pipeline. + +Pure functions — no GPU or model loading required. +Tests run in seconds on any machine. + +Note: sys.path for backend imports is handled by tests/conftest.py. +""" + +import pytest +import torch +import math +pedalboard = pytest.importorskip("pedalboard") + +from services.audio_dsp import ( + apply_effects_chain, + get_effect_chain, + list_effect_presets, + EFFECT_PRESETS, +) + + +def _make_test_audio(duration_s=1.0, sample_rate=24000) -> torch.Tensor: + """Create a test audio tensor with a simple sine wave.""" + t = torch.linspace(0, duration_s, int(duration_s * sample_rate)) + return torch.sin(2 * math.pi * 440 * t).unsqueeze(0) # 440 Hz sine, mono + + +class TestListEffectPresets: + def test_returns_all_presets(self): + presets = list_effect_presets() + assert len(presets) == 6 + ids = [p["id"] for p in presets] + assert "broadcast" in ids + assert "cinematic" in ids + assert "podcast" in ids + assert "raw" in ids + assert "warm" in ids + assert "bright" in ids + + def test_preset_has_required_fields(self): + for preset in list_effect_presets(): + assert "id" in preset + assert "label" in preset + assert "icon" in preset + assert "description" in preset + + +class TestGetEffectChain: + def test_broadcast_returns_chain(self): + chain = get_effect_chain("broadcast") + assert len(chain) > 0 + types = [fx["type"] for fx in chain] + assert "highpass" in types + assert "compressor" in types + assert "limiter" in types + + def test_cinematic_has_reverb(self): + chain = get_effect_chain("cinematic") + types = [fx["type"] for fx in chain] + assert "reverb" in types + + def test_raw_returns_empty(self): + chain = get_effect_chain("raw") + assert chain == [] + + def test_unknown_returns_empty(self): + chain = get_effect_chain("nonexistent") + assert chain == [] + + +class TestApplyEffectsChain: + def test_raw_preset_returns_unmodified(self): + audio = _make_test_audio() + result = apply_effects_chain(audio, sample_rate=24000, chain=[]) + assert torch.equal(audio, result) + + def test_broadcast_preset_returns_tensor(self): + audio = _make_test_audio() + chain = get_effect_chain("broadcast") + result = apply_effects_chain(audio, sample_rate=24000, chain=chain) + assert isinstance(result, torch.Tensor) + assert result.shape == audio.shape + + def test_cinematic_preset_returns_tensor(self): + audio = _make_test_audio() + chain = get_effect_chain("cinematic") + result = apply_effects_chain(audio, sample_rate=24000, chain=chain) + assert isinstance(result, torch.Tensor) + assert result.shape == audio.shape + + def test_podcast_preset_returns_tensor(self): + audio = _make_test_audio() + chain = get_effect_chain("podcast") + result = apply_effects_chain(audio, sample_rate=24000, chain=chain) + assert isinstance(result, torch.Tensor) + assert result.shape == audio.shape + + def test_warm_preset_returns_tensor(self): + audio = _make_test_audio() + chain = get_effect_chain("warm") + result = apply_effects_chain(audio, sample_rate=24000, chain=chain) + assert isinstance(result, torch.Tensor) + assert result.shape == audio.shape + + def test_bright_preset_returns_tensor(self): + audio = _make_test_audio() + chain = get_effect_chain("bright") + result = apply_effects_chain(audio, sample_rate=24000, chain=chain) + assert isinstance(result, torch.Tensor) + assert result.shape == audio.shape + + def test_all_presets_produce_output(self): + """Smoke test: every preset processes without error.""" + audio = _make_test_audio() + for preset_id in EFFECT_PRESETS: + chain = get_effect_chain(preset_id) + result = apply_effects_chain(audio, sample_rate=24000, chain=chain) + assert isinstance(result, torch.Tensor) + assert result.shape[0] == 1 # mono + + def test_clipping_prevention(self): + """Output should not exceed [-1.0, 1.0] range after limiter presets.""" + audio = _make_test_audio() + for preset_id in ("broadcast", "podcast", "bright"): + chain = get_effect_chain(preset_id) + result = apply_effects_chain(audio, sample_rate=24000, chain=chain) + assert result.abs().max() <= 1.0 + + def test_different_presets_produce_different_output(self): + """Different presets should produce audibly different output.""" + audio = _make_test_audio(duration_s=2.0) + results = {} + for preset_id in ("broadcast", "cinematic", "raw"): + chain = get_effect_chain(preset_id) + results[preset_id] = apply_effects_chain(audio, sample_rate=24000, chain=chain) + # broadcast and cinematic should differ from raw (unprocessed) + assert not torch.equal(results["broadcast"], results["raw"]) + assert not torch.equal(results["cinematic"], results["raw"])