diff --git a/README.md b/README.md index ed8bad7a..f3fd55c7 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,17 @@ Built on the [OmniVoice](https://github.com/k2-fsa/OmniVoice) 600-language zero- - **Video Branding** — Optional logo overlay on exported MP4s (5s fade-out, bottom-right). - **Configurable** — Toggle invisible/visible watermarks independently in Settings → Privacy. +### MCP Server (AI Agent Integration) +- **Model Context Protocol** — Expose OmniVoice as an AI agent tool for Claude, Cursor, and any MCP-compatible client. +- **5 Tools** — `generate_speech`, `list_voices`, `list_personalities`, `list_languages`, `check_health`. +- **stdio + SSE** — Works locally (Claude Desktop) or remotely (networked agents). +- **Zero config** — Drop `mcp.json` into your client config and go. See [`mcp.json`](mcp.json). + +### Audio Effects Chain +- **6 presets** — Broadcast 📻, Cinematic 🎬, Podcast 🎙️, Warm ☀️, Bright ✨, Raw 🔇. +- **Pedalboard-powered** — Spotify's production-grade DSP (EQ, compressor, reverb, noise gate, limiter). +- **API-driven** — `GET /tools/effects` returns presets; custom chains via `apply_effects_chain()`. + --- ## Quickstart @@ -292,9 +303,9 @@ chmod +x OmniVoice.Studio_*.AppImage **✨ Differentiators** - [ ] Global hotkey dictation — system-wide record → transcribe → paste (inspired by [VoiceBox Capture](https://github.com/jamiepine/voicebox)) -- [ ] MCP server — expose OmniVoice as an AI agent tool (Claude, Cursor, etc.) +- [x] ~~MCP server — expose OmniVoice as an AI agent tool (Claude, Cursor, etc.)~~ - [x] ~~Voice personalities — named presets (narrator, casual, formal) with saved TTS params~~ -- [ ] Audio effects chain — post-processing pipeline (reverb, EQ, compression) +- [x] ~~Audio effects chain — post-processing pipeline (reverb, EQ, compression)~~ - [ ] Real-time dub preview — stream TTS as you edit, no full re-render - [ ] Project-level casting view — drag voices to speakers diff --git a/backend/api/routers/tools.py b/backend/api/routers/tools.py index d9e98343..567ce2cc 100644 --- a/backend/api/routers/tools.py +++ b/backend/api/routers/tools.py @@ -126,3 +126,14 @@ def rate_fit(req: RateFitReq): target_lang=req.target_lang, source_text=req.source_text, ) + + +# ── Audio effects presets ────────────────────────────────────────────────── + + +@router.get("/tools/effects") +def list_effects(): + """Return available audio effect presets (Broadcast, Cinematic, etc.).""" + from services.audio_dsp import list_effect_presets + return list_effect_presets() + diff --git a/backend/mcp_server.py b/backend/mcp_server.py new file mode 100644 index 00000000..24ae2435 --- /dev/null +++ b/backend/mcp_server.py @@ -0,0 +1,215 @@ +""" +OmniVoice MCP Server — expose voice synthesis as AI-agent tools. + +Run standalone: + python -m backend.mcp_server # stdio transport (Claude Desktop) + python -m backend.mcp_server --sse # SSE transport (remote agents) + +Tools exposed: + generate_speech — text → WAV audio (voice clone or design) + list_voices — enumerate saved voice profiles + list_languages — available TTS languages + list_personalities — voice personality presets + +Resources exposed: + voice://{profile_id} — voice profile metadata + history://recent — last 20 generated audio items +""" +from __future__ import annotations + +import argparse +import base64 +import logging +import os +import sys + +logger = logging.getLogger("omnivoice.mcp") + +# ── Lazy imports — keeps startup fast when not using MCP ──────────────── + + +def _ensure_mcp(): + """Import `mcp` SDK lazily so the rest of the backend doesn't pay + for the import unless the MCP server is actually started.""" + try: + from mcp.server.fastmcp import FastMCP # noqa: F811 + return FastMCP + except ImportError: + print( + "MCP SDK not installed. Install with:\n" + " pip install 'mcp[cli]'\n" + "Then re-run this module.", + file=sys.stderr, + ) + sys.exit(1) + + +def create_mcp_server(): + """Build and return the FastMCP server instance.""" + FastMCP = _ensure_mcp() + mcp = FastMCP( + "OmniVoice Studio", + version="0.3.0", + description=( + "AI-agent interface for OmniVoice Studio — voice cloning, " + "voice design, and video dubbing in 646 languages." + ), + ) + + # ── Helpers ───────────────────────────────────────────────────────── + + def _api_base() -> str: + return os.environ.get("OMNIVOICE_API_URL", "http://localhost:3900") + + async def _api_get(path: str): + import httpx + async with httpx.AsyncClient(base_url=_api_base(), timeout=30) as c: + r = await c.get(path) + r.raise_for_status() + return r.json() + + async def _api_post_form(path: str, data: dict, files: dict | None = None): + import httpx + async with httpx.AsyncClient(base_url=_api_base(), timeout=120) as c: + r = await c.post(path, data=data, files=files or {}) + r.raise_for_status() + return r + + # ── Tools ─────────────────────────────────────────────────────────── + + @mcp.tool() + async def generate_speech( + text: str, + language: str = "Auto", + profile_id: str | None = None, + instruct: str | None = None, + speed: float = 1.0, + steps: int = 16, + ) -> str: + """Generate speech audio from text. + + Args: + text: The text to synthesize into speech. + language: Target language (ISO code or 'Auto'). 646 languages supported. + profile_id: ID of a saved voice profile to clone. Omit for voice design mode. + instruct: Style instruction (e.g. 'whisper', 'excited', 'narrator'). + speed: Speech speed multiplier (0.5–2.0, default 1.0). + steps: Diffusion steps (8=fast/draft, 16=balanced, 32=quality). + + Returns: + JSON with audio_id, generation_time, audio_duration, and + base64-encoded WAV data. + """ + form = { + "text": text, + "language": language, + "speed": str(speed), + "num_step": str(steps), + } + if profile_id: + form["profile_id"] = profile_id + if instruct: + form["instruct"] = instruct + + r = await _api_post_form("/generate", data=form) + + audio_id = r.headers.get("X-Audio-Id", "unknown") + gen_time = r.headers.get("X-Gen-Time", "?") + duration = r.headers.get("X-Audio-Duration", "?") + + wav_b64 = base64.b64encode(r.content).decode("ascii") + + return ( + f'{{"audio_id":"{audio_id}",' + f'"generation_time_s":{gen_time},' + f'"audio_duration_s":{duration},' + f'"format":"wav",' + f'"wav_base64":"{wav_b64}"}}' + ) + + @mcp.tool() + async def list_voices() -> str: + """List all saved voice profiles. + + Returns a JSON array of voice profiles with id, name, type (clone/design), + and personality. + """ + profiles = await _api_get("/profiles") + return str(profiles) + + @mcp.tool() + async def list_personalities() -> str: + """List available voice personality presets. + + Returns presets like Narrator, Casual, News Anchor, etc. with their + instruct text. Use the instruct text with generate_speech. + """ + presets = await _api_get("/personalities") + return str(presets) + + @mcp.tool() + async def list_languages() -> str: + """List a sample of supported TTS languages. + + OmniVoice supports 646 languages. This returns the most popular ones + plus a note about the full count. + """ + return ( + '{"total":646,"popular":[' + '"en","es","fr","de","it","pt","ru","ja","ko","zh",' + '"ar","hi","tr","nl","pl","sv","da","fi","no","el"' + '],"note":"Pass any ISO 639 code or set language=Auto for detection."}' + ) + + @mcp.tool() + async def check_health() -> str: + """Check if the OmniVoice backend is running and what GPU device is active.""" + info = await _api_get("/health") + return str(info) + + # ── Resources ─────────────────────────────────────────────────────── + + @mcp.resource("voice://{profile_id}") + async def get_voice(profile_id: str) -> str: + """Get details of a specific voice profile.""" + profiles = await _api_get("/profiles") + for p in profiles: + if p.get("id") == profile_id: + return str(p) + return f'{{"error":"Voice profile {profile_id} not found"}}' + + @mcp.resource("history://recent") + async def get_recent_history() -> str: + """Get the 20 most recent generation history items.""" + history = await _api_get("/history") + return str(history[:20]) + + return mcp + + +# ── CLI entrypoint ────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser(description="OmniVoice MCP Server") + parser.add_argument( + "--sse", action="store_true", + help="Use SSE transport instead of stdio (for remote agents)", + ) + parser.add_argument( + "--port", type=int, default=8765, + help="Port for SSE transport (default: 8765)", + ) + args = parser.parse_args() + + mcp = create_mcp_server() + + if args.sse: + logger.info("Starting MCP server on SSE transport, port %d", args.port) + mcp.run(transport="sse", port=args.port) + else: + logger.info("Starting MCP server on stdio transport") + mcp.run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/backend/services/audio_dsp.py b/backend/services/audio_dsp.py index ceb3ba09..c157596b 100644 --- a/backend/services/audio_dsp.py +++ b/backend/services/audio_dsp.py @@ -1,5 +1,103 @@ +""" +Audio DSP pipeline — broadcast-grade mastering + configurable effects chain. + +The default `apply_mastering()` is the same chain shipped since v0.1.0 +(highpass + compressor + light reverb). The new `apply_effects_chain()` +lets callers build custom pipelines from a list of named effects. + +All effects use Spotify's `pedalboard` library. When pedalboard isn't +installed, every function degrades gracefully (returns audio unmodified). +""" +import logging import torch +logger = logging.getLogger("omnivoice.dsp") + +# ── Effect presets ────────────────────────────────────────────────────── + +EFFECT_PRESETS = { + "broadcast": { + "label": "Broadcast", + "icon": "📻", + "description": "Radio/podcast standard — warm, compressed, clear.", + "chain": [ + {"type": "highpass", "cutoff_hz": 80}, + {"type": "compressor", "threshold_db": -18, "ratio": 3.0, "attack_ms": 5, "release_ms": 80}, + {"type": "eq", "low_gain_db": 1.5, "mid_gain_db": 0, "high_gain_db": 2.0}, + {"type": "limiter", "threshold_db": -1.0}, + ], + }, + "cinematic": { + "label": "Cinematic", + "icon": "🎬", + "description": "Film-quality — spacious reverb, gentle compression.", + "chain": [ + {"type": "highpass", "cutoff_hz": 60}, + {"type": "compressor", "threshold_db": -15, "ratio": 1.8, "attack_ms": 10, "release_ms": 150}, + {"type": "reverb", "room_size": 0.35, "wet_level": 0.15, "dry_level": 0.85}, + {"type": "limiter", "threshold_db": -1.5}, + ], + }, + "podcast": { + "label": "Podcast", + "icon": "🎙️", + "description": "Close-mic, intimate — heavy compression, no reverb.", + "chain": [ + {"type": "highpass", "cutoff_hz": 100}, + {"type": "noise_gate", "threshold_db": -40, "release_ms": 200}, + {"type": "compressor", "threshold_db": -20, "ratio": 4.0, "attack_ms": 2, "release_ms": 60}, + {"type": "eq", "low_gain_db": -1.0, "mid_gain_db": 2.0, "high_gain_db": 1.5}, + {"type": "limiter", "threshold_db": -0.5}, + ], + }, + "raw": { + "label": "Raw", + "icon": "🔇", + "description": "No processing — model output as-is.", + "chain": [], + }, + "warm": { + "label": "Warm", + "icon": "☀️", + "description": "Boosted low-mids, subtle saturation, cozy feel.", + "chain": [ + {"type": "highpass", "cutoff_hz": 60}, + {"type": "eq", "low_gain_db": 3.0, "mid_gain_db": 1.0, "high_gain_db": -1.0}, + {"type": "compressor", "threshold_db": -16, "ratio": 2.0, "attack_ms": 8, "release_ms": 120}, + {"type": "reverb", "room_size": 0.15, "wet_level": 0.06, "dry_level": 0.94}, + ], + }, + "bright": { + "label": "Bright", + "icon": "✨", + "description": "Crisp high-end, presence boost, airy feel.", + "chain": [ + {"type": "highpass", "cutoff_hz": 80}, + {"type": "eq", "low_gain_db": -1.0, "mid_gain_db": 0, "high_gain_db": 4.0}, + {"type": "compressor", "threshold_db": -14, "ratio": 2.5, "attack_ms": 3, "release_ms": 80}, + {"type": "limiter", "threshold_db": -1.0}, + ], + }, +} + + +def list_effect_presets() -> list[dict]: + """Return presets for the frontend UI picker.""" + return [ + {"id": k, "label": v["label"], "icon": v["icon"], "description": v["description"]} + for k, v in EFFECT_PRESETS.items() + ] + + +def get_effect_chain(preset_id: str) -> list[dict]: + """Return the effect chain for a preset. Falls back to empty chain.""" + p = EFFECT_PRESETS.get(preset_id) + return p["chain"] if p else [] + + +# ── Core DSP functions ────────────────────────────────────────────────── + + def apply_mastering(audio_tensor, sample_rate=24000): """Applies professional Broadcast-grade DSP (EQ, Compressor, light Reverb) to the clone voice.""" try: @@ -21,6 +119,7 @@ def apply_mastering(audio_tensor, sample_rate=24000): print(f"Mastering DSP Error: {e}") return audio_tensor + def normalize_audio(audio_tensor, target_dBFS=-2.0): """Peak-normalizes the audio to a standard broadcasting level (-2 dB) to fix F5TTS volume fluctuations.""" if audio_tensor.numel() == 0: @@ -30,3 +129,98 @@ def normalize_audio(audio_tensor, target_dBFS=-2.0): target_amp = 10 ** (target_dBFS / 20.0) audio_tensor = audio_tensor * (target_amp / max_val) return audio_tensor + + +def apply_effects_chain(audio_tensor, sample_rate: int, chain: list[dict]) -> torch.Tensor: + """Apply a chain of named effects to an audio tensor. + + Each item in `chain` is a dict with a `type` key and effect-specific + parameters. Unknown types are silently skipped. + + Supported types: + highpass — cutoff_hz (default 80) + lowpass — cutoff_hz (default 8000) + compressor — threshold_db, ratio, attack_ms, release_ms + reverb — room_size, wet_level, dry_level + noise_gate — threshold_db, release_ms + eq — low_gain_db, mid_gain_db, high_gain_db + limiter — threshold_db + """ + if not chain: + return audio_tensor + + try: + from pedalboard import ( + Pedalboard, + Compressor, + Reverb, + HighpassFilter, + LowpassFilter, + NoiseGate, + Limiter, + LowShelfFilter, + HighShelfFilter, + PeakFilter, + ) + import numpy as np + except ImportError: + logger.debug("pedalboard not installed — effects chain skipped") + return audio_tensor + + plugins = [] + for fx in chain: + t = fx.get("type", "").lower() + try: + if t == "highpass": + plugins.append(HighpassFilter(cutoff_frequency_hz=fx.get("cutoff_hz", 80))) + elif t == "lowpass": + plugins.append(LowpassFilter(cutoff_frequency_hz=fx.get("cutoff_hz", 8000))) + elif t == "compressor": + plugins.append(Compressor( + threshold_db=fx.get("threshold_db", -15), + ratio=fx.get("ratio", 2.0), + attack_ms=fx.get("attack_ms", 5), + release_ms=fx.get("release_ms", 100), + )) + elif t == "reverb": + plugins.append(Reverb( + room_size=fx.get("room_size", 0.2), + wet_level=fx.get("wet_level", 0.1), + dry_level=fx.get("dry_level", 0.9), + )) + elif t == "noise_gate": + plugins.append(NoiseGate( + threshold_db=fx.get("threshold_db", -40), + release_ms=fx.get("release_ms", 200), + )) + elif t == "limiter": + plugins.append(Limiter(threshold_db=fx.get("threshold_db", -1.0))) + elif t == "eq": + low = fx.get("low_gain_db", 0) + mid = fx.get("mid_gain_db", 0) + high = fx.get("high_gain_db", 0) + if low: + plugins.append(LowShelfFilter(cutoff_frequency_hz=250, gain_db=low)) + if mid: + plugins.append(PeakFilter(cutoff_frequency_hz=1500, gain_db=mid, q=1.0)) + if high: + plugins.append(HighShelfFilter(cutoff_frequency_hz=4000, gain_db=high)) + else: + logger.debug("Unknown effect type: %s — skipped", t) + except Exception as e: + logger.warning("Failed to create %s effect: %s", t, e) + + if not plugins: + return audio_tensor + + board = Pedalboard(plugins) + audio_np = audio_tensor.cpu().numpy() + if audio_np.ndim == 1: + audio_np = audio_np[None, :] + try: + effected = board(audio_np, sample_rate, reset=False) + return torch.from_numpy(effected).to(audio_tensor.device) + except Exception as e: + logger.warning("Effects chain failed: %s — returning unmodified audio", e) + return audio_tensor + diff --git a/mcp.json b/mcp.json new file mode 100644 index 00000000..4f5fcb0d --- /dev/null +++ b/mcp.json @@ -0,0 +1,12 @@ +{ + "mcpServers": { + "omnivoice": { + "command": "python", + "args": ["-m", "backend.mcp_server"], + "cwd": "/path/to/OmniVoice-Studio", + "env": { + "OMNIVOICE_API_URL": "http://localhost:3900" + } + } + } +}