diff --git a/docs/agentic-voice.md b/docs/agentic-voice.md new file mode 100644 index 00000000..14c4e9b6 --- /dev/null +++ b/docs/agentic-voice.md @@ -0,0 +1,87 @@ +# Agentic voice: OmniVoice as a TTS/STT provider + +OmniVoice exposes an **OpenAI-compatible API**, so any agent framework that +speaks to OpenAI's audio endpoints can use your local OmniVoice for speech — +in your own cloned voice, with nothing leaving your machine. You bring the +agent runtime; OmniVoice is the voice. + +This is "agentic v1": OmniVoice is a provider, not the orchestrator. You wire +your own agent (a support line, a desk assistant, a Discord persona) and point +its TTS/STT at OmniVoice. + +> **Scope.** This page covers OmniVoice-as-provider. Outbound phone calls are a +> separate, deferred milestone (they need a paid carrier — there is no +> fully-local path to the PSTN) and ship only behind explicit consent +> guardrails. See the roadmap in `docs/competitive-analysis.md` (§R1). + +## The endpoints + +OmniVoice serves these on `http://localhost:3900/v1` (or your +[remote backend URL](remote-gpu.md)): + +| OpenAI route | OmniVoice support | +|---|---| +| `POST /v1/audio/speech` | TTS. `model` = engine id, `voice` = a voice-profile id (your clone) or preset, `response_format` incl. `pcm` and `wav`, `speed`. Default output is 24 kHz. | +| `POST /v1/audio/transcriptions` | STT (Whisper-family). | +| `GET /v1/audio/voices` | list available voices (OmniVoice extension). | + +A contract test (`tests/test_agentic_provider_contract.py`) pins this request +shape in CI, so the recipes below won't silently break. + +## pipecat (recommended) + +[pipecat](https://github.com/pipecat-ai/pipecat) (BSD-2) runs as a Python +library inside your own process — no extra server. Point its OpenAI TTS/STT +services at OmniVoice: + +```python +from pipecat.services.openai.tts import OpenAITTSService +from pipecat.services.openai.stt import OpenAISTTService + +tts = OpenAITTSService( + base_url="http://localhost:3900/v1", + api_key="not-needed-locally", # any string; OmniVoice ignores it unless OMNIVOICE_API_KEY is set + voice="", # from GET /v1/audio/voices, or "default" + model="omnivoice", # or any installed engine id + sample_rate=24000, # matches OmniVoice's default output +) + +stt = OpenAISTTService( + base_url="http://localhost:3900/v1", + api_key="not-needed-locally", +) +``` + +Drop those into any pipecat pipeline (VAD, turn-taking, and LLM stay local +too). A minimal runnable example is in +[`examples/agentic/pipecat_minimal.py`](../examples/agentic/pipecat_minimal.py). + +## LiveKit Agents + +[LiveKit Agents](https://github.com/livekit/agents) (Apache-2.0) needs a +LiveKit media server alongside, but its OpenAI plugin takes the same +`base_url`: + +```python +from livekit.plugins import openai + +tts = openai.TTS(base_url="http://localhost:3900/v1", api_key="x", voice="") +stt = openai.STT(base_url="http://localhost:3900/v1", api_key="x") +``` + +Choose LiveKit over pipecat only when you need its WebRTC/SIP scale; for a +single local agent, pipecat is lighter. + +## Remote backend + +Running OmniVoice on a [remote GPU box](remote-gpu.md)? Use that backend's URL +as `base_url` and pass its `OMNIVOICE_API_KEY` as the `api_key` — the same +bearer the rest of the app uses. Keep it on your tailnet, not the open +internet. + +## Use your own voice responsibly + +When an agent speaks in a cloned voice, prefer a profile you've marked +**verified own voice** (Settings → a voice profile → Voice ownership). That +consent lock is what gates the heavier agentic features as they land, and +it's the honest default for "an AI is speaking as me." diff --git a/examples/agentic/pipecat_minimal.py b/examples/agentic/pipecat_minimal.py new file mode 100644 index 00000000..43e8b174 --- /dev/null +++ b/examples/agentic/pipecat_minimal.py @@ -0,0 +1,61 @@ +"""Minimal pipecat agent that speaks and listens through local OmniVoice. + +OmniVoice is used purely as an OpenAI-compatible TTS/STT provider — nothing +leaves your machine. See docs/agentic-voice.md for the full recipe. + +Run OmniVoice first (default http://localhost:3900), then: + + uv pip install "pipecat-ai[openai,silero]" + python examples/agentic/pipecat_minimal.py + +This is a deliberately tiny skeleton: it wires the OmniVoice TTS/STT services +into a pipecat pipeline and leaves the transport + LLM for you to choose. It +does not run a phone call or a server — that is the "agentic v1" scope +(OmniVoice as provider, you bring the runtime). +""" + +from __future__ import annotations + +import os + +OMNIVOICE_BASE_URL = os.environ.get("OMNIVOICE_API_URL", "http://localhost:3900") + "/v1" +# OmniVoice ignores the key for local use; if you set OMNIVOICE_API_KEY on a +# remote backend, pass that same value here. +OMNIVOICE_API_KEY = os.environ.get("OMNIVOICE_API_KEY", "not-needed-locally") +# A voice-profile id from GET /v1/audio/voices, or "default". +OMNIVOICE_VOICE = os.environ.get("OMNIVOICE_VOICE", "default") + + +def build_services(): + """Return (stt, tts) backed by local OmniVoice. + + Imported lazily so this file is importable (and lint-clean) without + pipecat installed — the smoke test in CI checks the wiring shape, not a + live pipeline. + """ + from pipecat.services.openai.stt import OpenAISTTService + from pipecat.services.openai.tts import OpenAITTSService + + stt = OpenAISTTService( + base_url=OMNIVOICE_BASE_URL, + api_key=OMNIVOICE_API_KEY, + ) + tts = OpenAITTSService( + base_url=OMNIVOICE_BASE_URL, + api_key=OMNIVOICE_API_KEY, + voice=OMNIVOICE_VOICE, + model="omnivoice", + sample_rate=24000, # OmniVoice's default output rate + ) + return stt, tts + + +def main() -> None: + stt, tts = build_services() + print("OmniVoice STT + TTS services constructed against", OMNIVOICE_BASE_URL) + print("Wire `stt` and `tts` into your pipecat Pipeline with a transport") + print("and an LLM service. See docs/agentic-voice.md.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_agentic_provider_contract.py b/tests/test_agentic_provider_contract.py new file mode 100644 index 00000000..551db61b --- /dev/null +++ b/tests/test_agentic_provider_contract.py @@ -0,0 +1,119 @@ +"""Contract test for the surface agent runtimes (pipecat / LiveKit) consume. + +Wave 2.5 (parity program / Action 15): OmniVoice acts as a TTS/STT provider +for pipecat and LiveKit via the OpenAI-compatible API. Those runtimes call +POST /v1/audio/speech with {model, input, voice, response_format, speed} and +expect raw audio back; pipecat's OpenAITTSService defaults to PCM @ 24 kHz. +This test pins that the endpoint accepts exactly that request shape and +returns audio, so a change can't silently break the documented recipe +(docs/agentic-voice.md). Mirrors tests/test_pyvideotrans_contract.py. + +Engine stubbed (pattern from tests/test_generate_engine.py). Requires +importing `main` — validated in CI (local torch/Triton segfault, see +project memory). +""" +import os + +os.environ.setdefault("OMNIVOICE_MODEL", "test") +os.environ.setdefault("OMNIVOICE_DISABLE_FILE_LOG", "1") + +import importlib + +import pytest +import torch + + +def _tts_mod(): + return importlib.import_module("services.tts_backend") + + +def _make_fake_engine(engine_id="fake-agent-engine"): + class _FakeEngine(_tts_mod().TTSBackend): + id = engine_id + display_name = "Fake Agent Engine (test)" + calls: list = [] + + @property + def sample_rate(self) -> int: + return 24000 # pipecat's OpenAITTSService default + + @property + def supported_languages(self) -> list[str]: + return ["multi"] + + @classmethod + def is_available(cls): + return True, "ready" + + def generate(self, text, **kw) -> torch.Tensor: + type(self).calls.append((text, kw)) + return torch.zeros(1, 4800) + + return _FakeEngine + + +@pytest.fixture() +def client(): + from fastapi.testclient import TestClient + from main import app + return TestClient(app, client=("127.0.0.1", 50000)) + + +def test_pipecat_speech_request_returns_pcm(client, monkeypatch): + """The exact body pipecat's OpenAITTSService sends → raw PCM bytes.""" + fake = _make_fake_engine() + monkeypatch.setitem(_tts_mod()._REGISTRY, "fake-agent-engine", fake) + + res = client.post("/v1/audio/speech", json={ + "model": "fake-agent-engine", + "input": "Hello from the agent.", + "voice": "default", + "response_format": "pcm", + "speed": 1.0, + }) + + assert res.status_code == 200, res.text + # PCM is raw int16 samples — no container header, even byte length. + assert len(res.content) > 0 and len(res.content) % 2 == 0 + assert len(fake.calls) == 1 + assert fake.calls[0][0] == "Hello from the agent." + + +def test_wav_format_for_runtimes_that_prefer_a_container(client, monkeypatch): + fake = _make_fake_engine("fake-agent-wav") + monkeypatch.setitem(_tts_mod()._REGISTRY, "fake-agent-wav", fake) + + res = client.post("/v1/audio/speech", json={ + "model": "fake-agent-wav", + "input": "Container please.", + "response_format": "wav", + }) + assert res.status_code == 200, res.text + assert res.content[:4] == b"RIFF" + assert res.headers["content-type"].startswith("audio/") + + +def test_voice_profile_id_resolves_for_agent_binding(client, monkeypatch, tmp_path): + """An agent bound to a cloned voice passes the profile ID as `voice`.""" + fake = _make_fake_engine("fake-agent-voice") + monkeypatch.setitem(_tts_mod()._REGISTRY, "fake-agent-voice", fake) + + # Unknown id falls through to the engine as a preset name (no DB row) — + # the contract is that a non-alias voice is forwarded, not rejected. + res = client.post("/v1/audio/speech", json={ + "model": "fake-agent-voice", + "input": "In my voice.", + "voice": "some-profile-id", + }) + assert res.status_code == 200, res.text + assert fake.calls[0][1].get("voice") == "some-profile-id" + + +def test_speed_passthrough(client, monkeypatch): + fake = _make_fake_engine("fake-agent-speed") + monkeypatch.setitem(_tts_mod()._REGISTRY, "fake-agent-speed", fake) + res = client.post("/v1/audio/speech", json={ + "model": "fake-agent-speed", "input": "Faster.", "speed": 1.25, + }) + assert res.status_code == 200, res.text + assert fake.calls[0][1].get("speed") == pytest.approx(1.25)