fix(audio): resample MLX outputs to the declared sample rate

This commit is contained in:
kapelame
2026-09-14 11:29:38 -04:00
parent eaf8bb9538
commit 68653ca408
4 changed files with 72 additions and 3 deletions
+16 -2
View File
@@ -1782,7 +1782,14 @@ class MLXAudioBackend(TTSBackend):
audio = getattr(result, "audio", result)
if hasattr(audio, "numpy"):
audio = audio.numpy()
pieces.append(np.asarray(audio, dtype=np.float32))
audio = np.asarray(audio, dtype=np.float32)
sr = getattr(result, "sample_rate", self.sample_rate)
if sr != self.sample_rate:
import torchaudio
audio = torchaudio.functional.resample(
torch.from_numpy(audio), sr, self.sample_rate,
).numpy()
pieces.append(audio)
except TypeError:
# Some engines don't accept lang_code / ref_audio. Retry with
# only the universal kwargs.
@@ -1791,7 +1798,14 @@ class MLXAudioBackend(TTSBackend):
audio = getattr(result, "audio", result)
if hasattr(audio, "numpy"):
audio = audio.numpy()
pieces.append(np.asarray(audio, dtype=np.float32))
audio = np.asarray(audio, dtype=np.float32)
sr = getattr(result, "sample_rate", self.sample_rate)
if sr != self.sample_rate:
import torchaudio
audio = torchaudio.functional.resample(
torch.from_numpy(audio), sr, self.sample_rate,
).numpy()
pieces.append(audio)
if not pieces:
raise RuntimeError(f"mlx-audio ({self._model_id}) produced no audio")
+2 -1
View File
@@ -43,7 +43,8 @@ HF repo id. The env var overrides the persisted UI choice.
## Behaviour notes
- Output is 24 kHz mono for most hosted models.
- Output is 24 kHz mono. Results from models with a different native rate
(such as Dia at 44.1 kHz) are resampled before stitching and export.
- **Cloning works only with the `csm` model** — it is the only curated model
confirmed to accept a reference clip. Other models silently ignore
reference audio, so the engine reports cloning support only when CSM is
+1
View File
@@ -64,6 +64,7 @@ def backend(monkeypatch, tts_backend):
"""An MLXAudioBackend with the model pre-loaded, so no mlx import happens."""
be = tts_backend.MLXAudioBackend.__new__(tts_backend.MLXAudioBackend)
be._model_id = "mlx-community/Qwen3-TTS-12Hz-1.7B-VoiceDesign-4bit"
be._sr = 24000
be._model = _FakeModel()
monkeypatch.setattr(be, "_ensure_loaded", lambda: None)
return be
+53
View File
@@ -0,0 +1,53 @@
"""MLX results must match the adapter's declared rate before stitching/export."""
import importlib
from types import SimpleNamespace
import numpy as np
import pytest
@pytest.fixture
def backend(monkeypatch):
cls = importlib.import_module("services.tts_backend").MLXAudioBackend
be = cls.__new__(cls)
be._model_id = "mlx-community/Dia-1.6B"
be._sr = 24000
monkeypatch.setattr(be, "_ensure_loaded", lambda: None)
return be
def _tone(rate):
return np.sin(2 * np.pi * 440 * np.arange(rate) / rate).astype(np.float32)
@pytest.mark.parametrize("rate", [24000, 44100, 48000])
@pytest.mark.parametrize("fallback", [False, True])
def test_generated_audio_keeps_its_duration_and_pitch(backend, rate, fallback):
def generate(**kwargs):
if fallback and "voice" in kwargs:
raise TypeError("unexpected keyword argument 'voice'")
yield SimpleNamespace(audio=_tone(rate), sample_rate=rate)
backend._model = SimpleNamespace(generate=generate)
declared_rate = backend.sample_rate # callers can read this before generate
wav = backend.generate("Hello", voice="speaker")
assert backend.sample_rate == declared_rate
assert wav.shape == (1, declared_rate)
spectrum = np.abs(np.fft.rfft(wav[0].numpy()))
assert np.fft.rfftfreq(wav.shape[-1], 1 / declared_rate)[spectrum.argmax()] == 440
def test_each_piece_is_resampled_before_concatenation(backend):
def generate(**kwargs):
for rate in (44100, 24000):
yield SimpleNamespace(audio=_tone(rate), sample_rate=rate)
backend._model = SimpleNamespace(generate=generate)
assert backend.generate("Two pieces").shape == (1, 48000)
def test_raw_audio_without_rate_metadata_keeps_the_existing_contract(backend):
audio = _tone(24000)
backend._model = SimpleNamespace(generate=lambda **kwargs: iter([audio]))
np.testing.assert_array_equal(backend.generate("Hello")[0].numpy(), audio)