fix(audio): resample MLX outputs to the declared sample rate
This commit is contained in:
@@ -1782,7 +1782,14 @@ class MLXAudioBackend(TTSBackend):
|
||||
audio = getattr(result, "audio", result)
|
||||
if hasattr(audio, "numpy"):
|
||||
audio = audio.numpy()
|
||||
pieces.append(np.asarray(audio, dtype=np.float32))
|
||||
audio = np.asarray(audio, dtype=np.float32)
|
||||
sr = getattr(result, "sample_rate", self.sample_rate)
|
||||
if sr != self.sample_rate:
|
||||
import torchaudio
|
||||
audio = torchaudio.functional.resample(
|
||||
torch.from_numpy(audio), sr, self.sample_rate,
|
||||
).numpy()
|
||||
pieces.append(audio)
|
||||
except TypeError:
|
||||
# Some engines don't accept lang_code / ref_audio. Retry with
|
||||
# only the universal kwargs.
|
||||
@@ -1791,7 +1798,14 @@ class MLXAudioBackend(TTSBackend):
|
||||
audio = getattr(result, "audio", result)
|
||||
if hasattr(audio, "numpy"):
|
||||
audio = audio.numpy()
|
||||
pieces.append(np.asarray(audio, dtype=np.float32))
|
||||
audio = np.asarray(audio, dtype=np.float32)
|
||||
sr = getattr(result, "sample_rate", self.sample_rate)
|
||||
if sr != self.sample_rate:
|
||||
import torchaudio
|
||||
audio = torchaudio.functional.resample(
|
||||
torch.from_numpy(audio), sr, self.sample_rate,
|
||||
).numpy()
|
||||
pieces.append(audio)
|
||||
|
||||
if not pieces:
|
||||
raise RuntimeError(f"mlx-audio ({self._model_id}) produced no audio")
|
||||
|
||||
@@ -43,7 +43,8 @@ HF repo id. The env var overrides the persisted UI choice.
|
||||
|
||||
## Behaviour notes
|
||||
|
||||
- Output is 24 kHz mono for most hosted models.
|
||||
- Output is 24 kHz mono. Results from models with a different native rate
|
||||
(such as Dia at 44.1 kHz) are resampled before stitching and export.
|
||||
- **Cloning works only with the `csm` model** — it is the only curated model
|
||||
confirmed to accept a reference clip. Other models silently ignore
|
||||
reference audio, so the engine reports cloning support only when CSM is
|
||||
|
||||
@@ -64,6 +64,7 @@ def backend(monkeypatch, tts_backend):
|
||||
"""An MLXAudioBackend with the model pre-loaded, so no mlx import happens."""
|
||||
be = tts_backend.MLXAudioBackend.__new__(tts_backend.MLXAudioBackend)
|
||||
be._model_id = "mlx-community/Qwen3-TTS-12Hz-1.7B-VoiceDesign-4bit"
|
||||
be._sr = 24000
|
||||
be._model = _FakeModel()
|
||||
monkeypatch.setattr(be, "_ensure_loaded", lambda: None)
|
||||
return be
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""MLX results must match the adapter's declared rate before stitching/export."""
|
||||
import importlib
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def backend(monkeypatch):
|
||||
cls = importlib.import_module("services.tts_backend").MLXAudioBackend
|
||||
be = cls.__new__(cls)
|
||||
be._model_id = "mlx-community/Dia-1.6B"
|
||||
be._sr = 24000
|
||||
monkeypatch.setattr(be, "_ensure_loaded", lambda: None)
|
||||
return be
|
||||
|
||||
|
||||
def _tone(rate):
|
||||
return np.sin(2 * np.pi * 440 * np.arange(rate) / rate).astype(np.float32)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rate", [24000, 44100, 48000])
|
||||
@pytest.mark.parametrize("fallback", [False, True])
|
||||
def test_generated_audio_keeps_its_duration_and_pitch(backend, rate, fallback):
|
||||
def generate(**kwargs):
|
||||
if fallback and "voice" in kwargs:
|
||||
raise TypeError("unexpected keyword argument 'voice'")
|
||||
yield SimpleNamespace(audio=_tone(rate), sample_rate=rate)
|
||||
|
||||
backend._model = SimpleNamespace(generate=generate)
|
||||
declared_rate = backend.sample_rate # callers can read this before generate
|
||||
wav = backend.generate("Hello", voice="speaker")
|
||||
|
||||
assert backend.sample_rate == declared_rate
|
||||
assert wav.shape == (1, declared_rate)
|
||||
spectrum = np.abs(np.fft.rfft(wav[0].numpy()))
|
||||
assert np.fft.rfftfreq(wav.shape[-1], 1 / declared_rate)[spectrum.argmax()] == 440
|
||||
|
||||
|
||||
def test_each_piece_is_resampled_before_concatenation(backend):
|
||||
def generate(**kwargs):
|
||||
for rate in (44100, 24000):
|
||||
yield SimpleNamespace(audio=_tone(rate), sample_rate=rate)
|
||||
|
||||
backend._model = SimpleNamespace(generate=generate)
|
||||
assert backend.generate("Two pieces").shape == (1, 48000)
|
||||
|
||||
|
||||
def test_raw_audio_without_rate_metadata_keeps_the_existing_contract(backend):
|
||||
audio = _tone(24000)
|
||||
backend._model = SimpleNamespace(generate=lambda **kwargs: iter([audio]))
|
||||
np.testing.assert_array_equal(backend.generate("Hello")[0].numpy(), audio)
|
||||
Reference in New Issue
Block a user