fix(cloning): retain selected passage volume
This commit is contained in:
@@ -785,8 +785,10 @@ class OmniVoice(PreTrainedModel):
|
|||||||
# than the trim thresholds but real is recovered below, so this is
|
# than the trim thresholds but real is recovered below, so this is
|
||||||
# the only remaining hard failure for a reference clip.
|
# the only remaining hard failure for a reference clip.
|
||||||
validate_clone_reference(ref_wav, ref_rms)
|
validate_clone_reference(ref_wav, ref_rms)
|
||||||
|
input_gain = 1.0
|
||||||
if 0 < ref_rms < 0.1:
|
if 0 < ref_rms < 0.1:
|
||||||
ref_wav = ref_wav * 0.1 / ref_rms
|
input_gain = 0.1 / ref_rms
|
||||||
|
ref_wav = ref_wav * input_gain
|
||||||
|
|
||||||
ref_duration = ref_wav.size(-1) / self.sampling_rate
|
ref_duration = ref_wav.size(-1) / self.sampling_rate
|
||||||
if ref_text is not None and ref_duration > 20.0:
|
if ref_text is not None and ref_duration > 20.0:
|
||||||
@@ -881,6 +883,9 @@ class OmniVoice(PreTrainedModel):
|
|||||||
chunk_size = self.audio_tokenizer.config.hop_length
|
chunk_size = self.audio_tokenizer.config.hop_length
|
||||||
clip_size = int(ref_wav.size(-1) % chunk_size)
|
clip_size = int(ref_wav.size(-1) % chunk_size)
|
||||||
ref_wav = ref_wav[:, :-clip_size] if clip_size > 0 else ref_wav
|
ref_wav = ref_wav[:, :-clip_size] if clip_size > 0 else ref_wav
|
||||||
|
aligned_rms = torch.sqrt(torch.mean(torch.square(ref_wav))).item()
|
||||||
|
validate_clone_reference(ref_wav, aligned_rms)
|
||||||
|
selected_ref_rms = aligned_rms / input_gain
|
||||||
ref_audio_tokens = self.audio_tokenizer.encode(
|
ref_audio_tokens = self.audio_tokenizer.encode(
|
||||||
ref_wav.unsqueeze(0).to(self.audio_tokenizer.device),
|
ref_wav.unsqueeze(0).to(self.audio_tokenizer.device),
|
||||||
).audio_codes.squeeze(
|
).audio_codes.squeeze(
|
||||||
@@ -893,7 +898,7 @@ class OmniVoice(PreTrainedModel):
|
|||||||
return VoiceClonePrompt(
|
return VoiceClonePrompt(
|
||||||
ref_audio_tokens=ref_audio_tokens,
|
ref_audio_tokens=ref_audio_tokens,
|
||||||
ref_text=ref_text,
|
ref_text=ref_text,
|
||||||
ref_rms=ref_rms,
|
ref_rms=selected_ref_rms,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _decode_and_post_process(
|
def _decode_and_post_process(
|
||||||
|
|||||||
@@ -171,6 +171,25 @@ def test_speech_aware_bound_prefers_quiet_voice_over_loud_non_speech(monkeypatch
|
|||||||
assert model.audio_tokenizer.seen_peak < 0.1
|
assert model.audio_tokenizer.seen_peak < 0.1
|
||||||
|
|
||||||
|
|
||||||
|
def test_selected_passage_rms_undoes_input_gain(monkeypatch):
|
||||||
|
model = _model(reject_tokenization=False)
|
||||||
|
model._asr_pipe = object()
|
||||||
|
model.transcribe = lambda candidate: (
|
||||||
|
"Selected speech." if float(candidate[0].abs().mean()) > 0.1 else ""
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"omnivoice.models.omnivoice.remove_silence_safe",
|
||||||
|
lambda audio, *_args, **_kwargs: audio,
|
||||||
|
)
|
||||||
|
audio = torch.full((1, 30 * 24_000), 0.01)
|
||||||
|
audio[:, 15 * 24_000 :] = 0.02
|
||||||
|
|
||||||
|
prompt = model.create_voice_clone_prompt((audio, 24_000), ref_text=None)
|
||||||
|
|
||||||
|
assert prompt.ref_text == "Selected speech."
|
||||||
|
assert prompt.ref_rms == pytest.approx(0.02)
|
||||||
|
|
||||||
|
|
||||||
def test_far_late_speech_at_auto_select_limit_survives_with_five_asr_calls(monkeypatch):
|
def test_far_late_speech_at_auto_select_limit_survives_with_five_asr_calls(monkeypatch):
|
||||||
model = _model(reject_tokenization=False)
|
model = _model(reject_tokenization=False)
|
||||||
model.sampling_rate = 100
|
model.sampling_rate = 100
|
||||||
|
|||||||
Reference in New Issue
Block a user