fix(tts): normalize complete signed ranges without rewriting chains

This commit is contained in:
Palash Debnath
2026-09-07 11:04:44 +05:30
parent da26a240e2
commit b8ff2721a5
3 changed files with 48 additions and 10 deletions
+23 -9
View File
@@ -228,23 +228,37 @@ _RANGE_FORM = {
#: ASCII tilde, wave dash, fullwidth tilde — Japanese and Korean IMEs emit the
#: latter two, so all three have to match.
#:
#: The neighbour guards block digits/decimal marks (so a longer number is never
#: split) and ASCII letters (so a product code like "AB20~30CD" is left alone),
#: but deliberately allow everything else: CJK writes its unit right against
#: the digits — "20~30초", "20〜30分", "2030秒" — and a \w guard would reject
#: exactly the cases this rule exists for.
#: Match complete signed/decimal endpoints; reject partial numbers and product
#: codes while allowing adjacent CJK units. Guard all tilde forms so malformed
#: chains cannot be partially rewritten, including when their separators have
#: whitespace around them.
_RANGE_MARKS = "~\u301c\uff5e"
_RANGE_ENDPOINT = r"[+-]?(?:\d{1,6}(?:\.\d{1,6})?|\.\d{1,6})"
_NUM_RANGE_RE = re.compile(
r"(?<![\d.,])(?<![A-Za-z])(\d{1,6})\s*[~\u301c\uff5e]\s*(\d{1,6})"
r"(?![\d.,])(?![A-Za-z])"
rf"(?<![\d.,A-Za-z+{_RANGE_MARKS}-])({_RANGE_ENDPOINT})"
rf"\s*[{_RANGE_MARKS}]\s*({_RANGE_ENDPOINT})"
rf"(?![\d.,A-Za-z+{_RANGE_MARKS}-])"
)
def _speak_number_ranges(text: str, lang: str) -> str:
"""``20~30`` → ``20에서 30``. No-op where the spoken form isn't verified."""
"""Speak complete tilde ranges only for languages with a verified form."""
form = _RANGE_FORM.get(lang)
if not form:
return text
return _NUM_RANGE_RE.sub(lambda m: form.format(a=m.group(1), b=m.group(2)), text)
def replace(match: re.Match) -> str:
before, after = match.start() - 1, match.end()
while before >= 0 and text[before].isspace():
before -= 1
while after < len(text) and text[after].isspace():
after += 1
if ((before >= 0 and text[before] in _RANGE_MARKS)
or (after < len(text) and text[after] in _RANGE_MARKS)):
return match.group(0)
return form.format(a=match.group(1), b=match.group(2))
return _NUM_RANGE_RE.sub(replace, text)
# ── Universal safety filters (all languages) ─────────────────────────────────
+3
View File
@@ -23,6 +23,9 @@ ignores.
Everything you type in the text box reaches the active engine **verbatim**
the pipeline goes out of its way not to break tags:
- Tilde-separated integer, signed, and decimal ranges get a spoken separator in
English, Korean, Japanese, and Chinese; malformed chains and product codes
are left unchanged.
- The text-normalization pass (numbers, abbreviations) skips every `[…]` span
(`backend/services/text_normalization.py`).
- The long-text chunker never cuts inside a bracket tag
+22 -1
View File
@@ -142,7 +142,6 @@ _UNCHANGED_CASES = [
("Korean", "2026-09-05 회의"), # date
("Korean", "010-1234-5678"), # phone number
("Korean", "AB20~30CD"), # product code, not a range
("Korean", "1.20~30.5"), # decimals either side
("Japanese", "そうですね〜"), # tilde not between digits
("Vietnamese", "20~30 giây"), # no verified spoken form
(None, "20~30초"), # no language given
@@ -450,3 +449,25 @@ def test_longform_cache_key_tracks_normalization_toggle(tmp_path, monkeypatch):
assert path_on != path_off
assert not was_cached_off
assert seen_off == ["Dr. Smith has 2 cats"] # raw text with the toggle off
@pytest.mark.parametrize("raw,expected", [
("-20~-10°C", "-20에서 -10°C"),
("0.5~1.0초", "0.5에서 1.0초"),
("1.20~30.5", "1.20에서 30.5"),
("+.5 ~ +1.0", "+.5에서 +1.0"),
("-0.5+1.25", "-0.5에서 +1.25"),
])
def test_complete_signed_decimal_ranges(raw, expected):
assert normalize_text(raw, "ko") == expected
assert normalize_text(expected, "ko") == expected
@pytest.mark.parametrize("raw", [
"20~30~40", "20 ~ 30 ~ 40", "20〜3040", "20 ~ 3040",
"1.2.3~4.5", "1,000~2,000", "--20~-10", "20~++30",
"AB-20~30CD", "1234567~20", "20~1234567", "1.1234567~2",
"[20~30]", "[0.5~1.0]",
])
def test_malformed_or_protected_ranges_remain_unchanged(raw):
assert normalize_text(raw, "ko") == raw