fix(tts): normalize complete signed ranges without rewriting chains
This commit is contained in:
@@ -228,23 +228,37 @@ _RANGE_FORM = {
|
||||
#: ASCII tilde, wave dash, fullwidth tilde — Japanese and Korean IMEs emit the
|
||||
#: latter two, so all three have to match.
|
||||
#:
|
||||
#: The neighbour guards block digits/decimal marks (so a longer number is never
|
||||
#: split) and ASCII letters (so a product code like "AB20~30CD" is left alone),
|
||||
#: but deliberately allow everything else: CJK writes its unit right against
|
||||
#: the digits — "20~30초", "20〜30分", "20~30秒" — and a \w guard would reject
|
||||
#: exactly the cases this rule exists for.
|
||||
#: Match complete signed/decimal endpoints; reject partial numbers and product
|
||||
#: codes while allowing adjacent CJK units. Guard all tilde forms so malformed
|
||||
#: chains cannot be partially rewritten, including when their separators have
|
||||
#: whitespace around them.
|
||||
_RANGE_MARKS = "~\u301c\uff5e"
|
||||
_RANGE_ENDPOINT = r"[+-]?(?:\d{1,6}(?:\.\d{1,6})?|\.\d{1,6})"
|
||||
_NUM_RANGE_RE = re.compile(
|
||||
r"(?<![\d.,])(?<![A-Za-z])(\d{1,6})\s*[~\u301c\uff5e]\s*(\d{1,6})"
|
||||
r"(?![\d.,])(?![A-Za-z])"
|
||||
rf"(?<![\d.,A-Za-z+{_RANGE_MARKS}-])({_RANGE_ENDPOINT})"
|
||||
rf"\s*[{_RANGE_MARKS}]\s*({_RANGE_ENDPOINT})"
|
||||
rf"(?![\d.,A-Za-z+{_RANGE_MARKS}-])"
|
||||
)
|
||||
|
||||
|
||||
def _speak_number_ranges(text: str, lang: str) -> str:
|
||||
"""``20~30`` → ``20에서 30``. No-op where the spoken form isn't verified."""
|
||||
"""Speak complete tilde ranges only for languages with a verified form."""
|
||||
form = _RANGE_FORM.get(lang)
|
||||
if not form:
|
||||
return text
|
||||
return _NUM_RANGE_RE.sub(lambda m: form.format(a=m.group(1), b=m.group(2)), text)
|
||||
|
||||
def replace(match: re.Match) -> str:
|
||||
before, after = match.start() - 1, match.end()
|
||||
while before >= 0 and text[before].isspace():
|
||||
before -= 1
|
||||
while after < len(text) and text[after].isspace():
|
||||
after += 1
|
||||
if ((before >= 0 and text[before] in _RANGE_MARKS)
|
||||
or (after < len(text) and text[after] in _RANGE_MARKS)):
|
||||
return match.group(0)
|
||||
return form.format(a=match.group(1), b=match.group(2))
|
||||
|
||||
return _NUM_RANGE_RE.sub(replace, text)
|
||||
|
||||
|
||||
# ── Universal safety filters (all languages) ─────────────────────────────────
|
||||
|
||||
@@ -23,6 +23,9 @@ ignores.
|
||||
Everything you type in the text box reaches the active engine **verbatim** —
|
||||
the pipeline goes out of its way not to break tags:
|
||||
|
||||
- Tilde-separated integer, signed, and decimal ranges get a spoken separator in
|
||||
English, Korean, Japanese, and Chinese; malformed chains and product codes
|
||||
are left unchanged.
|
||||
- The text-normalization pass (numbers, abbreviations) skips every `[…]` span
|
||||
(`backend/services/text_normalization.py`).
|
||||
- The long-text chunker never cuts inside a bracket tag
|
||||
|
||||
@@ -142,7 +142,6 @@ _UNCHANGED_CASES = [
|
||||
("Korean", "2026-09-05 회의"), # date
|
||||
("Korean", "010-1234-5678"), # phone number
|
||||
("Korean", "AB20~30CD"), # product code, not a range
|
||||
("Korean", "1.20~30.5"), # decimals either side
|
||||
("Japanese", "そうですね〜"), # tilde not between digits
|
||||
("Vietnamese", "20~30 giây"), # no verified spoken form
|
||||
(None, "20~30초"), # no language given
|
||||
@@ -450,3 +449,25 @@ def test_longform_cache_key_tracks_normalization_toggle(tmp_path, monkeypatch):
|
||||
assert path_on != path_off
|
||||
assert not was_cached_off
|
||||
assert seen_off == ["Dr. Smith has 2 cats"] # raw text with the toggle off
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw,expected", [
|
||||
("-20~-10°C", "-20에서 -10°C"),
|
||||
("0.5~1.0초", "0.5에서 1.0초"),
|
||||
("1.20~30.5", "1.20에서 30.5"),
|
||||
("+.5 ~ +1.0", "+.5에서 +1.0"),
|
||||
("-0.5~+1.25", "-0.5에서 +1.25"),
|
||||
])
|
||||
def test_complete_signed_decimal_ranges(raw, expected):
|
||||
assert normalize_text(raw, "ko") == expected
|
||||
assert normalize_text(expected, "ko") == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [
|
||||
"20~30~40", "20 ~ 30 ~ 40", "20〜30~40", "20 ~ 30~40",
|
||||
"1.2.3~4.5", "1,000~2,000", "--20~-10", "20~++30",
|
||||
"AB-20~30CD", "1234567~20", "20~1234567", "1.1234567~2",
|
||||
"[20~30]", "[0.5~1.0]",
|
||||
])
|
||||
def test_malformed_or_protected_ranges_remain_unchanged(raw):
|
||||
assert normalize_text(raw, "ko") == raw
|
||||
|
||||
Reference in New Issue
Block a user