Added Thai National ID Number (TNIN) recognizer (#1713)

CHANGELOG.md

@@ -672,4 +672,5 @@ New endpoint for deanonymizing encrypted entities by the anonymizer.
 ### Fixed
 - Fixed an issue where the CreditCardRecognizer regex could incorrectly identify 13-digit Unix timestamps as credit card numbers. Validated that 13 digit numbers that start with `1` and have no separators (e.g. `1748503543012`) are not flagged as credit cards.
 - Enhance NlpEngineProvider with validation methods for NLP engines, configuration, and conf file path.
-- Added Korean Resident Registration Number (RRN) recognizer (KrRrnRecognizer).
+- Added Korean Resident Registration Number (RRN) recognizer (KrRrnRecognizer).
+- Added Thai National ID Number (TNIN) recognizer (ThTninRecognizer).

docs/supported_entities.md

@@ -100,6 +100,11 @@ For more information, refer to the [adding new recognizers documentation](analyz
 |------------|---------------------------------------------------------------------------------------------------------|------------------------------------------|
 | KR_RRN     | The Korean Resident Registration Number (RRN) is a 13-digit number issued to all Korean residents. | Pattern match, context and custom logic. |
 
+### Thai
+| FieldType  | Description                                                                                             | Detection Method                         |
+|------------|---------------------------------------------------------------------------------------------------------|------------------------------------------|
+| TH_TNIN    | The Thai National ID Number (TNIN) is a unique 13-digit number issued to all Thai residents. | Pattern match, context and custom logic. |
+
 ## Adding a custom PII entity
 
 See [this documentation](analyzer/adding_recognizers.md) for instructions on how to add a new Recognizer for a new type of PII entity.

presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml

@@ -159,6 +159,12 @@ recognizers:
     type: predefined
     enabled: false
 
+  - name: ThTninRecognizer
+    supported_languages:
+    - th
+    type: predefined
+    enabled: false
+
   - name: CryptoRecognizer
     type: predefined
 

presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py

@@ -48,6 +48,9 @@ from .country_specific.singapore.sg_uen_recognizer import SgUenRecognizer
 from .country_specific.spain.es_nie_recognizer import EsNieRecognizer
 from .country_specific.spain.es_nif_recognizer import EsNifRecognizer
 
+# Thai recognizers
+from .country_specific.thai.th_tnin_recognizer import ThTninRecognizer
+
 # UK recognizers
 from .country_specific.uk.uk_nhs_recognizer import NhsRecognizer
 from .country_specific.uk.uk_nino_recognizer import UkNinoRecognizer
@@ -146,4 +149,5 @@ __all__ = [
     "UkNinoRecognizer",
     "AzureHealthDeidRecognizer",
     "KrRrnRecognizer",
+    "ThTninRecognizer",
 ]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/thai/__init__.py

@@ -0,0 +1,7 @@
+"""Thai-specific recognizers."""
+
+from .th_tnin_recognizer import ThTninRecognizer
+
+__all__ = [
+    "ThTninRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/thai/th_tnin_recognizer.py

@@ -0,0 +1,141 @@
+from typing import List, Optional, Tuple, Union
+
+from presidio_analyzer import EntityRecognizer, Pattern, PatternRecognizer
+
+
+class ThTninRecognizer(PatternRecognizer):
+    """
+    Recognize Thai National ID Number (TNIN).
+
+    The Thai National ID Number (TNIN) is a unique 13-digit number
+    issued to all Thai residents.
+
+    The format is N1N2N3N4N5N6N7N8N9N10N11N12N13 where:
+    - N1-N12 are the main digits
+    - N13 is a check digit calculated using the preceding 12 digits
+
+    Validation rules:
+    - Must be exactly 13 digits
+    - First digit (N1) cannot be 0
+    - Second digit (N2) cannot be 0
+    - Second and third digits (N2N3) cannot be: 28, 29, 59, 68, 69, 78, 79,
+      87, 88, 89, 97, 98, 99
+      These second and third digits in Thai ID number correspond to Thai provinces,
+      so we exclude non-existent or unassigned combinations in Thailand's
+      administrative division system. See ISO 3166-2:TH for reference.
+    - The 13th digit is a checksum computed modulo 11 from the first 12 digits
+
+    Checksum algorithm:
+    - Label first 12 digits N1…N12 (left to right)
+    - Compute S = 13·N1 + 12·N2 + … + 2·N12
+    - Let x = S mod 11
+    - Then check digit N13 = (11 − x) mod 10
+    - Equivalently: if x ≤ 1 then N13 = 1 − x; otherwise N13 = 11 − x
+
+    Reference: https://th.wikipedia.org/wiki/เลขประจำตัวประชาชนไทย
+               https://th.wikipedia.org/wiki/ISO_3166-2:TH
+
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    :param replacement_pairs: List of tuples with potential replacement values
+    for different strings to be used during pattern matching.
+    """
+
+    PATTERNS = [
+        Pattern(
+            "TNIN (Medium)",
+            r"\b[1-9](?:[134][0-9]|[25][0134567]|[67][01234567]|[89][0123456])\d{10}\b",
+            0.5,
+        )
+    ]
+
+    CONTEXT = [
+        "Thai National ID",
+        "Thai ID Number",
+        "TNIN",
+        "เลขประจำตัวประชาชน",
+        "เลขบัตรประชาชน",
+        "รหัสปชช",
+    ]
+
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "th",
+        supported_entity: str = "TH_TNIN",
+        replacement_pairs: Optional[List[Tuple[str, str]]] = None,
+    ):
+        self.replacement_pairs = replacement_pairs if replacement_pairs else []
+
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str) -> Union[bool, None]:
+        """
+        Validate the pattern logic e.g., by running checksum on a detected pattern.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool or None, indicating whether the validation was successful.
+        """
+        # Pre-processing before validation checks
+        sanitized_value = EntityRecognizer.sanitize_value(
+            pattern_text, self.replacement_pairs
+        )
+
+        # Check if the sanitized value has the correct length (13 digits)
+        if len(sanitized_value) != 13:
+            return False
+
+        # Check if all characters are digits
+        if not sanitized_value.isdigit():
+            return False
+
+        # Validate TNIN checksum (format validation is handled by regex)
+        return self._validate_checksum(sanitized_value)
+
+
+    def _validate_checksum(self, tnin: str) -> bool:
+        """
+        Validate the checksum of Thai TNIN.
+
+        Checksum algorithm:
+        - Label first 12 digits N1…N12 (left to right)
+        - Compute S = 13·N1 + 12·N2 + … + 2·N12
+        - Let x = S mod 11
+        - Then check digit N13 = (11 − x) mod 10
+
+        :param tnin: The TNIN to validate
+        :return: True if checksum is valid, False otherwise
+        """
+        # Calculate weighted sum: 13*N1 + 12*N2 + ... + 2*N12
+        weights = list(range(13, 1, -1))  # [13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2]
+
+        total_sum = 0
+        for i in range(12):  # First 12 digits
+            total_sum += weights[i] * int(tnin[i])
+
+        # Calculate x = S mod 11
+        x = total_sum % 11
+
+        # Calculate expected check digit: (11 - x) mod 10
+        if x <= 1:
+            expected_check_digit = 1 - x
+        else:
+            expected_check_digit = 11 - x
+
+        # Compare with actual check digit
+        actual_check_digit = int(tnin[12])
+
+        return expected_check_digit == actual_check_digit

presidio-analyzer/tests/test_th_tnin_recognizer.py

@@ -0,0 +1,162 @@
+"""Tests for Thai National ID (TNIN) recognizer."""
+
+import pytest
+from presidio_analyzer.predefined_recognizers import ThTninRecognizer
+
+from tests import assert_result_within_score_range
+
+
+@pytest.fixture(scope="module")
+def recognizer():
+    """Create a Thai TNIN recognizer instance for testing."""
+    return ThTninRecognizer()
+
+
+@pytest.fixture(scope="module")
+def entities():
+    """Return the Thai TNIN entity type for testing."""
+    return ["TH_TNIN"]
+
+
+@pytest.mark.parametrize(
+    "text, expected_len, expected_positions, expected_score_ranges",
+    [
+        # Valid TNINs - should get high scores due to format validation and checksum
+        # Note: These are example TNINs created for testing purposes
+        # with valid checksums
+        ("1234567890121", 1, ((0, 13),), ((0.5, 1.0),),),
+        ("2345678901234", 1, ((0, 13),), ((0.5, 1.0),),),
+        ("3456789012347", 1, ((0, 13),), ((0.5, 1.0),),),
+        ("4567890123459", 1, ((0, 13),), ((0.5, 1.0),),),
+        ("5678901234560", 1, ((0, 13),), ((0.5, 1.0),),),
+
+        # Valid TNINs in sentences
+        ("My Thai ID is 1234567890121", 1, ((14, 27),), ((0.5, 1.0),),),
+        ("TNIN: 2345678901234", 1, ((6, 19),), ((0.5, 1.0),),),
+        ("เลขประจำตัวประชาชน: 3456789012347", 1, ((20, 33),), ((0.5, 1.0),),),
+
+        # Invalid TNINs - wrong length
+        ("123456789012", 0, (), (),),  # 12 digits
+        ("12345678901234", 0, (), (),),  # 14 digits
+
+        # Invalid TNINs - non-digits
+        ("123456789012a", 0, (), (),),  # contains letter
+        ("123456789012 ", 0, (), (),),  # contains space
+
+        # Invalid TNINs - format violations (first digit is 0)
+        ("0234567890124", 0, (), (),),
+        ("0034567890124", 0, (), (),),
+
+        # Invalid TNINs - format violations (second digit is 0)
+        ("1034567890124", 0, (), (),),
+        ("1304567890124", 0, (), (),),
+
+        # Invalid TNINs - forbidden second-third combinations
+        ("1284567890124", 0, (), (),),  # 28 is forbidden
+        ("1294567890124", 0, (), (),),  # 29 is forbidden
+        ("1594567890124", 0, (), (),),  # 59 is forbidden
+        ("1684567890124", 0, (), (),),  # 68 is forbidden
+        ("1694567890124", 0, (), (),),  # 69 is forbidden
+        ("1784567890124", 0, (), (),),  # 78 is forbidden
+        ("1794567890124", 0, (), (),),  # 79 is forbidden
+        ("1874567890124", 0, (), (),),  # 87 is forbidden
+        ("1884567890124", 0, (), (),),  # 88 is forbidden
+        ("1894567890124", 0, (), (),),  # 89 is forbidden
+        ("1974567890124", 0, (), (),),  # 97 is forbidden
+        ("1984567890124", 0, (), (),),  # 98 is forbidden
+        ("1994567890124", 0, (), (),),  # 99 is forbidden
+
+        # Invalid TNINs - checksum failures (but valid format)
+        # These have valid format but wrong checksum digit
+        ("1234567890123", 0, (), (),),  # should be 1, not 3
+        ("2345678901235", 0, (), (),),  # should be 4, not 5
+        ("3456789012346", 0, (), (),),  # should be 7, not 6
+
+        # Edge cases
+        ("0000000000000", 0, (), (),),  # all zeros (invalid format)
+        ("1111111111111", 0, (), (),),  # all ones (valid format but wrong checksum)
+
+        # Context enhancement tests
+        ("Thai National ID 1234567890121", 1, ((17, 30),), ((0.5, 1.0),),),
+        ("เลขบัตรประชาชน 2345678901234", 1, ((15, 28),), ((0.5, 1.0),),),
+    ],
+)
+def test_when_tnin_in_text_then_all_tnins_found(
+    text,
+    expected_len,
+    expected_positions,
+    expected_score_ranges,
+    recognizer,
+    entities,
+    max_score,
+):
+    """Test that Thai TNIN recognizer correctly identifies TNINs.
+
+    Test various text contexts including valid TNINs, invalid formats,
+    and different languages to ensure proper recognition.
+    """
+    results = recognizer.analyze(text, entities)
+    assert len(results) == expected_len
+
+    for res, (st_pos, fn_pos), (st_score, fn_score) in zip(
+        results, expected_positions, expected_score_ranges
+    ):
+        if fn_score == "max":
+            fn_score = max_score
+        assert_result_within_score_range(
+            res, entities[0], st_pos, fn_pos, st_score, fn_score
+        )
+
+
+def test_validate_result_with_valid_tnin(recognizer):
+    """Test validate_result method with valid TNINs."""
+    # These should return True (valid)
+    assert recognizer.validate_result("1234567890121") is True
+    assert recognizer.validate_result("2345678901234") is True
+    assert recognizer.validate_result("3456789012347") is True
+
+
+def test_validate_result_with_invalid_format(recognizer):
+    """Test validate_result method with invalid format TNINs."""
+    # These should return False (invalid format)
+    assert recognizer.validate_result("0234567890124") is False  # starts with 0
+    assert recognizer.validate_result("1034567890124") is False  # second digit is 0
+    assert recognizer.validate_result("1284567890124") is False  # forbidden 28
+    assert recognizer.validate_result("1294567890124") is False  # forbidden 29
+
+
+def test_validate_result_with_wrong_length(recognizer):
+    """Test validate_result method with wrong length."""
+    # These should return False (wrong length)
+    assert recognizer.validate_result("123456789012") is False  # 12 digits
+    assert recognizer.validate_result("12345678901234") is False  # 14 digits
+
+
+def test_validate_result_with_non_digits(recognizer):
+    """Test validate_result method with non-digit characters."""
+    # These should return False (non-digits)
+    assert recognizer.validate_result("123456789012a") is False  # contains letter
+    assert recognizer.validate_result("123456789012 ") is False  # contains space
+
+
+def test_context_words(recognizer):
+    """Test that context words are properly set."""
+    expected_context = [
+        "Thai National ID",
+        "Thai ID Number",
+        "TNIN",
+        "เลขประจำตัวประชาชน",
+        "เลขบัตรประชาชน",
+        "รหัสปชช",
+    ]
+    assert recognizer.context == expected_context
+
+
+def test_supported_entity(recognizer):
+    """Test that supported entity is correctly set."""
+    assert recognizer.supported_entities == ["TH_TNIN"]
+
+
+def test_supported_language(recognizer):
+    """Test that supported language is correctly set."""
+    assert recognizer.supported_language == "th"