upstream-archive/presidio
1
0
Fork 0
mirror of https://github.com/data-privacy-stack/presidio.git synced 2026-09-21 05:27:53 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into replace-spacy-hf-pipeline

Browse Source
This commit is contained in:
Omri Mendels
2026-08-03 14:54:38 +03:00
committed by GitHub
parent 5da436d0c6 6116c0685c
commit d78e82612d
18 changed files with 538 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
### Analyzer
#### Added
- Added `UuidRecognizer` (generic, entity type `UUID`) to detect UUIDs in the standard 8-4-4-4-12 hyphenated hexadecimal format, covering RFC 4122 versions 1-5 and RFC 9562 versions 6-8. Validates version and variant nibbles and filters the nil UUID to reduce false positives.
- South African ID number (`ZA_ID_NUMBER`) recognizer for the 13-digit national identity number, using pattern matching, context words, birth-date validation, and Luhn checksum validation. Disabled by default.
- South African recognizers for `ZA_PASSPORT`, `ZA_INCOME_TAX_NUMBER`, `ZA_DRIVER_LICENSE`, `ZA_VAT_NUMBER`, `ZA_COMPANY_REGISTRATION`, `ZA_TRAFFIC_REGISTER_NUMBER`, `ZA_LICENSE_PLATE`, `ZA_MOBILE_NUMBER`, and `ZA_TELEPHONE_NUMBER`. All disabled by default.
- Added `NoOpNlpEngine` for configurations that do not require NLP engine artifacts, enabling standalone recognizers such as `HuggingFaceNerRecognizer` to run without a spaCy or Stanza model (#2071) (Thanks @ultramancode)
docs/supported_entities.md
+2
View File
@@ -25,6 +25,7 @@ For more information, refer to the [adding new recognizers documentation](analyz
|PHONE_NUMBER|A telephone number. The `PhoneRecognizer` can be extended programmatically for country-specific detection by configuring `supported_regions` and `supported_entity` (e.g. Philippines: `PhoneRecognizer(supported_regions=["PH"], supported_entity="PH_MOBILE_NUMBER")`, Turkey: `PhoneRecognizer(supported_regions=["TR"], supported_entity="TR_PHONE_NUMBER")`)|Custom logic, pattern match and context|
|MEDICAL_LICENSE|Common medical license numbers.|Pattern match, context and checksum|
|URL|A URL (Uniform Resource Locator), unique identifier used to locate a resource on the Internet|Pattern match, context and top level url validation|
|UUID|A Universally Unique Identifier (UUID) in the standard 8-4-4-4-12 hyphenated hexadecimal format. Covers RFC 4122 versions 1-5 and RFC 9562 versions 6-8; the nil UUID (all zeros) is excluded as a non-identifying sentinel.|Pattern match, validation of version/variant nibbles, and context|
### USA
@@ -131,6 +132,7 @@ For more information, refer to the [adding new recognizers documentation](analyz
|FieldType|Description|Detection Method|
|--- |--- |--- |
|CA_SIN|A Canadian Social Insurance Number (SIN) is a 9-digit number issued by Employment and Social Development Canada (ESDC) to administer government programs. The last digit is a Luhn check digit. SINs starting with 0 or 8 are reserved and not issued.|Pattern match, context, and checksum|
|CA_POSTAL_CODE|A Canadian postal code in the standard A1A 1A1 format (letter-digit-letter, space, digit-letter-digit). The letters D, F, I, O, Q, U are never used in any letter position; W and Z are additionally excluded from the first letter position.|Pattern match and context|
### Sweden
| FieldType | Description | Detection Method |
e2e-tests/tests/test_api_anonymizer.py
+11 -11
View File
@@ -31,8 +31,8 @@ def test_given_anonymize_called_with_valid_request_then_expected_valid_response_
{
"text": "hello world, my name is ANONYMIZED. My number is: 03445****",
"items": [
{"operator": "mask", "entity_type": "PHONE_NUMBER", "start": 50, "end": 59, "text":"03445****"},
{"operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text":"ANONYMIZED"}
{"operator": "mask", "entity_type": "PHONE_NUMBER", "start": 50, "end": 59, "text":"03445****", "score": 0.95},
{"operator": "replace", "entity_type": "NAME", "start": 24, "end": 34, "text":"ANONYMIZED", "score": 0.8}
]
}
"""
@@ -146,7 +146,7 @@ def test_given_decrypt_called_with_encrypted_text_then_decrypted_text_returned()
response_status, response_content = deanonymize(json.dumps(request_body))
expected_response = """{"text": "text_for_encryption", "items": [{"start": 0, "end": 19, "operator":"decrypt", "text": "text_for_encryption","entity_type":"NUMBER"}]}"""
expected_response = """{"text": "text_for_encryption", "items": [{"start": 0, "end": 19, "operator":"decrypt", "text": "text_for_encryption","entity_type":"NUMBER", "score": null}]}"""
assert response_status == 200
assert equal_json_strings(expected_response, response_content)
@@ -298,8 +298,8 @@ def test_keep_name():
{
"text": "hello world, my name is Jane Doe. My number is: <PHONE_NUMBER>",
"items": [
{"operator": "replace", "entity_type": "PHONE_NUMBER", "start": 48, "end": 62, "text":"<PHONE_NUMBER>"},
{"operator": "keep", "entity_type": "NAME", "start": 24, "end": 32, "text":"Jane Doe"}
{"operator": "replace", "entity_type": "PHONE_NUMBER", "start": 48, "end": 62, "text":"<PHONE_NUMBER>", "score": 0.95},
{"operator": "keep", "entity_type": "NAME", "start": 24, "end": 32, "text":"Jane Doe", "score": 0.8}
]
}
"""
@@ -330,8 +330,8 @@ def test_overlapping_keep_first():
{
"text": "I'm George Washington<LOCATION>",
"items": [
{"operator": "replace", "entity_type": "LOCATION", "start": 21, "end": 31, "text":"<LOCATION>"},
{"operator": "keep", "entity_type": "NAME", "start": 4, "end": 21, "text":"George Washington"}
{"operator": "replace", "entity_type": "LOCATION", "start": 21, "end": 31, "text":"<LOCATION>", "score": 0.8},
{"operator": "keep", "entity_type": "NAME", "start": 4, "end": 21, "text":"George Washington", "score": 0.8}
]
}
"""
@@ -362,8 +362,8 @@ def test_overlapping_keep_second():
{
"text": "I'm <NAME>Washington Square Park",
"items": [
{"operator": "keep", "entity_type": "LOCATION", "start": 10, "end": 32, "text":"Washington Square Park"},
{"operator": "replace", "entity_type": "NAME", "start": 4, "end": 10, "text":"<NAME>"}
{"operator": "keep", "entity_type": "LOCATION", "start": 10, "end": 32, "text":"Washington Square Park", "score": 0.8},
{"operator": "replace", "entity_type": "NAME", "start": 4, "end": 10, "text":"<NAME>", "score": 0.8}
]
}
"""
@@ -393,8 +393,8 @@ def test_overlapping_keep_both():
{
"text": "I'm George WashingtonWashington Square Park",
"items": [
{"operator": "keep", "entity_type": "LOCATION", "start": 21, "end": 43, "text":"Washington Square Park"},
{"operator": "keep", "entity_type": "NAME", "start": 4, "end": 21, "text":"George Washington"}
{"operator": "keep", "entity_type": "LOCATION", "start": 21, "end": 43, "text":"Washington Square Park", "score": 0.8},
{"operator": "keep", "entity_type": "NAME", "start": 4, "end": 21, "text":"George Washington", "score": 0.8}
]
}
"""
e2e-tests/tests/test_api_e2e_integration_flows.py
+4 -4
View File
@@ -56,7 +56,7 @@ def test_given_text_with_pii_then_analyze_and_anonymize_successfully():
"analyzer_results": analyzer_data,
}
expected_response = """{"text": "<PERSON> drivers license is AC43****", "items": [{"operator": "mask", "entity_type": "US_DRIVER_LICENSE", "start": 28, "end": 36, "text": "AC43****"}, {"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>"}]}"""
expected_response = """{"text": "<PERSON> drivers license is AC43****", "items": [{"operator": "mask", "entity_type": "US_DRIVER_LICENSE", "start": 28, "end": 36, "text": "AC43****", "score": 0.6499999999999999}, {"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>", "score": 0.85}]}"""
anonymize_and_assert(anonymizer_request, expected_response)
@@ -96,7 +96,7 @@ def test_given_a_correct_analyze_input_high_threashold_then_anonymize_partially(
"analyzer_results": analyzer_data,
}
expected_response = """{"text": "<PERSON> drivers license is AC432223", "items": [{"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>"}]}"""
expected_response = """{"text": "<PERSON> drivers license is AC432223", "items": [{"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>", "score": 0.85}]}"""
anonymize_and_assert(anonymizer_request, expected_response)
@@ -142,7 +142,7 @@ def test_given_a_correct_analyze_input_with_high_threshold_and_unmatched_entitie
"analyzer_results": analyzer_data,
}
expected_response = """{"text": "<PERSON> drivers license is AC432223", "items": [{"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>"}]}"""
expected_response = """{"text": "<PERSON> drivers license is AC432223", "items": [{"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>", "score": 0.85}]}"""
anonymize_and_assert(anonymizer_request, expected_response)
@@ -177,7 +177,7 @@ def test_given_an_unknown_entity_then_anonymize_uses_defaults():
"analyzer_results": analyzer_data,
}
expected_response = """{"text": "<PERSON> drivers license is <US_DRIVER_LICENSE>", "items": [{"operator": "replace", "entity_type": "US_DRIVER_LICENSE", "start": 28, "end": 47, "text": "<US_DRIVER_LICENSE>"}, {"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>"}]}"""
expected_response = """{"text": "<PERSON> drivers license is <US_DRIVER_LICENSE>", "items": [{"operator": "replace", "entity_type": "US_DRIVER_LICENSE", "start": 28, "end": 47, "text": "<US_DRIVER_LICENSE>", "score": 0.6499999999999999}, {"operator": "replace", "entity_type": "PERSON", "start": 0, "end": 8, "text": "<PERSON>", "score": 0.85}]}"""
anonymize_and_assert(anonymizer_request, expected_response)
presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml
+11
View File
@@ -451,6 +451,9 @@ recognizers:
- name: MacAddressRecognizer
type: predefined
- name: UuidRecognizer
type: predefined
- name: PhoneRecognizer
type: predefined
@@ -575,3 +578,11 @@ recognizers:
type: predefined
enabled: false
country_code: ca
- name: CaPostalCodeRecognizer
supported_languages:
- en
- fr
type: predefined
enabled: false
country_code: ca
presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py
+4
View File
@@ -11,6 +11,7 @@ from .country_specific.australia.au_medicare_recognizer import AuMedicareRecogni
from .country_specific.australia.au_tfn_recognizer import AuTfnRecognizer
# Canada recognizers
from .country_specific.canada.ca_postal_code_recognizer import CaPostalCodeRecognizer
from .country_specific.canada.ca_sin_recognizer import CaSinRecognizer
# Finland recognizers
@@ -172,6 +173,7 @@ from .generic.ip_recognizer import IpRecognizer
from .generic.mac_recognizer import MacAddressRecognizer
from .generic.phone_recognizer import PhoneRecognizer
from .generic.url_recognizer import UrlRecognizer
from .generic.uuid_recognizer import UuidRecognizer
# NER recognizers
from .ner.gliner_recognizer import GLiNERRecognizer
@@ -211,6 +213,7 @@ NLP_RECOGNIZERS = {
__all__ = [
"AbaRoutingRecognizer",
"CaPostalCodeRecognizer",
"CaSinRecognizer",
"CreditCardRecognizer",
"CryptoRecognizer",
@@ -221,6 +224,7 @@ __all__ = [
"NhsRecognizer",
"MedicalLicenseRecognizer",
"MacAddressRecognizer",
"UuidRecognizer",
"PhoneRecognizer",
"SgFinRecognizer",
"UrlRecognizer",
presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/canada/__init__.py
+2
View File
@@ -1,7 +1,9 @@
"""Canada-specific recognizers package."""
from .ca_postal_code_recognizer import CaPostalCodeRecognizer
from .ca_sin_recognizer import CaSinRecognizer
__all__ = [
"CaPostalCodeRecognizer",
"CaSinRecognizer",
]
presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/canada/ca_postal_code_recognizer.py
+74
View File
@@ -0,0 +1,74 @@
"""Recognizer for Canadian postal codes."""
from typing import List, Optional
from presidio_analyzer import Pattern, PatternRecognizer
class CaPostalCodeRecognizer(PatternRecognizer):
"""Recognize Canadian postal codes using regex.
Format: A1A 1A1 (letter-digit-letter, space, digit-letter-digit). A stricter
pattern (higher score) matches the canonical spaced form; a weaker pattern
(lower score, relies on surrounding context to clear the analyzer's score
threshold) matches the same shape without a space, since that form is more
prone to false positives against unrelated 6-character alphanumeric codes.
Canada Post's official format is uppercase, but this recognizer relies on
the base class's default case-insensitive matching since PII may appear
in lowercase in real-world text. The letters D, F, I, O, Q, U are never
used in any letter position. W and Z are additionally excluded from the
first letter position (reserved for future expansion).
Reference: https://www.canadapost-postescanada.ca/cpc/en/support/articles/addressing-guidelines/postal-codes.page
:param patterns: List of patterns to be used by this recognizer
:param context: List of context words to increase confidence in detection
:param supported_language: Language this recognizer supports
:param supported_entity: The entity this recognizer can detect
"""
COUNTRY_CODE = "ca"
PATTERNS = [
Pattern(
"CA Postal Code (strict, with space)",
r"\b[ABCEGHJ-NPRSTVXY]\d[ABCEGHJ-NPRSTV-Z] \d[ABCEGHJ-NPRSTV-Z]\d\b",
0.3,
),
Pattern(
"CA Postal Code (weak, no space)",
r"\b[ABCEGHJ-NPRSTVXY]\d[ABCEGHJ-NPRSTV-Z]\d[ABCEGHJ-NPRSTV-Z]\d\b",
0.1,
),
]
CONTEXT = [
"postal code",
"postcode",
"zip",
"canada",
"ontario",
"quebec",
"alberta",
"british columbia",
# French equivalents
"code postal",
]
def __init__(
self,
patterns: Optional[List[Pattern]] = None,
context: Optional[List[str]] = None,
supported_language: str = "en",
supported_entity: str = "CA_POSTAL_CODE",
name: Optional[str] = None,
):
patterns = patterns if patterns else self.PATTERNS
context = context if context else self.CONTEXT
super().__init__(
supported_entity=supported_entity,
patterns=patterns,
context=context,
supported_language=supported_language,
name=name,
)
presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/__init__.py
+2
View File
@@ -8,6 +8,7 @@ from .ip_recognizer import IpRecognizer
from .mac_recognizer import MacAddressRecognizer
from .phone_recognizer import PhoneRecognizer
from .url_recognizer import UrlRecognizer
from .uuid_recognizer import UuidRecognizer
__all__ = [
"CreditCardRecognizer",
@@ -18,4 +19,5 @@ __all__ = [
"PhoneRecognizer",
"UrlRecognizer",
"MacAddressRecognizer",
"UuidRecognizer",
]
presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/uuid_recognizer.py
+86
View File
@@ -0,0 +1,86 @@
from typing import List, Optional
from presidio_analyzer import Pattern, PatternRecognizer
class UuidRecognizer(PatternRecognizer):
"""
Recognize UUID (Universally Unique Identifier) using regex.
Supports the standard 8-4-4-4-12 hyphenated hexadecimal format for
RFC 4122 UUID versions 1-5 and RFC 9562 versions 6-8.
Note: the nil UUID (00000000-0000-0000-0000-000000000000) is explicitly
excluded as a non-identifying sentinel value.
ref:
- https://datatracker.ietf.org/doc/html/rfc4122
- https://datatracker.ietf.org/doc/html/rfc9562
"""
PATTERNS = [
Pattern(
"UUID (hyphenated)",
r"\b[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}\b",
0.5,
),
]
CONTEXT = ["uuid", "guid", "unique identifier"]
# Valid version nibble values: 1-8 (versions 1-5 per RFC 4122;
# versions 6, 7, 8 per RFC 9562)
VALID_VERSIONS = {"1", "2", "3", "4", "5", "6", "7", "8"}
# Valid variant bits (RFC 4122 variant): first hex digit of the
# 4th group must be 8, 9, a, or b
VALID_VARIANT_PREFIXES = {"8", "9", "a", "b"}
NIL_UUID = "00000000-0000-0000-0000-000000000000"
def __init__(
self,
patterns: Optional[List[Pattern]] = None,
context: Optional[List[str]] = None,
supported_language: str = "en",
supported_entity: str = "UUID",
name: Optional[str] = None,
):
patterns = patterns if patterns else self.PATTERNS
context = context if context else self.CONTEXT
super().__init__(
supported_entity=supported_entity,
patterns=patterns,
context=context,
supported_language=supported_language,
name=name,
)
def invalidate_result(self, pattern_text: str) -> bool:
"""
Check if the pattern text is a valid UUID format.
Validates the RFC 4122/9562 version nibble and variant bits to
reduce false positives from arbitrary hex-hyphen strings, and
filters out the nil UUID (all zeros), which is a well-known
sentinel value rather than an identifying value.
:param pattern_text: Text detected as pattern by regex
:return: True if invalidated (invalid or non-identifying UUID)
"""
if pattern_text.lower() == self.NIL_UUID:
return True
groups = pattern_text.split("-")
if len(groups) != 5 or any(not g for g in groups):
return True
version_nibble = groups[2][0].lower()
if version_nibble not in self.VALID_VERSIONS:
return True
variant_nibble = groups[3][0].lower()
if variant_nibble not in self.VALID_VARIANT_PREFIXES:
return True
return False
presidio-analyzer/tests/test_ca_postal_code_recognizer.py
+88
View File
@@ -0,0 +1,88 @@
import pytest
from presidio_analyzer.predefined_recognizers import CaPostalCodeRecognizer
from tests.assertions import assert_result_within_score_range
@pytest.fixture(scope="module")
def recognizer():
return CaPostalCodeRecognizer()
@pytest.fixture(scope="module")
def entities():
return ["CA_POSTAL_CODE"]
@pytest.mark.parametrize(
"text, expected_len, expected_positions, expected_score_ranges",
[
# fmt: off
# Valid, with space (strict pattern, higher score)
("K1A 0A1", 1, ((0, 7),), ((0.3, 0.3),),),
# Valid, no space (weak pattern, lower score)
("K1A0A1", 1, ((0, 6),), ((0.1, 0.1),),),
# Valid, lowercase, with space
("k1a 0a1", 1, ((0, 7),), ((0.3, 0.3),),),
# Valid, mixed case, with space
("K1a 0A1", 1, ((0, 7),), ((0.3, 0.3),),),
# Valid, leading zero digit (rural FSA), with space
("K0A 0A1", 1, ((0, 7),), ((0.3, 0.3),),),
# Valid: W in third-letter position (LDU, not first letter), with space
("K1A 1W1", 1, ((0, 7),), ((0.3, 0.3),),),
# Valid: Z in third-letter position (LDU, not first letter), with space
("K1A 1Z1", 1, ((0, 7),), ((0.3, 0.3),),),
# Embedded in text, with space
("My postal code is K1A 0A1 thanks", 1, ((18, 25),), ((0.3, 0.3),),),
# Multiple postal codes, with space
("From K1A 0A1 to M5V 3A8", 2, ((5, 12), (16, 23)), ((0.3, 0.3), (0.3, 0.3)),), # noqa: E501
# Invalid: D as first letter
("D1A 1A1", 0, (), (),),
# Invalid: F as first letter
("F1A 1A1", 0, (), (),),
# Invalid: I as first letter
("I1A 1A1", 0, (), (),),
# Invalid: O as first letter
("O1A 1A1", 0, (), (),),
# Invalid: Q as first letter
("Q1A 1A1", 0, (), (),),
# Invalid: U as first letter
("U1A 1A1", 0, (), (),),
# Invalid: W as first letter (valid elsewhere)
("W1A 1A1", 0, (), (),),
# Invalid: Z as first letter (valid elsewhere)
("Z1A 1A1", 0, (), (),),
# Invalid: D in FSA third-letter position
("K1D 1A1", 0, (), (),),
# Invalid: D in LDU letter position
("K1A 1D1", 0, (), (),),
# Invalid: starts with digit
("1A1 1A1", 0, (), (),),
# No match across a newline
("K1A\n0A1", 0, (), (),),
# No partial match embedded in a larger alphanumeric token
("XK1A0A1Y", 0, (), (),),
# No match, empty string
("", 0, (), (),),
# Two-space gap does not match either pattern (not a valid postal code)
("K1A 0A1", 0, (), (),),
# fmt: on
],
)
def test_when_postal_code_in_text_then_all_ca_postal_codes_found(
text,
expected_len,
expected_positions,
expected_score_ranges,
recognizer,
entities,
):
results = recognizer.analyze(text, entities)
assert len(results) == expected_len
for res, (st_pos, fn_pos), (st_score, fn_score) in zip(
results, expected_positions, expected_score_ranges
):
assert_result_within_score_range(
res, entities[0], st_pos, fn_pos, st_score, fn_score
)
presidio-analyzer/tests/test_recognizer_registry.py
+6 -2
View File
@@ -51,11 +51,15 @@ def mock_recognizer_registry():
def test_when_get_recognizers_then_all_recognizers_returned(mock_recognizer_registry):
registry = mock_recognizer_registry
count_before_loading = len(registry.get_recognizers(language="en", all_fields=True))
registry.load_predefined_recognizers()
recognizers = registry.get_recognizers(language="en", all_fields=True)
# 1 custom recognizer in english + 28 predefined - 11 disabled
assert len(recognizers) == 1 + 28 - 11
# Loading predefined recognizers should add EN recognizers, and the new
# UuidRecognizer should be among them. Avoid asserting an exact count,
# since that count changes whenever a recognizer is added or removed.
assert len(recognizers) > count_before_loading
assert any(type(rec).__name__ == "UuidRecognizer" for rec in recognizers)
def test_when_get_recognizers_then_return_all_fields(mock_recognizer_registry):
presidio-analyzer/tests/test_uuid_recognizer.py
+113
View File
@@ -0,0 +1,113 @@
import pytest
from presidio_analyzer.predefined_recognizers import UuidRecognizer
from tests import assert_result_within_score_range
@pytest.fixture(scope="module")
def recognizer():
"""Return a UuidRecognizer instance for testing."""
return UuidRecognizer()
@pytest.fixture(scope="module")
def entities():
"""Return the entity list this recognizer supports."""
return ["UUID"]
@pytest.mark.parametrize(
"text, expected_len, expected_positions, expected_score_ranges",
[
# fmt: off
# Version 4 (random) - most common
("Request ID: 550e8400-e29b-41d4-a716-446655440000",
1, ((12, 48),), ((0.5, 0.5),)),
("User UUID: 6fa459ea-ee8a-3ca4-894e-db77e160355e",
1, ((11, 47),), ((0.5, 0.5),)),
# Version 1 (time-based)
("Trace: f47ac10b-58cc-1372-8567-0e02b2c3d479",
1, ((7, 43),), ((0.5, 0.5),)),
# Version 2 (DCE Security, RFC 4122)
("DCE UUID: 550e8400-e29b-21d4-a716-446655440000",
1, ((10, 46),), ((0.5, 0.5),)),
# Version 3 (MD5 name-based, RFC 4122)
("6ba7b810-9dad-31d1-80b4-00c04fd430c8",
1, ((0, 36),), ((0.5, 0.5),)),
# Version 5 (SHA-1 name-based)
("Object id 74738ff5-5367-5958-9aee-98fffdcd1876 created",
1, ((10, 46),), ((0.5, 0.5),)),
# Version 6 (reordered time-based, RFC 9562)
("Sortable ID: 1ec9414c-232a-6b00-b3c8-9e6bdeced846",
1, ((13, 49),), ((0.5, 0.5),)),
# Version 7 (timestamp-based, RFC 9562)
("New record: 018f4f8e-9a3b-7c3d-8e9f-1a2b3c4d5e6f",
1, ((12, 48),), ((0.5, 0.5),)),
# Version 8 (vendor/implementation-specific, RFC 9562)
("Custom UUID: 550e8400-e29b-81d4-a716-446655440000",
1, ((13, 49),), ((0.5, 0.5),)),
# Uppercase
("GUID: 550E8400-E29B-41D4-A716-446655440000",
1, ((6, 42),), ((0.5, 0.5),)),
# With context keywords
("unique identifier: 550e8400-e29b-41d4-a716-446655440000",
1, ((19, 55),), ((0.5, "max"),)),
("The guid is 6fa459ea-ee8a-3ca4-894e-db77e160355e",
1, ((12, 48),), ((0.5, "max"),)),
# Multiple UUIDs
(
"IDs: 550e8400-e29b-41d4-a716-446655440000 and "
"f47ac10b-58cc-1372-8567-0e02b2c3d479",
2,
((5, 41), (46, 82)),
((0.5, 0.5), (0.5, 0.5)),
),
# Invalid cases - should not match
("Not a UUID: 550e8400-e29b-41d4-a716",
0, (), ()), # Too short
("Invalid: zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz",
0, (), ()), # Invalid hex
("Nil UUID: 00000000-0000-0000-0000-000000000000",
0, (), ()), # Nil UUID filtered
("Bad version: 550e8400-e29b-01d4-a716-446655440000",
0, (), ()), # Invalid version nibble (0)
("Bad version: 550e8400-e29b-91d4-a716-446655440000",
0, (), ()), # Invalid version nibble (9)
("Bad variant: 550e8400-e29b-41d4-1716-446655440000",
0, (), ()), # Invalid variant nibble
# fmt: on
],
)
def test_when_uuids_then_succeed(
text,
expected_len,
expected_positions,
expected_score_ranges,
recognizer,
entities,
max_score,
):
"""Verify UuidRecognizer detects valid UUIDs and rejects invalid ones."""
results = recognizer.analyze(text, entities)
assert len(results) == expected_len
assert len(expected_positions) == expected_len
assert len(expected_score_ranges) == expected_len
for res, (st_pos, fn_pos), (st_score, fn_score) in zip(
results, expected_positions, expected_score_ranges
):
if fn_score == "max":
fn_score = max_score
assert_result_within_score_range(
res, entities[0], st_pos, fn_pos, st_score, fn_score
)
presidio-anonymizer/presidio_anonymizer/core/engine_base.py
+1
View File
@@ -73,6 +73,7 @@ class EngineBase(ABC):
entity.entity_type,
changed_text,
operator_metadata.operator_name,
score=getattr(entity, "score", None),
)
engine_result.add_item(result_item)
presidio-anonymizer/presidio_anonymizer/entities/engine/pii_entity.py
+7 -3
View File
@@ -22,11 +22,15 @@ class PIIEntity(ABC):
def __repr__(self):
"""Return a string representation of the object."""
return (
f"start: {self.start}"
f"end: {self.end},"
repr_str = (
f"start: {self.start}, "
f"end: {self.end}, "
f"entity_type: {self.entity_type}"
)
score = getattr(self, "score", None)
if score is not None:
repr_str = repr_str + f", score: {score}"
return repr_str
def __gt__(self, other):
"""Check one entity is greater then other by the text end index."""
presidio-anonymizer/presidio_anonymizer/entities/engine/result/operator_result.py
+7 -2
View File
@@ -1,4 +1,4 @@
from typing import Dict
from typing import Dict, Optional
from presidio_anonymizer.entities import PIIEntity
@@ -13,10 +13,12 @@ class OperatorResult(PIIEntity):
entity_type: str,
text: str = None,
operator: str = None,
score: Optional[float] = None,
):
PIIEntity.__init__(self, start, end, entity_type)
self.text = text
self.operator = operator
self.score = score
def __repr__(self):
"""Return a string representation of the object."""
@@ -24,7 +26,7 @@ class OperatorResult(PIIEntity):
def to_dict(self) -> Dict:
"""Return object as Dict."""
return self.__dict__
return dict(self.__dict__)
def __str__(self):
"""Return a string representation of the object."""
@@ -58,6 +60,7 @@ class OperatorResult(PIIEntity):
"entity_type":"PERSON",
"text":"resulted_text",
"operator":"encrypt",
"score": 0.85
}
"""
start = json.get("start")
@@ -65,10 +68,12 @@ class OperatorResult(PIIEntity):
entity_type = json.get("entity_type")
text = json.get("text")
operator = json.get("operator")
score = json.get("score")
return cls(
start=start,
end=end,
entity_type=entity_type,
text=text,
operator=operator,
score=float(score) if score is not None else None,
)
presidio-anonymizer/tests/integration/test_anonymize_engine.py
+102 -13
View File
@@ -1,4 +1,5 @@
import re
import json
import pytest
@@ -27,7 +28,7 @@ def test_given_url_at_the_end_then_we_redact_is_successfully():
]
expected_result = (
'{"text": "The url is ", "items": [{"start": 11, "end": 11, "entity_type": '
'"URL", "text": "", "operator": "redact"}]}'
'"URL", "text": "", "operator": "redact", "score": 1.0}]}'
)
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
@@ -63,9 +64,9 @@ def test_given_name_and_phone_number_then_we_anonymize_correctly():
expected_result = (
'{"text": "hello world, my name is ********. My number is: '
'03-******4", "items": [{"start": 48, "end": 57, "entity_type": '
'"PHONE_NUMBER", "text": "03-******", "operator": "mask"}, '
'"PHONE_NUMBER", "text": "03-******", "operator": "mask", "score": 0.95}, '
'{"start": 24, "end": 32, "entity_type": "NAME", '
'"text": "********", "operator": "mask"}]}'
'"text": "********", "operator": "mask", "score": 0.8}]}'
)
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
@@ -85,9 +86,9 @@ def test_given_name_and_phone_number_without_anonymizers_then_we_use_default():
'{"text": "hello world, my name is <NAME>. My number is: '
'<PHONE_NUMBER>4", "items": [{"start": 46, "end": 60, '
'"entity_type": "PHONE_NUMBER", "text": "<PHONE_NUMBER>", '
'"operator": "replace"}, {"start": 24, "end": 30, '
'"operator": "replace", "score": 0.95}, {"start": 24, "end": 30, '
'"entity_type": "NAME", "text": "<NAME>", '
'"operator": "replace"}]}'
'"operator": "replace", "score": 0.8}]}'
)
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
@@ -106,9 +107,9 @@ def test_given_redact_and_replace_then_we_anonymize_successfully():
'{"text": "hello world, my name is . My number is: '
'<PHONE_NUMBER>4", "items": [{"start": 40, "end": 54, '
'"entity_type": "PHONE_NUMBER", "text": "<PHONE_NUMBER>", '
'"operator": "replace"}, {"start": 24, "end": 24, '
'"operator": "replace", "score": 0.95}, {"start": 24, "end": 24, '
'"entity_type": "NAME", "text": "", "operator": '
'"redact"}]}'
'"redact", "score": 0.8}]}'
)
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
@@ -128,13 +129,13 @@ def test_given_intersecting_entities_then_we_anonymize_correctly():
'{"text": "hello world, my name is <FULL_NAME><LAST_NAME> My '
'number is: <PHONE_NUMBER><SSN>4", "items": [{"start": 75, '
'"end": 80, "entity_type": "SSN", "text": "<SSN>", '
'"operator": "replace"}, {"start": 61, "end": 75, '
'"operator": "replace", "score": 0.8}, {"start": 61, "end": 75, '
'"entity_type": "PHONE_NUMBER", "text": "<PHONE_NUMBER>", '
'"operator": "replace"}, {"start": 35, "end": 46, '
'"operator": "replace", "score": 0.95}, {"start": 35, "end": 46, '
'"entity_type": "LAST_NAME", "text": "<LAST_NAME>", '
'"operator": "replace"}, {"start": 24, "end": 35, '
'"operator": "replace", "score": 0.6}, {"start": 24, "end": 35, '
'"entity_type": "FULL_NAME", "text": "<FULL_NAME>", '
'"operator": "replace"}]}'
'"operator": "replace", "score": 0.6}]}'
)
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
@@ -149,7 +150,7 @@ def test_given_intersecting_the_same_entities_then_we_anonymize_correctly():
expected_result = (
'{"text": "hello world, my name is <FULL_NAME> My number is: 03-4453334", '
'"items": [{"start": 24, "end": 35, "entity_type": "FULL_NAME",'
' "text": "<FULL_NAME>", "operator": "replace"}]}'
' "text": "<FULL_NAME>", "operator": "replace", "score": 0.6}]}'
)
run_engine_and_validate(text, anonymizer_config, analyzer_results, expected_result)
@@ -318,7 +319,6 @@ def test_when_hash_with_user_provided_salt_then_hash_is_reproducible():
)
def test_hash_with_known_salt_produces_expected_output(text, salt, hash_type, expected_hash):
"""Test that hashing with a known salt produces expected deterministic output."""
from presidio_anonymizer import AnonymizerEngine
params = {"hash_type": hash_type, "salt": salt}
anonymizer_config = {"DEFAULT": OperatorConfig("hash", params)}
@@ -339,6 +339,95 @@ def test_hash_with_known_salt_produces_expected_output(text, salt, hash_type, ex
assert result2.items[0].text == expected_hash
def test_given_single_entity_then_score_is_propagated_to_result():
"""Score from analyzer result appears on the corresponding OperatorResult."""
text = "My name is Jane Doe"
anonymizer_config = {"PERSON": OperatorConfig("replace")}
analyzer_results = [
RecognizerResult(start=11, end=19, score=0.85, entity_type="PERSON"),
]
engine = AnonymizerEngine()
result = engine.anonymize(text, analyzer_results, anonymizer_config)
assert result.items[0].score == 0.85
def test_given_multiple_entities_then_each_score_is_propagated_correctly():
"""Each OperatorResult carries the score of its originating RecognizerResult."""
text = "My name is Jane Doe. My number is 034453334"
anonymizer_config = {}
analyzer_results = [
RecognizerResult(start=11, end=19, score=0.8, entity_type="NAME"),
RecognizerResult(start=34, end=43, score=0.95, entity_type="PHONE_NUMBER"),
]
engine = AnonymizerEngine()
result = engine.anonymize(text, analyzer_results, anonymizer_config)
result_by_type = {item.entity_type: item for item in result.items}
assert result_by_type["NAME"].score == 0.8
assert result_by_type["PHONE_NUMBER"].score == 0.95
@pytest.mark.parametrize(
"operator_name,operator_params",
[
("replace", {}),
("redact", {}),
("mask", {"masking_char": "*", "chars_to_mask": 4, "from_end": False}),
("hash", {}),
],
)
def test_given_different_operators_then_score_is_always_propagated(
operator_name, operator_params
):
"""Score is propagated regardless of which operator is applied."""
text = "My name is Jane Doe"
anonymizer_config = {"NAME": OperatorConfig(operator_name, operator_params)}
analyzer_results = [
RecognizerResult(start=11, end=19, score=0.75, entity_type="NAME"),
]
engine = AnonymizerEngine()
result = engine.anonymize(text, analyzer_results, anonymizer_config)
assert result.items[0].score == 0.75
def test_given_conflicting_entities_then_winning_entity_score_is_preserved():
"""When conflict resolution drops an entity, the surviving entity keeps its own score."""
text = "hello world, my name is Jane Doe"
anonymizer_config = {}
analyzer_results = [
RecognizerResult(start=24, end=32, score=0.6, entity_type="FULL_NAME"),
RecognizerResult(start=24, end=28, score=0.9, entity_type="FIRST_NAME"), # loses
RecognizerResult(start=24, end=30, score=0.8, entity_type="NAME"), # loses
]
engine = AnonymizerEngine()
result = engine.anonymize(text, analyzer_results, anonymizer_config)
assert len(result.items) == 1
assert result.items[0].entity_type == "FULL_NAME"
assert result.items[0].score == 0.6 # its own score, not the score of dropped entities
def test_given_score_in_result_then_it_is_present_in_json_output():
"""Score is included in to_json() serialized output."""
text = "My name is Jane Doe"
anonymizer_config = {}
analyzer_results = [
RecognizerResult(start=11, end=19, score=0.85, entity_type="NAME"),
]
engine = AnonymizerEngine()
result = engine.anonymize(text, analyzer_results, anonymizer_config)
output = json.loads(result.to_json())
assert output["items"][0]["score"] == pytest.approx(0.85)
def run_engine_and_validate(
text: str, anonymizers_config, analyzer_results, expected_result
):
presidio-anonymizer/tests/test_pii_entity.py
+17
View File
@@ -0,0 +1,17 @@
from presidio_anonymizer.entities import PIIEntity
def test_pii_entity_repr_includes_score_when_present():
entity = PIIEntity(start=0, end=5, entity_type="PERSON")
entity.score = 0.85
result = repr(entity)
assert "score: 0.85" in result
assert "start: 0" in result
assert "entity_type: PERSON" in result
def test_pii_entity_repr_omits_score_when_absent():
entity = PIIEntity(start=0, end=5, entity_type="PERSON")
result = repr(entity)
assert "score" not in result
assert "start: 0" in result