Recognizers refactor (continuing #1640) (#1669) (#1670)

CHANGELOG.md

@@ -634,4 +634,4 @@ New endpoint for deanonymizing encrypted entities by the anonymizer.
 
 ### Fixed
 - Fixed an issue where the CreditCardRecognizer regex could incorrectly identify 13-digit Unix timestamps as credit card numbers. Validated that 13 digit numbers that start with `1` and have no separators (e.g. `1748503543012`) are not flagged as credit cards.
-- Enhance NlpEngineProvider with validation methods for NLP engines, configuration, and conf file path.
+- Enhance NlpEngineProvider with validation methods for NLP engines, configuration, and conf file path.

CONTRIBUTING.md

@@ -41,6 +41,42 @@ Adding a new recognizer is a great way to improve Presidio. A new capability to
 
 Best practices for developing recognizers [are described here](docs/analyzer/developing_recognizers.md). Please follow these guidelines when proposing new recognizers.
 
+#### Contributing a New Predefined Recognizer
+
+To contribute a new predefined recognizer to Presidio Analyzer:
+
+1. **Choose the correct folder:**
+   - If your recognizer is specific to a country or region, add it under `presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/<country>/`.
+   - For globally applicable recognizers, use `generic/`.
+   - For recognizers based on NLP engines, use `nlp_engine_recognizers/`.
+   - For standalone NER models, use `ner/`.
+   - For third-party integrations, use `third_party/`.
+
+
+2. **Add your recognizer class** in the appropriate folder.
+
+   - **If your recognizer uses regex patterns:**
+     - Make regex patterns as specific as possible to minimize false positives.
+     - Document the source or reference for any new regex logic (e.g., link to a standard, documentation, or example dataset) in the code as a comment.
+
+3. **Add your recognizer to the configuration:**
+   - Add your recognizer to `presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml`.
+   - For country-specific recognizers, set `enabled: false` by default in the YAML configuration.
+
+3. **Update imports:** Add your recognizer to `presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py` so it is available for import and backward compatibility.
+
+4. **Update `__all__`:** Add your recognizer class name to the `__all__` list in the same `__init__.py` file.
+
+5. **Testing:**
+   - Ensure all existing tests pass.
+   - Add or update tests for your new recognizer.
+
+6. **Documentation:**
+   - If your recognizer supports a new entity, consider updating the [supported entities list](docs/analyzer/supported_entities.md).
+   - Follow the [best practices for recognizer development](docs/analyzer/developing_recognizers.md) and [adding recognizers](docs/analyzer/adding_recognizers.md).
+
+For more details, see the [predefined recognizers README](presidio-analyzer/presidio_analyzer/predefined_recognizers/README.md).
+
 ### Fixing Bugs and improving the code
 
 Please review the open [issues on Github](https://github.com/microsoft/presidio/issues) for known bugs and feature requests. We sometimes add 'good first issue' labels on those we believe are simpler, and 'advanced' labels on those which require more work or multiple changes across the solution.

presidio-analyzer/Pipfile

@@ -0,0 +1,11 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+
+[requires]
+python_version = "3.12"

presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py

@@ -1,51 +1,83 @@
 """Predefined recognizers package. Holds all the default recognizers."""
 
-from presidio_analyzer.predefined_recognizers.transformers_recognizer import (
+
+# Australia recognizers
+from presidio_analyzer.predefined_recognizers.nlp_engine_recognizers.transformers_recognizer import (  #noqa: E501
     TransformersRecognizer,
 )
 
-from .aba_routing_recognizer import AbaRoutingRecognizer
-from .ahds_recognizer import AzureHealthDeidRecognizer
-from .au_abn_recognizer import AuAbnRecognizer
-from .au_acn_recognizer import AuAcnRecognizer
-from .au_medicare_recognizer import AuMedicareRecognizer
-from .au_tfn_recognizer import AuTfnRecognizer
-from .azure_ai_language import AzureAILanguageRecognizer
-from .credit_card_recognizer import CreditCardRecognizer
-from .crypto_recognizer import CryptoRecognizer
-from .date_recognizer import DateRecognizer
-from .email_recognizer import EmailRecognizer
-from .es_nie_recognizer import EsNieRecognizer
-from .es_nif_recognizer import EsNifRecognizer
-from .fi_personal_identity_code_recognizer import FiPersonalIdentityCodeRecognizer
-from .gliner_recognizer import GLiNERRecognizer
-from .iban_recognizer import IbanRecognizer
-from .in_aadhaar_recognizer import InAadhaarRecognizer
-from .in_pan_recognizer import InPanRecognizer
-from .in_passport_recognizer import InPassportRecognizer
-from .in_vehicle_registration_recognizer import InVehicleRegistrationRecognizer
-from .in_voter_recognizer import InVoterRecognizer
-from .ip_recognizer import IpRecognizer
-from .it_driver_license_recognizer import ItDriverLicenseRecognizer
-from .it_fiscal_code_recognizer import ItFiscalCodeRecognizer
-from .it_identity_card_recognizer import ItIdentityCardRecognizer
-from .it_passport_recognizer import ItPassportRecognizer
-from .it_vat_code import ItVatCodeRecognizer
-from .medical_license_recognizer import MedicalLicenseRecognizer
-from .phone_recognizer import PhoneRecognizer
-from .pl_pesel_recognizer import PlPeselRecognizer
-from .sg_fin_recognizer import SgFinRecognizer
-from .sg_uen_recognizer import SgUenRecognizer
-from .spacy_recognizer import SpacyRecognizer
-from .stanza_recognizer import StanzaRecognizer
-from .uk_nhs_recognizer import NhsRecognizer
-from .uk_nino_recognizer import UkNinoRecognizer
-from .url_recognizer import UrlRecognizer
-from .us_bank_recognizer import UsBankRecognizer
-from .us_driver_license_recognizer import UsLicenseRecognizer
-from .us_itin_recognizer import UsItinRecognizer
-from .us_passport_recognizer import UsPassportRecognizer
-from .us_ssn_recognizer import UsSsnRecognizer
+from .country_specific.australia.au_abn_recognizer import AuAbnRecognizer
+from .country_specific.australia.au_acn_recognizer import AuAcnRecognizer
+from .country_specific.australia.au_medicare_recognizer import AuMedicareRecognizer
+from .country_specific.australia.au_tfn_recognizer import AuTfnRecognizer
+
+# Finland recognizers
+from .country_specific.finland.fi_personal_identity_code_recognizer import (
+    FiPersonalIdentityCodeRecognizer,
+)
+from .country_specific.india import (
+    InVehicleRegistrationRecognizer,
+)
+
+# India recognizers
+from .country_specific.india.in_aadhaar_recognizer import InAadhaarRecognizer
+from .country_specific.india.in_pan_recognizer import InPanRecognizer
+from .country_specific.india.in_passport_recognizer import InPassportRecognizer
+from .country_specific.india.in_voter_recognizer import InVoterRecognizer
+
+# Italy recognizers
+from .country_specific.italy.it_driver_license_recognizer import (
+    ItDriverLicenseRecognizer,
+)
+from .country_specific.italy.it_fiscal_code_recognizer import ItFiscalCodeRecognizer
+from .country_specific.italy.it_identity_card_recognizer import ItIdentityCardRecognizer
+from .country_specific.italy.it_passport_recognizer import ItPassportRecognizer
+from .country_specific.italy.it_vat_code import ItVatCodeRecognizer
+
+# Poland recognizers
+from .country_specific.poland.pl_pesel_recognizer import PlPeselRecognizer
+
+# Singapore recognizers
+from .country_specific.singapore.sg_fin_recognizer import SgFinRecognizer
+from .country_specific.singapore.sg_uen_recognizer import SgUenRecognizer
+
+# Spain recognizers
+from .country_specific.spain.es_nie_recognizer import EsNieRecognizer
+from .country_specific.spain.es_nif_recognizer import EsNifRecognizer
+
+# UK recognizers
+from .country_specific.uk.uk_nhs_recognizer import NhsRecognizer
+from .country_specific.uk.uk_nino_recognizer import UkNinoRecognizer
+
+# US recognizers
+from .country_specific.us.aba_routing_recognizer import AbaRoutingRecognizer
+from .country_specific.us.medical_license_recognizer import MedicalLicenseRecognizer
+from .country_specific.us.us_bank_recognizer import UsBankRecognizer
+from .country_specific.us.us_driver_license_recognizer import UsLicenseRecognizer
+from .country_specific.us.us_itin_recognizer import UsItinRecognizer
+from .country_specific.us.us_passport_recognizer import UsPassportRecognizer
+from .country_specific.us.us_ssn_recognizer import UsSsnRecognizer
+
+# Generic recognizers
+from .generic.credit_card_recognizer import CreditCardRecognizer
+from .generic.crypto_recognizer import CryptoRecognizer
+from .generic.date_recognizer import DateRecognizer
+from .generic.email_recognizer import EmailRecognizer
+from .generic.iban_recognizer import IbanRecognizer
+from .generic.ip_recognizer import IpRecognizer
+from .generic.phone_recognizer import PhoneRecognizer
+from .generic.url_recognizer import UrlRecognizer
+
+# NER recognizers
+from .ner.gliner_recognizer import GLiNERRecognizer
+
+# NLP Engine recognizers
+from .nlp_engine_recognizers.spacy_recognizer import SpacyRecognizer
+from .nlp_engine_recognizers.stanza_recognizer import StanzaRecognizer
+from .third_party.ahds_recognizer import AzureHealthDeidRecognizer
+
+# Third-party recognizers
+from .third_party.azure_ai_language import AzureAILanguageRecognizer
 
 PREDEFINED_RECOGNIZERS = [
     "PhoneRecognizer",

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/__init__.py

@@ -0,0 +1 @@
+"""Country-specific recognizers package."""

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/__init__.py

@@ -0,0 +1,13 @@
+"""Australia-specific recognizers."""
+
+from .au_abn_recognizer import AuAbnRecognizer
+from .au_acn_recognizer import AuAcnRecognizer
+from .au_medicare_recognizer import AuMedicareRecognizer
+from .au_tfn_recognizer import AuTfnRecognizer
+
+__all__ = [
+    "AuAbnRecognizer",
+    "AuAcnRecognizer",
+    "AuMedicareRecognizer",
+    "AuTfnRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/finland/__init__.py

@@ -0,0 +1,7 @@
+"""Finland-specific recognizers."""
+
+from .fi_personal_identity_code_recognizer import FiPersonalIdentityCodeRecognizer
+
+__all__ = [
+    "FiPersonalIdentityCodeRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/__init__.py

@@ -0,0 +1,15 @@
+"""India-specific recognizers."""
+
+from .in_aadhaar_recognizer import InAadhaarRecognizer
+from .in_pan_recognizer import InPanRecognizer
+from .in_passport_recognizer import InPassportRecognizer
+from .in_vehicle_registration_recognizer import InVehicleRegistrationRecognizer
+from .in_voter_recognizer import InVoterRecognizer
+
+__all__ = [
+    "InAadhaarRecognizer",
+    "InPanRecognizer",
+    "InVoterRecognizer",
+    "InVehicleRegistrationRecognizer",
+    "InPassportRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/__init__.py

@@ -0,0 +1,15 @@
+"""Italy-specific recognizers."""
+
+from .it_driver_license_recognizer import ItDriverLicenseRecognizer
+from .it_fiscal_code_recognizer import ItFiscalCodeRecognizer
+from .it_identity_card_recognizer import ItIdentityCardRecognizer
+from .it_passport_recognizer import ItPassportRecognizer
+from .it_vat_code import ItVatCodeRecognizer
+
+__all__ = [
+    "ItFiscalCodeRecognizer",
+    "ItDriverLicenseRecognizer",
+    "ItIdentityCardRecognizer",
+    "ItPassportRecognizer",
+    "ItVatCodeRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/poland/__init__.py

@@ -0,0 +1,7 @@
+"""Poland-specific recognizers."""
+
+from .pl_pesel_recognizer import PlPeselRecognizer
+
+__all__ = [
+    "PlPeselRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/__init__.py

@@ -0,0 +1,9 @@
+"""Singapore-specific recognizers package."""
+
+from .sg_fin_recognizer import SgFinRecognizer
+from .sg_uen_recognizer import SgUenRecognizer
+
+__all__ = [
+    "SgUenRecognizer",
+    "SgFinRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/__init__.py

@@ -0,0 +1,9 @@
+"""Spain-specific recognizers package."""
+
+from .es_nie_recognizer import EsNieRecognizer
+from .es_nif_recognizer import EsNifRecognizer
+
+__all__ = [
+    "EsNifRecognizer",
+    "EsNieRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/__init__.py

@@ -0,0 +1,9 @@
+"""UK-specific recognizers package."""
+
+from .uk_nhs_recognizer import NhsRecognizer
+from .uk_nino_recognizer import UkNinoRecognizer
+
+__all__ = [
+    "NhsRecognizer",
+    "UkNinoRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/__init__.py

@@ -0,0 +1,19 @@
+"""US-specific recognizers package."""
+
+from .aba_routing_recognizer import AbaRoutingRecognizer
+from .medical_license_recognizer import MedicalLicenseRecognizer
+from .us_bank_recognizer import UsBankRecognizer
+from .us_driver_license_recognizer import UsLicenseRecognizer
+from .us_itin_recognizer import UsItinRecognizer
+from .us_passport_recognizer import UsPassportRecognizer
+from .us_ssn_recognizer import UsSsnRecognizer
+
+__all__ = [
+    "MedicalLicenseRecognizer",
+    "UsItinRecognizer",
+    "UsBankRecognizer",
+    "UsLicenseRecognizer",
+    "UsPassportRecognizer",
+    "AbaRoutingRecognizer",
+    "UsSsnRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/__init__.py

@@ -0,0 +1,19 @@
+"""Generic recognizers package."""
+
+from .credit_card_recognizer import CreditCardRecognizer
+from .crypto_recognizer import CryptoRecognizer
+from .email_recognizer import EmailRecognizer
+from .iban_recognizer import IbanRecognizer
+from .ip_recognizer import IpRecognizer
+from .phone_recognizer import PhoneRecognizer
+from .url_recognizer import UrlRecognizer
+
+__all__ = [
+    "CreditCardRecognizer",
+    "CryptoRecognizer",
+    "EmailRecognizer",
+    "IbanRecognizer",
+    "IpRecognizer",
+    "PhoneRecognizer",
+    "UrlRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_recognizer.py

@@ -11,7 +11,7 @@ from presidio_analyzer import (
     RecognizerResult,
 )
 from presidio_analyzer.nlp_engine import NlpArtifacts
-from presidio_analyzer.predefined_recognizers.iban_patterns import (
+from presidio_analyzer.predefined_recognizers.generic.iban_patterns import (
     BOS,
     EOS,
     regex_per_country,

presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/phone_recognizer.py

@@ -68,11 +68,11 @@ class PhoneRecognizer(LocalRecognizer):
                 text, region, leniency=self.leniency
             ):
                 try:
-                    parsed_number = phonenumbers.parse(text[match.start:match.end])
+                    parsed_number = phonenumbers.parse(text[match.start : match.end])
                     region = phonenumbers.region_code_for_number(parsed_number)
                     results += [
-                    self._get_recognizer_result(match, text, region, nlp_artifacts)
-                ]
+                        self._get_recognizer_result(match, text, region, nlp_artifacts)
+                    ]
                 except NumberParseException:
                     results += [
                         self._get_recognizer_result(match, text, region, nlp_artifacts)

presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/__init__.py

@@ -0,0 +1,7 @@
+"""NER-based recognizers package."""
+
+from .gliner_recognizer import GLiNERRecognizer
+
+__all__ = [
+    "GLiNERRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py

@@ -79,8 +79,7 @@ class GLiNERRecognizer(LocalRecognizer):
                     entity: entity for entity in supported_entities
                 }
 
-        logger.info("Using entity mapping %s",
-                    json.dumps(entity_mapping, indent=2))
+        logger.info("Using entity mapping %s", json.dumps(entity_mapping, indent=2))
         supported_entities = list(set(self.model_to_presidio_entity_mapping.values()))
         self.model_name = model_name
         self.map_location = map_location
@@ -156,7 +155,7 @@ class GLiNERRecognizer(LocalRecognizer):
         return recognizer_results
 
     def __create_input_labels(self, entities):
-        """Append the entities requested by the user to the list of labels if it's not there.""" # noqa: E501
+        """Append the entities requested by the user to the list of labels if it's not there."""  # noqa: E501
         labels = self.gliner_labels
         for entity in entities:
             if (

presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/__init__.py

@@ -0,0 +1,11 @@
+"""NLP engine recognizers package."""
+
+from .spacy_recognizer import SpacyRecognizer
+from .stanza_recognizer import StanzaRecognizer
+from .transformers_recognizer import TransformersRecognizer
+
+__all__ = [
+    "SpacyRecognizer",
+    "StanzaRecognizer",
+    "TransformersRecognizer",
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/stanza_recognizer.py

@@ -1,4 +1,4 @@
-from presidio_analyzer.predefined_recognizers import SpacyRecognizer
+from presidio_analyzer.predefined_recognizers.nlp_engine_recognizers.spacy_recognizer import SpacyRecognizer #noqa E501
 
 
 class StanzaRecognizer(SpacyRecognizer):

presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py

@@ -1,6 +1,8 @@
 import logging
 
-from presidio_analyzer.predefined_recognizers.spacy_recognizer import SpacyRecognizer
+from presidio_analyzer.predefined_recognizers.nlp_engine_recognizers.spacy_recognizer import (  #noqa E501
+    SpacyRecognizer,
+)
 
 logger = logging.getLogger("presidio-analyzer")
 

presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/__init__.py

@@ -0,0 +1,9 @@
+"""Third-party recognizers package."""
+
+from .ahds_recognizer import AzureHealthDeidRecognizer
+from .azure_ai_language import AzureAILanguageRecognizer
+
+__all__ = [
+    "AzureAILanguageRecognizer",
+    "AzureHealthDeidRecognizer"
+]

presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_ai_language.py

@@ -25,7 +25,7 @@ class AzureAILanguageRecognizer(RemoteRecognizer):
         ta_client: Optional["TextAnalyticsClient"] = None,
         azure_ai_key: Optional[str] = None,
         azure_ai_endpoint: Optional[str] = None,
-        **kwargs
+        **kwargs,
     ):
         """
         Wrap the PII detection in Azure AI Language.
@@ -47,7 +47,7 @@ class AzureAILanguageRecognizer(RemoteRecognizer):
             supported_language=supported_language,
             name="Azure AI Language PII",
             version="5.2.0",
-            **kwargs
+            **kwargs,
         )
 
         is_available = bool(TextAnalyticsClient)

presidio-analyzer/tests/test_ahds_recognizer.py

@@ -5,18 +5,25 @@ from unittest.mock import MagicMock
 import pytest
 import dotenv
 
-from azure.health.deidentification import DeidentificationClient
-from azure.health.deidentification.models import (
-    DeidentificationContent,
-    DeidentificationOperationType,
-    DeidentificationResult,
-)
-from azure.identity import DefaultAzureCredential
-
 
 from presidio_analyzer.predefined_recognizers import AzureHealthDeidRecognizer
 
 
+@pytest.fixture(scope="module")
+def import_modules():
+    pytest.importorskip("azure-identity")
+    pytest.importskip("azure-health-deidentification")
+    from azure.health.deidentification import DeidentificationClient
+    from azure.health.deidentification.models import (
+        DeidentificationContent,
+        DeidentificationOperationType,
+        DeidentificationResult,
+    )
+    from azure.identity import DefaultAzureCredential
+
+
+    
+
 def requires_env_vars():
     dotenv.load_dotenv()
     endpoint = os.getenv("AHDS_ENDPOINT", "")
@@ -26,7 +33,7 @@ def requires_env_vars():
     )
 
 @requires_env_vars()
-def test_get_supported_entities():
+def test_get_supported_entities(import_modules):
     recognizer = AzureHealthDeidRecognizer()
     supported_entities = recognizer.get_supported_entities()
     assert isinstance(supported_entities, list)
@@ -34,7 +41,7 @@ def test_get_supported_entities():
 
 
 @requires_env_vars()
-def test_analyze_name():
+def test_analyze_name(import_modules):
     recognizer = AzureHealthDeidRecognizer()
     text = "Patient name is John Doe."
     results = recognizer.analyze(text)
@@ -42,7 +49,7 @@ def test_analyze_name():
 
 
 @requires_env_vars()
-def test_analyze_email():
+def test_analyze_email(import_modules):
     recognizer = AzureHealthDeidRecognizer()
     text = "Contact: john.doe@example.com"
     results = recognizer.analyze(text)
@@ -50,7 +57,7 @@ def test_analyze_email():
 
 
 @requires_env_vars()
-def test_analyze_multiple_entities():
+def test_analyze_multiple_entities(import_modules):
     """Test that multiple entities are recognized in a single text."""
     recognizer = AzureHealthDeidRecognizer()
     text = "Dr. Smith met Jane Doe on 2023-01-01. Email: jane@example.com"
@@ -62,7 +69,7 @@ def test_analyze_multiple_entities():
 
 
 @requires_env_vars()
-def test_analyze_with_supported_entities_filter():
+def test_analyze_with_supported_entities_filter(import_modules):
     recognizer = AzureHealthDeidRecognizer(supported_entities=["EMAIL"])
     text = "Contact: john.doe@example.com"
     results = recognizer.analyze(text)

presidio-analyzer/tests/test_certificate_number_recognizer.py

@@ -1,7 +1,7 @@
 import pytest
 
 from tests import assert_result
-from presidio_analyzer.predefined_recognizers.medical_license_recognizer import MedicalLicenseRecognizer
+from presidio_analyzer.predefined_recognizers.country_specific.us import MedicalLicenseRecognizer
 
 
 @pytest.fixture(scope="module")

presidio-analyzer/tests/test_iban_recognizer.py

@@ -1,7 +1,7 @@
 import pytest
 
 from tests import assert_result
-from presidio_analyzer.predefined_recognizers.iban_recognizer import IbanRecognizer
+from presidio_analyzer.predefined_recognizers.generic.iban_recognizer import IbanRecognizer
 
 
 @pytest.fixture(scope="module")

presidio-analyzer/tests/test_phone_recognizer.py

@@ -1,6 +1,6 @@
 import pytest
 
-from presidio_analyzer.predefined_recognizers.phone_recognizer import PhoneRecognizer
+from presidio_analyzer.predefined_recognizers.generic.phone_recognizer import PhoneRecognizer
 from tests import assert_result, assert_result_with_textual_explanation