From ef587d0d53dc86d3679768100156ea5adb3bde8d Mon Sep 17 00:00:00 2001 From: Omri Mendels Date: Sun, 27 Jul 2025 14:29:37 +0300 Subject: [PATCH] Recognizers refactor (continuing #1640) (#1669) (#1670) --- CHANGELOG.md | 2 +- CONTRIBUTING.md | 36 ++++++ presidio-analyzer/Pipfile | 11 ++ .../predefined_recognizers/__init__.py | 118 +++++++++++------- .../country_specific/__init__.py | 1 + .../country_specific/australia/__init__.py | 13 ++ .../australia}/au_abn_recognizer.py | 0 .../australia}/au_acn_recognizer.py | 0 .../australia}/au_medicare_recognizer.py | 0 .../australia}/au_tfn_recognizer.py | 0 .../country_specific/finland/__init__.py | 7 ++ .../fi_personal_identity_code_recognizer.py | 0 .../country_specific/india/__init__.py | 15 +++ .../india}/in_aadhaar_recognizer.py | 0 .../india}/in_pan_recognizer.py | 0 .../india}/in_passport_recognizer.py | 0 .../in_vehicle_registration_recognizer.py | 0 .../india}/in_voter_recognizer.py | 0 .../country_specific/italy/__init__.py | 15 +++ .../italy}/it_driver_license_recognizer.py | 0 .../italy}/it_fiscal_code_recognizer.py | 0 .../italy}/it_identity_card_recognizer.py | 0 .../italy}/it_passport_recognizer.py | 0 .../italy}/it_vat_code.py | 0 .../country_specific/poland/__init__.py | 7 ++ .../poland}/pl_pesel_recognizer.py | 0 .../country_specific/singapore/__init__.py | 9 ++ .../singapore}/sg_fin_recognizer.py | 0 .../singapore}/sg_uen_recognizer.py | 0 .../country_specific/spain/__init__.py | 9 ++ .../spain}/es_nie_recognizer.py | 0 .../spain}/es_nif_recognizer.py | 0 .../country_specific/uk/__init__.py | 9 ++ .../uk}/uk_nhs_recognizer.py | 0 .../uk}/uk_nino_recognizer.py | 0 .../country_specific/us/__init__.py | 19 +++ .../us}/aba_routing_recognizer.py | 0 .../us}/medical_license_recognizer.py | 0 .../us}/us_bank_recognizer.py | 0 .../us}/us_driver_license_recognizer.py | 0 .../us}/us_itin_recognizer.py | 0 .../us}/us_passport_recognizer.py | 0 .../us}/us_ssn_recognizer.py | 0 .../generic/__init__.py | 19 +++ .../{ => generic}/credit_card_recognizer.py | 0 .../{ => generic}/crypto_recognizer.py | 0 .../{ => generic}/date_recognizer.py | 0 .../{ => generic}/email_recognizer.py | 0 .../{ => generic}/iban_patterns.py | 0 .../{ => generic}/iban_recognizer.py | 2 +- .../{ => generic}/ip_recognizer.py | 0 .../{ => generic}/phone_recognizer.py | 6 +- .../{ => generic}/url_recognizer.py | 0 .../predefined_recognizers/ner/__init__.py | 7 ++ .../{ => ner}/gliner_recognizer.py | 5 +- .../nlp_engine_recognizers/__init__.py | 11 ++ .../spacy_recognizer.py | 0 .../stanza_recognizer.py | 2 +- .../transformers_recognizer.py | 4 +- .../third_party/__init__.py | 9 ++ .../{ => third_party}/ahds_recognizer.py | 0 .../{ => third_party}/azure_ai_language.py | 4 +- .../tests/test_ahds_recognizer.py | 33 +++-- .../test_certificate_number_recognizer.py | 2 +- .../tests/test_iban_recognizer.py | 2 +- .../tests/test_phone_recognizer.py | 2 +- 66 files changed, 308 insertions(+), 71 deletions(-) create mode 100644 presidio-analyzer/Pipfile create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/__init__.py create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/australia}/au_abn_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/australia}/au_acn_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/australia}/au_medicare_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/australia}/au_tfn_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/finland/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/finland}/fi_personal_identity_code_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/india}/in_aadhaar_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/india}/in_pan_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/india}/in_passport_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/india}/in_vehicle_registration_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/india}/in_voter_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/italy}/it_driver_license_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/italy}/it_fiscal_code_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/italy}/it_identity_card_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/italy}/it_passport_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/italy}/it_vat_code.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/poland/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/poland}/pl_pesel_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/singapore}/sg_fin_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/singapore}/sg_uen_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/spain}/es_nie_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/spain}/es_nif_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/uk}/uk_nhs_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/uk}/uk_nino_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/us}/aba_routing_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/us}/medical_license_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/us}/us_bank_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/us}/us_driver_license_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/us}/us_itin_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/us}/us_passport_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => country_specific/us}/us_ssn_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/credit_card_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/crypto_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/date_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/email_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/iban_patterns.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/iban_recognizer.py (98%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/ip_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/phone_recognizer.py (96%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => generic}/url_recognizer.py (100%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => ner}/gliner_recognizer.py (97%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => nlp_engine_recognizers}/spacy_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => nlp_engine_recognizers}/stanza_recognizer.py (79%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => nlp_engine_recognizers}/transformers_recognizer.py (87%) create mode 100644 presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/__init__.py rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => third_party}/ahds_recognizer.py (100%) rename presidio-analyzer/presidio_analyzer/predefined_recognizers/{ => third_party}/azure_ai_language.py (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af291829..a8883ac6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -634,4 +634,4 @@ New endpoint for deanonymizing encrypted entities by the anonymizer. ### Fixed - Fixed an issue where the CreditCardRecognizer regex could incorrectly identify 13-digit Unix timestamps as credit card numbers. Validated that 13 digit numbers that start with `1` and have no separators (e.g. `1748503543012`) are not flagged as credit cards. -- Enhance NlpEngineProvider with validation methods for NLP engines, configuration, and conf file path. \ No newline at end of file +- Enhance NlpEngineProvider with validation methods for NLP engines, configuration, and conf file path. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6a6b398b..3d73f02e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,6 +41,42 @@ Adding a new recognizer is a great way to improve Presidio. A new capability to Best practices for developing recognizers [are described here](docs/analyzer/developing_recognizers.md). Please follow these guidelines when proposing new recognizers. +#### Contributing a New Predefined Recognizer + +To contribute a new predefined recognizer to Presidio Analyzer: + +1. **Choose the correct folder:** + - If your recognizer is specific to a country or region, add it under `presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific//`. + - For globally applicable recognizers, use `generic/`. + - For recognizers based on NLP engines, use `nlp_engine_recognizers/`. + - For standalone NER models, use `ner/`. + - For third-party integrations, use `third_party/`. + + +2. **Add your recognizer class** in the appropriate folder. + + - **If your recognizer uses regex patterns:** + - Make regex patterns as specific as possible to minimize false positives. + - Document the source or reference for any new regex logic (e.g., link to a standard, documentation, or example dataset) in the code as a comment. + +3. **Add your recognizer to the configuration:** + - Add your recognizer to `presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml`. + - For country-specific recognizers, set `enabled: false` by default in the YAML configuration. + +3. **Update imports:** Add your recognizer to `presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py` so it is available for import and backward compatibility. + +4. **Update `__all__`:** Add your recognizer class name to the `__all__` list in the same `__init__.py` file. + +5. **Testing:** + - Ensure all existing tests pass. + - Add or update tests for your new recognizer. + +6. **Documentation:** + - If your recognizer supports a new entity, consider updating the [supported entities list](docs/analyzer/supported_entities.md). + - Follow the [best practices for recognizer development](docs/analyzer/developing_recognizers.md) and [adding recognizers](docs/analyzer/adding_recognizers.md). + +For more details, see the [predefined recognizers README](presidio-analyzer/presidio_analyzer/predefined_recognizers/README.md). + ### Fixing Bugs and improving the code Please review the open [issues on Github](https://github.com/microsoft/presidio/issues) for known bugs and feature requests. We sometimes add 'good first issue' labels on those we believe are simpler, and 'advanced' labels on those which require more work or multiple changes across the solution. diff --git a/presidio-analyzer/Pipfile b/presidio-analyzer/Pipfile new file mode 100644 index 000000000..645a67eae --- /dev/null +++ b/presidio-analyzer/Pipfile @@ -0,0 +1,11 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] + +[requires] +python_version = "3.12" diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py index 5a04505d4..ce0b2f96c 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py @@ -1,51 +1,83 @@ """Predefined recognizers package. Holds all the default recognizers.""" -from presidio_analyzer.predefined_recognizers.transformers_recognizer import ( + +# Australia recognizers +from presidio_analyzer.predefined_recognizers.nlp_engine_recognizers.transformers_recognizer import ( #noqa: E501 TransformersRecognizer, ) -from .aba_routing_recognizer import AbaRoutingRecognizer -from .ahds_recognizer import AzureHealthDeidRecognizer -from .au_abn_recognizer import AuAbnRecognizer -from .au_acn_recognizer import AuAcnRecognizer -from .au_medicare_recognizer import AuMedicareRecognizer -from .au_tfn_recognizer import AuTfnRecognizer -from .azure_ai_language import AzureAILanguageRecognizer -from .credit_card_recognizer import CreditCardRecognizer -from .crypto_recognizer import CryptoRecognizer -from .date_recognizer import DateRecognizer -from .email_recognizer import EmailRecognizer -from .es_nie_recognizer import EsNieRecognizer -from .es_nif_recognizer import EsNifRecognizer -from .fi_personal_identity_code_recognizer import FiPersonalIdentityCodeRecognizer -from .gliner_recognizer import GLiNERRecognizer -from .iban_recognizer import IbanRecognizer -from .in_aadhaar_recognizer import InAadhaarRecognizer -from .in_pan_recognizer import InPanRecognizer -from .in_passport_recognizer import InPassportRecognizer -from .in_vehicle_registration_recognizer import InVehicleRegistrationRecognizer -from .in_voter_recognizer import InVoterRecognizer -from .ip_recognizer import IpRecognizer -from .it_driver_license_recognizer import ItDriverLicenseRecognizer -from .it_fiscal_code_recognizer import ItFiscalCodeRecognizer -from .it_identity_card_recognizer import ItIdentityCardRecognizer -from .it_passport_recognizer import ItPassportRecognizer -from .it_vat_code import ItVatCodeRecognizer -from .medical_license_recognizer import MedicalLicenseRecognizer -from .phone_recognizer import PhoneRecognizer -from .pl_pesel_recognizer import PlPeselRecognizer -from .sg_fin_recognizer import SgFinRecognizer -from .sg_uen_recognizer import SgUenRecognizer -from .spacy_recognizer import SpacyRecognizer -from .stanza_recognizer import StanzaRecognizer -from .uk_nhs_recognizer import NhsRecognizer -from .uk_nino_recognizer import UkNinoRecognizer -from .url_recognizer import UrlRecognizer -from .us_bank_recognizer import UsBankRecognizer -from .us_driver_license_recognizer import UsLicenseRecognizer -from .us_itin_recognizer import UsItinRecognizer -from .us_passport_recognizer import UsPassportRecognizer -from .us_ssn_recognizer import UsSsnRecognizer +from .country_specific.australia.au_abn_recognizer import AuAbnRecognizer +from .country_specific.australia.au_acn_recognizer import AuAcnRecognizer +from .country_specific.australia.au_medicare_recognizer import AuMedicareRecognizer +from .country_specific.australia.au_tfn_recognizer import AuTfnRecognizer + +# Finland recognizers +from .country_specific.finland.fi_personal_identity_code_recognizer import ( + FiPersonalIdentityCodeRecognizer, +) +from .country_specific.india import ( + InVehicleRegistrationRecognizer, +) + +# India recognizers +from .country_specific.india.in_aadhaar_recognizer import InAadhaarRecognizer +from .country_specific.india.in_pan_recognizer import InPanRecognizer +from .country_specific.india.in_passport_recognizer import InPassportRecognizer +from .country_specific.india.in_voter_recognizer import InVoterRecognizer + +# Italy recognizers +from .country_specific.italy.it_driver_license_recognizer import ( + ItDriverLicenseRecognizer, +) +from .country_specific.italy.it_fiscal_code_recognizer import ItFiscalCodeRecognizer +from .country_specific.italy.it_identity_card_recognizer import ItIdentityCardRecognizer +from .country_specific.italy.it_passport_recognizer import ItPassportRecognizer +from .country_specific.italy.it_vat_code import ItVatCodeRecognizer + +# Poland recognizers +from .country_specific.poland.pl_pesel_recognizer import PlPeselRecognizer + +# Singapore recognizers +from .country_specific.singapore.sg_fin_recognizer import SgFinRecognizer +from .country_specific.singapore.sg_uen_recognizer import SgUenRecognizer + +# Spain recognizers +from .country_specific.spain.es_nie_recognizer import EsNieRecognizer +from .country_specific.spain.es_nif_recognizer import EsNifRecognizer + +# UK recognizers +from .country_specific.uk.uk_nhs_recognizer import NhsRecognizer +from .country_specific.uk.uk_nino_recognizer import UkNinoRecognizer + +# US recognizers +from .country_specific.us.aba_routing_recognizer import AbaRoutingRecognizer +from .country_specific.us.medical_license_recognizer import MedicalLicenseRecognizer +from .country_specific.us.us_bank_recognizer import UsBankRecognizer +from .country_specific.us.us_driver_license_recognizer import UsLicenseRecognizer +from .country_specific.us.us_itin_recognizer import UsItinRecognizer +from .country_specific.us.us_passport_recognizer import UsPassportRecognizer +from .country_specific.us.us_ssn_recognizer import UsSsnRecognizer + +# Generic recognizers +from .generic.credit_card_recognizer import CreditCardRecognizer +from .generic.crypto_recognizer import CryptoRecognizer +from .generic.date_recognizer import DateRecognizer +from .generic.email_recognizer import EmailRecognizer +from .generic.iban_recognizer import IbanRecognizer +from .generic.ip_recognizer import IpRecognizer +from .generic.phone_recognizer import PhoneRecognizer +from .generic.url_recognizer import UrlRecognizer + +# NER recognizers +from .ner.gliner_recognizer import GLiNERRecognizer + +# NLP Engine recognizers +from .nlp_engine_recognizers.spacy_recognizer import SpacyRecognizer +from .nlp_engine_recognizers.stanza_recognizer import StanzaRecognizer +from .third_party.ahds_recognizer import AzureHealthDeidRecognizer + +# Third-party recognizers +from .third_party.azure_ai_language import AzureAILanguageRecognizer PREDEFINED_RECOGNIZERS = [ "PhoneRecognizer", diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/__init__.py new file mode 100644 index 000000000..d0c82e11c --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/__init__.py @@ -0,0 +1 @@ +"""Country-specific recognizers package.""" diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/__init__.py new file mode 100644 index 000000000..3821c180d --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/__init__.py @@ -0,0 +1,13 @@ +"""Australia-specific recognizers.""" + +from .au_abn_recognizer import AuAbnRecognizer +from .au_acn_recognizer import AuAcnRecognizer +from .au_medicare_recognizer import AuMedicareRecognizer +from .au_tfn_recognizer import AuTfnRecognizer + +__all__ = [ + "AuAbnRecognizer", + "AuAcnRecognizer", + "AuMedicareRecognizer", + "AuTfnRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/au_abn_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/au_abn_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/au_abn_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/au_abn_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/au_acn_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/au_acn_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/au_acn_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/au_acn_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/au_medicare_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/au_medicare_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/au_medicare_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/au_medicare_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/au_tfn_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/au_tfn_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/au_tfn_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/australia/au_tfn_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/finland/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/finland/__init__.py new file mode 100644 index 000000000..a1ace0736 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/finland/__init__.py @@ -0,0 +1,7 @@ +"""Finland-specific recognizers.""" + +from .fi_personal_identity_code_recognizer import FiPersonalIdentityCodeRecognizer + +__all__ = [ + "FiPersonalIdentityCodeRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/fi_personal_identity_code_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/finland/fi_personal_identity_code_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/fi_personal_identity_code_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/finland/fi_personal_identity_code_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/__init__.py new file mode 100644 index 000000000..20d9921f8 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/__init__.py @@ -0,0 +1,15 @@ +"""India-specific recognizers.""" + +from .in_aadhaar_recognizer import InAadhaarRecognizer +from .in_pan_recognizer import InPanRecognizer +from .in_passport_recognizer import InPassportRecognizer +from .in_vehicle_registration_recognizer import InVehicleRegistrationRecognizer +from .in_voter_recognizer import InVoterRecognizer + +__all__ = [ + "InAadhaarRecognizer", + "InPanRecognizer", + "InVoterRecognizer", + "InVehicleRegistrationRecognizer", + "InPassportRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/in_aadhaar_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_aadhaar_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/in_aadhaar_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_aadhaar_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/in_pan_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_pan_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/in_pan_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_pan_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/in_passport_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_passport_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/in_passport_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_passport_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/in_vehicle_registration_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_vehicle_registration_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/in_vehicle_registration_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_vehicle_registration_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/in_voter_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_voter_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/in_voter_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/india/in_voter_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/__init__.py new file mode 100644 index 000000000..377ea19fe --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/__init__.py @@ -0,0 +1,15 @@ +"""Italy-specific recognizers.""" + +from .it_driver_license_recognizer import ItDriverLicenseRecognizer +from .it_fiscal_code_recognizer import ItFiscalCodeRecognizer +from .it_identity_card_recognizer import ItIdentityCardRecognizer +from .it_passport_recognizer import ItPassportRecognizer +from .it_vat_code import ItVatCodeRecognizer + +__all__ = [ + "ItFiscalCodeRecognizer", + "ItDriverLicenseRecognizer", + "ItIdentityCardRecognizer", + "ItPassportRecognizer", + "ItVatCodeRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/it_driver_license_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_driver_license_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/it_driver_license_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_driver_license_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/it_fiscal_code_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_fiscal_code_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/it_fiscal_code_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_fiscal_code_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/it_identity_card_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_identity_card_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/it_identity_card_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_identity_card_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/it_passport_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_passport_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/it_passport_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_passport_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/it_vat_code.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_vat_code.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/it_vat_code.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/italy/it_vat_code.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/poland/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/poland/__init__.py new file mode 100644 index 000000000..ef571c039 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/poland/__init__.py @@ -0,0 +1,7 @@ +"""Poland-specific recognizers.""" + +from .pl_pesel_recognizer import PlPeselRecognizer + +__all__ = [ + "PlPeselRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/pl_pesel_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/poland/pl_pesel_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/pl_pesel_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/poland/pl_pesel_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/__init__.py new file mode 100644 index 000000000..564995714 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/__init__.py @@ -0,0 +1,9 @@ +"""Singapore-specific recognizers package.""" + +from .sg_fin_recognizer import SgFinRecognizer +from .sg_uen_recognizer import SgUenRecognizer + +__all__ = [ + "SgUenRecognizer", + "SgFinRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/sg_fin_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/sg_fin_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/sg_fin_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/sg_fin_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/sg_uen_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/sg_uen_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/sg_uen_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/singapore/sg_uen_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/__init__.py new file mode 100644 index 000000000..2b1cded8e --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/__init__.py @@ -0,0 +1,9 @@ +"""Spain-specific recognizers package.""" + +from .es_nie_recognizer import EsNieRecognizer +from .es_nif_recognizer import EsNifRecognizer + +__all__ = [ + "EsNifRecognizer", + "EsNieRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/es_nie_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/es_nie_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/es_nie_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/es_nie_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/es_nif_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/es_nif_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/es_nif_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/spain/es_nif_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/__init__.py new file mode 100644 index 000000000..03d800be9 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/__init__.py @@ -0,0 +1,9 @@ +"""UK-specific recognizers package.""" + +from .uk_nhs_recognizer import NhsRecognizer +from .uk_nino_recognizer import UkNinoRecognizer + +__all__ = [ + "NhsRecognizer", + "UkNinoRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/uk_nhs_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/uk_nhs_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/uk_nhs_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/uk_nhs_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/uk_nino_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/uk_nino_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/uk_nino_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/uk/uk_nino_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/__init__.py new file mode 100644 index 000000000..79fbcac8f --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/__init__.py @@ -0,0 +1,19 @@ +"""US-specific recognizers package.""" + +from .aba_routing_recognizer import AbaRoutingRecognizer +from .medical_license_recognizer import MedicalLicenseRecognizer +from .us_bank_recognizer import UsBankRecognizer +from .us_driver_license_recognizer import UsLicenseRecognizer +from .us_itin_recognizer import UsItinRecognizer +from .us_passport_recognizer import UsPassportRecognizer +from .us_ssn_recognizer import UsSsnRecognizer + +__all__ = [ + "MedicalLicenseRecognizer", + "UsItinRecognizer", + "UsBankRecognizer", + "UsLicenseRecognizer", + "UsPassportRecognizer", + "AbaRoutingRecognizer", + "UsSsnRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/aba_routing_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/aba_routing_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/aba_routing_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/aba_routing_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/medical_license_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/medical_license_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/medical_license_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/medical_license_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/us_bank_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_bank_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/us_bank_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_bank_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/us_driver_license_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_driver_license_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/us_driver_license_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_driver_license_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/us_itin_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_itin_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/us_itin_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_itin_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/us_passport_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_passport_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/us_passport_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_passport_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/us_ssn_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_ssn_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/us_ssn_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/us/us_ssn_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/__init__.py new file mode 100644 index 000000000..adc6e0804 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/__init__.py @@ -0,0 +1,19 @@ +"""Generic recognizers package.""" + +from .credit_card_recognizer import CreditCardRecognizer +from .crypto_recognizer import CryptoRecognizer +from .email_recognizer import EmailRecognizer +from .iban_recognizer import IbanRecognizer +from .ip_recognizer import IpRecognizer +from .phone_recognizer import PhoneRecognizer +from .url_recognizer import UrlRecognizer + +__all__ = [ + "CreditCardRecognizer", + "CryptoRecognizer", + "EmailRecognizer", + "IbanRecognizer", + "IpRecognizer", + "PhoneRecognizer", + "UrlRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/credit_card_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/credit_card_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/credit_card_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/credit_card_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/crypto_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/crypto_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/crypto_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/crypto_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/date_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/date_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/date_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/date_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/email_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/email_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/email_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/email_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_patterns.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_patterns.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_patterns.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_patterns.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_recognizer.py similarity index 98% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_recognizer.py index 9cf62ea51..aaa661aff 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/iban_recognizer.py @@ -11,7 +11,7 @@ from presidio_analyzer import ( RecognizerResult, ) from presidio_analyzer.nlp_engine import NlpArtifacts -from presidio_analyzer.predefined_recognizers.iban_patterns import ( +from presidio_analyzer.predefined_recognizers.generic.iban_patterns import ( BOS, EOS, regex_per_country, diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/ip_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/ip_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/ip_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/ip_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/phone_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/phone_recognizer.py similarity index 96% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/phone_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/phone_recognizer.py index 01c30848e..95121f33d 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/phone_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/phone_recognizer.py @@ -68,11 +68,11 @@ class PhoneRecognizer(LocalRecognizer): text, region, leniency=self.leniency ): try: - parsed_number = phonenumbers.parse(text[match.start:match.end]) + parsed_number = phonenumbers.parse(text[match.start : match.end]) region = phonenumbers.region_code_for_number(parsed_number) results += [ - self._get_recognizer_result(match, text, region, nlp_artifacts) - ] + self._get_recognizer_result(match, text, region, nlp_artifacts) + ] except NumberParseException: results += [ self._get_recognizer_result(match, text, region, nlp_artifacts) diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/url_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/url_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/url_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/url_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/__init__.py new file mode 100644 index 000000000..b8fbbc221 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/__init__.py @@ -0,0 +1,7 @@ +"""NER-based recognizers package.""" + +from .gliner_recognizer import GLiNERRecognizer + +__all__ = [ + "GLiNERRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/gliner_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py similarity index 97% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/gliner_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py index 3bee43049..13523f5c0 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/gliner_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py @@ -79,8 +79,7 @@ class GLiNERRecognizer(LocalRecognizer): entity: entity for entity in supported_entities } - logger.info("Using entity mapping %s", - json.dumps(entity_mapping, indent=2)) + logger.info("Using entity mapping %s", json.dumps(entity_mapping, indent=2)) supported_entities = list(set(self.model_to_presidio_entity_mapping.values())) self.model_name = model_name self.map_location = map_location @@ -156,7 +155,7 @@ class GLiNERRecognizer(LocalRecognizer): return recognizer_results def __create_input_labels(self, entities): - """Append the entities requested by the user to the list of labels if it's not there.""" # noqa: E501 + """Append the entities requested by the user to the list of labels if it's not there.""" # noqa: E501 labels = self.gliner_labels for entity in entities: if ( diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/__init__.py new file mode 100644 index 000000000..42b395e74 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/__init__.py @@ -0,0 +1,11 @@ +"""NLP engine recognizers package.""" + +from .spacy_recognizer import SpacyRecognizer +from .stanza_recognizer import StanzaRecognizer +from .transformers_recognizer import TransformersRecognizer + +__all__ = [ + "SpacyRecognizer", + "StanzaRecognizer", + "TransformersRecognizer", +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/spacy_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/spacy_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/spacy_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/spacy_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/stanza_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/stanza_recognizer.py similarity index 79% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/stanza_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/stanza_recognizer.py index 1152cbeca..c90401221 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/stanza_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/stanza_recognizer.py @@ -1,4 +1,4 @@ -from presidio_analyzer.predefined_recognizers import SpacyRecognizer +from presidio_analyzer.predefined_recognizers.nlp_engine_recognizers.spacy_recognizer import SpacyRecognizer #noqa E501 class StanzaRecognizer(SpacyRecognizer): diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/transformers_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py similarity index 87% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/transformers_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py index 2f1109325..83ce4d1f0 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/transformers_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py @@ -1,6 +1,8 @@ import logging -from presidio_analyzer.predefined_recognizers.spacy_recognizer import SpacyRecognizer +from presidio_analyzer.predefined_recognizers.nlp_engine_recognizers.spacy_recognizer import ( #noqa E501 + SpacyRecognizer, +) logger = logging.getLogger("presidio-analyzer") diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/__init__.py new file mode 100644 index 000000000..9714fbaa0 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/__init__.py @@ -0,0 +1,9 @@ +"""Third-party recognizers package.""" + +from .ahds_recognizer import AzureHealthDeidRecognizer +from .azure_ai_language import AzureAILanguageRecognizer + +__all__ = [ + "AzureAILanguageRecognizer", + "AzureHealthDeidRecognizer" +] diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/ahds_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/ahds_recognizer.py similarity index 100% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/ahds_recognizer.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/ahds_recognizer.py diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/azure_ai_language.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_ai_language.py similarity index 99% rename from presidio-analyzer/presidio_analyzer/predefined_recognizers/azure_ai_language.py rename to presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_ai_language.py index 17c60f613..57b1debbe 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/azure_ai_language.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/third_party/azure_ai_language.py @@ -25,7 +25,7 @@ class AzureAILanguageRecognizer(RemoteRecognizer): ta_client: Optional["TextAnalyticsClient"] = None, azure_ai_key: Optional[str] = None, azure_ai_endpoint: Optional[str] = None, - **kwargs + **kwargs, ): """ Wrap the PII detection in Azure AI Language. @@ -47,7 +47,7 @@ class AzureAILanguageRecognizer(RemoteRecognizer): supported_language=supported_language, name="Azure AI Language PII", version="5.2.0", - **kwargs + **kwargs, ) is_available = bool(TextAnalyticsClient) diff --git a/presidio-analyzer/tests/test_ahds_recognizer.py b/presidio-analyzer/tests/test_ahds_recognizer.py index f0d597610..ca8412e5d 100644 --- a/presidio-analyzer/tests/test_ahds_recognizer.py +++ b/presidio-analyzer/tests/test_ahds_recognizer.py @@ -5,18 +5,25 @@ from unittest.mock import MagicMock import pytest import dotenv -from azure.health.deidentification import DeidentificationClient -from azure.health.deidentification.models import ( - DeidentificationContent, - DeidentificationOperationType, - DeidentificationResult, -) -from azure.identity import DefaultAzureCredential - from presidio_analyzer.predefined_recognizers import AzureHealthDeidRecognizer +@pytest.fixture(scope="module") +def import_modules(): + pytest.importorskip("azure-identity") + pytest.importskip("azure-health-deidentification") + from azure.health.deidentification import DeidentificationClient + from azure.health.deidentification.models import ( + DeidentificationContent, + DeidentificationOperationType, + DeidentificationResult, + ) + from azure.identity import DefaultAzureCredential + + + + def requires_env_vars(): dotenv.load_dotenv() endpoint = os.getenv("AHDS_ENDPOINT", "") @@ -26,7 +33,7 @@ def requires_env_vars(): ) @requires_env_vars() -def test_get_supported_entities(): +def test_get_supported_entities(import_modules): recognizer = AzureHealthDeidRecognizer() supported_entities = recognizer.get_supported_entities() assert isinstance(supported_entities, list) @@ -34,7 +41,7 @@ def test_get_supported_entities(): @requires_env_vars() -def test_analyze_name(): +def test_analyze_name(import_modules): recognizer = AzureHealthDeidRecognizer() text = "Patient name is John Doe." results = recognizer.analyze(text) @@ -42,7 +49,7 @@ def test_analyze_name(): @requires_env_vars() -def test_analyze_email(): +def test_analyze_email(import_modules): recognizer = AzureHealthDeidRecognizer() text = "Contact: john.doe@example.com" results = recognizer.analyze(text) @@ -50,7 +57,7 @@ def test_analyze_email(): @requires_env_vars() -def test_analyze_multiple_entities(): +def test_analyze_multiple_entities(import_modules): """Test that multiple entities are recognized in a single text.""" recognizer = AzureHealthDeidRecognizer() text = "Dr. Smith met Jane Doe on 2023-01-01. Email: jane@example.com" @@ -62,7 +69,7 @@ def test_analyze_multiple_entities(): @requires_env_vars() -def test_analyze_with_supported_entities_filter(): +def test_analyze_with_supported_entities_filter(import_modules): recognizer = AzureHealthDeidRecognizer(supported_entities=["EMAIL"]) text = "Contact: john.doe@example.com" results = recognizer.analyze(text) diff --git a/presidio-analyzer/tests/test_certificate_number_recognizer.py b/presidio-analyzer/tests/test_certificate_number_recognizer.py index 5be9024a0..59186936e 100644 --- a/presidio-analyzer/tests/test_certificate_number_recognizer.py +++ b/presidio-analyzer/tests/test_certificate_number_recognizer.py @@ -1,7 +1,7 @@ import pytest from tests import assert_result -from presidio_analyzer.predefined_recognizers.medical_license_recognizer import MedicalLicenseRecognizer +from presidio_analyzer.predefined_recognizers.country_specific.us import MedicalLicenseRecognizer @pytest.fixture(scope="module") diff --git a/presidio-analyzer/tests/test_iban_recognizer.py b/presidio-analyzer/tests/test_iban_recognizer.py index ebcdae483..8111b6693 100644 --- a/presidio-analyzer/tests/test_iban_recognizer.py +++ b/presidio-analyzer/tests/test_iban_recognizer.py @@ -1,7 +1,7 @@ import pytest from tests import assert_result -from presidio_analyzer.predefined_recognizers.iban_recognizer import IbanRecognizer +from presidio_analyzer.predefined_recognizers.generic.iban_recognizer import IbanRecognizer @pytest.fixture(scope="module") diff --git a/presidio-analyzer/tests/test_phone_recognizer.py b/presidio-analyzer/tests/test_phone_recognizer.py index 1003eef36..9abe7f15f 100644 --- a/presidio-analyzer/tests/test_phone_recognizer.py +++ b/presidio-analyzer/tests/test_phone_recognizer.py @@ -1,6 +1,6 @@ import pytest -from presidio_analyzer.predefined_recognizers.phone_recognizer import PhoneRecognizer +from presidio_analyzer.predefined_recognizers.generic.phone_recognizer import PhoneRecognizer from tests import assert_result, assert_result_with_textual_explanation