mirror of
https://github.com/data-privacy-stack/presidio.git
synced 2026-08-03 00:30:44 -05:00
2107f051c6
* add revised example for transformer recognizer * update documentation index to point to new example * Update docs/samples/python/transformers_recognizer/transformer_recognizer.py Co-authored-by: Omri Mendels <omri374@users.noreply.github.com> * linting, docstrings, readibility * externalize text splitting parameters * document class input variables * add documentaiton to the sample * Update docs/samples/python/transformers_recognizer/transformer_recognizer.md Co-authored-by: Omri Mendels <omri374@users.noreply.github.com> * Update docs/samples/python/transformers_recognizer/transformer_recognizer.md Co-authored-by: Omri Mendels <omri374@users.noreply.github.com> * Update docs/samples/python/transformers_recognizer/transformer_recognizer.md Co-authored-by: Omri Mendels <omri374@users.noreply.github.com> * update documentation Co-authored-by: Omri Mendels <omri374@users.noreply.github.com> Co-authored-by: Sharon Hart <sharonh.dev@gmail.com>
112 lines
3.9 KiB
Python
112 lines
3.9 KiB
Python
STANFORD_COFIGURATION = {
|
|
'DEFAULT_MODEL_PATH': "StanfordAIMI/stanford-deidentifier-base",
|
|
'PRESIDIO_SUPPORTED_ENTITIES': [
|
|
"LOCATION",
|
|
"PERSON",
|
|
"ORGANIZATION",
|
|
"AGE",
|
|
"ID",
|
|
"PHONE_NUMBER",
|
|
"EMAIL",
|
|
"DATE_TIME",
|
|
|
|
],
|
|
'LABELS_TO_IGNORE': ["O"],
|
|
'DEFAULT_EXPLANATION': "Identified as {} by transformers's Named Entity Recognition",
|
|
'SUB_WORD_AGGREGATION': 'simple',
|
|
'DATASET_TO_PRESIDIO_MAPPING': {"DATE": "DATE_TIME",
|
|
"DOCTOR": "PERSON",
|
|
"PATIENT": "PERSON",
|
|
"HOSPITAL": "LOCATION",
|
|
"MEDICALRECORD": "ID",
|
|
"IDNUM": "ID",
|
|
"ORGANIZATION": "ORGANIZATION",
|
|
"ZIP": "O",
|
|
"PHONE": "PHONE_NUMBER",
|
|
"USERNAME": "O",
|
|
"STREET": "O",
|
|
"PROFESSION": "O",
|
|
"COUNTRY": "LOCATION",
|
|
"LOCATION-OTHER": "LOCATION",
|
|
"FAX": "PHONE_NUMBER",
|
|
"EMAIL": "EMAIL",
|
|
"STATE": "LOCATION",
|
|
"DEVICE": "O",
|
|
"ORG": "ORGANIZATION",
|
|
"AGE": "O"
|
|
},
|
|
'MODEL_TO_PRESIDIO_MAPPING': {
|
|
"PER": "PERSON",
|
|
"PERSON": "PERSON",
|
|
"LOC": "LOCATION",
|
|
"ORG": "ORGANIZATION",
|
|
"AGE": "AGE",
|
|
"ID": "ID",
|
|
"PATIENT": "PERSON",
|
|
"HCW": "PERSON",
|
|
"HOSPITAL": "LOCATION",
|
|
"PATORG": "ORGANIZATION",
|
|
"DATE": "DATE_TIME",
|
|
"PHONE": "PHONE_NUMBER",
|
|
"VENDOR": "ORGANIZATION"
|
|
},
|
|
'CHUNK_OVERLAP_SIZE': 40,
|
|
'CHUNK_SIZE': 600
|
|
}
|
|
|
|
|
|
BERT_DEID_CONFIGURATION = {'PRESIDIO_SUPPORTED_ENTITIES': [
|
|
"LOCATION",
|
|
"PERSON",
|
|
"ORGANIZATION",
|
|
"AGE",
|
|
"ID",
|
|
"PHONE_NUMBER",
|
|
"EMAIL",
|
|
"DATE_TIME",
|
|
],
|
|
'DEFAULT_MODEL_PATH': "obi/deid_roberta_i2b2",
|
|
'LABELS_TO_IGNORE': ["O"],
|
|
'DEFAULT_EXPLANATION': "Identified as {} by transformers's Named Entity Recognition",
|
|
'SUB_WORD_AGGREGATION': 'simple',
|
|
|
|
'DATASET_TO_PRESIDIO_MAPPING': {"DATE": "DATE_TIME",
|
|
"DOCTOR": "PERSON",
|
|
"PATIENT": "PERSON",
|
|
"HOSPITAL": "ORGANIZATION",
|
|
"MEDICALRECORD": "ID",
|
|
"IDNUM": "ID",
|
|
"ORGANIZATION": "ORGANIZATION",
|
|
"ZIP": "O",
|
|
"PHONE": "PHONE_NUMBER",
|
|
"USERNAME": "O",
|
|
"STREET": "O",
|
|
"PROFESSION": "O",
|
|
"COUNTRY": "LOCATION",
|
|
"LOCATION-OTHER": "LOCATION",
|
|
"FAX": "PHONE_NUMBER",
|
|
"EMAIL": "EMAIL",
|
|
"STATE": "LOCATION",
|
|
"DEVICE": "O",
|
|
"ORG": "ORGANIZATION",
|
|
"AGE": "O"
|
|
},
|
|
|
|
'MODEL_TO_PRESIDIO_MAPPING': {
|
|
"PER": "PERSON",
|
|
"LOC": "LOCATION",
|
|
"ORG": "ORGANIZATION",
|
|
"AGE": "AGE",
|
|
"ID": "ID",
|
|
"EMAIL": "EMAIL",
|
|
"PATIENT": "PERSON",
|
|
"STAFF": "PERSON",
|
|
"HOSP": "ORGANIZATION",
|
|
"PATORG": "ORGANIZATION",
|
|
"DATE": "DATE_TIME",
|
|
"PHONE": "PHONE_NUMBER",
|
|
},
|
|
'CHUNK_OVERLAP_SIZE': 40,
|
|
'CHUNK_SIZE': 600
|
|
}
|