presidio/docs/samples/python/batch_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "gothic-trademark",
   "metadata": {},
   "source": [
    "# Run Presidio on structured / semi-structured data\n",
    "\n",
    "This sample shows how Presidio could be potentially extended to handle the anonymization of a table or data frame.\n",
    "It introduces methods for the analysis and anonymization of both lists and dicts. \n",
    "\n",
    "In this example we create two classes which implement the base Presidio classes:\n",
    "1. `BatchAnalyzerEngine(AnalyzerEngine)`: for the presidio-anlyzer side\n",
    "2. `BatchAnonymizerEngine(AnonymizerEngine`): for the presidio-anonymizer side\n",
    "\n",
    "In addition, we create a `dataclass` (`DictAnalyzerResult`) to serve as the data transfer object between the two.\n",
    "\n",
    "Note: this sample input here is a Pandas DataFrame, but it can be used in other scenarios such as querying SQL data or using Spark DataFrames."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "roman-allergy",
   "metadata": {},
   "source": [
    "### Set up imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "extensive-greensboro",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List, Optional, Dict, Union, Iterator, Iterable\n",
    "import collections\n",
    "from dataclasses import dataclass\n",
    "import pprint\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from presidio_analyzer import AnalyzerEngine, RecognizerResult\n",
    "from presidio_anonymizer import AnonymizerEngine\n",
    "from presidio_anonymizer.entities.engine.result import EngineResult\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "specialized-durham",
   "metadata": {},
   "source": [
    "### Set up classes for batch processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "metropolitan-atlantic",
   "metadata": {},
   "source": [
    "1. Presidio Analyzer: Batch mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "medium-ridge",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class DictAnalyzerResult:\n",
    "    \"\"\"Hold the analyzer results per value or list of values.\"\"\"\n",
    "    key: str\n",
    "    value: Union[str, List[str]]\n",
    "    recognizer_results: Union[List[RecognizerResult], List[List[RecognizerResult]]]\n",
    "\n",
    "\n",
    "class BatchAnalyzerEngine(AnalyzerEngine):\n",
    "    \"\"\"\n",
    "    Class inheriting from AnalyzerEngine and adds the funtionality to analyze lists or dictionaries.\n",
    "    \"\"\"\n",
    "    \n",
    "    def analyze_list(self, list_of_texts: Iterable[str], **kwargs) -> List[List[RecognizerResult]]:\n",
    "        \"\"\"\n",
    "        Analyze an iterable of strings\n",
    "        \n",
    "        :param list_of_texts: An iterable containing strings to be analyzed.\n",
    "        :param kwargs: Additional parameters for the `AnalyzerEngine.analyze` method.\n",
    "        \"\"\"\n",
    "        \n",
    "        list_results = []\n",
    "        for text in list_of_texts:\n",
    "            results = self.analyze(text=text, **kwargs) if isinstance(text, str) else []\n",
    "            list_results.append(results)\n",
    "        return list_results\n",
    "\n",
    "    def analyze_dict(\n",
    "     self, input_dict: Dict[str, Union[object, Iterable[object]]], **kwargs) -> Iterator[DictAnalyzerResult]:\n",
    "        \"\"\"\n",
    "        Analyze a dictionary of keys (strings) and values (either object or Iterable[object]). \n",
    "        Non-string values are returned as is.\n",
    "        \n",
    "        :param input_dict: The input dictionary for analysis\n",
    "        :param kwargs: Additional keyword arguments for the `AnalyzerEngine.analyze` method\n",
    "        \"\"\"\n",
    "        \n",
    "        for key, value in input_dict.items():\n",
    "            if not value:\n",
    "                results = []\n",
    "            else:\n",
    "                if isinstance(value, str):\n",
    "                    results: List[RecognizerResult] = self.analyze(text=value, **kwargs)\n",
    "                elif isinstance(value, collections.Iterable):\n",
    "                    results: List[List[RecognizerResult]] = self.analyze_list(\n",
    "                                list_of_texts=value, \n",
    "                                **kwargs)\n",
    "                else:\n",
    "                    results = []\n",
    "            yield DictAnalyzerResult(key=key, value=value, recognizer_results=results)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "obvious-fifty",
   "metadata": {},
   "source": [
    "2. Presidio Anonymizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "outstanding-celebration",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchAnonymizerEngine(AnonymizerEngine):\n",
    "    \"\"\"\n",
    "    Class inheriting from the AnonymizerEngine and adding additional functionality \n",
    "    for anonymizing lists or dictionaries.\n",
    "    \"\"\"\n",
    "    \n",
    "    def anonymize_list(\n",
    "        self, \n",
    "        texts:List[str], \n",
    "        recognizer_results_list: List[List[RecognizerResult]], \n",
    "        **kwargs\n",
    "    ) -> List[EngineResult]:\n",
    "        \"\"\"\n",
    "        Anonymize a list of strings.\n",
    "        \n",
    "        :param texts: List containing the texts to be anonymized (original texts)\n",
    "        :param recognizer_results_list: A list of lists of RecognizerResult, \n",
    "        the output of the AnalyzerEngine on each text in the list.\n",
    "        :param kwargs: Additional kwargs for the `AnonymizerEngine.anonymize` method\n",
    "        \"\"\"\n",
    "        return_list = []\n",
    "        for text, recognizer_results in zip(texts, recognizer_results_list):\n",
    "            if isinstance(text,str):\n",
    "                res = self.anonymize(text=text,analyzer_results=recognizer_results,**kwargs)\n",
    "                return_list.append(res.text)\n",
    "            else:\n",
    "                return_list.append(text)\n",
    "\n",
    "        return return_list\n",
    "\n",
    "\n",
    "    def anonymize_dict(self, analyzer_results: Iterator[DictAnalyzerResult],**kwargs) -> Dict[str, str]:\n",
    "\n",
    "        \"\"\"\n",
    "        Anonymize values in a dictionary.\n",
    "        \n",
    "        :param analyzer_results: Iterator of `DictAnalyzerResult` \n",
    "        containing the output of the AnalyzerEngine.analyze_dict on the input text.\n",
    "        :param kwargs: Additional kwargs for the `AnonymizerEngine.anonymize` method\n",
    "        \"\"\"\n",
    "        \n",
    "        return_dict = {}\n",
    "        for result in analyzer_results:\n",
    "            if isinstance(result.value, str):\n",
    "                resp = self.anonymize(text=result.value, analyzer_results=result.recognizer_results, **kwargs)\n",
    "                return_dict[result.key] = resp.text\n",
    "            elif isinstance(result.value, collections.Iterable):\n",
    "                anonymize_respones = self.anonymize_list(texts=result.value,\n",
    "                                                         recognizer_results_list=result.recognizer_results, \n",
    "                                                         **kwargs)\n",
    "                return_dict[result.key] = anonymize_respones \n",
    "            else:\n",
    "                return_dict[result.key] = result.value\n",
    "\n",
    "        return return_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fiscal-affair",
   "metadata": {},
   "source": [
    "## Example using sample data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bright-maple",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\"name phrase\",\"phone number phrase\", \"integer\", \"boolean\" ]\n",
    "sample_data = [\n",
    "        ('Morris likes this','Please call 212-555-1234 after 2pm', 1, True),\n",
    "        ('You should talk to Mike','his number is 978-428-7111', 2, False),\n",
    "        ('Mary had a little startup','Phone number: 202-342-1234', 3, False)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "russian-proceeding",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name phrase</th>\n",
       "      <th>phone number phrase</th>\n",
       "      <th>integer</th>\n",
       "      <th>boolean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Morris likes this</td>\n",
       "      <td>Please call 212-555-1234 after 2pm</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You should talk to Mike</td>\n",
       "      <td>his number is 978-428-7111</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mary had a little startup</td>\n",
       "      <td>Phone number: 202-342-1234</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 name phrase                 phone number phrase  integer  \\\n",
       "0          Morris likes this  Please call 212-555-1234 after 2pm        1   \n",
       "1    You should talk to Mike          his number is 978-428-7111        2   \n",
       "2  Mary had a little startup          Phone number: 202-342-1234        3   \n",
       "\n",
       "   boolean  \n",
       "0     True  \n",
       "1    False  \n",
       "2    False  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create Pandas DataFrame\n",
    "df  = pd.DataFrame(sample_data,columns=columns)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "brazilian-punch",
   "metadata": {},
   "outputs": [],
   "source": [
    "# DataFrame to dict\n",
    "df_dict = df.to_dict(orient=\"list\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fixed-commerce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'boolean': [True, False, False],\n",
      " 'integer': [1, 2, 3],\n",
      " 'name phrase': ['Morris likes this',\n",
      "                 'You should talk to Mike',\n",
      "                 'Mary had a little startup'],\n",
      " 'phone number phrase': ['Please call 212-555-1234 after 2pm',\n",
      "                         'his number is 978-428-7111',\n",
      "                         'Phone number: 202-342-1234']}\n"
     ]
    }
   ],
   "source": [
    "pprint.pprint(df_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "verified-spirituality",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_analyzer = BatchAnalyzerEngine()\n",
    "batch_anonymizer = BatchAnonymizerEngine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "narrative-freeze",
   "metadata": {},
   "outputs": [],
   "source": [
    "analyzer_results = batch_analyzer.analyze_dict(df_dict, language=\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "rural-month",
   "metadata": {},
   "outputs": [],
   "source": [
    "anonymizer_results = batch_anonymizer.anonymize_dict(analyzer_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "acute-mauritius",
   "metadata": {},
   "outputs": [],
   "source": [
    "scrubbed_df = pd.DataFrame(anonymizer_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "irish-phoenix",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name phrase</th>\n",
       "      <th>phone number phrase</th>\n",
       "      <th>integer</th>\n",
       "      <th>boolean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;PERSON&gt; likes this</td>\n",
       "      <td>Please call &lt;PHONE_NUMBER&gt; after &lt;DATE_TIME&gt;</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You should talk to &lt;PERSON&gt;</td>\n",
       "      <td>his number is &lt;PHONE_NUMBER&gt;</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;PERSON&gt; had a little startup</td>\n",
       "      <td>Phone number: &lt;PHONE_NUMBER&gt;</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     name phrase  \\\n",
       "0            <PERSON> likes this   \n",
       "1    You should talk to <PERSON>   \n",
       "2  <PERSON> had a little startup   \n",
       "\n",
       "                            phone number phrase  integer  boolean  \n",
       "0  Please call <PHONE_NUMBER> after <DATE_TIME>        1     True  \n",
       "1                  his number is <PHONE_NUMBER>        2    False  \n",
       "2                  Phone number: <PHONE_NUMBER>        3    False  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scrubbed_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "conceptual-responsibility",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PyCharm (presidio-research)",
   "language": "python",
   "name": "pycharm-c8930cf3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}