mirror of
https://github.com/data-privacy-stack/presidio.git
synced 2026-07-23 11:20:55 -05:00
570 lines
28 KiB
JSON
570 lines
28 KiB
JSON
{
|
|
"$schema": "http://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"parameters": {
|
|
"dataFactoryName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Data Factory name"
|
|
},
|
|
"defaultValue": "[concat('presidioadf', uniqueString(resourceGroup().id))]"
|
|
},
|
|
"AzureBlobStorage_accountName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Azure Storage Account Name"
|
|
},
|
|
"defaultValue": "[concat('presidio', uniqueString(resourceGroup().id))]"
|
|
},
|
|
"AzureBlobStorage_cotainerName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Azure Storage Container Name"
|
|
},
|
|
"defaultValue": "presidio"
|
|
},
|
|
"AzureKeyVault_name": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Azure Key Vault Name"
|
|
},
|
|
"defaultValue": "[concat('presidioakv', uniqueString(resourceGroup().id))]"
|
|
},
|
|
"AzureKeyVault_storageAccessKeySecretName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name of storage access key secert in Key Vault"
|
|
},
|
|
"defaultValue": "access-token"
|
|
},
|
|
"AzureKeyVault_storageSASSecretName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name of storage SAS token secert in Key Vault"
|
|
},
|
|
"defaultValue": "sas-token"
|
|
},
|
|
"Analyzer_appName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Analyzer App Service Name"
|
|
},
|
|
"defaultValue": "[concat('presidio-analyzer', uniqueString(resourceGroup().id))]"
|
|
},
|
|
"Anonymizer_appName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Anonymizer App Service Name"
|
|
},
|
|
"defaultValue": "[concat('presidio-anonymizer', uniqueString(resourceGroup().id))]"
|
|
},
|
|
"accountSasProperties": {
|
|
"type": "object",
|
|
"defaultValue": {
|
|
"signedServices": "b",
|
|
"signedPermission": "rw",
|
|
"signedExpiry": "[dateTimeAdd(utcNow('u'), 'P3D')]",
|
|
"signedResourceTypes": "o"
|
|
}
|
|
}
|
|
},
|
|
"variables": {
|
|
"AzureBlobStorage_connectionString": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('AzureBlobStorage_accountName'), ';EndpointSuffix=core.windows.net')]",
|
|
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('dataFactoryName'))]",
|
|
"Dataset_url": "https://raw.githubusercontent.com/data-privacy-stack/presidio-research/master/tests/data/generated_large.txt",
|
|
"Analyzer_url": "[concat('https://', parameters('Analyzer_appName'), '.azurewebsites.net/analyze')]",
|
|
"Anonymizer_url": "[concat('https://', parameters('Anonymizer_appName'), '.azurewebsites.net/anonymize')]",
|
|
"AzureKeyVault_baseUrl": "[concat('https://', parameters('AzureKeyVault_name'), '.vault.azure.net/')]"
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "presidio-app-services",
|
|
"properties": {
|
|
"mode": "Incremental",
|
|
"templateLink": {
|
|
"uri": "https://raw.githubusercontent.com/data-privacy-stack/presidio/main/docs/samples/deployments/app-service/presidio-services.json",
|
|
"contentVersion": "1.0.0.0"
|
|
},
|
|
"parameters": {
|
|
"imageRedactor": {
|
|
"value": false
|
|
},
|
|
"analyzerWebAppName": {
|
|
"value": "[parameters('Analyzer_appName')]"
|
|
},
|
|
"anonymizerWebAppName": {
|
|
"value": "[parameters('Anonymizer_appName')]"
|
|
},
|
|
"location": {
|
|
"value": "[resourceGroup().location]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Storage/storageAccounts",
|
|
"apiVersion": "2018-07-01",
|
|
"name": "[parameters('AzureBlobStorage_accountName')]",
|
|
"location": "[resourceGroup().location]",
|
|
"tags": {
|
|
"displayName": "[parameters('AzureBlobStorage_accountName')]"
|
|
},
|
|
"sku": {
|
|
"name": "Standard_LRS"
|
|
},
|
|
"kind": "StorageV2",
|
|
"resources": [
|
|
{
|
|
"type": "blobServices/containers",
|
|
"apiVersion": "2018-03-01-preview",
|
|
"name": "[concat('default/', parameters('AzureBlobStorage_cotainerName'))]",
|
|
"dependsOn": [
|
|
"[parameters('AzureBlobStorage_accountName')]"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.DataFactory/factories",
|
|
"apiVersion": "2018-06-01",
|
|
"name": "[parameters('dataFactoryName')]",
|
|
"location": "[resourceGroup().location]",
|
|
"properties": {},
|
|
"identity": {
|
|
"type": "SystemAssigned"
|
|
},
|
|
"resources": [
|
|
{
|
|
"name": "[concat(parameters('dataFactoryName'), '/AzureBlobStorage')]",
|
|
"type": "Microsoft.DataFactory/factories/linkedServices",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"annotations": [],
|
|
"type": "AzureBlobStorage",
|
|
"typeProperties": {
|
|
"connectionString": "[variables('AzureBlobStorage_connectionString')]",
|
|
"accountKey": {
|
|
"type": "AzureKeyVaultSecret",
|
|
"store": {
|
|
"referenceName": "StorageSecretsKeyVault",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"secretName": "[parameters('AzureKeyVault_storageAccessKeySecretName')]"
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[parameters('dataFactoryName')]",
|
|
"[concat(variables('factoryId'), '/linkedServices/StorageSecretsKeyVault')]"
|
|
]
|
|
},
|
|
{
|
|
"name": "[concat(parameters('dataFactoryName'), '/datasetGithub')]",
|
|
"type": "Microsoft.DataFactory/factories/linkedServices",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"annotations": [],
|
|
"type": "RestService",
|
|
"typeProperties": {
|
|
"url": "[variables('Dataset_url')]",
|
|
"enableServerCertificateValidation": true,
|
|
"authenticationType": "Anonymous"
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[parameters('dataFactoryName')]"
|
|
]
|
|
},
|
|
{
|
|
"name": "[concat(parameters('dataFactoryName'), '/StorageSecretsKeyVault')]",
|
|
"type": "Microsoft.DataFactory/factories/linkedServices",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"annotations": [],
|
|
"type": "AzureKeyVault",
|
|
"typeProperties": {
|
|
"baseUrl": "[variables('AzureKeyVault_baseUrl')]"
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[parameters('dataFactoryName')]"
|
|
]
|
|
},
|
|
{
|
|
"name": "[concat(parameters('dataFactoryName'), '/GithubDataSet')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"description": "JSON Dataset on GitHub",
|
|
"linkedServiceName": {
|
|
"referenceName": "datasetGithub",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"annotations": [],
|
|
"type": "RestResource",
|
|
"typeProperties": {},
|
|
"schema": []
|
|
},
|
|
"dependsOn": [
|
|
"[parameters('dataFactoryName')]",
|
|
"[concat(variables('factoryId'), '/linkedServices/datasetGithub')]"
|
|
]
|
|
},
|
|
{
|
|
"name": "[concat(parameters('dataFactoryName'), '/BlobStorageDataset')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"description": "JSON Dataset on Azure Blob Storage",
|
|
"linkedServiceName": {
|
|
"referenceName": "AzureBlobStorage",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"annotations": [],
|
|
"type": "Json",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "AzureBlobStorageLocation",
|
|
"fileName": "sentances.json",
|
|
"folderPath": "dataset",
|
|
"container": "[parameters('AzureBlobStorage_cotainerName')]"
|
|
}
|
|
},
|
|
"schema": {}
|
|
},
|
|
"dependsOn": [
|
|
"[parameters('dataFactoryName')]",
|
|
"[concat(variables('factoryId'), '/linkedServices/AzureBlobStorage')]"
|
|
]
|
|
},
|
|
{
|
|
"name": "[concat(parameters('dataFactoryName'), '/Anonymize')]",
|
|
"type": "Microsoft.DataFactory/factories/pipelines",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"activities": [
|
|
{
|
|
"name": "GetDataSet",
|
|
"description": "Get the data set from GitHub to Azure Blob Storage",
|
|
"type": "Copy",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"source": {
|
|
"type": "RestSource",
|
|
"httpRequestTimeout": "00:01:40",
|
|
"requestInterval": "00.00:00:00.010",
|
|
"requestMethod": "GET"
|
|
},
|
|
"sink": {
|
|
"type": "JsonSink",
|
|
"storeSettings": {
|
|
"type": "AzureBlobStorageWriteSettings"
|
|
},
|
|
"formatSettings": {
|
|
"type": "JsonWriteSettings"
|
|
}
|
|
},
|
|
"enableStaging": false,
|
|
"translator": {
|
|
"type": "TabularTranslator",
|
|
"mappings": [
|
|
{
|
|
"source": {
|
|
"path": "$['full_text']"
|
|
},
|
|
"sink": {
|
|
"path": "full_text"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"inputs": [
|
|
{
|
|
"referenceName": "GithubDataSet",
|
|
"type": "DatasetReference",
|
|
"parameters": {}
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"referenceName": "BlobStorageDataset",
|
|
"type": "DatasetReference",
|
|
"parameters": {}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "LoadSet",
|
|
"description": "Load the data set to array for processing",
|
|
"type": "Lookup",
|
|
"dependsOn": [
|
|
{
|
|
"activity": "GetDataSet",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
}
|
|
],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"source": {
|
|
"type": "JsonSource",
|
|
"storeSettings": {
|
|
"type": "AzureBlobStorageReadSettings",
|
|
"recursive": false,
|
|
"enablePartitionDiscovery": false
|
|
},
|
|
"formatSettings": {
|
|
"type": "JsonReadSettings"
|
|
}
|
|
},
|
|
"dataset": {
|
|
"referenceName": "BlobStorageDataset",
|
|
"type": "DatasetReference",
|
|
"parameters": {}
|
|
},
|
|
"firstRowOnly": false
|
|
}
|
|
},
|
|
{
|
|
"name": "SaveSet",
|
|
"description": "Save each json document as a separate blob",
|
|
"type": "ForEach",
|
|
"dependsOn": [
|
|
{
|
|
"activity": "LoadSet",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
},
|
|
{
|
|
"activity": "GetSASToken",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
}
|
|
],
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"items": {
|
|
"value": "@activity('LoadSet').output.value",
|
|
"type": "Expression"
|
|
},
|
|
"activities": [
|
|
{
|
|
"name": "PresidioAnalyze",
|
|
"description": "Analyze text with presidio",
|
|
"type": "WebActivity",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"url": {
|
|
"value": "[variables('Analyzer_url')]",
|
|
"type": "Expression"
|
|
},
|
|
"method": "POST",
|
|
"headers": {
|
|
"Content-Type": "application/json"
|
|
},
|
|
"body": {
|
|
"value": "@concat('{\"text\":\"',replace(item().full_text, '\"', '\\\"'),'\",\"language\":\"en\"}')",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "PresidioAnonymize",
|
|
"type": "WebActivity",
|
|
"description": "Anonymize text with presidio",
|
|
"dependsOn": [
|
|
{
|
|
"activity": "PresidioAnalyze",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
}
|
|
],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"url": {
|
|
"value": "[variables('Anonymizer_url')]",
|
|
"type": "Expression"
|
|
},
|
|
"method": "POST",
|
|
"headers": {
|
|
"Content-Type": "application/json"
|
|
},
|
|
"body": {
|
|
"value": "@concat('{\"text\":\"',replace(item().full_text, '\"', '\\\"'), '\",\"anonymizers\": {\"DEFAULT\": { \"type\": \"replace\", \"new_value\": \"ANONYMIZED\" }},\"analyzer_results\": ', activity('PresidioAnalyze').output. Response,'}')",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "UploadBlob",
|
|
"type": "WebActivity",
|
|
"dependsOn": [
|
|
{
|
|
"activity": "PresidioAnonymize",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
}
|
|
],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"url": {
|
|
"value": "[concat('@concat(''https://', parameters('AzureBlobStorage_accountName'), '.blob.core.windows.net/', parameters('AzureBlobStorage_cotainerName'), '/output/'', ''file'', string(rand(0,1000)), ''.txt?'', activity(''GetSASToken'').output.value)')]",
|
|
"type": "Expression"
|
|
},
|
|
"method": "PUT",
|
|
"headers": {
|
|
"x-ms-blob-type": "BlockBlob"
|
|
},
|
|
"body": {
|
|
"value": "@activity('PresidioAnonymize').output.result",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "GetSASToken",
|
|
"description": "Get storage account SAS Token from Azure Key Vault",
|
|
"type": "WebActivity",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"url": "[concat(variables('AzureKeyVault_baseUrl'), 'secrets/', parameters('AzureKeyVault_storageSASSecretName'), '?api-version=7.0')]",
|
|
"method": "GET",
|
|
"headers": {},
|
|
"authentication": {
|
|
"type": "MSI",
|
|
"resource": "https://vault.azure.net"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"annotations": [],
|
|
"lastPublishTime": "2021-03-15T17:44:33Z"
|
|
},
|
|
"dependsOn": [
|
|
"[parameters('dataFactoryName')]",
|
|
"[concat(variables('factoryId'), '/datasets/GithubDataSet')]",
|
|
"[concat(variables('factoryId'), '/datasets/BlobStorageDataset')]"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.KeyVault/vaults",
|
|
"apiVersion": "2018-02-14",
|
|
"name": "[parameters('AzureKeyVault_name')]",
|
|
"location": "[resourceGroup().location]",
|
|
"tags": {
|
|
"displayName": "[parameters('AzureKeyVault_name')]"
|
|
},
|
|
"dependsOn": [
|
|
"[concat('Microsoft.DataFactory/factories/', parameters('dataFactoryName'))]"
|
|
],
|
|
"properties": {
|
|
"enabledForDeployment": true,
|
|
"enabledForTemplateDeployment": true,
|
|
"enabledForDiskEncryption": true,
|
|
"tenantId": "[subscription().tenantId]",
|
|
"accessPolicies": [
|
|
{
|
|
"tenantId": "[subscription().tenantId]",
|
|
"objectId": "[reference(concat('Microsoft.DataFactory/factories/', parameters('dataFactoryName')), '2018-06-01', 'Full').identity.principalId]",
|
|
"permissions": {
|
|
"keys": [
|
|
"get"
|
|
],
|
|
"secrets": [
|
|
"list",
|
|
"get",
|
|
"set"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"sku": {
|
|
"name": "standard",
|
|
"family": "A"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"apiVersion": "2018-02-14",
|
|
"type": "Microsoft.KeyVault/vaults/secrets",
|
|
"dependsOn": [
|
|
"[concat('Microsoft.KeyVault/vaults/', parameters('AzureKeyVault_name'))]",
|
|
"[concat('Microsoft.Storage/storageAccounts/', parameters('AzureBlobStorage_accountName'))]"
|
|
],
|
|
"name": "[concat(parameters('AzureKeyVault_name'), '/', parameters('AzureKeyVault_storageSASSecretName'))]",
|
|
"properties": {
|
|
"value": "[listAccountSas(parameters('AzureBlobStorage_accountName'), '2018-07-01', parameters('accountSasProperties')).accountSasToken]"
|
|
}
|
|
},
|
|
{
|
|
"apiVersion": "2018-02-14",
|
|
"type": "Microsoft.KeyVault/vaults/secrets",
|
|
"dependsOn": [
|
|
"[concat('Microsoft.KeyVault/vaults/', parameters('AzureKeyVault_name'))]",
|
|
"[concat('Microsoft.Storage/storageAccounts/', parameters('AzureBlobStorage_accountName'))]"
|
|
],
|
|
"name": "[concat(parameters('AzureKeyVault_name'), '/', parameters('AzureKeyVault_storageAccessKeySecretName'))]",
|
|
"properties": {
|
|
"value": "[listKeys(parameters('AzureBlobStorage_accountName'), '2018-07-01').keys[0].value]"
|
|
}
|
|
}
|
|
]
|
|
} |