mirror of
https://github.com/data-privacy-stack/presidio.git
synced 2026-08-05 01:30:43 -05:00
417 lines
12 KiB
YAML
417 lines
12 KiB
YAML
name: CI
|
|
|
|
on:
|
|
pull_request:
|
|
branches: [ main ]
|
|
push:
|
|
branches: [ main ]
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions: read-all
|
|
|
|
env:
|
|
REGISTRY_NAME: ${{ secrets.ACR_NAME }}.azurecr.io
|
|
TAG: gha${{ github.run_number }}
|
|
|
|
jobs:
|
|
lint:
|
|
name: Linting
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install ruff
|
|
run: pip install ruff
|
|
|
|
- name: Run ruff check
|
|
run: ruff check
|
|
|
|
dependency-review:
|
|
name: Dependency Review
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'pull_request'
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Dependency Review
|
|
uses: actions/dependency-review-action@v4
|
|
with:
|
|
fail-on-severity: low
|
|
allow-licenses: MIT, Apache-2.0, BSD-3-Clause
|
|
comment-summary-in-pr: on-failure
|
|
|
|
test:
|
|
name: Test ${{ matrix.component.name }} (Python ${{ matrix.python-version }})
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
python-version: ['3.10', '3.11', '3.12', '3.13']
|
|
component:
|
|
- name: 'Analyzer'
|
|
path: 'presidio-analyzer'
|
|
extras: '--all-extras'
|
|
spacy-models: 'en_core_web_lg en_core_web_sm'
|
|
- name: 'Anonymizer'
|
|
path: 'presidio-anonymizer'
|
|
extras: ''
|
|
spacy-models: ''
|
|
- name: 'Image Redactor'
|
|
path: 'presidio-image-redactor'
|
|
extras: ''
|
|
spacy-models: 'en_core_web_lg'
|
|
- name: 'CLI'
|
|
path: 'presidio-cli'
|
|
extras: ''
|
|
spacy-models: ''
|
|
- name: 'Structured'
|
|
path: 'presidio-structured'
|
|
extras: ''
|
|
spacy-models: ''
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
cache: 'pip'
|
|
cache-dependency-path: ${{ matrix.component.path }}/pyproject.toml
|
|
|
|
- name: Install Poetry
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install poetry
|
|
|
|
- name: Install system dependencies for Image Redactor
|
|
if: matrix.component.name == 'Image Redactor'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install tesseract-ocr -y
|
|
|
|
- name: Install dependencies
|
|
working-directory: ${{ matrix.component.path }}
|
|
run: |
|
|
poetry install ${{ matrix.component.extras }}
|
|
|
|
- name: Install presidio-analyzer for Image Redactor
|
|
if: matrix.component.name == 'Image Redactor'
|
|
working-directory: ${{ matrix.component.path }}
|
|
run: |
|
|
poetry run pip install -e ../presidio-analyzer/.
|
|
|
|
- name: Download spaCy models
|
|
if: matrix.component.spacy-models != ''
|
|
working-directory: ${{ matrix.component.path }}
|
|
run: |
|
|
for model in ${{ matrix.component.spacy-models }}; do
|
|
poetry run python -m spacy download $model
|
|
done
|
|
|
|
- name: Run unit tests
|
|
working-directory: ${{ matrix.component.path }}
|
|
run: |
|
|
poetry run pytest -vv --tb=short
|
|
|
|
- name: Build wheel package
|
|
working-directory: ${{ matrix.component.path }}
|
|
run: |
|
|
pip install build
|
|
python -m build --wheel
|
|
|
|
build-platform-images:
|
|
name: Build ${{ matrix.image }} (${{ matrix.platform }})
|
|
runs-on: ${{ matrix.runner }}
|
|
needs: [lint]
|
|
if: github.ref == 'refs/heads/main'
|
|
permissions:
|
|
id-token: write
|
|
contents: read
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- image: presidio-anonymizer
|
|
platform: linux/amd64
|
|
runner: ubuntu-latest
|
|
- image: presidio-analyzer
|
|
platform: linux/amd64
|
|
runner: ubuntu-latest
|
|
- image: presidio-image-redactor
|
|
platform: linux/amd64
|
|
runner: ubuntu-latest
|
|
- image: presidio-anonymizer
|
|
platform: linux/arm64
|
|
runner: ubuntu-24.04-arm
|
|
- image: presidio-analyzer
|
|
platform: linux/arm64
|
|
runner: ubuntu-24.04-arm
|
|
- image: presidio-image-redactor
|
|
platform: linux/arm64
|
|
runner: ubuntu-24.04-arm
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Azure Login using OIDC
|
|
uses: azure/login@v2
|
|
with:
|
|
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
|
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
|
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
|
|
|
- name: Log in to Azure Container Registry
|
|
run: az acr login --name ${{ secrets.ACR_NAME }}
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Build and Push ${{ matrix.image }} for ${{ matrix.platform }}
|
|
run: |
|
|
# Create platform-specific tag
|
|
PLATFORM_TAG=$(echo "${{ matrix.platform }}" | sed 's/\//-/g')
|
|
docker buildx build \
|
|
--platform ${{ matrix.platform }} \
|
|
--push \
|
|
--tag ${{ env.REGISTRY_NAME }}/${{ matrix.image }}:${{ env.TAG }}-${PLATFORM_TAG} \
|
|
--cache-from type=registry,ref=${{ env.REGISTRY_NAME }}/public/${{ matrix.image }}:latest \
|
|
--cache-to type=inline \
|
|
--attest type=sbom \
|
|
--attest type=provenance,mode=max \
|
|
./${{ matrix.image }}
|
|
env:
|
|
REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
|
|
TAG: ${{ env.TAG }}
|
|
|
|
create-manifests:
|
|
name: Create Multi-Platform Manifests
|
|
runs-on: ubuntu-latest
|
|
needs: build-platform-images
|
|
if: github.ref == 'refs/heads/main'
|
|
permissions:
|
|
id-token: write
|
|
contents: read
|
|
steps:
|
|
- name: Azure Login using OIDC
|
|
uses: azure/login@v2
|
|
with:
|
|
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
|
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
|
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
|
|
|
- name: Log in to Azure Container Registry
|
|
run: az acr login --name ${{ secrets.ACR_NAME }}
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Create all multi-platform manifests
|
|
run: |
|
|
IMAGES=("presidio-anonymizer" "presidio-analyzer" "presidio-image-redactor")
|
|
|
|
for image in "${IMAGES[@]}"; do
|
|
echo "Creating manifest for $image"
|
|
docker buildx imagetools create \
|
|
--tag ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }} \
|
|
${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-amd64 \
|
|
${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-arm64
|
|
done
|
|
env:
|
|
REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
|
|
TAG: ${{ env.TAG }}
|
|
|
|
e2e-tests:
|
|
name: E2E Tests (${{ matrix.platform }})
|
|
runs-on: ${{ matrix.runner }}
|
|
needs: create-manifests
|
|
if: github.ref == 'refs/heads/main'
|
|
permissions:
|
|
id-token: write
|
|
contents: read
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- platform: linux/amd64
|
|
runner: ubuntu-latest
|
|
- platform: linux/arm64
|
|
runner: ubuntu-24.04-arm
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Azure Login using OIDC
|
|
uses: azure/login@v2
|
|
with:
|
|
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
|
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
|
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
|
|
|
- name: Log in to Azure Container Registry
|
|
run: az acr login --name ${{ secrets.ACR_NAME }}
|
|
|
|
- name: Pull Presidio images from ACR
|
|
run: |
|
|
export DOCKER_DEFAULT_PLATFORM=${{ matrix.platform }}
|
|
docker compose -f docker-compose.yml pull
|
|
env:
|
|
REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
|
|
TAG: :${{ env.TAG }}
|
|
|
|
- name: Start Presidio cluster
|
|
run: |
|
|
export DOCKER_DEFAULT_PLATFORM=${{ matrix.platform }}
|
|
docker compose -f docker-compose.yml up -d
|
|
env:
|
|
REGISTRY_NAME: ${{ env.REGISTRY_NAME }}
|
|
TAG: :${{ env.TAG }}
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Cache E2E dependencies
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/pip
|
|
e2e-tests/env
|
|
key: ${{ runner.os }}-${{ matrix.platform }}-e2e-${{ hashFiles('e2e-tests/requirements.txt') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-${{ matrix.platform }}-e2e-
|
|
|
|
- name: Install E2E test dependencies
|
|
working-directory: e2e-tests
|
|
run: |
|
|
python -m venv env
|
|
source env/bin/activate
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
python -m spacy download en_core_web_lg
|
|
|
|
- name: Wait for services to be ready
|
|
run: sleep 60
|
|
|
|
- name: Run E2E tests
|
|
working-directory: e2e-tests
|
|
run: |
|
|
source env/bin/activate
|
|
pytest -vv --tb=short
|
|
|
|
- name: Capture Docker logs on failure (${{ matrix.platform }})
|
|
if: failure()
|
|
run: |
|
|
echo "Capturing Docker logs for platform: ${{ matrix.platform }}"
|
|
docker compose -f docker-compose.yml logs
|
|
|
|
- name: Stop Docker containers
|
|
if: always()
|
|
run: |
|
|
docker compose -f docker-compose.yml down
|
|
|
|
# Local build and test jobs for external contributors (no secrets required)
|
|
local-build-and-test:
|
|
name: Local Build and E2E Tests (${{ matrix.platform }})
|
|
runs-on: ${{ matrix.runner }}
|
|
needs: [lint]
|
|
if: github.ref != 'refs/heads/main'
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- platform: linux/amd64
|
|
runner: ubuntu-latest
|
|
- platform: linux/arm64
|
|
runner: ubuntu-24.04-arm
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v5
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Build Presidio images locally
|
|
run: |
|
|
# Build all images locally without pushing
|
|
docker compose -f docker-compose.yml build
|
|
env:
|
|
REGISTRY_NAME: 'mcr.microsoft.com'
|
|
IMAGE_PREFIX: ''
|
|
TAG: gha${{ github.run_number }}
|
|
|
|
- name: Start Presidio cluster locally
|
|
run: |
|
|
docker compose -f docker-compose.yml up -d
|
|
env:
|
|
REGISTRY_NAME: 'mcr.microsoft.com'
|
|
IMAGE_PREFIX: ''
|
|
TAG: gha${{ github.run_number }}
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Cache E2E dependencies
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/pip
|
|
e2e-tests/env
|
|
key: ${{ runner.os }}-${{ matrix.platform }}-local-e2e-${{ hashFiles('e2e-tests/requirements.txt') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-${{ matrix.platform }}-local-e2e-
|
|
|
|
- name: Install E2E test dependencies
|
|
working-directory: e2e-tests
|
|
run: |
|
|
python -m venv env
|
|
source env/bin/activate
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
python -m spacy download en_core_web_lg
|
|
|
|
- name: Wait for services to be ready
|
|
run: sleep 60
|
|
|
|
- name: Run E2E tests
|
|
working-directory: e2e-tests
|
|
run: |
|
|
source env/bin/activate
|
|
pytest -vv --tb=short
|
|
|
|
- name: Capture Docker logs on failure
|
|
if: failure()
|
|
run: |
|
|
echo "Capturing Docker logs for local build"
|
|
docker compose -f docker-compose.yml logs
|
|
|
|
- name: Stop Docker containers
|
|
if: always()
|
|
run: |
|
|
docker compose -f docker-compose.yml down
|