Coverage for presidio_analyzer / input_validation / yaml_recognizer_models.py: 95%
198 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-03-29 09:03 +0000
1"""Pydantic models for YAML recognizer configurations."""
3from typing import Any, Dict, List, Optional, Type, Union
5from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
7from presidio_analyzer.input_validation import validate_language_codes
8from presidio_analyzer.recognizer_registry.recognizers_loader_utils import (
9 PredefinedRecognizerNotFoundError,
10 RecognizerListLoader,
11)
14class LanguageContextConfig(BaseModel):
15 """Configuration for language-specific validation with context words.
17 :param language: Language code (e.g., 'en', 'es')
18 :param context: Context words for this language
19 """
21 language: str = Field(..., description="Language code (e.g., 'en', 'es')")
22 context: Optional[List[str]] = Field(
23 default=None, description="Context words for this language"
24 )
26 @field_validator("language")
27 @classmethod
28 def validate_language_code(cls, v: str) -> str:
29 """Validate language code format."""
30 validate_language_codes([v])
31 return v
34class BaseRecognizerConfig(BaseModel):
35 """Base validation for all recognizer configuration types.
37 :param name: Instance name used in analysis results. Defaults to class name.
38 :param class_name: Python class name for lookup. If not provided, uses 'name'.
39 :param enabled: Whether the recognizer is enabled
40 :param type: Type of recognizer (predefined/custom)
41 :param supported_language: Single supported language (legacy)
42 :param supported_languages: Multiple supported languages with optional context.
43 Passing multiple languages will result in multiple actual
44 recognizers initialized in Presidio.
45 :param context: context words. Context is best defined
46 in the language-specific configuration,
47 as it is language-dependent. If context is defined outside,
48 it should only work if the user passed one language
49 (either in supported_language or have a supported_languages with length 1).
50 :param supported_entity: Supported entity for this recognizer (legacy)
51 :param supported_entities: List of supported entities for this recognizer.
52 """
54 name: str = Field(..., description="Instance name for the recognizer")
55 class_name: Optional[str] = Field(
56 default=None,
57 description=(
58 "Python class name for predefined recognizers "
59 "(if different from instance name)"
60 ),
61 )
62 enabled: bool = Field(default=True, description="Whether the recognizer is enabled")
63 type: Optional[str] = Field(
64 default="predefined", description="Type of recognizer (predefined/custom)"
65 )
66 supported_language: Optional[str] = Field(
67 default=None, description="The language this recognizer supports"
68 )
69 supported_languages: Optional[Union[List[str], List[LanguageContextConfig]]] = (
70 Field(
71 default=None,
72 description="Multiple supported languages with optional context",
73 )
74 )
75 context: Optional[List[str]] = Field(
76 default=None, description="Global context words"
77 )
78 supported_entity: Optional[str] = Field(
79 default=None, description="Supported entity for this recognizer"
80 )
81 supported_entities: Optional[List[str]] = Field(
82 default=None, description="List of supported entities for this recognizer"
83 )
85 @field_validator("supported_language")
86 @classmethod
87 def validate_single_language(cls, v: Optional[str]) -> Optional[str]:
88 """Validate single language code format."""
89 validate_language_codes([v])
90 return v
92 @model_validator(mode="after")
93 def validate_language_configuration(self):
94 """Ensure proper language validation."""
95 if self.supported_language and self.supported_languages:
96 raise ValueError(
97 "Cannot specify both 'supported_language' and 'supported_languages'"
98 )
100 return self
102 @model_validator(mode="after")
103 def validate_entity_configuration(self):
104 """Ensure proper entity validation."""
105 # Check if user provided both (before we modify them)
106 user_provided_both = (
107 self.supported_entity is not None and self.supported_entities is not None
108 )
110 if user_provided_both:
111 raise ValueError(
112 f"Recognizer {self.name} has both "
113 "'supported_entity' and 'supported_entities' specified."
114 )
116 return self
118 @model_validator(mode="after")
119 def validate_context_configuration(self):
120 """Validate context configuration according to language settings."""
121 # Check if global context is defined
122 if self.context:
123 # Global context is only valid if we have exactly one language
124 if self.supported_languages and len(self.supported_languages) > 1:
125 raise ValueError(
126 "Global context can only be used with a single language. "
127 "For multiple languages, define context in "
128 "language-specific configurations."
129 "Example: "
130 " supported_languages: "
131 " - language: en "
132 " context: [credit, card, visa, mastercard] "
133 " - language: es "
134 " context: [tarjeta, credito, visa, mastercard] "
135 )
136 return self
139class PredefinedRecognizerConfig(BaseRecognizerConfig):
140 """Configuration for predefined recognizers."""
142 type: str = Field(default="predefined", description="Type of recognizer")
144 @model_validator(mode="after")
145 def validate_predefined_recognizer_exists(self):
146 """Validate that the predefined recognizer class actually exists."""
147 recognizer_class_name = self.class_name if self.class_name else self.name
148 try:
149 RecognizerListLoader.get_existing_recognizer_cls(recognizer_class_name)
150 except PredefinedRecognizerNotFoundError as e:
151 raise ValueError(
152 f"Predefined recognizer '{recognizer_class_name}' not found: {str(e)}"
153 ) from e
154 return self
157class HuggingFaceRecognizerConfig(PredefinedRecognizerConfig):
158 """Configuration specifically for HuggingFace NER models."""
160 model_config = ConfigDict(extra="forbid")
162 model_name: Optional[str] = Field(None, description="HuggingFace model name")
163 tokenizer_name: Optional[str] = Field(
164 None, description="HuggingFace tokenizer name"
165 )
166 label_mapping: Optional[Dict[str, str]] = Field(None, description="Label mapping")
167 threshold: Optional[float] = Field(None, description="Confidence threshold")
168 aggregation_strategy: Optional[str] = Field(
169 None, description="Aggregation strategy"
170 )
171 chunk_overlap: Optional[int] = Field(None, description="Chunk overlap")
172 chunk_size: Optional[int] = Field(None, description="Chunk size")
173 device: Optional[Union[str, int]] = Field(None, description="Device (cpu/gpu)")
174 label_prefixes: Optional[List[str]] = Field(
175 default=None, description="Prefixes to strip from labels (e.g. B-, I-)"
176 )
179class CustomRecognizerConfig(BaseRecognizerConfig):
180 """Configuration for custom pattern-based recognizers."""
182 type: str = Field(default="custom", description="Type of recognizer")
183 supported_entity: str = Field(
184 ..., description="Entity type this recognizer detects"
185 )
186 patterns: Optional[List[Dict[str, Any]]] = Field(
187 default=None, description="List of patterns"
188 )
189 context: Optional[List[str]] = Field(
190 default=None, description="Global context words"
191 )
192 deny_list: Optional[List[str]] = Field(
193 default=None, description="Words to deny/exclude"
194 )
195 deny_list_score: Optional[float] = Field(
196 default=0.0, ge=0.0, le=1.0, description="Deny list score"
197 )
199 # Language validation (legacy and new formats)
200 supported_language: Optional[str] = Field(
201 default=None, description="Single supported language (legacy)"
202 )
203 supported_languages: Optional[Union[List[str], List[LanguageContextConfig]]] = (
204 Field(
205 default=None,
206 description="Multiple supported languages with optional context",
207 )
208 )
210 model_config = ConfigDict(arbitrary_types_allowed=True)
212 @model_validator(mode="before")
213 @classmethod
214 def check_predefined_name_conflict(cls, data: Any) -> Any:
215 """Check if custom recognizer name conflicts with predefined recognizer.
217 This validation runs BEFORE field validation to provide a clearer error message
218 when someone tries to use a predefined recognizer name for a custom recognizer.
219 """
220 if isinstance(data, dict):
221 name = data.get("name")
222 if name:
223 try:
224 RecognizerListLoader.get_existing_recognizer_cls(name)
225 # If we reach here, the recognizer IS predefined, so raise an error
226 raise ValueError(
227 f"Recognizer '{name}' conflicts with a predefined "
228 f"recognizer. "
229 f"Custom recognizers cannot use the same name "
230 f"as predefined recognizers. "
231 f"Either use type: 'predefined' or choose a different name "
232 f"for your custom recognizer."
233 )
234 except PredefinedRecognizerNotFoundError:
235 pass
236 return data
238 @field_validator("patterns")
239 @classmethod
240 def validate_patterns(cls, patterns: Optional[List[Dict]]) -> Optional[List[Dict]]:
241 """Validate single language code format.
243 :param patterns: List of patterns
244 """
245 if patterns and not isinstance(patterns, list):
246 raise ValueError(f"Patterns should be a list: {patterns}")
248 for pattern in patterns:
249 if not isinstance(pattern, dict):
250 raise ValueError(f"Pattern should be a dict: {pattern}")
251 if "name" not in pattern:
252 raise ValueError(f"Pattern should contain a name field: {pattern}")
253 if "regex" not in pattern:
254 raise ValueError(f"Pattern should contain a regex field: {pattern}")
255 if "score" not in pattern:
256 raise ValueError(f"Pattern should contain a score field: {pattern}")
257 if not isinstance(pattern["score"], (int, float)):
258 raise ValueError(f"Pattern score should be a float: {pattern}")
259 if not (0.0 <= pattern["score"] <= 1.0):
260 raise ValueError(f"Pattern score should be between 0 and 1: {pattern}")
261 return patterns
263 @model_validator(mode="after")
264 def validate_patterns_or_deny_list(self):
265 """Ensure custom recognizer has at least patterns or deny_list."""
266 if not self.patterns and not self.deny_list:
267 raise ValueError(
268 "Custom recognizer must have at least one of 'patterns' or 'deny_list'"
269 )
270 return self
273class RecognizerRegistryConfig(BaseModel):
274 """Complete validation for the recognizer registry."""
276 supported_languages: Optional[List[str]] = Field(
277 default=None, description="List of supported languages"
278 )
279 global_regex_flags: int = Field(default=26, description="Global regex flags")
280 recognizers: List[
281 Union[
282 HuggingFaceRecognizerConfig,
283 PredefinedRecognizerConfig,
284 CustomRecognizerConfig,
285 str,
286 ]
287 ] = Field(default_factory=list, description="List of recognizer configurations")
289 model_config = ConfigDict(extra="forbid")
291 @field_validator("supported_languages")
292 @classmethod
293 def validate_language_codes(
294 cls, languages: Optional[List[str]]
295 ) -> Optional[List[str]]:
296 """Validate language codes format."""
298 # Allow None or empty list for cases where languages will be inferred
299 if languages is None:
300 return None
302 if len(languages) == 0:
303 return []
305 validate_language_codes(languages)
306 return languages
308 @model_validator(mode="after")
309 def validate_languages_for_custom_recognizers(self):
310 """Validate that custom recognizers have language configuration."""
311 # If we have custom recognizers, we need language configuration somewhere
312 custom_recognizers = [
313 rec for rec in self.recognizers if isinstance(rec, CustomRecognizerConfig)
314 ]
315 for recognizer in custom_recognizers:
316 if not recognizer.supported_language and not recognizer.supported_languages:
317 # If no language config on recognizer, we need global languages
318 if not self.supported_languages:
319 raise ValueError(
320 f"Language configuration missing for custom recognizer "
321 f"'{recognizer.name}': "
322 "Either specify 'supported_languages' "
323 "on the recognizer or provide "
324 "global 'supported_languages' in the "
325 "registry configuration."
326 )
328 return self
330 @model_validator(mode="after")
331 def validate_recognizers_not_empty(self):
332 """Ensure recognizers list is not empty after all defaults are applied."""
333 if not self.recognizers:
334 raise ValueError(
335 "The 'recognizers' field must contain at least one recognizer. "
336 "Found an empty recognizers list."
337 )
338 return self
340 @field_validator("recognizers", mode="before")
341 @classmethod
342 def parse_recognizers(
343 cls, recognizers: List[Union[Dict[str, Any], str]]
344 ) -> List[BaseRecognizerConfig]:
345 """Parse recognizers from various input formats without duplication."""
346 if recognizers is None:
347 raise ValueError(
348 "Configuration error: 'recognizers' is required. "
349 "Please provide a list of recognizers in the configuration."
350 )
352 if not isinstance(recognizers, list):
353 raise ValueError("Recognizers must be a list")
355 if len(recognizers) == 0:
356 raise ValueError(
357 "The 'recognizers' field must contain at least one recognizer. "
358 "Found an empty recognizers list."
359 )
361 parsed_recognizers = []
362 for recognizer in recognizers:
363 if isinstance(recognizer, str):
364 parsed_recognizers.append(recognizer)
365 continue
367 if isinstance(recognizer, dict):
368 recognizer_type = recognizer.get("type")
370 # Validate conflicting custom-only fields if explicitly predefined
371 if recognizer_type == "predefined" and (
372 "patterns" in recognizer or "deny_list" in recognizer
373 ):
374 raise ValueError(
375 f"Recognizer '{recognizer.get('name')}' is marked "
376 f"as 'predefined' but contains 'patterns' or 'deny_list' "
377 f"which are only valid for custom recognizers. "
378 f"Either use type: 'custom' or remove these fields."
379 )
381 if not recognizer_type:
382 if "patterns" in recognizer or "deny_list" in recognizer:
383 recognizer_type = "custom"
384 recognizer_name = recognizer.get("name")
385 if recognizer_name:
386 cls.__check_if_predefined(recognizer_name)
387 else:
388 recognizer_type = "predefined"
389 recognizer["type"] = recognizer_type
391 if recognizer_type == "predefined":
392 # Determine config model based on recognizer class_name or name.
393 recognizer_class_name = recognizer.get("class_name")
394 recognizer_name = recognizer.get("name")
395 # Prioritize class_name for lookup
396 # (e.g., custom instance of HuggingFaceNerRecognizer)
397 config_model_key = recognizer_class_name or recognizer_name
399 config_model = CONFIG_MODEL_MAP.get(
400 config_model_key, PredefinedRecognizerConfig
401 )
403 parsed_recognizers.append(config_model(**recognizer))
404 elif recognizer_type == "custom":
405 parsed_recognizers.append(CustomRecognizerConfig(**recognizer))
406 else:
407 raise ValueError(
408 f"Invalid recognizer type: {recognizer_type}. "
409 f"Must be 'predefined' or 'custom'."
410 )
411 continue
413 parsed_recognizers.append(recognizer)
415 return parsed_recognizers
417 @classmethod
418 def __check_if_predefined(cls, recognizer_name: Optional[Any]) -> None:
419 try:
420 RecognizerListLoader.get_existing_recognizer_cls(recognizer_name)
421 raise ValueError(
422 f"Recognizer '{recognizer_name}' conflicts with a predefined "
423 f"recognizer. "
424 f"Custom recognizers cannot use the same name "
425 f"as predefined recognizers. "
426 f"Either use type: 'predefined' or choose a different name "
427 f"for your custom recognizer."
428 )
429 except PredefinedRecognizerNotFoundError:
430 pass
432 @model_validator(mode="after")
433 def validate_language_presence(self):
434 """Ensure custom recognizers define languages if no global languages are set."""
435 if self.recognizers and (
436 not self.supported_languages or len(self.supported_languages) == 0
437 ):
438 any_language_defined = False
439 custom_without_language_present = False
440 for r in self.recognizers:
441 if isinstance(r, (PredefinedRecognizerConfig, CustomRecognizerConfig)):
442 if (r.supported_language and r.supported_language.strip()) or (
443 r.supported_languages and len(r.supported_languages) > 0
444 ):
445 any_language_defined = True
446 if (
447 isinstance(r, CustomRecognizerConfig)
448 and not r.supported_language
449 and not r.supported_languages
450 ):
451 custom_without_language_present = True
453 if custom_without_language_present and not any_language_defined:
454 raise ValueError(
455 "Language configuration missing for custom recognizer(s): "
456 "provide 'supported_languages' at registry level "
457 "or specify languages for each custom recognizer."
458 )
459 return self
462# Map specific recognizer classes to their dedicated config models
463# This allows for modular expansion without polluting the base config
464CONFIG_MODEL_MAP: Dict[str, Type[BaseModel]] = {
465 "HuggingFaceNerRecognizer": HuggingFaceRecognizerConfig,
466}