Coverage for presidio_analyzer / chunkers / text_chunker_provider.py: 89%

1"""Factory provider for creating text chunkers from configuration.""" 

2 

3import logging 

4from typing import Any, Dict, Optional, Type 

5 

6from presidio_analyzer.chunkers.base_chunker import BaseTextChunker 

7from presidio_analyzer.chunkers.character_based_text_chunker import ( 

8 CharacterBasedTextChunker, 

9) 

10 

11logger = logging.getLogger("presidio-analyzer") 

12 

13# Registry mapping chunker type names to classes 

14_CHUNKER_REGISTRY: Dict[str, Type[BaseTextChunker]] = { 

15 "character": CharacterBasedTextChunker, 

16} 

17 

18 

19class TextChunkerProvider: 

20 """Create text chunkers from configuration. 

21 

22 :param chunker_configuration: Dict with chunker_type and optional params. 

23 Example:: 

24 

25 {"chunker_type": "character", "chunk_size": 300, "chunk_overlap": 75} 

26 

27 If no configuration provided, uses character-based chunker with default params 

28 tuned for boundary coverage (chunk_size=250, chunk_overlap=50). 

29 """ 

30 

31 def __init__( 

32 self, 

33 chunker_configuration: Optional[Dict[str, Any]] = None, 

34 ): 

35 # Default to a safe overlap to avoid boundary losses for cross-chunk entities. 

36 self.chunker_configuration = chunker_configuration or { 

37 "chunker_type": "character", 

38 "chunk_size": 250, 

39 "chunk_overlap": 50, 

40 } 

41 

42 def create_chunker(self) -> BaseTextChunker: 

43 """Create a text chunker instance from configuration.""" 

44 config = self.chunker_configuration.copy() 

45 chunker_type = config.pop("chunker_type", "character") 

46 

47 if chunker_type not in _CHUNKER_REGISTRY: 

48 raise ValueError( 

49 f"Unknown chunker_type '{chunker_type}'. " 

50 f"Available: {list(_CHUNKER_REGISTRY.keys())}" 

51 ) 

52 

53 chunker_class = _CHUNKER_REGISTRY[chunker_type] 

54 try: 

55 return chunker_class(**config) 

56 except TypeError as exc: 

57 raise ValueError( 

58 f"Invalid configuration for chunker_type '{chunker_type}': {config}" 

59 ) from exc 

60