Coverage for presidio_analyzer / chunkers / character_based_text_chunker.py: 100%

1"""Character-based text chunker with word boundary preservation. 

2 

3Based on gliner-spacy implementation: 

4https://github.com/theirstory/gliner-spacy/blob/main/gliner_spacy/pipeline.py#L60-L96 

5""" 

6import logging 

7from typing import Iterable, List, Tuple 

8 

9from presidio_analyzer.chunkers.base_chunker import BaseTextChunker, TextChunk 

10 

11logger = logging.getLogger("presidio-analyzer") 

12 

13 

14WORD_BOUNDARY_CHARS: Tuple[str, ...] = (" ", "\n") 

15 

16 

17class CharacterBasedTextChunker(BaseTextChunker): 

18 """Character-based text chunker with word boundary preservation.""" 

19 

20 def __init__( 

21 self, 

22 chunk_size: int = 250, 

23 chunk_overlap: int = 50, 

24 boundary_chars: Iterable[str] | None = None, 

25 ): 

26 """Initialize the character-based text chunker. 

27 

28 Note: Chunks may slightly exceed chunk_size to preserve complete words. 

29 When this occurs, the actual overlap may vary from the specified value. 

30 

31 :param chunk_size: Target maximum characters per chunk (must be > 0) 

32 :param chunk_overlap: Target characters to overlap between chunks 

33 (must be >= 0 and < chunk_size) 

34 :param boundary_chars: Characters that count as word boundaries. 

35 Defaults to space/newline to keep current behavior. 

36 """ 

37 if chunk_size <= 0: 

38 logger.error("Invalid chunk_size: %d. Must be greater than 0.", chunk_size) 

39 raise ValueError("chunk_size must be greater than 0") 

40 if chunk_overlap < 0 or chunk_overlap >= chunk_size: 

41 logger.error( 

42 "Invalid chunk_overlap. Must be non-negative and less than chunk_size" 

43 ) 

44 raise ValueError( 

45 "chunk_overlap must be non-negative and less than chunk_size" 

46 ) 

47 

48 self._chunk_size = chunk_size 

49 self._chunk_overlap = chunk_overlap 

50 # Allow callers to tune boundaries 

51 # (e.g., punctuation, tabs) without changing defaults. 

52 self._boundary_chars: Tuple[str, ...] = ( 

53 tuple(boundary_chars) if boundary_chars is not None else WORD_BOUNDARY_CHARS 

54 ) 

55 

56 @property 

57 def chunk_size(self) -> int: 

58 """Get the chunk size. 

59 

60 :return: The chunk size 

61 """ 

62 return self._chunk_size 

63 

64 @property 

65 def chunk_overlap(self) -> int: 

66 """Get the chunk overlap. 

67 

68 :return: The chunk overlap 

69 """ 

70 return self._chunk_overlap 

71 

72 @property 

73 def boundary_chars(self) -> Tuple[str, ...]: 

74 """Characters treated as word boundaries when extending chunks.""" 

75 

76 return self._boundary_chars 

77 

78 def chunk(self, text: str) -> List[TextChunk]: 

79 """Split text into overlapping chunks at word boundaries. 

80 

81 Chunks are extended to the nearest word boundary (space or newline) 

82 to avoid splitting words. This means chunks may slightly exceed 

83 chunk_size. For texts without spaces (e.g., CJK languages), chunks 

84 may extend to end of text. 

85 

86 :param text: The input text to chunk 

87 :return: List of TextChunk objects with text and position information 

88 """ 

89 if not text: 

90 logger.debug("Empty text provided, returning empty chunk list") 

91 return [] 

92 

93 logger.debug( 

94 "Chunking text: length=%d, chunk_size=%d, overlap=%d", 

95 len(text), 

96 self._chunk_size, 

97 self._chunk_overlap, 

98 ) 

99 

100 chunks = [] 

101 start = 0 

102 

103 while start < len(text): 

104 # Calculate end position 

105 end = ( 

106 start + self._chunk_size 

107 if start + self._chunk_size < len(text) 

108 else len(text) 

109 ) 

110 

111 # Extend to complete word boundary (space or newline by default) 

112 while end < len(text) and text[end] not in self._boundary_chars: 

113 end += 1 

114 

115 chunks.append(TextChunk(text=text[start:end], start=start, end=end)) 

116 

117 # Move start position with overlap (stop if we've covered all text) 

118 if end >= len(text): 

119 break 

120 start = end - self._chunk_overlap 

121 

122 logger.debug("Created %d chunks from text", len(chunks)) 

123 return chunks