Coverage for presidio_analyzer / nlp_engine / nlp_engine.py: 91%

1from abc import ABC, abstractmethod 

2from typing import Iterable, Iterator, List, Tuple 

3 

4from presidio_analyzer.nlp_engine import NlpArtifacts 

5 

6 

7class NlpEngine(ABC): 

8 """ 

9 NlpEngine is an abstraction layer over the nlp module. 

10 

11 It provides NLP preprocessing functionality as well as other queries 

12 on tokens. 

13 """ 

14 

15 @abstractmethod 

16 def load(self) -> None: 

17 """Load the NLP model.""" 

18 

19 @abstractmethod 

20 def is_loaded(self) -> bool: 

21 """Return True if the model is already loaded.""" 

22 

23 @abstractmethod 

24 def process_text(self, text: str, language: str) -> NlpArtifacts: 

25 """Execute the NLP pipeline on the given text and language.""" 

26 

27 @abstractmethod 

28 def process_batch( 

29 self, 

30 texts: Iterable[str], 

31 language: str, 

32 batch_size: int = 1, 

33 n_process: int = 1, 

34 **kwargs, 

35 ) -> Iterator[Tuple[str, NlpArtifacts]]: 

36 """Execute the NLP pipeline on a batch of texts. 

37 

38 Returns a tuple of (text, NlpArtifacts) 

39 """ 

40 

41 @abstractmethod 

42 def is_stopword(self, word: str, language: str) -> bool: 

43 """ 

44 Return true if the given word is a stop word. 

45 

46 (within the given language) 

47 """ 

48 

49 @abstractmethod 

50 def is_punct(self, word: str, language: str) -> bool: 

51 """ 

52 Return true if the given word is a punctuation word. 

53 

54 (within the given language) 

55 """ 

56 

57 @abstractmethod 

58 def get_supported_entities(self) -> List[str]: 

59 """Return the supported entities for this NLP engine.""" 

60 pass 

61 

62 @abstractmethod 

63 def get_supported_languages(self) -> List[str]: 

64 """Return the supported languages for this NLP engine.""" 

65 pass