Coverage for presidio_cli / config.py: 95%

1import os 

2from typing import Optional 

3 

4import pathspec 

5import yaml 

6from presidio_analyzer import AnalyzerEngine 

7 

8 

9class PresidioCLIConfigError(Exception): 

10 """Represents an error in the configuration file.""" 

11 

12 pass 

13 

14 

15class PresidioCLIConfig(object): 

16 """Represents Presidio CLI configuration file. Reads the file and transforms the contents to class fields.""" # noqa: E501 

17 

18 def __init__( 

19 self, content: Optional[str] = None, file: Optional[str] = None 

20 ) -> None: 

21 assert (content is None) ^ (file is None) 

22 self.ignore = None 

23 self.locale = None 

24 self.analyzer = AnalyzerEngine() 

25 self.threshold = 0 

26 self.language = "en" 

27 self.allow_list = [] 

28 if file is not None: 

29 with open(file) as f: 

30 content = f.read() 

31 self.parse(content) 

32 self.validate() 

33 

34 def is_file_ignored(self, filepath: str) -> bool: 

35 """ 

36 Check if file should be processed by the Analyzer or ignored. 

37 

38 :param filepath: Path of file to be processed. 

39 """ 

40 return self.ignore and filepath and self.ignore.match_file(filepath) 

41 

42 def is_text_file(self, filepath: str) -> bool: 

43 """Detect if file is a not a binary file.Based on https://stackoverflow.com/a/7392391. 

44 

45 :param filepath: Path of the configuration file. 

46 """ 

47 

48 # Try to read the file as UTF-8. 

49 # In case some invalid UTF-8 characters are found, 

50 # an exception is caught and the file is not going 

51 # to be processed. 

52 try: 

53 with open(filepath, newline="", encoding="utf-8") as f: 

54 _ = f.read() 

55 except UnicodeDecodeError: 

56 return False 

57 

58 textchars = bytearray( 

59 {7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F} 

60 ) 

61 with open(filepath, "rb") as f: 

62 # return true if it's not a binary 

63 return not bool(f.read(1024).translate(None, textchars)) 

64 

65 def extend(self, base_config: "PresidioCLIConfig") -> None: 

66 """ 

67 In case the configuration file is based on another file, append detected entities from base config to current config and overwrite language and ignored files with contents of base config. 

68 

69 :param base_config: PresidioCLIConfig object 

70 """ # noqa: E501 

71 assert isinstance(base_config, PresidioCLIConfig) 

72 

73 # Create list with unique entries 

74 if base_config.entities is not None: 

75 self.entities = list(set(base_config.entities + self.entities)) 

76 

77 if base_config.ignore is not None: 

78 self.ignore = base_config.ignore 

79 

80 if base_config.language is not None: 

81 self.language = base_config.language 

82 

83 def parse(self, raw_content: str) -> None: 

84 """ 

85 Read the content of YAML file and save the properties in class fields. 

86 

87 :param raw_content: String with the contents of configuration file. 

88 """ 

89 try: 

90 conf = yaml.safe_load(raw_content) 

91 except Exception as e: 

92 raise PresidioCLIConfigError("invalid config: %s" % e) 

93 

94 if not isinstance(conf, dict): 

95 raise PresidioCLIConfigError("invalid config: not a dict") 

96 

97 self.entities = conf.get("entities", {}) 

98 

99 if self.entities == {}: 

100 self.entities = self.analyzer.get_supported_entities() 

101 

102 if "threshold" in conf: 

103 if not 0 <= float(self.threshold) <= 1: 

104 raise PresidioCLIConfigError( 

105 f"Invalid threshold value: {self.threshold}. " 

106 f"Threshold must be between 0 and 1" 

107 ) 

108 self.threshold = float(conf["threshold"]) 

109 if "allow" in conf: 

110 self.allow_list = conf["allow"] 

111 if "language" in conf: 

112 self.language = conf["language"] 

113 if "extends" in conf: 

114 path = get_extended_config_file(conf["extends"]) 

115 base = PresidioCLIConfig(file=path) 

116 try: 

117 self.extend(base) 

118 except Exception as e: 

119 raise PresidioCLIConfigError("invalid config: %s" % e) 

120 

121 if "ignore" in conf: 

122 if not isinstance(conf["ignore"], str): 

123 raise PresidioCLIConfigError( 

124 "invalid config: ignore should contain file patterns" 

125 ) 

126 self.ignore = pathspec.PathSpec.from_lines( 

127 "gitwildmatch", conf["ignore"].splitlines() 

128 ) 

129 

130 if "locale" in conf: 

131 if not isinstance(conf["locale"], str): 

132 raise PresidioCLIConfigError( 

133 "invalid config: locale should be a string" 

134 ) 

135 self.locale = conf["locale"] 

136 

137 def validate(self) -> None: 

138 """Check if entities requested to be detected in input file are supported by Presidio.""" # noqa: E501 

139 for id in self.entities: 

140 try: 

141 assert id in self.analyzer.get_supported_entities() 

142 except Exception: 

143 raise PresidioCLIConfigError("invalid config: no such entity %s" % id) 

144 

145 

146def get_extended_config_file(name: str) -> str: 

147 """ 

148 Check if the configuration file is one of sample configs or if it is a file supplied by the user. 

149 

150 :param name: file name 

151 :return: Full path of configuration file 

152 """ # noqa: E501 

153 # Is it a standard conf shipped with yamllint... 

154 if "/" not in name: 

155 std_conf = os.path.join( 

156 os.path.dirname(os.path.realpath(__file__)), "conf", name + ".yaml" 

157 ) 

158 

159 if os.path.isfile(std_conf): 

160 return std_conf 

161 

162 # or a custom conf on filesystem? 

163 return name