pyctags.exuberant

40 """ 41 Wraps the Exuberant Ctags program. U{http://ctags.sourceforge.net} 42 43 The B{generate_tags} and B{generate_tagfile} methods will accept custom command line parameters for exuberant ctags via the generator_options keyword dict. 44 The Exuberant Ctags output flags (-f and -o) are reserved for internal use and will trigger an exception. 45 """ 46 __version_opt = "--version" 47 __list_kinds_opt = "--list-kinds" 48 __argless_args = ["--version", "--help", "--license", "--list-languages", 49 "-a", "-B", "-e", "-F", "-n", "-N", "-R", "-u", "-V", "-w", "-x"] 50 __default_opts = {"-L" : "-", "-f" : "-"} 51 __exuberant_id = "exuberant ctags" 52 __supported_versions = ["5.7", "5.6b1"] 53 __warning_str = ": Warning:" 54

55 - def __init__(self, *args, **kwargs):

56 """ 57 Wraps the Exuberant Ctags program. 58 - B{Keyword Arguments:} 59 - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path 60 - B{files:} (sequence) files to process with ctags 61 """ 62 valid_kwargs = ['tag_program', 'files'] 63 validator.validate(kwargs.keys(), valid_kwargs) 64 65 self.version = None 66 """ Exuberant ctags version number.""" 67 self.language_info = None 68 """ Exuberant ctags supported language parsing features.""" 69 70 ctags_base.__init__(self, *args, **kwargs)

71

72 - def __process_kinds_list(self, kinds_list):

73 """ Slice n dice the --list-kinds output from exuberant ctags.""" 74 d = dict() 75 key = "" 76 for k in kinds_list: 77 if len(k): 78 if k[0].isspace(): 79 if len(key): 80 kind_info = k.strip().split(' ') 81 if len(kind_info) > 2: 82 raise ValueError("Kind information is in an unexpected format.") 83 d[key][kind_info[0]] = kind_info[1] 84 else: 85 key = k.strip().lower() 86 if key not in d: 87 d[key] = dict() 88 89 return d

90 91

92 - def _query_tag_generator(self, path):

93 """ 94 Gets Exuberant Ctags program information. 95 @raise ValueError: No valid ctags executable set. 96 @raise TypeError: Executable is not Exuberant Ctags. 97 """ 98 99 shell_str = path + ' ' + self.__version_opt 100 101 p = subprocess.Popen(shell_str, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 102 (out, err) = p.communicate() 103 outstr = out.decode() 104 if outstr.lower().find(self.__exuberant_id) < 0: 105 raise TypeError("Executable file " + self._executable_path + " is not Exuberant Ctags") 106 107 comma = outstr.find(',') 108 self.version = outstr[len(self.__exuberant_id):comma].strip() 109 110 if self.version not in self.__supported_versions: 111 print("Version %s of Exuberant Ctags isn't known to work, but might." % (self.version)) 112 113 # find out what this version of ctags supports in terms of language and kinds of tags 114 p = subprocess.Popen(path + ' ' + self.__list_kinds_opt, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 115 (out, err) = p.communicate() 116 117 self.language_info = self.__process_kinds_list(out.decode().splitlines())

118

119 - def _dict_to_args(self, gen_opts):

120 """ 121 Converts from a dict with command line arguments to a string to feed exuberant ctags on the comand line. 122 @param gen_opts: command line arguments, key=argument, value=setting 123 @type gen_opts: dict 124 @rtype: str 125 """ 126 127 # because yargs sounds like a pirate 128 yargs = "" 129 for k, v in gen_opts.items(): 130 if k in self.__argless_args: 131 yargs += k + ' ' 132 continue 133 if k[0:2] == '--': 134 # long opt 135 yargs += k + '=' + v 136 elif k[0] == '-': 137 # short opt 138 yargs += k + ' ' + v + ' ' 139 140 return yargs

141

142 - def _prepare_to_generate(self, kw):

143 """ 144 Prepares parameters to be passed to exuberant ctags. 145 @returns: tuple (generator_options_dict, files_str) 146 """ 147 input_file_override = False 148 149 self.warnings = list() 150 if 'generator_options' in kw: 151 if '-f' in kw['generator_options'] or '-o' in kw['generator_options']: 152 raise ValueError("The options -f and -o are used internally.") 153 if '-L' in kw['generator_options']: 154 input_file_override = True 155 156 if 'tag_program' in kw: 157 if self.ctags_executable(kw['tag_program']): 158 self._executable_path = kw['tag_program'] 159 160 if 'files' in kw: 161 self._file_list = list(kw['files']) 162 163 if not self._executable_path: 164 if self.ctags_executable('ctags'): 165 self._executable_path = 'ctags' 166 else: 167 raise ValueError("No ctags executable set.") 168 169 gen_opts = copy(self.__default_opts) 170 if 'generator_options' in kw: 171 gen_opts.update(kw['generator_options']) 172 173 file_list = '' 174 if not input_file_override: 175 for f in self._file_list: 176 file_list += f + os.linesep 177 178 return (gen_opts, file_list)

179 180

181 - def generate_tags(self, **kwargs):

182 """ 183 Parses source files into list of tags. 184 - B{Keyword Arguments:} 185 - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path 186 - B{files:} (sequence) files to process with ctags 187 - B{generator_options:} (dict) command-line options to pass to ctags program 188 @returns: strings output by exuberant ctags 189 @rtype: list 190 @raise ValueError: ctags executable path not set, fails execution 191 """ 192 valid_kwargs = ['tag_program', 'files', 'generator_options'] 193 validator.validate(kwargs.keys(), valid_kwargs) 194 195 (gen_opts, file_list) = self._prepare_to_generate(kwargs) 196 tag_args = self._dict_to_args(gen_opts) 197 198 self.command_line = self._executable_path + ' ' + tag_args 199 p = subprocess.Popen(self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 200 (out, err) = p.communicate(input=file_list.encode()) 201 202 if p.returncode != 0: 203 raise ValueError("Ctags execution did not complete, return value: " + p.returncode + ".\nCommand line: " + self.command_line) 204 205 results = out.decode("utf-8").splitlines() 206 207 if sys.platform == 'win32': 208 # check for warning strings in output 209 if self._executable_path.rfind("/") >= 0: 210 shortname = self._executable_path[self._executable_path.rfind("/"):] 211 elif self._executable_path.rfind("\\") >= 0: 212 shortname = self._executable_path[self._executable_path.rfind("\\"):] 213 else: 214 shortname = self._executable_path 215 216 idxs = [] 217 i = 0 218 for r in results: 219 if r.find(shortname + self.__warning_str) == 0: 220 idxs.append(i) 221 i += 1 222 223 # reverse the list so we don't mess up index numbers as we're removing them 224 idxs.sort(reverse=True) 225 for i in idxs: 226 self.warnings.append(results.pop(i)) 227 else: 228 self.warnings = err.decode("utf-8").splitlines() 229 230 return results

231

232 - def generate_tagfile(self, output_file, **kwargs):

233 """ 234 Generates tag file from list of files. 235 - B{Keyword Arguments:} 236 - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path 237 - B{files:} (sequence) files to process with ctags 238 - B{generator_options:} (dict) options to pass to ctags program 239 @param output_file: File name and location to write tagfile. 240 @type output_file: str 241 @returns: file written 242 @rtype: boolean 243 @raise ValueError: ctags executable path not set or output file isn't valid 244 245 """ 246 valid_kwargs = ['tag_program', 'files', 'generator_options'] 247 validator.validate(kwargs.keys(), valid_kwargs) 248 249 # exuberant ctags 5.7 chops 'def' off the beginning of variables, if it starts with def 250 _default_output_file = 'tags' 251 252 if 'generator_options' in kwargs: 253 if '-e' in kwargs['generator_options']: 254 _default_output_file.upper() 255 256 if output_file: 257 if output_file != "-": 258 if os.path.isdir(output_file): 259 output_file = os.path.join(output_file, _default_output_file) 260 else: 261 (head, tail) = os.path.split(output_file) 262 if len(head) == 0 and len(tail) == 0: 263 raise ValueError("No output file set") 264 if len(head) != 0: 265 if not os.path.isdir(head): 266 raise ValueError("Output directory " + head + " does not exist.") 267 else: 268 raise ValueError("No output file set") 269 270 (gen_opts, file_list) = self._prepare_to_generate(kwargs) 271 gen_opts['-f'] = '"' + output_file + '"' 272 tag_args = self._dict_to_args(gen_opts) 273 274 self.command_line = self._executable_path + ' ' + tag_args 275 p = subprocess.Popen(self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 276 (out, err) = p.communicate(input=file_list.encode()) 277 if sys.platform == 'win32': 278 self.warnings = out.decode("utf-8").splitlines() 279 else: 280 self.warnings = err.decode("utf-8").splitlines() 281 282 if (p.returncode == 0): 283 return True 284 return False

285

286 - def generate_object(self, **kwargs):

287 """ 288 Parses source files into a ctags_file instance. 289 This method exists to avoid storing ctags generated data in an intermediate form before parsing. 290 291 According to python documentation, this mechanism could deadlock due to other OS pipe buffers filling and blocking the child process. 292 U{http://docs.python.org/library/subprocess.html} 293 - B{Keyword Arguments:} 294 - B{tag_program:} (str) path to ctags executable, or name of a ctags program in path 295 - B{files:} (sequence) files to process with ctags 296 - B{generator_options:} (dict) options to pass to ctags program 297 - B{harvesters:} (list) list of harvester data classes for ctags_file to use while parsing 298 @returns: generated instance of ctags_file on success, None on failure 299 @rtype: (ctags_file or None) 300 @raise ValueError: ctags executable path not set 301 """ 302 valid_kwargs = ['tag_program', 'files', 'generator_options', 'harvesters'] 303 validator.validate(kwargs.keys(), valid_kwargs) 304 305 (gen_opts, file_list) = self._prepare_to_generate(kwargs) 306 tag_args = self._dict_to_args(gen_opts) 307 308 tagfile = ctags_file() 309 310 harvesters = list() 311 if 'harvesters' in kwargs: 312 harvesters = kwargs['harvesters'] 313 314 tagfile.feed_init(harvesters=harvesters) 315 316 self.command_line = self._executable_path + ' ' + tag_args 317 p = subprocess.Popen(self.command_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 318 p.stdin.write(file_list.encode()) 319 320 # is this the cleanest way to do this? it makes the program execute, but I haven't found another way 321 p.stdin.close() 322 323 if sys.platform == "win32": 324 if self._executable_path.rfind("/") >= 0: 325 shortname = self._executable_path[self._executable_path.rfind("/"):] 326 elif self._executable_path.rfind("\\") >= 0: 327 shortname = self._executable_path[self._executable_path.rfind("\\"):] 328 else: 329 shortname = self._executable_path 330 331 332 while p.poll() is None: 333 line = p.stdout.readline().decode("utf-8") 334 if not len(line): 335 continue 336 if sys.platform == 'win32' and line.find(shortname + self.__warning_str) == 0: 337 self.warnings.append(line) 338 else: 339 tagfile.feed_line(line) 340 341 # process the remaining buffer 342 for line in p.stdout.read().decode("utf-8").splitlines(): 343 if not len(line): 344 continue 345 if sys.platform == 'win32' and line.find(shortname + self.__warning_str) == 0: 346 self.warnings.append(line) 347 else: 348 tagfile.feed_line(line) 349 350 if sys.platform != 'win32': 351 self.warnings = p.stderr.read().decode("utf-8").splitlines() 352 353 tagfile.feed_finish() 354 355 if p.returncode == 0: 356 return tagfile 357 else: 358 return None

Source Code for Module pyctags.exuberant