mrv.doc.base

24 """Encapsulates all functionality required to create sphinx/epydoc documentaiton""" 25 26 #{ Configuration 27 forbidden_dirs = ['test', 'ext', 'doc', '.'] 28 29 # PATHS 30 source_dir = make_path('source') 31 source_dl_dir = source_dir / 'download' 32 33 build_dir = make_path('build') 34 35 html_dir = build_dir / 'html' 36 downloads_dir = html_dir / '_downloads' 37 38 # EPYDOC 39 epydoc_show_source = 'yes' 40 epydoc_modules = """modules: unittest 41 modules: ../mrv,../mrv/ext/networkx/networkx,../mrv/ext/pyparsing/src,../mrv/ext/pydot""" 42 43 epydoc_exclude = "mrv.test,mrv.cmd.ipythonstartup" 44 45 # BOOTSTRAPPING 46 # To be set by derived types in order to define the root package name that 47 # shouldbe imported 48 package_name = None 49 50 # DYNAMICALLY ADJUSTED MEMBERS 51 # These members will be adjusted after reading the current project's 52 # information 53 pinfo = None 54 rootmodule = None 55 epydoc_cfg = """[epydoc] 56 name: %s 57 url: %s 58 59 sourcecode: %s 60 %s 61 62 exclude: %s 63 output: html""" 64 65 #} END configuration 66

67 - def __init__(self, sphinx=True, sphinx_autogen=True, coverage=True, epydoc=True, base_dir='.', *args):

68 """Initialize the instance 69 70 :param sphinx: If True, sphinx documentation will be produced 71 :param coverage: If True, the coverage report will be generated 72 :param epydoc: If True, epydoc documentation will be generated""" 73 if self.pinfo is None: 74 self._retrieve_project_info(base_dir) 75 # END asssure project info is set 76 77 self._sphinx = sphinx 78 self._sphinx_autogen = sphinx_autogen 79 self._coverage = coverage 80 self._epydoc = epydoc 81 82 self._base_dir = make_path(base_dir) 83 84 # We assume to be in the project's doc directory, otherwise we cannot 85 # automatically handle the project information 86 if self._base_dir.abspath().basename() != 'doc': 87 raise EnvironmentError("Basedirectory needs to be the 'doc' directory, not %s" % self._base_dir.abspath()) 88 89 90 self._project_dir = make_path(self._base_dir / "..")

91 92 @classmethod

93 - def _apply_epydoc_config(cls):

94 """Read package info configuration and apply it""" 95 assert cls.pinfo is not None 96 dcon = getattr(cls.pinfo, 'doc_config', dict()) 97 for k,v in dcon.items(): 98 if k.startswith('epydoc'): 99 setattr(cls, k, v) 100 # END apply project info 101 102 cls.epydoc_cfg = cls.epydoc_cfg % (cls.pinfo.project_name, 103 cls.pinfo.url, 104 cls.epydoc_show_source, 105 cls.epydoc_modules, 106 cls.epydoc_exclude)

107 108 109 #{ Public Interface 110 111 @classmethod

112 - def remove_version_info(cls, idstring, basedir='.'):

113 """Remove the version info file if it exists""" 114 try: 115 os.remove(cls.version_file_name(idstring, basedir)) 116 except OSError: 117 pass

118 # END exception handling 119 120 @classmethod

121 - def version_file_name(cls, idstring, basedir='.'):

122 """:return: filename at which to write the version file with the given id""" 123 return make_path(os.path.join(basedir, "%s.version_info" % idstring))

124 125 @classmethod

126 - def write_version(cls, idstring, basedir='.'):

127 """Writes a version file containing the rootmodule's version info. 128 This allows to verify that the version of the individual parts, like 129 epydoc and sphinx are still matching""" 130 version_string = "version_info = (%i, %i, %i, '%s', %i)" % cls.pinfo.version 131 open(cls.version_file_name(idstring, basedir), 'wb').write(version_string)

132 133 @classmethod

134 - def check_version(cls, opid, idstring, basedir='.'):

135 """Checks whether the current version info matches with the stored version info 136 as retrieved from idstring. 137 If there is no such info or if the version matches exactly, do nothing. 138 Otherwise raise an environment error to tell the user to rebuild the 139 respective part of the documentation""" 140 vlocals = dict() 141 vfile = cls.version_file_name(idstring, basedir) 142 if not os.path.isfile(vfile): 143 return 144 145 execfile(vfile, vlocals) 146 vinfo = vlocals['version_info'] 147 if vinfo != cls.pinfo.version: 148 msg = "Documentation target named '%s' at version %s requires '%s' ( last built at %s ) to be rebuild" % (opid, str(cls.pinfo.version), idstring, str(vinfo)) 149 raise EnvironmentError(msg)

150 # END raise exception 151 152 @classmethod

153 - def parser(cls):

154 """:return: OptionParser instance suitable to parse commandline arguments 155 with which to initialize our instance""" 156 usage = """%prog [options] 157 158 Make documentation or remove the generated files.""" 159 parser = optparse.OptionParser(usage=usage) 160 161 hlp = "Specify the name of the package to import, defaults to 'mrv'" 162 parser.add_option('-p', '--package', dest='package_name', help=hlp) 163 164 hlp = """Specifies to build sphinx documentation""" 165 parser.add_option('-s', '--sphinx', dest='sphinx', type='int',default=1, 166 help=hlp, metavar='STATE') 167 168 hlp = """If specified, sphinx API docuementation will be generated""" 169 parser.add_option('-a', '--sphinx-autogen', dest='sphinx_autogen', type='int', default=1, 170 help=hlp, metavar='STATE') 171 172 hlp = """Specifies epydoc documentation""" 173 parser.add_option('-e', '--epydoc', dest='epydoc', type='int', default=1, 174 help=hlp, metavar='STATE') 175 176 hlp = """Specifies a coverage report. It will be referenced from within the 177 sphinx documentation""" 178 parser.add_option('-c', '--coverage', dest='coverage', type='int', default=1, 179 help=hlp, metavar='STATE') 180 181 return parser

182 183 @classmethod

184 - def root_dir(cls, basedir='.'):

185 """ 186 :return: path which includes our package - if it would be in the sys.path, 187 we should be able to import it 188 :param basedir: we expect to be in the root/doc path of the project - if this is not 189 the case, the basedir can be adjusted accordingly to 'virtually' chdir into the 190 doc directory""" 191 return ospd(os.path.realpath(os.path.abspath(basedir)))

192 193 @classmethod

194 - def makedoc(cls, args):

195 """Produce the actual docs using this type""" 196 p = cls.parser() 197 198 hlp = """If specified, previously generated files will be removed. Works in conjunction 199 with the other flags, which default to True, hence %prog --clean will remove all 200 generated files by default""" 201 p.add_option('--clean', dest='clean', action='store_true', default=False, help=hlp) 202 203 options, args = p.parse_args(args) 204 clean = options.clean 205 del(options.clean) 206 207 # commandline overrides class configuration 208 cls.package_name = options.package_name or cls.package_name 209 # assume mrv, and assert it really is in our root path 210 default_package = 'mrv' 211 if cls.package_name is None and os.path.isdir(os.path.join(cls.root_dir(), default_package)): 212 cls.package_name = default_package 213 #END handle default 214 215 if cls.package_name is None: 216 p.error("Please specify the --package that should be imported") 217 #END assure package is set 218 del(options.package_name) 219 220 dgen = cls(*args, **options.__dict__) 221 if clean: 222 dgen.clean() 223 else: 224 dgen.generate()

225 # END handle mode 226

227 - def generate(self):

228 """Geneate the documentation according to our configuration 229 230 :note: respects the options given during construction""" 231 if self._coverage: 232 self._make_coverage() 233 234 if self._epydoc: 235 self._make_epydoc() 236 237 if self._sphinx: 238 self._make_sphinx_index() 239 if self._sphinx_autogen: 240 self._make_sphinx_autogen() 241 # END generate autogen 242 self._make_sphinx()

243 # END make sphinx 244

245 - def clean(self):

246 """Clean the generated files by removing them 247 :note: Must respect the options the same way as done by the ``generate`` 248 method""" 249 if self._coverage: 250 self.remove_version_info('coverage') 251 bdd = self.build_downloads_dir() 252 csdd = self.source_downloads_coverage_dir() 253 coverage_dir = make_path(self._project_dir / cmd.tmrv_coverage_dir) 254 255 # delete all files we copied from the coverage dir 256 if coverage_dir.isdir(): 257 for fpath in coverage_dir.files(): 258 tfpath = bdd / fpath.basename() 259 if tfpath.isfile(): 260 tfpath.remove() 261 # END remove file 262 # END for each coverage file to remove 263 # END if coverage directory exists 264 265 try: 266 shutil.rmtree(csdd) 267 except OSError: 268 pass 269 # END exceptionhandlint 270 271 # END clean coverage 272 273 if self._epydoc: 274 self.remove_version_info('epydoc') 275 try: 276 shutil.rmtree(self.epydoc_target_dir()) 277 except OSError: 278 pass 279 # END ignore errors if directory doesnt exist 280 # END clean epydoc 281 282 if self._sphinx: 283 self.remove_version_info('sphinx') 284 ip = self.index_rst_path() 285 iph = ip+'.header' 286 # only remove index.rst if it appears we are generating it using 287 # header and footer 288 if iph.isfile() and ip.isfile(): 289 ip.remove() 290 # END remove generated index 291 292 out_dir = self.html_output_dir() 293 dt_dir = self.doctrees_dir() 294 agp = self.autogen_output_dir() 295 for dir in (agp, out_dir, dt_dir): 296 if dir.isdir(): 297 shutil.rmtree(dir)

298 # END remove html dir 299 # END for each directory 300 # END clean sphinx 301 #} END public interface 302 303 #{ Paths 304

305 - def base_dir(self):

306 """:return: Path containing all documentation sources and output files""" 307 return self._base_dir

308

309 - def set_base_dir(self, base_dir):

310 """Set the base directory to the given value 311 :return: self""" 312 self._base_dir = Path(base_dir) 313 return self

314

315 - def index_rst_path(self):

316 """:return: Path to index rst file""" 317 return self._base_dir / self.source_dir / "index.rst"

318

319 - def build_downloads_dir(self):

320 """:return: Path to the build downloads directory""" 321 return self._base_dir / self.downloads_dir

322

323 - def source_downloads_dir(self):

324 """:return: Path to the source downloads directory""" 325 return self._base_dir / self.source_dl_dir

326

327 - def source_downloads_coverage_dir(self):

328 """:return: Path to coverage related downloads""" 329 return self.source_downloads_dir() / 'coverage'

330

331 - def epydoc_target_dir(self):

332 """:return: Path to directory to which epydoc will write its output""" 333 return self.html_output_dir() / 'generated' / 'api'

334

335 - def html_output_dir(self):

336 """:return: html directory to receive all output""" 337 return self._base_dir / self.html_dir

338

339 - def autogen_output_dir(self):

340 """:return: directory to which sphinx-autogen will write its output to""" 341 return self._base_dir / self.source_dir / 'generated'

342

343 - def doctrees_dir(self):

344 """:return: Path to doctrees directory to which sphinx writes some files""" 345 return self._base_dir / self.build_dir / 'doctrees'

346

347 - def mrv_bin_path(self):

348 """:return: Path to mrv binary""" 349 import mrv.cmd.base 350 return mrv.cmd.base.find_mrv_script('mrv')

351

352 - def tmrv_bin_path(self):

353 """:return: Path to tmrv binary""" 354 import mrv.cmd.base 355 return mrv.cmd.base.find_mrv_script('tmrv')

356 357 #} END paths 358 359 #{ Utilities

360 - def _mrv_maya_version(self):

361 """:return: maya version with which mrv subcommands should be started with""" 362 import mrv.cmd.base 363 try: 364 return mrv.cmd.base.available_maya_versions()[-1] 365 except IndexError: 366 print >> sys.stderr, "No maya version available, trying without" 367 import mrv.cmd 368 return mrv.cmd.mrv_nomaya_flag

369 #END handle no maya available 370

371 - def _call_python_script(self, *args, **kwargs):

372 """Wrapper of subprocess.call which assumes that we call a python script. 373 On windows, the python interpreter needs to be called directly 374 :raise EnvironmentError: if the called had a non-0 return value""" 375 if sys.platform.startswith('win'): 376 args[0].insert(0, "python") 377 # END handle windows 378 cmd = ' '.join(str(i) for i in args[0]) 379 print cmd 380 rval = subprocess.call(*args, **kwargs) 381 382 if rval: 383 raise EnvironmentError("Call to %s failed with status %i" % (args[0][0], rval))

384 # END handle call error 385 #} END utilities 386 387 #{ Protected Interface 388 389 @classmethod

390 - def _retrieve_project_info(cls, base_dir='.'):

391 """Store the project information of the actual project in our class members 392 for later use 393 :note: must be called exactly once""" 394 if cls.pinfo is not None: 395 return cls.pinfo 396 #END handle cached pinfo 397 398 if cls.package_name is None: 399 raise ValueError("Package name needs to be set, but was None") 400 #END assure package is set 401 402 # Even though we could use the mrv.pinfo module, which is the top-level 403 # package info, we should prefer to start a search based on our current 404 # directory as the user must call us from his own doc-path, from which 405 # we can conclude quite a lot 406 rootpath = cls.root_dir(base_dir) 407 sys.path.append(rootpath) 408 409 try: 410 cls.rootmodule = __import__(cls.package_name) 411 except ImportError: 412 raise EnvironmentError("Root package %s could not be imported" % cls.package_name) 413 # END handle import 414 415 pinfo_package = "%s.info" % cls.package_name 416 try: 417 cls.pinfo = __import__(pinfo_package, fromlist=['']) 418 except ImportError: 419 raise EnvironmentError("Project information module %r could not be imported:" % pinfo_package)

420 # END handle import 421 422

423 - def _make_sphinx_index(self):

424 """Generate the index.rst file according to the modules and packages we 425 actually have""" 426 import mrv 427 428 indexpath = self.index_rst_path() 429 index_header = indexpath+'.header' 430 if not index_header.isfile(): 431 return 432 # END handle header doesn't exist 433 434 ifp = open(indexpath, 'wb') 435 # write header 436 ifp.write(index_header.bytes()) 437 438 # write api index 439 if self._sphinx_autogen: 440 basepath = self._base_dir / ".." / self.pinfo.root_package 441 rootmodule = basepath.abspath().basename() 442 for root, dirs, files in os.walk(basepath): 443 remove_dirs = list() 444 for dirname in dirs: 445 if dirname in self.forbidden_dirs: 446 remove_dirs.append(dirname) 447 # END for each forbidden dir 448 # END for each directory 449 450 for dirname in remove_dirs: 451 del(dirs[dirs.index(dirname)]) 452 # END for each dirname to remove 453 454 for fname in files: 455 if not fname.endswith('.py') or fname.startswith('_'): 456 continue 457 filepath = os.path.join(root, fname) 458 459 # + 1 as there is a trailing path separator 460 modulepath = "%s.%s" % (rootmodule, filepath[len(basepath)+1:-3].replace(os.path.sep, '.')) 461 ifp.write("\t%s\n" % modulepath) 462 # END for each file 463 # END for each file 464 # END generate api index 465 466 # finalize it, write the footer 467 ifp.write((indexpath+'.footer').bytes()) 468 ifp.close()

469

470 - def _make_coverage(self):

471 """Generate a coverage report and make it available as download""" 472 tmrvpath = self.tmrv_bin_path() 473 474 # for some reason, the html output can only be generated if the current 475 # working dir is in the project root. Its something within nose's coverage 476 # module apparently 477 prevcwd = os.getcwd() 478 os.chdir(self._project_dir) 479 480 try: 481 rval = self._call_python_script([tmrvpath, str(self._mrv_maya_version()), 482 "%s=%s" % (cmd.tmrv_coverage_flag, self.pinfo.root_package)]) 483 finally: 484 os.chdir(prevcwd) 485 # END handle cwd 486 487 if rval: 488 raise SystemError("tmrv reported failure") 489 # END handle return value 490 491 bdd = self.build_downloads_dir() 492 csdd = self.source_downloads_coverage_dir() 493 for dir in (bdd, csdd): 494 if not dir.isdir(): 495 dir.makedirs() 496 # END if dir doesnt exist, create it 497 # END for each directory 498 499 # coverage was generated into the current working dir 500 # index goes to downloads in the source directory as it is referenced 501 # by the docs 502 coverage_dir = make_path(self._project_dir / cmd.tmrv_coverage_dir) 503 cindex = coverage_dir / 'index.html' 504 shutil.copy(cindex, csdd) 505 506 # all coverage html files go to the downlods directory 507 for html in coverage_dir.files(): 508 shutil.copy(html, bdd) 509 # END for each html 510 511 self.write_version('coverage')

512 513

514 - def _make_sphinx_autogen(self):

515 """Instruct sphinx to generate the autogen rst files""" 516 # will have to run it in a separate process for maya support 517 mrvpath = self.mrv_bin_path() 518 519 # note: the mrv import resolves the site-packages for us which does not 520 # happen on osx for some reason 521 code = "import mrv; import sphinx.ext.autosummary.generate as sas; sas.main()" 522 agp = self.autogen_output_dir() 523 524 # make sure its clean, otherwise we will reprocess the same files 525 if agp.isdir(): 526 shutil.rmtree(agp) 527 agp.makedirs() 528 # END handle existing directory 529 530 # make sure the instance will actually find our info file, and not 531 # its own one, otherwise it cannot make the necessary imports 532 os.environ['MRV_INFO_DIR'] = os.path.dirname(self.pinfo.__file__) 533 args = [mrvpath, str(self._mrv_maya_version()), '-c', code, 534 '-o', agp, 535 self.index_rst_path()] 536 537 self._call_python_script(args) 538 539 # it could be that there was absolutely no autogenerated part in the 540 # index, hence it didn't write any rst files for it 541 if not agp.isdir(): 542 print >> sys.stderr, "WARNING: No autogenerated rst files written to %s" % agp.abspath() 543 return 544 #END handle no autogen docs 545 546 # POST PROCESS 547 ############## 548 # Add :api:module.name which gets picked up by extapi, inserting a 549 # epydoc link to the respective file. 550 for rstfile in agp.files("*.rst"): 551 # insert module link 552 lines = rstfile.lines() 553 modulename = lines[0][6:-2] # skip `\n 554 lines.insert(2, ":api:`%s`\n" % modulename) 555 556 # insert :api: links to the autoclasses 557 i = 0 558 l = len(lines) 559 while i < l: 560 line = lines[i] 561 if line.startswith('.. autoclass'): 562 classname = line[line.rfind(' ')+1:-1] # skip newline 563 l += 1 564 lines.insert(i, ':api:`%s.%s`\n\n' % (modulename, classname)) 565 i += 1 566 # END if we have a class 567 i += 1 568 # END for each line 569 570 rstfile.write_lines(lines)

571 # END for each rst to process 572

573 - def _sphinx_args(self):

574 """:return: list of arguments to be used when calling sphinx from the commandline 575 :note: directories of all kinds will be handled by the caller""" 576 # we don't need "" around the values as we don't use a shell 577 return ['-c', 'import sys, mrv, sphinx.cmdline; sphinx.cmdline.main(sys.argv)', 578 '-b', 'html', 579 '-D', 'latex_paper_size=a4', 580 '-D', 'latex_paper_size=letter', 581 '-D', 'project=%s' % self.pinfo.project_name, 582 '-D', 'copyright=%s' % self.pinfo.author, 583 '-D', 'version=%s' % "%i.%i" % self.pinfo.version[:2], 584 '-D', 'release=%s' % "%i.%i.%i-%s" % self.pinfo.version[:4]]

585

586 - def _make_sphinx(self):

587 """Generate the sphinx documentation""" 588 self.check_version('sphinx', 'epydoc') 589 self.check_version('sphinx', 'coverage') 590 591 mrvpath = self.mrv_bin_path() 592 out_dir = self.html_output_dir() 593 594 for dir in (self.source_dir, out_dir): 595 if not dir.isdir(): 596 dir.makedirs() 597 # END assure directory exists 598 # END for each directory 599 600 pathargs = ['-d', self.doctrees_dir(), self.source_dir, out_dir] 601 602 args = [mrvpath, str(self._mrv_maya_version())] + self._sphinx_args() + pathargs 603 604 self._call_python_script(args) 605 606 self.write_version('sphinx')

607

608 - def _make_epydoc(self):

609 """Generate epydoc documentation""" 610 try: 611 import epydoc 612 except ImportError: 613 raise ImportError("Epydoc could not be imported, please make sure it is available in your PYTHONPATH") 614 #END handle epydoc installation 615 616 self._apply_epydoc_config() 617 618 # start epydocs in a separate process 619 # as maya support is required 620 epytarget = self.epydoc_target_dir() 621 if not epytarget.isdir(): 622 epytarget.makedirs() 623 # END assure directory exists 624 625 # SETUP MONKEYPATCH 626 def visit_paragraph(this, node): 627 """Epydoc patch - will be applied on demand""" 628 if this.summary is not None: 629 # found a paragraph after the first one 630 this.other_docs = True 631 raise docutils.nodes.NodeFound('Found summary') 632 633 summary_pieces = [] 634 635 # Extract the first sentence. 636 for child in node: 637 if isinstance(child, docutils.nodes.Text): 638 m = this._SUMMARY_RE.match(child) 639 if m: 640 summary_pieces.append(docutils.nodes.Text(m.group(1))) 641 other = child[m.end():] 642 if other and not other.isspace(): 643 this.other_docs = True 644 break 645 summary_pieces.append(child) 646 647 summary_doc = this.document.copy() # shallow copy 648 summary_para = node.copy() # shallow copy 649 summary_doc[:] = [summary_para] 650 summary_para[:] = summary_pieces 651 this.summary = ParsedRstDocstring(summary_doc)

652 #END monkaypatch method 653 654 # write epydoc.cfg file temporarily 655 epydoc_cfg_file = "epydoc.cfg" 656 open(epydoc_cfg_file, 'wb').write(self.epydoc_cfg) 657 658 args = ['epydoc', '-q', '-q', '--debug', '--config', epydoc_cfg_file, '-o', str(epytarget)] 659 660 print "Launching in-process epydoc: ", " ".join(args) 661 origargs = sys.argv[:] 662 del(sys.argv[:]) 663 sys.argv.extend(args) 664 try: 665 try: 666 # apply patch 667 import docutils.nodes 668 import epydoc.markup.restructuredtext 669 from epydoc.markup.restructuredtext import ParsedRstDocstring 670 epydoc.markup.restructuredtext._SummaryExtractor.visit_paragraph = visit_paragraph 671 672 import epydoc.cli 673 epydoc.cli.cli() 674 except: 675 import pdb 676 print >> sys.stderr, "Epydoc encountered an exception - use pdb to figure out which code it couldn't handle" 677 pdb.post_mortem(sys.exc_info()[-1]) 678 raise 679 #END ease debugging 680 finally: 681 os.remove(epydoc_cfg_file) 682 del(sys.argv[:]) 683 sys.argv.extend(origargs) 684 # END handle epydoc config file 685 686 self.write_version('epydoc')

Source Code for Module mrv.doc.base