Ticket #6495: trac_6495_separate_inventory.patch
File trac_6495_separate_inventory.patch, 18.7 KB (added by , 9 years ago) |
---|
-
doc/common/builder.py
# HG changeset patch # User Volker Braun <vbraun.name@gmail.com> # Date 1359716667 0 # Node ID 3d16efbe90bcc31567f6c87945c849beb0bd3dc9 # Parent 94f8c642b47ebf1d885cb25ea99fc9f461fc7b44 Use a separate output directory for the object inventory. This prevents a race in intersphinx where object.inv files are written to and read by parallel processes, which causes random errors. Also, filter unnecessary messages and line buffer the necessary ones. diff --git a/doc/common/builder.py b/doc/common/builder.py
a b 1 1 #!/usr/bin/env python 2 """ 3 The documentation builder 4 5 It is the starting point for building documentation, and is 6 responsible to figure out what to build and with which options. The 7 actual documentation build for each individual document is then done 8 in a subprocess call to sphinx, see :func:`builder_helper`. 9 10 * The builder can be configured in build_options.py 11 * The sphinx subprocesses are configured in conf.py 12 """ 13 2 14 import glob, logging, optparse, os, shutil, subprocess, sys, textwrap 3 15 4 16 #We remove the current directory from sys.path right away … … 17 29 # from build_options.py. 18 30 execfile(os.path.join(os.getenv('SAGE_ROOT'), 'devel', 'sage', 'doc', 'common' , 'build_options.py')) 19 31 20 ##########################################21 # Utility Functions #22 ##########################################23 def copytree(src, dst, symlinks=False, ignore_errors=False):24 """25 Recursively copy a directory tree using copy2().26 27 The destination directory must not already exist.28 If exception(s) occur, an Error is raised with a list of reasons.29 30 If the optional symlinks flag is true, symbolic links in the31 source tree result in symbolic links in the destination tree; if32 it is false, the contents of the files pointed to by symbolic33 links are copied.34 35 XXX Consider this example code rather than the ultimate tool.36 37 """38 names = os.listdir(src)39 mkdir(dst)40 errors = []41 for name in names:42 srcname = os.path.join(src, name)43 dstname = os.path.join(dst, name)44 try:45 if symlinks and os.path.islink(srcname):46 linkto = os.readlink(srcname)47 os.symlink(linkto, dstname)48 elif os.path.isdir(srcname):49 copytree(srcname, dstname, symlinks)50 else:51 shutil.copy2(srcname, dstname)52 # XXX What about devices, sockets etc.?53 except (IOError, os.error) as why:54 errors.append((srcname, dstname, str(why)))55 # catch the Error from the recursive copytree so that we can56 # continue with other files57 except shutil.Error as err:58 errors.extend(err.args[0])59 try:60 shutil.copystat(src, dst)61 except OSError as why:62 errors.extend((src, dst, str(why)))63 if errors and not ignore_errors:64 raise shutil.Error(errors)65 66 32 67 33 ########################################## 68 34 # Parallel Building Ref Manual # … … 73 39 format = args[2] 74 40 kwds = args[3] 75 41 args = args[4:] 42 if format == 'inventory': # you must not use the inventory to build the inventory 43 kwds['use_multidoc_inventory'] = False 76 44 getattr(ReferenceSubBuilder(doc, lang), format)(*args, **kwds) 77 45 78 46 ########################################## … … 84 52 Returns a function which builds the documentation for 85 53 output type type. 86 54 """ 87 def f(self ):55 def f(self, *args, **kwds): 88 56 output_dir = self._output_dir(type) 89 57 os.chdir(self.dir) 90 58 … … 94 62 # WEBSITESPHINXOPTS is either empty or " -A hide_pdf_links=1 " 95 63 options += WEBSITESPHINXOPTS 96 64 97 build_command = 'sphinx-build' 65 if kwds.get('use_multidoc_inventory', True): 66 options += ' -D multidoc_first_pass=0' 67 else: 68 options += ' -D multidoc_first_pass=1' 69 70 build_command = 'python '+os.path.join(SAGE_DOC, 'common', 'custom-sphinx-build.py') 98 71 build_command += ' -b %s -d %s %s %s %s'%(type, self._doctrees_dir(), 99 72 options, self.dir, 100 73 output_dir) 101 logger. warning(build_command)74 logger.debug(build_command) 102 75 subprocess.call(build_command, shell=True) 103 76 104 logger.warning("Build finished. The built documents can be found in %s", output_dir) 77 logger.info("Build finished and can be found in %s", 78 output_dir.replace(SAGE_DOC+'/', '')) 105 79 106 80 f.is_output_format = True 107 81 return f … … 144 118 sage: b._output_dir('html') 145 119 '.../devel/sage/doc/output/html/en/tutorial' 146 120 """ 147 if type == "inventory": # put inventories in the html tree148 type = "html"149 121 d = os.path.join(SAGE_DOC, "output", type, self.lang, self.name) 150 122 mkdir(d) 151 123 return d … … 266 238 docs = self.get_all_documents() 267 239 refs = [x for x in docs if x.endswith('reference')] 268 240 others = [x for x in docs if not x.endswith('reference')] 241 269 242 # Build the reference manual twice to resolve references. That is, 270 243 # build once with the inventory builder to construct the intersphinx 271 244 # inventory files, and then build the second time for real. So the 272 245 # first build should be as fast as possible; 273 246 logger.warning("\nBuilding reference manual, first pass.\n") 274 global ALLSPHINXOPTS275 ALLSPHINXOPTS += ' -Q -D multidoc_first_pass=1'276 247 for document in refs: 277 248 getattr(get_builder(document), 'inventory')(*args, **kwds) 249 278 250 logger.warning("Building reference manual, second pass.\n") 279 ALLSPHINXOPTS = ALLSPHINXOPTS.replace(280 'multidoc_first_pass=1', 'multidoc_first_pass=0')281 ALLSPHINXOPTS = ALLSPHINXOPTS.replace('-Q', '-q') + ' '282 251 for document in refs: 283 252 getattr(get_builder(document), name)(*args, **kwds) 284 253 … … 286 255 from multiprocessing import Pool 287 256 pool = Pool(NUM_THREADS) 288 257 L = [(doc, name, kwds) + args for doc in others] 289 # map_async, with get to provide a timeout, handles 290 # KeyboardInterrupt correctly. apply_async does not, so don't 291 # use it. 292 pool.map_async(build_other_doc, L).get(99999) 258 # map_async handles KeyboardInterrupt correctly. Plain map and 259 # apply_async does not, so don't use it. 260 pool.map_async(build_other_doc, L, 1).get(99999) 293 261 pool.close() 294 262 pool.join() 295 263 logger.warning("Elapsed time: %.1f seconds."%(time.time()-start)) … … 335 303 """ 336 304 DocBuilder.html(self) 337 305 html_output_dir = self._output_dir('html') 338 copytree(html_output_dir, 339 os.path.realpath(os.path.join(html_output_dir, '..')), 340 ignore_errors=False) 341 306 for f in os.listdir(html_output_dir): 307 src = os.path.join(html_output_dir, f) 308 dst = os.path.join(html_output_dir, '..', f) 309 if os.path.isdir(src): 310 shutil.rmtree(dst, ignore_errors=True) 311 shutil.copytree(src, dst) 312 else: 313 shutil.copy2(src, dst) 342 314 self.create_html_redirects() 343 315 344 316 def create_html_redirects(self): … … 458 430 sage: b._output_dir('html') 459 431 '.../devel/sage/doc/output/html/en/reference' 460 432 """ 461 if type == "inventory": # put inventories in the html tree462 type = "html"463 433 d = os.path.join(SAGE_DOC, "output", type, lang, self.name) 464 434 mkdir(d) 465 435 return d … … 477 447 from multiprocessing import Pool 478 448 pool = Pool(NUM_THREADS) 479 449 L = [(doc, lang, format, kwds) + args for doc in self.get_all_documents(refdir)] 480 # (See comment in AllBuilder._wrapper about using map_async 481 # instead of apply_async.) 482 pool.map_async(build_ref_doc, L).get(99999) 450 # (See comment in AllBuilder._wrapper about using map instead of apply.) 451 pool.map_async(build_ref_doc, L, 1).get(99999) 483 452 pool.close() 484 453 pool.join() 485 454 # The html refman must be build at the end to ensure correct … … 596 565 We add a component name if it's a subdirectory of the manual's 597 566 directory and contains a file named 'index.rst'. 598 567 568 We return the largest component (most subdirectory entries) 569 first since they will take the longest to build. 570 599 571 EXAMPLES:: 600 572 601 573 sage: import os, sys; sys.path.append(os.environ['SAGE_DOC']+'/common/'); import builder 602 574 sage: b = builder.ReferenceBuilder('reference') 603 575 sage: refdir = os.path.join(os.environ['SAGE_DOC'], 'en', b.name) 604 sage: b.get_all_documents(refdir)576 sage: sorted(b.get_all_documents(refdir)) 605 577 ['reference/algebras', 'reference/arithgroup', ..., 'reference/tensor'] 606 578 """ 607 579 documents = [] 608 580 609 581 for doc in os.listdir(refdir): 610 if os.path.exists(os.path.join(refdir, doc, 'index.rst')): 611 documents.append(os.path.join(self.name, doc)) 582 directory = os.path.join(refdir, doc) 583 if os.path.exists(os.path.join(directory, 'index.rst')): 584 n = len(os.listdir(directory)) 585 documents.append((-n, os.path.join(self.name, doc))) 612 586 613 return sorted(documents)587 return [ doc[1] for doc in sorted(documents) ] 614 588 615 589 616 590 class ReferenceSubBuilder(DocBuilder): … … 677 651 _sage = os.path.join(self.dir, '_sage') 678 652 if os.path.exists(_sage): 679 653 logger.info("Copying over custom .rst files from %s ...", _sage) 680 copytree(_sage, os.path.join(self.dir, 'sage'))654 shutil.copytree(_sage, os.path.join(self.dir, 'sage')) 681 655 682 656 getattr(DocBuilder, build_type)(self, *args, **kwds) 683 657 … … 1413 1387 if options.warn_links: 1414 1388 ALLSPHINXOPTS += "-n " 1415 1389 1416 1417 1390 # Make sure common/static exists. 1418 1391 mkdir(os.path.join(SAGE_DOC, 'common', 'static')) 1419 1392 -
doc/common/conf.py
diff --git a/doc/common/conf.py b/doc/common/conf.py
a b 110 110 111 111 def set_intersphinx_mappings(app): 112 112 """ 113 Add reference's objects.inv to intersphinx if not compiling reference113 Add precompiled inventory (the objects.inv) 114 114 """ 115 refpath = get_doc_abspath('output/html/en/reference') 116 invpath = get_doc_abspath('output/inventory/en/reference') 117 if app.config.multidoc_first_pass == 1 or \ 118 not (os.path.exists(refpath) and os.path.exists(invpath)): 119 app.config.intersphinx_mapping = {} 120 return 115 121 app.config.intersphinx_mapping = intersphinx_mapping 116 refpath = 'output/html/en/reference/' 117 if not app.srcdir.endswith('reference'): 118 app.config.intersphinx_mapping[get_doc_abspath(refpath)] = get_doc_abspath(refpath+'objects.inv') 122 123 def add(subdoc=''): 124 src = os.path.join(refpath, subdoc) if subdoc else refpath 125 dst = os.path.join(invpath, subdoc, 'objects.inv') 126 app.config.intersphinx_mapping[src] = dst 127 128 add() 129 for directory in os.listdir(os.path.join(invpath)): 130 if os.path.isdir(os.path.join(invpath, directory)): 131 add(directory) 132 133 119 134 pythonversion = sys.version.split(' ')[0] 120 135 # Python and Sage trac ticket shortcuts. For example, :trac:`7549` . 121 136 … … 609 624 app.connect('autodoc-process-docstring', process_dollars) 610 625 app.connect('autodoc-process-docstring', process_inherited) 611 626 app.connect('autodoc-skip-member', skip_member) 612 627 613 628 # When building the standard docs, app.srcdir is set to SAGE_DOC + 614 629 # 'LANGUAGE/DOCNAME', but when doing introspection, app.srcdir is 615 630 # set to a temporary directory. We don't want to use intersphinx, … … 625 640 app.connect('builder-inited', set_intersphinx_mappings) 626 641 app.connect('builder-inited', sphinx.ext.intersphinx.load_mappings) 627 642 app.connect('builder-inited', nitpick_patch_config) 628 # Minimize GAP/libGAP RAM usage when we build the docs629 from sage.interfaces.gap import set_gap_memory_pool_size630 set_gap_memory_pool_size(0) # will be rounded up to 1M631 643 -
new file doc/common/custom-sphinx-build.py
diff --git a/doc/common/custom-sphinx-build.py b/doc/common/custom-sphinx-build.py new file mode 100644
- + 1 """ 2 This is Sage's version of the sphinx-build script 3 4 Enhancements are: 5 6 * import the Sage library to access the docstrings, otherwise doc 7 buliding doesn't work. 8 9 * redirect stdout to our own logger, and remove some unwanted chatter. 10 """ 11 12 import os 13 import sys 14 import re 15 16 17 # override the fancy multi-line formatting 18 def term_width_line(text): 19 return text + '\n' 20 21 import sphinx.util.console 22 sphinx.util.console.term_width_line = term_width_line 23 24 25 26 27 useless_chatter = ( 28 re.compile('^$'), 29 re.compile('^Running Sphinx v'), 30 re.compile('^loading intersphinx inventory from '), 31 re.compile('^Compiling a sub-document'), 32 re.compile('^updating environment: 0 added, 0 changed, 0 removed'), 33 re.compile('^looking for now-outdated files... none found'), 34 re.compile('^no targets are out of date.'), 35 re.compile('^building \[.*\]: targets for 0 source files that are out of date'), 36 re.compile('^loading pickled environment... done'), 37 re.compile('^loading cross citations... done \([0-9]* citations\).') 38 ) 39 40 41 class SageSphinxLogger(object): 42 """ 43 This implements the file object interface to serve as sys.stdout 44 replacement. 45 """ 46 ansi_color = re.compile(r'\x1b\[[0-9;]*m') 47 ansi_reset = re.compile(r'\x1b\[39;49;00m') 48 prefix_len = 9 49 50 def __init__(self, stream, prefix): 51 self._stream = stream 52 self._color = stream.isatty() 53 prefix = prefix[0:self.prefix_len] 54 prefix = ('[{0:'+str(self.prefix_len)+'}]').format(prefix) 55 color = { 1:'darkgreen', 2:'red' } 56 color = color.get(stream.fileno(), 'lightgray') 57 self._prefix = sphinx.util.console.colorize(color, prefix) 58 59 60 def _filter_out(self, line): 61 line = re.sub(self.ansi_color, '', line) 62 for regex in useless_chatter: 63 if regex.match(line) is not None: 64 return True 65 return False 66 67 def _log_line(self, line): 68 if self._filter_out(line): 69 return 70 line = self._prefix + ' ' + line.strip() + '\n' 71 if not self._color: 72 line = self.ansi_color.sub('', line) 73 self._stream.write(line) 74 self._stream.flush() 75 76 _line_buffer = '' 77 78 def _break_long_lines(self): 79 """ 80 Break text that has been formated with string.ljust() back 81 into individual lines. Return partial output. Do nothing if 82 the filter rule matches, otherwise subsequent lines would be 83 not filtered out. 84 """ 85 if self._filter_out(self._line_buffer): 86 return 87 cols = sphinx.util.console._tw 88 lines = [] 89 buf = self._line_buffer 90 while len(buf) > cols: 91 lines.append(buf[0:cols]) 92 buf = buf[cols:] 93 lines.append(buf) 94 self._line_buffer = '\n'.join(lines) 95 96 def _write(self, string): 97 self._line_buffer += string 98 #self._break_long_lines() 99 lines = self._line_buffer.splitlines() 100 for i, line in enumerate(lines): 101 last = (i == len(lines)-1) 102 if last and not self._line_buffer.endswith('\n'): 103 self._line_buffer = line 104 return 105 self._log_line(line) 106 self._line_buffer = '' 107 108 109 # file object interface follows 110 111 closed = False 112 encoding = None 113 mode = 'w' 114 name = '<log>' 115 newlines = None 116 softspace = 0 117 118 def isatty(self): 119 return True 120 121 def close(self): 122 if self._line_buffer != '': 123 self._log_line(self._line_buffer) 124 self._line_buffer = '' 125 126 def flush(self): 127 self._stream.flush() 128 129 def write(self, str): 130 try: 131 self._write(str) 132 except: 133 import traceback 134 traceback.print_exc(file=self._stream) 135 136 def writelines(self, sequence): 137 for line in sequence: 138 self.write(line) 139 140 141 output_dir = sys.argv[-1] 142 sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir)) 143 sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir)) 144 145 146 147 # pull in the Sage library 148 import sage.all 149 150 # Minimize GAP/libGAP RAM usage when we build the docs 151 from sage.interfaces.gap import set_gap_memory_pool_size 152 set_gap_memory_pool_size(0) # will be rounded up to 1M 153 154 if __name__ == '__main__': 155 from sphinx.cmdline import main 156 sys.exit(main(sys.argv)) 157 -
doc/common/multidocs.py
diff --git a/doc/common/multidocs.py b/doc/common/multidocs.py
a b 3 3 multi documentation in Sphinx 4 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 5 6 This is a slightly hacked-up version of the Sphinx-multidoc plugin 7 6 8 The goal of this extension is to manage a multi documentation in Sphinx. 7 9 To be able to compile Sage's huge documentation in parallel, the 8 10 documentation is cut into a bunch of independent documentations called … … 18 20 - the javascript index; 19 21 - the citations. 20 22 """ 21 import cPickle, os, sys 23 import cPickle, os, sys, shutil 22 24 import sphinx 23 25 from sphinx.util.console import bold 24 26 … … 69 71 env.all_docs.update(newalldoc) 70 72 # needed by env.check_consistency (sphinx.environement, line 1734) 71 73 for ind in newalldoc: 72 env.metadata[ind] = {} 74 # treat subdocument source as orphaned file and don't complain 75 env.metadata[ind] = {'orphan'} 73 76 # merge the citations 74 77 newcite = {} 75 78 for ind, (path, tag) in docenv.citations.iteritems(): … … 255 258 app.builder.info(bold('linking _static directory.')) 256 259 static_dir = os.path.join(app.builder.outdir, '_static') 257 260 master_static_dir = os.path.join('..', '_static') 258 if not os.path.isdir(static_dir): 259 os.symlink(master_static_dir, static_dir) 261 if os.path.exists(static_dir): 262 if os.path.isdir(static_dir) and not os.path.islink(static_dir): 263 shutil.rmtree(static_dir) 264 else: 265 os.unlink(static_dir) 266 os.symlink(master_static_dir, static_dir) 260 267 261 268 app.builder.copy_static_files = link_static_files 262 269