Ticket #6495: trac_6495_separate_inventory.patch

File trac_6495_separate_inventory.patch, 18.7 KB (added by vbraun, 6 years ago)

Initial patch

  • doc/common/builder.py

    # HG changeset patch
    # User Volker Braun <vbraun.name@gmail.com>
    # Date 1359716667 0
    # Node ID 3d16efbe90bcc31567f6c87945c849beb0bd3dc9
    # Parent  94f8c642b47ebf1d885cb25ea99fc9f461fc7b44
    Use a separate output directory for the object inventory. This
    prevents a race in intersphinx where object.inv files are written to
    and read by parallel processes, which causes random errors. Also,
    filter unnecessary messages and line buffer the necessary ones.
    
    diff --git a/doc/common/builder.py b/doc/common/builder.py
    a b  
    11#!/usr/bin/env python
     2"""
     3The documentation builder
     4
     5It is the starting point for building documentation, and is
     6responsible to figure out what to build and with which options. The
     7actual documentation build for each individual document is then done
     8in a subprocess call to sphinx, see :func:`builder_helper`.
     9
     10* The builder can be configured in build_options.py
     11* The sphinx subprocesses are configured in conf.py
     12"""
     13
    214import glob, logging, optparse, os, shutil, subprocess, sys, textwrap
    315
    416#We remove the current directory from sys.path right away
     
    1729# from build_options.py.
    1830execfile(os.path.join(os.getenv('SAGE_ROOT'), 'devel', 'sage', 'doc', 'common' , 'build_options.py'))
    1931
    20 ##########################################
    21 #          Utility Functions             #
    22 ##########################################
    23 def copytree(src, dst, symlinks=False, ignore_errors=False):
    24     """
    25     Recursively copy a directory tree using copy2().
    26 
    27     The destination directory must not already exist.
    28     If exception(s) occur, an Error is raised with a list of reasons.
    29 
    30     If the optional symlinks flag is true, symbolic links in the
    31     source tree result in symbolic links in the destination tree; if
    32     it is false, the contents of the files pointed to by symbolic
    33     links are copied.
    34 
    35     XXX Consider this example code rather than the ultimate tool.
    36 
    37     """
    38     names = os.listdir(src)
    39     mkdir(dst)
    40     errors = []
    41     for name in names:
    42         srcname = os.path.join(src, name)
    43         dstname = os.path.join(dst, name)
    44         try:
    45             if symlinks and os.path.islink(srcname):
    46                 linkto = os.readlink(srcname)
    47                 os.symlink(linkto, dstname)
    48             elif os.path.isdir(srcname):
    49                 copytree(srcname, dstname, symlinks)
    50             else:
    51                 shutil.copy2(srcname, dstname)
    52             # XXX What about devices, sockets etc.?
    53         except (IOError, os.error) as why:
    54             errors.append((srcname, dstname, str(why)))
    55         # catch the Error from the recursive copytree so that we can
    56         # continue with other files
    57         except shutil.Error as err:
    58             errors.extend(err.args[0])
    59     try:
    60         shutil.copystat(src, dst)
    61     except OSError as why:
    62         errors.extend((src, dst, str(why)))
    63     if errors and not ignore_errors:
    64         raise shutil.Error(errors)
    65 
    6632
    6733##########################################
    6834#      Parallel Building Ref Manual      #
     
    7339    format = args[2]
    7440    kwds = args[3]
    7541    args = args[4:]
     42    if format == 'inventory':  # you must not use the inventory to build the inventory
     43        kwds['use_multidoc_inventory'] = False
    7644    getattr(ReferenceSubBuilder(doc, lang), format)(*args, **kwds)
    7745
    7846##########################################
     
    8452    Returns a function which builds the documentation for
    8553    output type type.
    8654    """
    87     def f(self):
     55    def f(self, *args, **kwds):
    8856        output_dir = self._output_dir(type)
    8957        os.chdir(self.dir)
    9058
     
    9462            # WEBSITESPHINXOPTS is either empty or " -A hide_pdf_links=1 "
    9563            options += WEBSITESPHINXOPTS
    9664
    97         build_command = 'sphinx-build'
     65        if kwds.get('use_multidoc_inventory', True):
     66            options += ' -D multidoc_first_pass=0'
     67        else:
     68            options += ' -D multidoc_first_pass=1'
     69
     70        build_command = 'python '+os.path.join(SAGE_DOC, 'common', 'custom-sphinx-build.py')
    9871        build_command += ' -b %s -d %s %s %s %s'%(type, self._doctrees_dir(),
    9972                                                  options, self.dir,
    10073                                                  output_dir)
    101         logger.warning(build_command)
     74        logger.debug(build_command)
    10275        subprocess.call(build_command, shell=True)
    10376
    104         logger.warning("Build finished.  The built documents can be found in %s", output_dir)
     77        logger.info("Build finished and can be found in %s",
     78                    output_dir.replace(SAGE_DOC+'/', ''))
    10579       
    10680    f.is_output_format = True
    10781    return f
     
    144118            sage: b._output_dir('html')
    145119            '.../devel/sage/doc/output/html/en/tutorial'
    146120        """
    147         if type == "inventory": # put inventories in the html tree
    148             type = "html"
    149121        d = os.path.join(SAGE_DOC, "output", type, self.lang, self.name)
    150122        mkdir(d)
    151123        return d
     
    266238        docs = self.get_all_documents()
    267239        refs = [x for x in docs if x.endswith('reference')]
    268240        others = [x for x in docs if not x.endswith('reference')]
     241
    269242        # Build the reference manual twice to resolve references.  That is,
    270243        # build once with the inventory builder to construct the intersphinx
    271244        # inventory files, and then build the second time for real.  So the
    272245        # first build should be as fast as possible;
    273246        logger.warning("\nBuilding reference manual, first pass.\n")
    274         global ALLSPHINXOPTS
    275         ALLSPHINXOPTS += ' -Q -D multidoc_first_pass=1'
    276247        for document in refs:
    277248            getattr(get_builder(document), 'inventory')(*args, **kwds)
     249
    278250        logger.warning("Building reference manual, second pass.\n")
    279         ALLSPHINXOPTS = ALLSPHINXOPTS.replace(
    280             'multidoc_first_pass=1', 'multidoc_first_pass=0')
    281         ALLSPHINXOPTS = ALLSPHINXOPTS.replace('-Q', '-q') + ' '
    282251        for document in refs:
    283252            getattr(get_builder(document), name)(*args, **kwds)
    284253
     
    286255        from multiprocessing import Pool
    287256        pool = Pool(NUM_THREADS)
    288257        L = [(doc, name, kwds) + args for doc in others]
    289         # map_async, with get to provide a timeout, handles
    290         # KeyboardInterrupt correctly. apply_async does not, so don't
    291         # use it.
    292         pool.map_async(build_other_doc, L).get(99999)
     258        # map_async handles KeyboardInterrupt correctly. Plain map and
     259        # apply_async does not, so don't use it.
     260        pool.map_async(build_other_doc, L, 1).get(99999)
    293261        pool.close()
    294262        pool.join()
    295263        logger.warning("Elapsed time: %.1f seconds."%(time.time()-start))
     
    335303        """
    336304        DocBuilder.html(self)
    337305        html_output_dir = self._output_dir('html')
    338         copytree(html_output_dir,
    339                  os.path.realpath(os.path.join(html_output_dir, '..')),
    340                  ignore_errors=False)
    341 
     306        for f in os.listdir(html_output_dir):
     307            src = os.path.join(html_output_dir, f)
     308            dst = os.path.join(html_output_dir, '..', f)
     309            if os.path.isdir(src):
     310                shutil.rmtree(dst, ignore_errors=True)
     311                shutil.copytree(src, dst)
     312            else:
     313                shutil.copy2(src, dst)
    342314        self.create_html_redirects()
    343315
    344316    def create_html_redirects(self):
     
    458430            sage: b._output_dir('html')
    459431            '.../devel/sage/doc/output/html/en/reference'
    460432        """
    461         if type == "inventory": # put inventories in the html tree
    462             type = "html"
    463433        d = os.path.join(SAGE_DOC, "output", type, lang, self.name)
    464434        mkdir(d)
    465435        return d
     
    477447            from multiprocessing import Pool
    478448            pool = Pool(NUM_THREADS)
    479449            L = [(doc, lang, format, kwds) + args for doc in self.get_all_documents(refdir)]
    480             # (See comment in AllBuilder._wrapper about using map_async
    481             # instead of apply_async.)
    482             pool.map_async(build_ref_doc, L).get(99999)
     450            # (See comment in AllBuilder._wrapper about using map instead of apply.)
     451            pool.map_async(build_ref_doc, L, 1).get(99999)
    483452            pool.close()
    484453            pool.join()
    485454            # The html refman must be build at the end to ensure correct
     
    596565        We add a component name if it's a subdirectory of the manual's
    597566        directory and contains a file named 'index.rst'.
    598567
     568        We return the largest component (most subdirectory entries)
     569        first since they will take the longest to build.
     570
    599571        EXAMPLES::
    600572
    601573            sage: import os, sys; sys.path.append(os.environ['SAGE_DOC']+'/common/'); import builder
    602574            sage: b = builder.ReferenceBuilder('reference')
    603575            sage: refdir = os.path.join(os.environ['SAGE_DOC'], 'en', b.name)
    604             sage: b.get_all_documents(refdir)
     576            sage: sorted(b.get_all_documents(refdir))
    605577            ['reference/algebras', 'reference/arithgroup', ..., 'reference/tensor']
    606578        """
    607579        documents = []
    608580
    609581        for doc in os.listdir(refdir):
    610             if os.path.exists(os.path.join(refdir, doc, 'index.rst')):
    611                 documents.append(os.path.join(self.name, doc))
     582            directory = os.path.join(refdir, doc)
     583            if os.path.exists(os.path.join(directory, 'index.rst')):
     584                n = len(os.listdir(directory))
     585                documents.append((-n, os.path.join(self.name, doc)))
    612586
    613         return sorted(documents)
     587        return [ doc[1] for doc in sorted(documents) ]
    614588
    615589
    616590class ReferenceSubBuilder(DocBuilder):
     
    677651        _sage = os.path.join(self.dir, '_sage')
    678652        if os.path.exists(_sage):
    679653            logger.info("Copying over custom .rst files from %s ...", _sage)
    680             copytree(_sage, os.path.join(self.dir, 'sage'))
     654            shutil.copytree(_sage, os.path.join(self.dir, 'sage'))
    681655               
    682656        getattr(DocBuilder, build_type)(self, *args, **kwds)
    683657   
     
    14131387    if options.warn_links:
    14141388        ALLSPHINXOPTS += "-n "
    14151389
    1416 
    14171390    # Make sure common/static exists.
    14181391    mkdir(os.path.join(SAGE_DOC, 'common', 'static'))
    14191392
  • doc/common/conf.py

    diff --git a/doc/common/conf.py b/doc/common/conf.py
    a b  
    110110
    111111def set_intersphinx_mappings(app):
    112112    """
    113     Add reference's objects.inv to intersphinx if not compiling reference
     113    Add precompiled inventory (the objects.inv)
    114114    """
     115    refpath = get_doc_abspath('output/html/en/reference')
     116    invpath = get_doc_abspath('output/inventory/en/reference')
     117    if app.config.multidoc_first_pass == 1 or \
     118            not (os.path.exists(refpath) and os.path.exists(invpath)):
     119        app.config.intersphinx_mapping = {}
     120        return
    115121    app.config.intersphinx_mapping = intersphinx_mapping
    116     refpath = 'output/html/en/reference/'
    117     if not app.srcdir.endswith('reference'):
    118         app.config.intersphinx_mapping[get_doc_abspath(refpath)] = get_doc_abspath(refpath+'objects.inv')
     122   
     123    def add(subdoc=''):
     124        src = os.path.join(refpath, subdoc) if subdoc else refpath
     125        dst = os.path.join(invpath, subdoc, 'objects.inv')
     126        app.config.intersphinx_mapping[src] = dst
     127       
     128    add()
     129    for directory in os.listdir(os.path.join(invpath)):
     130        if os.path.isdir(os.path.join(invpath, directory)):
     131            add(directory)
     132
     133
    119134pythonversion = sys.version.split(' ')[0]
    120135# Python and Sage trac ticket shortcuts. For example, :trac:`7549` .
    121136
     
    609624    app.connect('autodoc-process-docstring', process_dollars)
    610625    app.connect('autodoc-process-docstring', process_inherited)
    611626    app.connect('autodoc-skip-member', skip_member)
    612 
     627   
    613628    # When building the standard docs, app.srcdir is set to SAGE_DOC +
    614629    # 'LANGUAGE/DOCNAME', but when doing introspection, app.srcdir is
    615630    # set to a temporary directory.  We don't want to use intersphinx,
     
    625640        app.connect('builder-inited', set_intersphinx_mappings)
    626641        app.connect('builder-inited', sphinx.ext.intersphinx.load_mappings)
    627642        app.connect('builder-inited', nitpick_patch_config)
    628         # Minimize GAP/libGAP RAM usage when we build the docs
    629         from sage.interfaces.gap import set_gap_memory_pool_size
    630         set_gap_memory_pool_size(0)  # will be rounded up to 1M
    631643
  • new file doc/common/custom-sphinx-build.py

    diff --git a/doc/common/custom-sphinx-build.py b/doc/common/custom-sphinx-build.py
    new file mode 100644
    - +  
     1"""
     2This is Sage's version of the sphinx-build script
     3
     4Enhancements are:
     5
     6* import the Sage library to access the docstrings, otherwise doc
     7  buliding doesn't work.
     8
     9* redirect stdout to our own logger, and remove some unwanted chatter.
     10"""
     11
     12import os
     13import sys
     14import re
     15
     16
     17# override the fancy multi-line formatting
     18def term_width_line(text):
     19    return text + '\n'
     20
     21import sphinx.util.console
     22sphinx.util.console.term_width_line = term_width_line
     23
     24
     25
     26
     27useless_chatter = (
     28    re.compile('^$'),
     29    re.compile('^Running Sphinx v'),
     30    re.compile('^loading intersphinx inventory from '),
     31    re.compile('^Compiling a sub-document'),
     32    re.compile('^updating environment: 0 added, 0 changed, 0 removed'),
     33    re.compile('^looking for now-outdated files... none found'),
     34    re.compile('^no targets are out of date.'),
     35    re.compile('^building \[.*\]: targets for 0 source files that are out of date'),
     36    re.compile('^loading pickled environment... done'),
     37    re.compile('^loading cross citations... done \([0-9]* citations\).')
     38    )
     39
     40
     41class SageSphinxLogger(object):
     42    """
     43    This implements the file object interface to serve as sys.stdout
     44    replacement.
     45    """
     46    ansi_color = re.compile(r'\x1b\[[0-9;]*m')
     47    ansi_reset = re.compile(r'\x1b\[39;49;00m')
     48    prefix_len = 9
     49
     50    def __init__(self, stream, prefix):
     51        self._stream = stream
     52        self._color = stream.isatty()
     53        prefix = prefix[0:self.prefix_len]
     54        prefix = ('[{0:'+str(self.prefix_len)+'}]').format(prefix)
     55        color = { 1:'darkgreen', 2:'red' }
     56        color = color.get(stream.fileno(), 'lightgray')
     57        self._prefix = sphinx.util.console.colorize(color, prefix)
     58       
     59
     60    def _filter_out(self, line):
     61        line = re.sub(self.ansi_color, '', line)
     62        for regex in useless_chatter:
     63            if regex.match(line) is not None:
     64                return True
     65        return False
     66
     67    def _log_line(self, line):
     68        if self._filter_out(line):
     69            return
     70        line = self._prefix + ' ' + line.strip() + '\n'
     71        if not self._color:
     72            line = self.ansi_color.sub('', line)
     73        self._stream.write(line)
     74        self._stream.flush()
     75       
     76    _line_buffer = ''
     77   
     78    def _break_long_lines(self):
     79        """
     80        Break text that has been formated with string.ljust() back
     81        into individual lines.  Return partial output. Do nothing if
     82        the filter rule matches, otherwise subsequent lines would be
     83        not filtered out.
     84        """
     85        if self._filter_out(self._line_buffer):
     86            return
     87        cols = sphinx.util.console._tw
     88        lines = []
     89        buf = self._line_buffer
     90        while len(buf) > cols:
     91            lines.append(buf[0:cols])
     92            buf = buf[cols:]
     93        lines.append(buf)
     94        self._line_buffer = '\n'.join(lines)
     95
     96    def _write(self, string):
     97        self._line_buffer += string
     98        #self._break_long_lines()
     99        lines = self._line_buffer.splitlines()
     100        for i, line in enumerate(lines):
     101            last = (i == len(lines)-1)
     102            if last and not self._line_buffer.endswith('\n'):
     103                self._line_buffer = line
     104                return
     105            self._log_line(line)
     106        self._line_buffer = ''
     107               
     108
     109    # file object interface follows
     110
     111    closed = False
     112    encoding = None
     113    mode = 'w'
     114    name = '<log>'
     115    newlines = None
     116    softspace = 0
     117
     118    def isatty(self):
     119        return True
     120
     121    def close(self):
     122        if self._line_buffer != '':
     123            self._log_line(self._line_buffer)
     124            self._line_buffer = ''
     125
     126    def flush(self):
     127        self._stream.flush()
     128       
     129    def write(self, str):
     130        try:
     131            self._write(str)
     132        except:
     133            import traceback
     134            traceback.print_exc(file=self._stream)
     135       
     136    def writelines(self, sequence):
     137        for line in sequence:
     138            self.write(line)
     139   
     140
     141output_dir = sys.argv[-1]
     142sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir))
     143sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir))
     144
     145
     146
     147# pull in the Sage library
     148import sage.all
     149
     150# Minimize GAP/libGAP RAM usage when we build the docs
     151from sage.interfaces.gap import set_gap_memory_pool_size
     152set_gap_memory_pool_size(0)  # will be rounded up to 1M
     153
     154if __name__ == '__main__':
     155    from sphinx.cmdline import main
     156    sys.exit(main(sys.argv))
     157
  • doc/common/multidocs.py

    diff --git a/doc/common/multidocs.py b/doc/common/multidocs.py
    a b  
    33    multi documentation in Sphinx
    44    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    55
     6    This is a slightly hacked-up version of the Sphinx-multidoc plugin
     7
    68    The goal of this extension is to manage a multi documentation in Sphinx.
    79    To be able to compile Sage's huge documentation in parallel, the
    810    documentation is cut into a bunch of independent documentations called
     
    1820    - the javascript index;
    1921    - the citations.
    2022"""
    21 import cPickle, os, sys
     23import cPickle, os, sys, shutil
    2224import sphinx
    2325from sphinx.util.console import bold
    2426
     
    6971            env.all_docs.update(newalldoc)
    7072            # needed by env.check_consistency (sphinx.environement, line 1734)
    7173            for ind in newalldoc:
    72                 env.metadata[ind] = {}
     74                # treat subdocument source as orphaned file and don't complain
     75                env.metadata[ind] = {'orphan'}
    7376            # merge the citations
    7477            newcite = {}
    7578            for ind, (path, tag) in docenv.citations.iteritems():
     
    255258            app.builder.info(bold('linking _static directory.'))
    256259            static_dir = os.path.join(app.builder.outdir, '_static')
    257260            master_static_dir = os.path.join('..', '_static')
    258             if not os.path.isdir(static_dir):
    259                 os.symlink(master_static_dir, static_dir)
     261            if os.path.exists(static_dir):
     262                if os.path.isdir(static_dir) and not os.path.islink(static_dir):
     263                    shutil.rmtree(static_dir)
     264                else:
     265                    os.unlink(static_dir)
     266            os.symlink(master_static_dir, static_dir)
    260267
    261268        app.builder.copy_static_files = link_static_files
    262269