Ticket #6495: trac_6495_separate_inventory.patch

doc/common/builder.py

@@ -1 +1 @@
 #!/usr/bin/env python
+"""
+The documentation builder
+
+It is the starting point for building documentation, and is
+responsible to figure out what to build and with which options. The
+actual documentation build for each individual document is then done
+in a subprocess call to sphinx, see :func:`builder_helper`.
+
+* The builder can be configured in build_options.py
+* The sphinx subprocesses are configured in conf.py
+"""
+
 import glob, logging, optparse, os, shutil, subprocess, sys, textwrap
 
 #We remove the current directory from sys.path right away
@@ -17 +29 @@
 # from build_options.py.
 execfile(os.path.join(os.getenv('SAGE_ROOT'), 'devel', 'sage', 'doc', 'common' , 'build_options.py'))
 
-##########################################
-#          Utility Functions             #
-##########################################
-def copytree(src, dst, symlinks=False, ignore_errors=False):
-    """
-    Recursively copy a directory tree using copy2().
-
-    The destination directory must not already exist.
-    If exception(s) occur, an Error is raised with a list of reasons.
-
-    If the optional symlinks flag is true, symbolic links in the
-    source tree result in symbolic links in the destination tree; if
-    it is false, the contents of the files pointed to by symbolic
-    links are copied.
-
-    XXX Consider this example code rather than the ultimate tool.
-
-    """
-    names = os.listdir(src)
-    mkdir(dst)
-    errors = []
-    for name in names:
-        srcname = os.path.join(src, name)
-        dstname = os.path.join(dst, name)
-        try:
-            if symlinks and os.path.islink(srcname):
-                linkto = os.readlink(srcname)
-                os.symlink(linkto, dstname)
-            elif os.path.isdir(srcname):
-                copytree(srcname, dstname, symlinks)
-            else:
-                shutil.copy2(srcname, dstname)
-            # XXX What about devices, sockets etc.?
-        except (IOError, os.error) as why:
-            errors.append((srcname, dstname, str(why)))
-        # catch the Error from the recursive copytree so that we can
-        # continue with other files
-        except shutil.Error as err:
-            errors.extend(err.args[0])
-    try:
-        shutil.copystat(src, dst)
-    except OSError as why:
-        errors.extend((src, dst, str(why)))
-    if errors and not ignore_errors:
-        raise shutil.Error(errors)
-
 
 ##########################################
 #      Parallel Building Ref Manual      #
@@ -73 +39 @@
     format = args[2]
     kwds = args[3]
     args = args[4:]
+    if format == 'inventory':  # you must not use the inventory to build the inventory
+        kwds['use_multidoc_inventory'] = False
     getattr(ReferenceSubBuilder(doc, lang), format)(*args, **kwds)
 
 ##########################################
@@ -84 +52 @@
     Returns a function which builds the documentation for
     output type type.
     """
-    def f(self):
+    def f(self, *args, **kwds):
         output_dir = self._output_dir(type)
         os.chdir(self.dir)
 
@@ -94 +62 @@
             # WEBSITESPHINXOPTS is either empty or " -A hide_pdf_links=1 "
             options += WEBSITESPHINXOPTS
 
-        build_command = 'sphinx-build'
+        if kwds.get('use_multidoc_inventory', True):
+            options += ' -D multidoc_first_pass=0'
+        else:
+            options += ' -D multidoc_first_pass=1'
+
+        build_command = 'python '+os.path.join(SAGE_DOC, 'common', 'custom-sphinx-build.py')
         build_command += ' -b %s -d %s %s %s %s'%(type, self._doctrees_dir(),
                                                   options, self.dir,
                                                   output_dir)
-        logger.warning(build_command)
+        logger.debug(build_command)
         subprocess.call(build_command, shell=True)
 
-        logger.warning("Build finished.  The built documents can be found in %s", output_dir)
+        logger.info("Build finished and can be found in %s", 
+                    output_dir.replace(SAGE_DOC+'/', ''))
         
     f.is_output_format = True
     return f
@@ -144 +118 @@
             sage: b._output_dir('html')
             '.../devel/sage/doc/output/html/en/tutorial'
         """
-        if type == "inventory": # put inventories in the html tree
-            type = "html"
         d = os.path.join(SAGE_DOC, "output", type, self.lang, self.name)
         mkdir(d)
         return d
@@ -266 +238 @@
         docs = self.get_all_documents()
         refs = [x for x in docs if x.endswith('reference')]
         others = [x for x in docs if not x.endswith('reference')]
+
         # Build the reference manual twice to resolve references.  That is,
         # build once with the inventory builder to construct the intersphinx
         # inventory files, and then build the second time for real.  So the
         # first build should be as fast as possible;
         logger.warning("\nBuilding reference manual, first pass.\n")
-        global ALLSPHINXOPTS
-        ALLSPHINXOPTS += ' -Q -D multidoc_first_pass=1'
         for document in refs:
             getattr(get_builder(document), 'inventory')(*args, **kwds)
+
         logger.warning("Building reference manual, second pass.\n")
-        ALLSPHINXOPTS = ALLSPHINXOPTS.replace(
-            'multidoc_first_pass=1', 'multidoc_first_pass=0')
-        ALLSPHINXOPTS = ALLSPHINXOPTS.replace('-Q', '-q') + ' '
         for document in refs:
             getattr(get_builder(document), name)(*args, **kwds)
 
@@ -286 +255 @@
         from multiprocessing import Pool
         pool = Pool(NUM_THREADS)
         L = [(doc, name, kwds) + args for doc in others]
-        # map_async, with get to provide a timeout, handles
-        # KeyboardInterrupt correctly. apply_async does not, so don't
-        # use it.
-        pool.map_async(build_other_doc, L).get(99999)
+        # map_async handles KeyboardInterrupt correctly. Plain map and
+        # apply_async does not, so don't use it.
+        pool.map_async(build_other_doc, L, 1).get(99999)
         pool.close()
         pool.join()
         logger.warning("Elapsed time: %.1f seconds."%(time.time()-start))
@@ -335 +303 @@
         """
         DocBuilder.html(self)
         html_output_dir = self._output_dir('html')
-        copytree(html_output_dir,
-                 os.path.realpath(os.path.join(html_output_dir, '..')),
-                 ignore_errors=False)
-
+        for f in os.listdir(html_output_dir):
+            src = os.path.join(html_output_dir, f)
+            dst = os.path.join(html_output_dir, '..', f)
+            if os.path.isdir(src):
+                shutil.rmtree(dst, ignore_errors=True)
+                shutil.copytree(src, dst)
+            else:
+                shutil.copy2(src, dst)
         self.create_html_redirects()
 
     def create_html_redirects(self):
@@ -458 +430 @@
             sage: b._output_dir('html')
             '.../devel/sage/doc/output/html/en/reference'
         """
-        if type == "inventory": # put inventories in the html tree
-            type = "html"
         d = os.path.join(SAGE_DOC, "output", type, lang, self.name)
         mkdir(d)
         return d
@@ -477 +447 @@
             from multiprocessing import Pool
             pool = Pool(NUM_THREADS)
             L = [(doc, lang, format, kwds) + args for doc in self.get_all_documents(refdir)]
-            # (See comment in AllBuilder._wrapper about using map_async
-            # instead of apply_async.)
-            pool.map_async(build_ref_doc, L).get(99999)
+            # (See comment in AllBuilder._wrapper about using map instead of apply.)
+            pool.map_async(build_ref_doc, L, 1).get(99999)
             pool.close()
             pool.join()
             # The html refman must be build at the end to ensure correct
@@ -596 +565 @@
         We add a component name if it's a subdirectory of the manual's
         directory and contains a file named 'index.rst'.
 
+        We return the largest component (most subdirectory entries)
+        first since they will take the longest to build.
+
         EXAMPLES::
 
             sage: import os, sys; sys.path.append(os.environ['SAGE_DOC']+'/common/'); import builder
             sage: b = builder.ReferenceBuilder('reference')
             sage: refdir = os.path.join(os.environ['SAGE_DOC'], 'en', b.name)
-            sage: b.get_all_documents(refdir)
+            sage: sorted(b.get_all_documents(refdir))
             ['reference/algebras', 'reference/arithgroup', ..., 'reference/tensor']
         """
         documents = []
 
         for doc in os.listdir(refdir):
-            if os.path.exists(os.path.join(refdir, doc, 'index.rst')):
-                documents.append(os.path.join(self.name, doc))
+            directory = os.path.join(refdir, doc)
+            if os.path.exists(os.path.join(directory, 'index.rst')):
+                n = len(os.listdir(directory))
+                documents.append((-n, os.path.join(self.name, doc)))
 
-        return sorted(documents)
+        return [ doc[1] for doc in sorted(documents) ]
 
 
 class ReferenceSubBuilder(DocBuilder):
@@ -677 +651 @@
         _sage = os.path.join(self.dir, '_sage')
         if os.path.exists(_sage):
             logger.info("Copying over custom .rst files from %s ...", _sage)
-            copytree(_sage, os.path.join(self.dir, 'sage'))
+            shutil.copytree(_sage, os.path.join(self.dir, 'sage'))
                 
         getattr(DocBuilder, build_type)(self, *args, **kwds)
     
@@ -1413 +1387 @@
     if options.warn_links:
         ALLSPHINXOPTS += "-n "
 
-
     # Make sure common/static exists.
     mkdir(os.path.join(SAGE_DOC, 'common', 'static'))
 

doc/common/conf.py

@@ -110 +110 @@
 
 def set_intersphinx_mappings(app):
     """
-    Add reference's objects.inv to intersphinx if not compiling reference
+    Add precompiled inventory (the objects.inv)
     """
+    refpath = get_doc_abspath('output/html/en/reference')
+    invpath = get_doc_abspath('output/inventory/en/reference')
+    if app.config.multidoc_first_pass == 1 or \
+            not (os.path.exists(refpath) and os.path.exists(invpath)):
+        app.config.intersphinx_mapping = {}
+        return
     app.config.intersphinx_mapping = intersphinx_mapping
-    refpath = 'output/html/en/reference/'
-    if not app.srcdir.endswith('reference'):
-        app.config.intersphinx_mapping[get_doc_abspath(refpath)] = get_doc_abspath(refpath+'objects.inv')
+    
+    def add(subdoc=''):
+        src = os.path.join(refpath, subdoc) if subdoc else refpath
+        dst = os.path.join(invpath, subdoc, 'objects.inv')
+        app.config.intersphinx_mapping[src] = dst
+        
+    add()
+    for directory in os.listdir(os.path.join(invpath)):
+        if os.path.isdir(os.path.join(invpath, directory)):
+            add(directory)
+
+
 pythonversion = sys.version.split(' ')[0]
 # Python and Sage trac ticket shortcuts. For example, :trac:`7549` .
 
@@ -609 +624 @@
     app.connect('autodoc-process-docstring', process_dollars)
     app.connect('autodoc-process-docstring', process_inherited)
     app.connect('autodoc-skip-member', skip_member)
-
+    
     # When building the standard docs, app.srcdir is set to SAGE_DOC +
     # 'LANGUAGE/DOCNAME', but when doing introspection, app.srcdir is
     # set to a temporary directory.  We don't want to use intersphinx,
@@ -625 +640 @@
         app.connect('builder-inited', set_intersphinx_mappings)
         app.connect('builder-inited', sphinx.ext.intersphinx.load_mappings)
         app.connect('builder-inited', nitpick_patch_config)
-        # Minimize GAP/libGAP RAM usage when we build the docs
-        from sage.interfaces.gap import set_gap_memory_pool_size
-        set_gap_memory_pool_size(0)  # will be rounded up to 1M
 

new file doc/common/custom-sphinx-build.py

@@ +1 @@
+"""
+This is Sage's version of the sphinx-build script
+
+Enhancements are:
+
+* import the Sage library to access the docstrings, otherwise doc
+  buliding doesn't work.
+
+* redirect stdout to our own logger, and remove some unwanted chatter.
+"""
+
+import os
+import sys
+import re
+
+
+# override the fancy multi-line formatting
+def term_width_line(text):
+    return text + '\n'
+
+import sphinx.util.console
+sphinx.util.console.term_width_line = term_width_line
+
+
+
+
+useless_chatter = (
+    re.compile('^$'),
+    re.compile('^Running Sphinx v'),
+    re.compile('^loading intersphinx inventory from '),
+    re.compile('^Compiling a sub-document'),
+    re.compile('^updating environment: 0 added, 0 changed, 0 removed'),
+    re.compile('^looking for now-outdated files... none found'),
+    re.compile('^no targets are out of date.'),
+    re.compile('^building \[.*\]: targets for 0 source files that are out of date'),
+    re.compile('^loading pickled environment... done'),
+    re.compile('^loading cross citations... done \([0-9]* citations\).')
+    )
+
+
+class SageSphinxLogger(object):
+    """
+    This implements the file object interface to serve as sys.stdout
+    replacement.
+    """
+    ansi_color = re.compile(r'\x1b\[[0-9;]*m')
+    ansi_reset = re.compile(r'\x1b\[39;49;00m')
+    prefix_len = 9
+
+    def __init__(self, stream, prefix):
+        self._stream = stream
+        self._color = stream.isatty()
+        prefix = prefix[0:self.prefix_len]
+        prefix = ('[{0:'+str(self.prefix_len)+'}]').format(prefix)
+        color = { 1:'darkgreen', 2:'red' }
+        color = color.get(stream.fileno(), 'lightgray')
+        self._prefix = sphinx.util.console.colorize(color, prefix)
+        
+
+    def _filter_out(self, line):
+        line = re.sub(self.ansi_color, '', line)
+        for regex in useless_chatter:
+            if regex.match(line) is not None:
+                return True
+        return False
+
+    def _log_line(self, line):
+        if self._filter_out(line):
+            return
+        line = self._prefix + ' ' + line.strip() + '\n'
+        if not self._color:
+            line = self.ansi_color.sub('', line)
+        self._stream.write(line)
+        self._stream.flush()
+        
+    _line_buffer = ''
+    
+    def _break_long_lines(self):
+        """
+        Break text that has been formated with string.ljust() back
+        into individual lines.  Return partial output. Do nothing if
+        the filter rule matches, otherwise subsequent lines would be
+        not filtered out.
+        """
+        if self._filter_out(self._line_buffer):
+            return
+        cols = sphinx.util.console._tw
+        lines = []
+        buf = self._line_buffer
+        while len(buf) > cols:
+            lines.append(buf[0:cols])
+            buf = buf[cols:]
+        lines.append(buf)
+        self._line_buffer = '\n'.join(lines)
+
+    def _write(self, string):
+        self._line_buffer += string
+        #self._break_long_lines()
+        lines = self._line_buffer.splitlines()
+        for i, line in enumerate(lines):
+            last = (i == len(lines)-1)
+            if last and not self._line_buffer.endswith('\n'):
+                self._line_buffer = line
+                return
+            self._log_line(line)
+        self._line_buffer = ''
+                
+
+    # file object interface follows
+
+    closed = False
+    encoding = None
+    mode = 'w'
+    name = '<log>'
+    newlines = None
+    softspace = 0
+
+    def isatty(self):
+        return True
+
+    def close(self):
+        if self._line_buffer != '':
+            self._log_line(self._line_buffer)
+            self._line_buffer = ''
+
+    def flush(self):
+        self._stream.flush()
+        
+    def write(self, str):
+        try:
+            self._write(str)
+        except:
+            import traceback
+            traceback.print_exc(file=self._stream)
+        
+    def writelines(self, sequence):
+        for line in sequence:
+            self.write(line)
+    
+
+output_dir = sys.argv[-1]
+sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir))
+sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir))
+
+
+
+# pull in the Sage library
+import sage.all
+
+# Minimize GAP/libGAP RAM usage when we build the docs
+from sage.interfaces.gap import set_gap_memory_pool_size
+set_gap_memory_pool_size(0)  # will be rounded up to 1M
+
+if __name__ == '__main__':
+    from sphinx.cmdline import main
+    sys.exit(main(sys.argv))
+

doc/common/multidocs.py

@@ -3 +3 @@
     multi documentation in Sphinx
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+    This is a slightly hacked-up version of the Sphinx-multidoc plugin
+
     The goal of this extension is to manage a multi documentation in Sphinx.
     To be able to compile Sage's huge documentation in parallel, the
     documentation is cut into a bunch of independent documentations called
@@ -18 +20 @@
     - the javascript index;
     - the citations.
 """
-import cPickle, os, sys
+import cPickle, os, sys, shutil
 import sphinx
 from sphinx.util.console import bold
 
@@ -69 +71 @@
             env.all_docs.update(newalldoc)
             # needed by env.check_consistency (sphinx.environement, line 1734)
             for ind in newalldoc:
-                env.metadata[ind] = {}
+                # treat subdocument source as orphaned file and don't complain
+                env.metadata[ind] = {'orphan'}
             # merge the citations
             newcite = {}
             for ind, (path, tag) in docenv.citations.iteritems():
@@ -255 +258 @@
             app.builder.info(bold('linking _static directory.'))
             static_dir = os.path.join(app.builder.outdir, '_static')
             master_static_dir = os.path.join('..', '_static')
-            if not os.path.isdir(static_dir):
-                os.symlink(master_static_dir, static_dir)
+            if os.path.exists(static_dir):
+                if os.path.isdir(static_dir) and not os.path.islink(static_dir):
+                    shutil.rmtree(static_dir)
+                else:
+                    os.unlink(static_dir)
+            os.symlink(master_static_dir, static_dir)
 
         app.builder.copy_static_files = link_static_files