Ticket #6495: trac_6495-part4-interrupts.patch

File trac_6495-part4-interrupts.patch, 13.4 KB (added by jhpalmieri, 7 years ago)
  • doc/common/build_options.py

    # HG changeset patch
    # User J. H. Palmieri <palmieri@math.washington.edu>
    # Date 1337145563 25200
    # Node ID f6729d74be181606aa085cd59fb45b739bfe46ee
    # Parent  19b6ef9826f0be64b58339fa894e665d6cbcf389
    Handle interrupts when docbuilding in parallel.
    
    diff --git a/doc/common/build_options.py b/doc/common/build_options.py
    a b else: 
    1717#Note that this needs to have the doctrees dir   
    1818ALLSPHINXOPTS   = SPHINXOPTS + " " + PAPEROPTS + " "
    1919WEBSITESPHINXOPTS = ""
     20
     21# Number of threads to use for parallel-building the documentation.
     22NUM_THREADS = int(os.environ.get('SAGE_NUM_THREADS', 1))
  • doc/common/builder.py

    diff --git a/doc/common/builder.py b/doc/common/builder.py
    a b except ValueError: 
    1111from sage.misc.cachefunc import cached_method
    1212from sage.misc.misc import sage_makedirs as mkdir
    1313
    14 # Read options
     14# Load the options, including
     15#     SAGE_DOC, LANGUAGES, SPHINXOPTS, PAPER, OMIT,
     16#     PAPEROPTS, ALLSPHINXOPTS, NUM_THREADS, WEBSITESPHINXOPTS
     17# from build_options.py.
    1518execfile(os.path.join(os.getenv('SAGE_ROOT'), 'devel', 'sage', 'doc', 'common' , 'build_options.py'))
    1619
    1720##########################################
    def copytree(src, dst, symlinks=False, i 
    6467##########################################
    6568#      Parallel Building Ref Manual      #
    6669##########################################
    67 def build_ref_doc(doc, lang, format, *args, **kwds):
     70def build_ref_doc(args):
     71    doc = args[0]
     72    lang = args[1]
     73    format = args[2]
     74    kwds = args[3]
     75    args = args[4:]
    6876    getattr(ReferenceSubBuilder(doc, lang), format)(*args, **kwds)
    6977
    7078##########################################
    class DocBuilder(object): 
    226234##########################################
    227235#      Parallel Building Ref Manual      #
    228236##########################################
    229 def build_other_doc(document, name, *args, **kwds):
     237def build_other_doc(args):
     238    document = args[0]
     239    name = args[1]
     240    kwds = args[2]
     241    args = args[3:]
    230242    logger.warning("\nBuilding %s.\n" % document)
    231243    getattr(get_builder(document), name)(*args, **kwds)
    232244
    233 
    234245class AllBuilder(object):
    235246    """
    236247    A class used to build all of the documentation.
    class AllBuilder(object): 
    261272        # first build should be as fast as possible;
    262273        logger.warning("\nBuilding reference manual, first pass.\n")
    263274        global ALLSPHINXOPTS
    264         #ALLSPHINXOPTS += ' -Q -D multidoc_first_pass=1'
    265         ALLSPHINXOPTS += ' -D multidoc_first_pass=1'
     275        ALLSPHINXOPTS += ' -Q -D multidoc_first_pass=1'
    266276        for document in refs:
    267277            getattr(get_builder(document), 'inventory')(*args, **kwds)
    268278        logger.warning("Building reference manual, second pass.\n")
    class AllBuilder(object): 
    273283            getattr(get_builder(document), name)(*args, **kwds)
    274284
    275285        # build the other documents in parallel
    276         from multiprocessing import Pool, cpu_count
    277         pool = Pool(int(os.environ.get('SAGE_NUM_THREADS', 1)))
    278         for document in others:
    279             pool.apply_async(build_other_doc,
    280                              (document, name)+args, kwds)
     286        from multiprocessing import Pool
     287        pool = Pool(NUM_THREADS)
     288        L = [(doc, name, kwds) + args for doc in others]
     289        # map_async, with get to provide a timeout, handles
     290        # KeyboardInterrupt correctly. apply_async does not, so don't
     291        # use it.
     292        pool.map_async(build_other_doc, L).get(99999)
    281293        pool.close()
    282294        pool.join()
    283         logger.warning("Elapsed time = %s."%(time.time()-start))
    284         logger.warning("Done compiling the doc !")
     295        logger.warning("Elapsed time: %.1f seconds."%(time.time()-start))
     296        logger.warning("Done building the documentation!")
    285297
    286298    def get_all_documents(self):
    287299        """
    class ReferenceBuilder(AllBuilder): 
    382394        """
    383395        if type == "inventory": # put inventories in the html tree
    384396            type = "html"
    385         return mkdir(os.path.join(SAGE_DOC, "output", type, lang, self.name))
     397        d = os.path.join(SAGE_DOC, "output", type, lang, self.name)
     398        mkdir(d)
     399        return d
    386400
    387401    def _wrapper(self, format, *args, **kwds):
    388402        """
    class ReferenceBuilder(AllBuilder): 
    394408            if not os.path.exists(refdir):
    395409                continue
    396410            output_dir = self._output_dir(format, lang)
    397             from multiprocessing import Pool, cpu_count
    398             # Determine the number of threads from the environment variable
    399             # SAGE_NUM_THREADS.
    400             pool = Pool(int(os.environ.get('SAGE_NUM_THREADS', 1)))
    401 
    402             for doc in self.get_all_documents(refdir):
    403                 pool.apply_async(build_ref_doc,
    404                                  (doc, lang, format) + args, kwds)
     411            from multiprocessing import Pool
     412            pool = Pool(NUM_THREADS)
     413            L = [(doc, lang, format, kwds) + args for doc in self.get_all_documents(refdir)]
     414            # (See comment in AllBuilder._wrapper about using map_async
     415            # instead of apply_async.)
     416            pool.map_async(build_ref_doc, L).get(99999)
    405417            pool.close()
    406418            pool.join()
    407419            # The html refman must be build at the end to ensure correct
    class ReferenceBuilder(AllBuilder): 
    429441                         'pdf.png', 'plus.png', 'pygments.css',
    430442                         'sage.css', 'sageicon.png', 'sagelogo.png',
    431443                         'searchtools.js', 'sidebar.js', 'underscore.js']
    432                 try:
    433                     os.mkdir(os.path.join(output_dir, '_static'))
    434                 except OSError:
    435                     pass
     444                mkdir(os.path.join(output_dir, '_static'))
    436445                for f in static_files:
    437                     shutil.copyfile(os.path.join(website_dir, '_static', f),
    438                                     os.path.join(output_dir, '_static', f))
     446                    try:
     447                        shutil.copyfile(os.path.join(website_dir, '_static', f),
     448                                        os.path.join(output_dir, '_static', f))
     449                    except IOError: # original file does not exist
     450                        pass
    439451                # Now modify website's index.html page and write it
    440452                # to output_dir.
    441453                f = open(os.path.join(website_dir, 'index.html'))
    class ReferenceBuilder(AllBuilder): 
    452464                f = open(os.path.join(SAGE_DOC, lang, 'reference', 'index.rst'))
    453465                rst = f.read()
    454466                f.close()
    455                 # Replace rst links with html links.  There are two types:
     467                # Replace rst links with html links.  There are two forms:
    456468                #
    457                 # `blah`__
    458                 # __ link
     469                #   `blah`__    followed by __ LINK
    459470                #
    460                 # `blah <module/index.html>`_
     471                #   :doc:`blah <module/index>`
    461472                #
    462                 # For the second type, also change "module/index.html"
    463                 # to "module/module.pdf".
     473                # Change the first form to
     474                #
     475                #   <a href="LINK">blah</a>
     476                #
     477                # Change the second form to
     478                #
     479                #   <a href="module/module.pdf">blah <img src="_static/pdf.png" /></a>
     480                #
    464481                rst = re.sub('`([^`]*)`__\.\n\n__ (.*)',
    465482                                  r'<a href="\2">\1</a>.', rst)
    466                 rst = re.sub(r'`([^<]*?)\s+<(.*)/index\.html>`_',
     483                rst = re.sub(r':doc:`([^<]*?)\s+<(.*)/index>`',
    467484                             r'<a href="\2/\2.pdf">\1 <img src="_static/pdf.png" /></a>',
    468485                             rst)
     486                # Get rid of todolist and miscellaneous rst markup.
     487                rst = rst.replace('.. toctree::', '')
     488                rst = rst.replace(':maxdepth: 2', '')
     489                rst = rst.replace('todolist', '')
    469490                start = rst.find('=\n') + 1
    470491                end = rst.find('Table of Contents')
    471492                # Body: add paragraph <p> markup.
    472493                rst_body = rst[start:end]
    473494                rst_body = rst_body.replace('\n\n', '</p>\n<p>')
    474495                start = rst.find('Table of Contents') + 2*len('Table of Contents') + 1
    475                 end = rst.find('.. include:: footer.txt')
     496                # Don't include the indices.
     497                end = rst.find('Indices and Tables')
    476498                # TOC: change * to <li>, change rst headers to html headers.
    477499                rst_toc = rst[start:end]
    478500                rst_toc = rst_toc.replace('*', '<li>')
    for a webpage listing all of the documen 
    514536            sage: b = builder.ReferenceBuilder('reference')
    515537            sage: refdir = os.path.join(os.environ['SAGE_DOC'], 'en', b.name)
    516538            sage: b.get_all_documents(refdir)
     539            ['reference/algebras', 'reference/arithgroup', ..., 'reference/tensor']
    517540        """
    518541        documents = []
    519542
  • doc/common/inventory_builder.py

    diff --git a/doc/common/inventory_builder.py b/doc/common/inventory_builder.py
    a b  
    33    inventory builder
    44    ~~~~~~~~~~~~~~~~~
    55
    6     A customized HTML builder which only generate intersphinx "object.inv"
    7     inventory files and pickle files. The documentation files are not writen.
     6    A customized HTML builder which only generates intersphinx "object.inv"
     7    inventory files and pickle files. The documentation files are not written.
    88"""
    99from sphinx.builders.html import StandaloneHTMLBuilder
    1010from sphinx.util.console import bold
    1111
    1212class InventoryBuilder(StandaloneHTMLBuilder):
    1313    """
    14     A customized HTML builder which only generate intersphinx "object.inv"
    15     inventory files and pickle files. The documentation files are not writen.
     14    A customized HTML builder which only generates intersphinx "object.inv"
     15    inventory files and pickle files. The documentation files are not written.
    1616    """
    1717    name = "inventory"
    1818
    class InventoryBuilder(StandaloneHTMLBui 
    3535        This is just for making sure that some writer methods are indeed
    3636        deactivated.
    3737        """
    38         raise RuntimeError, "This function souldn't be called in \"%s\" builder"%(self.name)
     38        raise RuntimeError, "This function shouldn't be called in \"%s\" builder"%(self.name)
    3939
    4040    copy_image_files = removed_method_error
    4141    copy_download_files = removed_method_error
  • doc/common/multidocs.py

    diff --git a/doc/common/multidocs.py b/doc/common/multidocs.py
    a b  
    11# -*- coding: utf-8 -*-
    22"""
    3     multi documentations in Sphinx
    4     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     3    multi documentation in Sphinx
     4    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    55
    66    The goal of this extension is to manage a multi documentation in Sphinx.
    7     To be able to compile Sage huge documentation in parallel, the
    8     documentation is cut in a bunch of independent documentations called a
    9     "subdoc", which are compiled separately. There is a master document which
    10     points to all the subdocs. The intersphinx extension ensure that the
     7    To be able to compile Sage's huge documentation in parallel, the
     8    documentation is cut into a bunch of independent documentations called
     9    "subdocs", which are compiled separately. There is a master document which
     10    points to all the subdocs. The intersphinx extension ensures that the
    1111    cross-link between the subdocs are correctly resolved. However some work
    1212    is needed to build a global index. This is the goal of multidocs.
    1313
    14     More precisely this extension ensure the correct merging of
     14    More precisely this extension ensures the correct merging of
    1515    - the todo list if this extension is activated;
    1616    - the python indexes;
    1717    - the list of python modules;
    def merge_environment(app, env): 
    9292
    9393def get_env(app, curdoc):
    9494    """
    95     Get the environement of a sub-doc from the pickle
     95    Get the environment of a sub-doc from the pickle
    9696    """
    9797    from sphinx.application import ENV_PICKLE_FILENAME
    9898    filename = os.path.join(
    def fix_path_html(app, pagename, templat 
    168168    - search.html
    169169    - genindex.html
    170170    - py-modindex.html
    171     points to the right place, that is in
     171    point to the right place, that is in
    172172        reference/
    173173    instead of
    174174        reference/subdocument
    def write_citations(app, citations): 
    197197
    198198def fetch_citation(app, env):
    199199    """
    200     Fetch the global citation index from the refman to allows for cross
     200    Fetch the global citation index from the refman to allow for cross
    201201    references.
    202202    """
    203203    app.builder.info(bold('loading cross citations... '), nonl=1)
  • sage/misc/sagedoc.py

    diff --git a/sage/misc/sagedoc.py b/sage/misc/sagedoc.py
    a b TESTS: 
    1818Check that argspecs of extension function/methods appear correctly,
    1919see :trac:`12849`::
    2020   
    21     sage: docfilename = os.path.join(SAGE_ROOT, 'devel', 'sage', 'doc', 'output', 'html', 'en', 'reference', 'sage', 'symbolic', 'expression.html')
     21    sage: docfilename = os.path.join(SAGE_ROOT, 'devel', 'sage', 'doc', 'output', 'html', 'en', 'reference', 'calculus', 'sage', 'symbolic', 'expression.html')
    2222    sage: for line in open(docfilename):
    2323    ...       if "#sage.symbolic.expression.Expression.N" in line:
    2424    ...           print line