# HG changeset patch
# User Florent Hivert <Florent.Hivert@univ-rouen.fr>
# Date 1336416039 -7200
# Node ID 4d6d27806033232f68c18551b2e271bb177824e6
# Parent  ff684d217ab372f611772290497ebe4cf1b9c5f6
#6495: Build the reference manual incrementally
- Add a builder for generating object.inv and pickle files.
- Add an extension multidocs which merge the various indexes.

diff --git a/doc/common/builder.py b/doc/common/builder.py
--- a/doc/common/builder.py
+++ b/doc/common/builder.py
@@ -85,33 +85,13 @@ def copytree(src, dst, symlinks=False, i
 ##########################################
 #      Parallel Building Ref Manual      #
 ##########################################
-
-def build_ref_doc(doc, lang, format, output_dir, *args, **kwds):
-    # Build the reference manual for doc
-    static_dir = os.path.join(output_dir, 'reference', '_static')
-    bad_static = os.path.join(output_dir, doc, '_static')
-    # We need to remove the link to "_static" before generating the
-    # documentation, because the html doc builder writes to the
-    # _static directory.
-    if (os.path.isdir(static_dir) and os.path.isdir(bad_static)
-        and os.path.islink(bad_static)):
-        os.remove(bad_static)
+def build_ref_doc(doc, lang, format, *args, **kwds):
     getattr(ReferenceSubBuilder(doc, lang), format)(*args, **kwds)
-    # The standard Sphinx html build puts a copy of the "static"
-    # directory in reference/doc/_static.  For the reference manual,
-    # these are all identical to reference/_static, so delete the
-    # directory reference/doc/_static and replace it with a link to
-    # reference/_static.  This saves hundreds of megabytes of disk
-    # space.
-    if (os.path.isdir(static_dir) and os.path.isdir(bad_static)
-        and not os.path.islink(bad_static)):
-        shutil.rmtree(bad_static)
-    if os.path.isdir(static_dir):
-        os.symlink(static_dir, bad_static)
 
 ##########################################
 #             Builders                   #
 ##########################################
+
 def builder_helper(type):
     """
     Returns a function which builds the documentation for
@@ -171,6 +151,8 @@ class DocBuilder(object):
             sage: b._output_dir('html')
             '.../devel/sage/doc/output/html/en/tutorial'
         """
+        if type == "inventory": # put inventories in the html tree
+            type = "html"
         return mkdir(os.path.join(SAGE_DOC, "output", type, self.lang, self.name))
 
     def _doctrees_dir(self):
@@ -197,7 +179,7 @@ class DocBuilder(object):
             sage: import os, sys; sys.path.append(os.environ['SAGE_DOC']+'/common/'); import builder
             sage: b = builder.DocBuilder('tutorial')
             sage: b._output_formats()
-            ['changes', 'html', 'htmlhelp', 'json', 'latex', 'linkcheck', 'pickle', 'web']
+            ['changes', 'html', 'htmlhelp', 'inventory', 'json', 'latex', 'linkcheck', 'pickle', 'web']
 
         """
         #Go through all the attributes of self and check to
@@ -249,6 +231,15 @@ class DocBuilder(object):
     latex = builder_helper('latex')
     changes = builder_helper('changes')
     linkcheck = builder_helper('linkcheck')
+    # import the customized builder for object.inv files
+    inventory = builder_helper('inventory')
+
+##########################################
+#      Parallel Building Ref Manual      #
+##########################################
+def build_other_doc(document, name, *args, **kwds):
+    logger.warning("\nBuilding %s.\n" % document)
+    getattr(get_builder(document), name)(*args, **kwds)
 
 
 class AllBuilder(object):
@@ -270,27 +261,39 @@ class AllBuilder(object):
         This is the function which goes through all of the documents
         and does the actual building.
         """
+        import time
+        start = time.time()
         docs = self.get_all_documents()
         refs = [x for x in docs if x.endswith('reference')]
         others = [x for x in docs if not x.endswith('reference')]
-        for document in others:
-            logger.warning("\nBuilding %s.\n" % document)
-            getattr(get_builder(document), name)(*args, **kwds)
-        # Build the reference manual twice to resolve references.
-        # That is, build once to construct the intersphinx inventory
-        # files, and then build the second time for real.  So the
-        # first build should be as fast as possible; thus do an html
-        # build first.
+        # Build the reference manual twice to resolve references.  That is,
+        # build once with the inventory builder to construct the intersphinx
+        # inventory files, and then build the second time for real.  So the
+        # first build should be as fast as possible;
         logger.warning("\nBuilding reference manual, first pass.\n")
         global ALLSPHINXOPTS
-        ALLSPHINXOPTS += ' -Q '
+        #ALLSPHINXOPTS += ' -Q -D multidoc_first_pass=1'
+        ALLSPHINXOPTS += ' -D multidoc_first_pass=1'
         for document in refs:
-            getattr(get_builder(document), 'html')(*args, **kwds)
+            getattr(get_builder(document), 'inventory')(*args, **kwds)
         logger.warning("Building reference manual, second pass.\n")
+        ALLSPHINXOPTS = ALLSPHINXOPTS.replace(
+            'multidoc_first_pass=1', 'multidoc_first_pass=0')
         ALLSPHINXOPTS = ALLSPHINXOPTS.replace('-Q', '-q') + ' -a '
         for document in refs:
             getattr(get_builder(document), name)(*args, **kwds)
 
+        # build the other documents in parallel
+        from multiprocessing import Pool, cpu_count
+        pool = Pool(int(os.environ.get('SAGE_NUM_THREADS', 1)))
+        for document in others:
+            pool.apply_async(build_other_doc,
+                             (document, name)+args, kwds)
+        pool.close()
+        pool.join()
+        logger.warning("Elapsed time = %s."%(time.time()-start))
+        logger.warning("Done compiling the doc !")
+
     def get_all_documents(self):
         """
         Returns a list of all of the documents. A document is a directory within one of 
@@ -387,6 +390,8 @@ class ReferenceBuilder(AllBuilder):
             sage: b._output_dir('html')
             '.../devel/sage/doc/output/html/en/reference'
         """
+        if type == "inventory": # put inventories in the html tree
+            type = "html"
         return mkdir(os.path.join(SAGE_DOC, "output", type, lang, self.name))
 
     def _wrapper(self, format, *args, **kwds):
@@ -398,51 +403,21 @@ class ReferenceBuilder(AllBuilder):
             refdir = os.path.join(SAGE_DOC, lang, self.name)
             if not os.path.exists(refdir):
                 continue
-
             output_dir = self._output_dir(format, lang)
-            getattr(DocBuilder(self.name, lang), format)(*args, **kwds)
-
             from multiprocessing import Pool, cpu_count
             # Determine the number of threads from the environment variable
             # SAGE_NUM_THREADS.
             pool = Pool(int(os.environ.get('SAGE_NUM_THREADS', 1)))
+
             for doc in self.get_all_documents(refdir):
                 pool.apply_async(build_ref_doc,
-                                 (doc, lang, format,
-                                  os.path.split(output_dir)[0]) + args, kwds)
+                                 (doc, lang, format) + args, kwds)
             pool.close()
             pool.join()
-            if format == 'html':
-                # html build: combine the todo lists from the
-                # different modules.
-                todofile = os.path.join(output_dir, 'todolist', 'index.html')
-                old = open(todofile).read()
-                note = "The combined to do list is only available in the html version of the reference manual."
-                preamble = old.find(note)
-                postamble = preamble + len(note)
-                if preamble != -1:
-                    old_todofile = os.path.join(output_dir, 'todolist', 'index-old.html')
-                    shutil.move(todofile, old_todofile)
-                    new = open(todofile, 'w')
-                    new.write(old[:preamble])
-                    for f in os.listdir(output_dir):
-                        index = os.path.join(output_dir, f, 'index.html')
-                        if (f != 'todolist' and os.path.exists(index)):
-                            html = open(index).read()
-                            start = html.find('<div class="admonition-todo')
-                            end = html.find('<div class="section" id="indices-and-tables">')
-                            if start != -1:
-                                html = html[start:end].replace('sage/%s' %f,
-                                                               '../%s/sage/%s' % (f, f))
-                                new.write(html)
-                    new.write(old[postamble:])
-                    new.close()
+            # The html refman must be build at the end to ensure correct
+            # merging of indexes and inventories.
+            getattr(DocBuilder(self.name, lang), format)(*args, **kwds)
 
-                logger.warning('''
-Build finished.  The Sage reference manual can be found in
-
-  %s
-''' % (os.path.join(output_dir, 'index.html')))
             # PDF: we need to build master index file which lists all
             # of the PDF file.  So we create an html file, based on
             # the file index.html from the "website" target.
@@ -542,14 +517,21 @@ for a webpage listing all of the documen
         Returns a list of all reference manual components to build.
         We add a component name if it's a subdirectory of the manual's
         directory and contains a file named 'index.rst'.
+
+        EXAMPLES::
+
+            sage: import os, sys; sys.path.append(os.environ['SAGE_DOC']+'/common/'); import builder
+            sage: b = builder.ReferenceBuilder('reference')
+            sage: refdir = os.path.join(os.environ['SAGE_DOC'], 'en', b.name)
+            sage: b.get_all_documents(refdir)
         """
         documents = []
 
         for doc in os.listdir(refdir):
             if os.path.exists(os.path.join(refdir, doc, 'index.rst')):
                 documents.append(os.path.join(self.name, doc))
-        
-        return documents
+
+        return sorted(documents)
 
 
 class ReferenceSubBuilder(DocBuilder):
@@ -684,7 +666,7 @@ class ReferenceSubBuilder(DocBuilder):
         except IOError as err:
             logger.debug("Failed to open Sphinx environment: %s", err)
             pass
-                                           
+
     def update_mtimes(self):
         """
         Updates the modification times for ReST files in the Sphinx
@@ -853,7 +835,7 @@ class ReferenceSubBuilder(DocBuilder):
 
             sage: import os, sys; sys.path.append(os.environ['SAGE_DOC']+'/common/'); import builder
             sage: import builder
-            sage: builder.ReferenceBuilder("reference").auto_rest_filename("sage.combinat.partition")
+            sage: builder.ReferenceSubBuilder("reference").auto_rest_filename("sage.combinat.partition")
             '.../devel/sage/doc/en/reference/sage/combinat/partition.rst'
         """
         return self.dir + os.path.sep + module_name.replace('.',os.path.sep) + '.rst'
diff --git a/doc/common/conf.py b/doc/common/conf.py
--- a/doc/common/conf.py
+++ b/doc/common/conf.py
@@ -19,7 +19,8 @@ sys.path.append(get_doc_abspath('common'
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sage_autodoc',  'sphinx.ext.graphviz',
+extensions = ['inventory_builder', 'multidocs',
+              'sage_autodoc',  'sphinx.ext.graphviz',
               'sphinx.ext.inheritance_diagram', 'sphinx.ext.todo',
               'sphinx.ext.extlinks']
 # We do *not* fully initialize intersphinx since we call it by hand
@@ -206,6 +207,11 @@ if 'SAGE_DOC_JSMATH' in os.environ:
 # If false, no module index is generated.
 #html_use_modindex = True
 
+# A list of prefixes that are ignored for sorting the Python module index ( if
+# this is set to ['foo.'], then foo.bar is shown under B, not F). Works only
+# for the HTML builder currently.
+modindex_common_prefix = ['sage.']
+
 # If false, no index is generated.
 #html_use_index = True
 
diff --git a/doc/common/inventory_builder.py b/doc/common/inventory_builder.py
new file mode 100644
--- /dev/null
+++ b/doc/common/inventory_builder.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+    inventory builder
+    ~~~~~~~~~~~~~~~~~
+
+    A customized HTML builder which only generate intersphinx "object.inv"
+    inventory files and pickle files. The documentation files are not writen.
+"""
+from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.util.console import bold
+
+class InventoryBuilder(StandaloneHTMLBuilder):
+    """
+    A customized HTML builder which only generate intersphinx "object.inv"
+    inventory files and pickle files. The documentation files are not writen.
+    """
+    name = "inventory"
+
+    def write_doc(self, docname, doctree):
+        """
+        Don't write any doc
+        """
+
+    def finish(self):
+        """
+        Only write the inventory files.
+        """
+        self.write_buildinfo()
+        self.dump_inventory()
+
+    def removed_method_error(self):
+        """
+        Raise an error if this method is called.
+
+        This is just for making sure that some writer methods are indeed
+        deactivated.
+        """
+        raise RuntimeError, "This function souldn't be called in \"%s\" builder"%(self.name)
+
+    copy_image_files = removed_method_error
+    copy_download_files = removed_method_error
+    copy_static_files = removed_method_error
+    handle_finish = removed_method_error
+
+def setup(app):
+    app.add_builder(InventoryBuilder)
+
diff --git a/doc/common/multidocs.py b/doc/common/multidocs.py
new file mode 100644
--- /dev/null
+++ b/doc/common/multidocs.py
@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+"""
+    multi documentations in Sphinx
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    The goal of this extension is to manage a multi documentation in Sphinx.
+    To be able to compile Sage huge documentation in parallel, the
+    documentation is cut in a bunch of independent documentations called a
+    "subdoc", which are compiled separately. There is a master document which
+    points to all the subdocs. The intersphinx extension ensure that the
+    cross-link between the subdocs are correctly resolved. However some work
+    is needed to build a global index. This is the goal of multidocs.
+
+    More precisely this extension ensure the correct merging of
+    - the todo list if this extension is activated;
+    - the python indexes;
+    - the list of python modules;
+    - the javascript index;
+    - the citations.
+"""
+import cPickle, os, sys
+import sphinx
+from sphinx.util.console import bold
+
+
+CITE_FILENAME = 'citations.pickle'
+
+
+def merge_environment(app, env):
+    """
+    Merges the following attributes of the sub-docs environment into the main
+    environment:
+    - todo_all_todos              # ToDo's
+    - indexentries                # global python index
+    - all_docs                    # needed by the js index
+    - citations                   # citations
+
+    - domaindata['py']['modules'] # list of python modules
+    """
+    app.info(bold('Merging environment/index files...'))
+    for curdoc in app.env.config.multidocs_subdoc_list:
+        app.info("    %s:"%curdoc, nonl=1)
+        docenv = get_env(app, curdoc)
+        if docenv is not None:
+            fixpath = lambda path: os.path.join(curdoc, path)
+            app.info(" %s todos, %s index, %s citations"%(
+                    len(docenv.todo_all_todos),
+                    len(docenv.indexentries),
+                    len(docenv.citations)
+                    ), nonl=1)
+
+            # merge the todo links
+            for dct in docenv.todo_all_todos:
+                dct['docname']=fixpath(dct['docname'])
+            env.todo_all_todos += docenv.todo_all_todos
+            # merge the html index links
+            newindex = {}
+            for ind in docenv.indexentries:
+                if ind.startswith('sage/'):
+                    newind = fixpath(ind)
+                    newindex[newind] = docenv.indexentries[ind]
+                else:
+                    newindex[ind] = docenv.indexentries[ind]
+            env.indexentries.update(newindex)
+            # merge the all_docs links, needed by the js index
+            newalldoc = {}
+            for ind in docenv.all_docs:
+                newalldoc[fixpath(ind)]=docenv.all_docs[ind]
+            env.all_docs.update(newalldoc)
+            # needed by env.check_consistency (sphinx.environement, line 1734)
+            for ind in newalldoc:
+                env.metadata[ind] = {}
+            # merge the citations
+            newcite = {}
+            for ind, (path, tag) in docenv.citations.iteritems():
+                # TODO: Warn on conflicts
+                newcite[ind]=(fixpath(path), tag)
+            env.citations.update(newcite)
+            # merge the py:module indexes
+            newmodules = {}
+            for ind,(modpath,v1,v2,v3) in (
+                docenv.domaindata['py']['modules'].iteritems()):
+                newmodules[ind] = (fixpath(modpath),v1,v2,v3)
+            env.domaindata['py']['modules'].update(newmodules)
+            app.info(", %s modules"%(len(newmodules)))
+    app.info('... done (%s todos, %s index, %s citations, %s modules)'%(
+            len(env.todo_all_todos),
+            len(env.indexentries),
+            len(env.citations),
+            len(env.domaindata['py']['modules'])))
+    write_citations(app, env.citations)
+
+def get_env(app, curdoc):
+    """
+    Get the environement of a sub-doc from the pickle
+    """
+    from sphinx.application import ENV_PICKLE_FILENAME
+    filename = os.path.join(
+        app.env.doctreedir, curdoc, ENV_PICKLE_FILENAME)
+    try:
+        f = open(filename, 'rb')
+    except IOError:
+        app.info("")
+        app.warn("Unable to fetch %s "%filename)
+        return None
+    docenv = cPickle.load(f)
+    f.close()
+    return docenv
+
+def merge_js_index(app):
+    """
+    Merge the JS indexes of the sub-docs into the main JS index
+    """
+    app.info('')
+    app.info(bold('Merging js index files...'))
+    mapping = app.builder.indexer._mapping
+    for curdoc in app.env.config.multidocs_subdoc_list:
+        app.info("    %s:"%curdoc, nonl=1)
+        fixpath = lambda path: os.path.join(curdoc, path)
+        index = get_js_index(app, curdoc)
+        if index is not None:
+            # merge the mappings
+            app.info(" %s js index entries"%(len(index._mapping)))
+            for (ref, locs) in index._mapping.iteritems():
+                newmapping = set(map(fixpath, locs))
+                if ref in mapping:
+                    newmapping = mapping[ref] | newmapping
+                mapping[unicode(ref)] = newmapping
+            # merge the titles
+            titles = app.builder.indexer._titles
+            for (res, title) in index._titles.iteritems():
+                titles[fixpath(res)] = title
+            # TODO: merge indexer._objtypes, indexer._objnames as well
+
+            # Setup source symbolic links
+            dest = os.path.join(app.outdir, "_sources", curdoc)
+            if not os.path.exists(dest):
+                os.symlink(os.path.join("..", curdoc, "_sources"), dest)
+    app.info('... done (%s js index entries)'%(len(mapping)))
+    app.info(bold('Writing js search indexes...'), nonl=1)
+    return [] # no extra page to setup
+
+def get_js_index(app, curdoc):
+    """
+    Get the JS index of a sub-doc from the file
+    """
+    from sphinx.search import IndexBuilder, languages
+    # FIXME: find the correct lang
+    indexer = IndexBuilder(app.env, 'en',
+                           app.config.html_search_options)
+    indexfile = os.path.join(app.outdir, curdoc, 'searchindex.js')
+    try:
+        f = open(indexfile, 'rb')
+    except IOError:
+        app.info("")
+        app.warn("Unable to fetch %s "%indexfile)
+        return None
+    indexer.load(f, sphinx.search.js_index)
+    f.close()
+    return indexer
+
+
+mustbefixed = ['search', 'genindex', 'genindex-all'
+               'py-modindex', 'searchindex.js']
+def fix_path_html(app, pagename, templatename, ctx, event_arg):
+    """
+    Fixes the context so that the files
+    - search.html
+    - genindex.html
+    - py-modindex.html
+    points to the right place, that is in
+        reference/
+    instead of
+        reference/subdocument
+    """
+    # sphinx/builder/html.py line 702
+    # def pathto(otheruri, resource=False,
+    #            baseuri=self.get_target_uri(pagename)):
+    old_pathto = ctx['pathto']
+    def sage_pathto(otheruri, *args, **opts):
+        if otheruri in mustbefixed:
+            otheruri = os.path.join("..", otheruri)
+        return old_pathto(otheruri, *args, **opts)
+    ctx['pathto'] = sage_pathto
+
+
+def write_citations(app, citations):
+    """
+    Pickle the citation in a file
+    """
+    import cPickle
+    file = open(os.path.join(app.outdir, CITE_FILENAME), 'wb')
+    cPickle.dump(citations, file)
+    file.close()
+    app.info("Saved pickle file: %s"%CITE_FILENAME)
+
+
+def fetch_citation(app, env):
+    """
+    Fetch the global citation index from the refman to allows for cross
+    references.
+    """
+    app.builder.info(bold('loading cross citations... '), nonl=1)
+    filename = os.path.join(app.outdir, '..', CITE_FILENAME)
+    if not os.path.exists(filename):
+        return
+    import cPickle
+    file = open(filename, 'rb')
+    try:
+        cache = cPickle.load(file)
+    except:
+        app.warn("Cache file '%s' is corrupted; ignoring it..."% filename)
+        return
+    else:
+        app.builder.info("done (%s citations)."%len(cache))
+    finally:
+        file.close()
+    cite = env.citations
+    for ind, (path, tag) in cache.iteritems():
+        if ind not in cite: # don't override local citation
+            cite[ind]=(os.path.join("..", path), tag)
+
+def init_subdoc(app):
+    """
+    Init the merger depending on if we are compiling a subdoc or the master
+    doc itself.
+    """
+    if app.config.multidocs_is_master:
+        app.info(bold("Compiling the master document"))
+        app.connect('env-updated', merge_environment)
+        app.connect('html-collect-pages', merge_js_index)
+    else:
+        app.info(bold("Compiling a sub-document"))
+        app.connect('env-updated', fetch_citation)
+        app.connect('html-page-context', fix_path_html)
+
+        # Monkey patch copy_static_files to make a symlink to "../"
+        def link_static_files():
+            """
+            Instead of copying static files, make a link to the master static file.
+            See sphinx/builder/html.py line 536::
+
+                class StandaloneHTMLBuilder(Builder):
+                [...]
+                    def copy_static_files(self):
+                    [...]
+            """
+            app.builder.info(bold('linking _static directory.'))
+            static_dir = os.path.join(app.builder.outdir, '_static')
+            master_static_dir = os.path.join('..', '_static')
+            if not os.path.isdir(static_dir):
+                os.symlink(master_static_dir, static_dir)
+
+        app.builder.copy_static_files = link_static_files
+
+    if app.config.multidoc_first_pass == 1:
+        app.config.intersphinx_mapping = {}
+
+
+def setup(app):
+    app.add_config_value('multidocs_is_master', True, True)
+    app.add_config_value('multidocs_subdoc_list', [], True)
+    app.add_config_value('multidoc_first_pass', 0, False)   # 1 = deactivate the loading of the inventory
+    app.connect('builder-inited', init_subdoc)
diff --git a/doc/en/reference/conf.py b/doc/en/reference/conf.py
--- a/doc/en/reference/conf.py
+++ b/doc/en/reference/conf.py
@@ -61,9 +61,10 @@ latex_elements['preamble'] += r'''
 #Ignore all .rst in the _sage subdirectory
 exclude_trees = exclude_trees + ['_sage']
 
-# List of directories, relative to source directory, that shouldn't be
-# searched for source files.
-exclude_trees = exclude_trees + [
+multidocs_is_master = True
+
+# List of subdocs
+multidocs_subdoc_list = [
     'algebras',
     'arithgroup',
     'calculus',
@@ -100,7 +101,6 @@ exclude_trees = exclude_trees + [
     'notebook',
     'number_fields',
     'numerical',
-    'options',
     'padics',
     'parallel',
     'plane_curves',
@@ -114,12 +114,15 @@ exclude_trees = exclude_trees + [
     'rings',
     'rings_numerical',
     'rings_standard',
-    'sage',
-    'sagenb',
     'schemes',
     'semirings',
     'stats',
     'structure',
-    'tensor',
-    'todolist'
+    'tensor'
     ]
+
+# List of directories, relative to source directory, that shouldn't be
+# searched for source files.
+exclude_trees += multidocs_subdoc_list + [
+    'sage', 'sagenb', 'options'
+    ]
diff --git a/doc/en/reference/conf_sub.py b/doc/en/reference/conf_sub.py
--- a/doc/en/reference/conf_sub.py
+++ b/doc/en/reference/conf_sub.py
@@ -69,3 +69,5 @@ latex_documents = [
 
 #Ignore all .rst in the _sage subdirectory
 exclude_trees = exclude_trees + ['_sage']
+
+multidocs_is_master = False
diff --git a/doc/en/reference/footer.txt b/doc/en/reference/footer.txt
--- a/doc/en/reference/footer.txt
+++ b/doc/en/reference/footer.txt
@@ -1,12 +1,6 @@
-
-.. todolist::
-
 Indices and Tables
 ==================
 
-.. toctree::
-   :maxdepth: 1
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+* `Index <../genindex.html>`_
+* `Module Index <../py-modindex.html>`_
+* `Search Page <../search.html>`_
diff --git a/doc/en/reference/index.rst b/doc/en/reference/index.rst
--- a/doc/en/reference/index.rst
+++ b/doc/en/reference/index.rst
@@ -14,12 +14,12 @@ usage of Sage. The examples are all test
 Sage, and should produce exactly the same output as in this manual,
 except for line breaks.
 
-The Sage command line is briefly described in `The Sage Command Line 
-<cmd/index.html>`_, which lists the command line options. For more
+The Sage command line is briefly described in :doc:`The Sage Command Line 
+<cmd/index>`, which lists the command line options. For more
 details about the command line, see the Sage tutorial.
 
-The Sage graphical user interface is described in `The Sage Notebook 
-<notebook/index.html>`_. This graphical user interface is unusual in
+The Sage graphical user interface is described in :doc:`The Sage Notebook 
+<notebook/index>`. This graphical user interface is unusual in
 that it operates via your web browser. It provides you with Sage
 worksheets that you can edit and evaluate, which contain scalable
 typeset mathematics and beautiful antialiased images.
@@ -34,100 +34,109 @@ Enjoy Sage!
 Table of Contents
 =================
 
-   * `The Sage Command Line <cmd/index.html>`_
-   * `The Sage Notebook <notebook/index.html>`_
- 
+* :doc:`The Sage Command Line <cmd/index>`
+* :doc:`The Sage Notebook <notebook/index>`
+
 Calculus, Plotting
 ------------------
- 
-   * `Symbolic Calculus <calculus/index.html>`_
-   * `Constants <constants/index.html>`_
-   * `Functions <functions/index.html>`_
-   * `2D Graphics <plotting/index.html>`_
-   * `3D Graphics <plot3d/index.html>`_
- 
+
+* :doc:`Symbolic Calculus <calculus/index>`
+* :doc:`Constants <constants/index>`
+* :doc:`Functions <functions/index>`
+* :doc:`2D Graphics <plotting/index>`
+* :doc:`3D Graphics <plot3d/index>`
+
 Combinatorics, Discrete Mathematics
 -----------------------------------
- 
-   * `Combinatorics <combinat/index.html>`_
-   * `Graph Theory <graphs/index.html>`_
- 
+
+* :doc:`Combinatorics <combinat/index>`
+* :doc:`Graph Theory <graphs/index>`
+
 Structures, Coercion, Categories
 --------------------------------
 
-   * `Basic Structures <structure/index.html>`_
-   * `Coercion <coercion/index.html>`_
-   * `Category Theory and Categories <categories/index.html>`_
+* :doc:`Basic Structures <structure/index>`
+* :doc:`Coercion <coercion/index>`
+* :doc:`Category Theory and Categories <categories/index>`
 
 Rings, Fields, Algebras
 -----------------------
 
-   * `General Rings, Ideals, and Morphisms <rings/index.html>`_
-   * `Standard Commutative Rings <rings_standard/index.html>`_
-   * `Fixed and Arbitrary Precision Numerical Fields <rings_numerical/index.html>`_
-   * `Algebraic Number Fields <number_fields/index.html>`_
-   * `Function Fields <function_fields/index.html>`_
-   * `p-Adics <padics/index.html>`_
-   * `Polynomial Rings <polynomial_rings/index.html>`_
-   * `Power Series Rings <power_series/index.html>`_
-   * `Standard Semirings <semirings/index.html>`_
-   * `Algebras <algebras/index.html>`_
-   * `Quaternion Algebras <quat_algebras/index.html>`_
+* :doc:`General Rings, Ideals, and Morphisms <rings/index>`
+* :doc:`Standard Commutative Rings <rings_standard/index>`
+* :doc:`Fixed and Arbitrary Precision Numerical Fields <rings_numerical/index>`
+* :doc:`Algebraic Number Fields <number_fields/index>`
+* :doc:`Function Fields <function_fields/index>`
+* :doc:`p-Adics <padics/index>`
+* :doc:`Polynomial Rings <polynomial_rings/index>`
+* :doc:`Power Series Rings <power_series/index>`
+* :doc:`Standard Semirings <semirings/index>`
+* :doc:`Algebras <algebras/index>`
+* :doc:`Quaternion Algebras <quat_algebras/index>`
 
 Groups, Monoids, Matrices, Modules
 ----------------------------------
 
-   * `Groups <groups/index.html>`_
-   * `Monoids <monoids/index.html>`_
-   * `Matrices and Spaces of Matrices <matrices/index.html>`_
-   * `Modules <modules/index.html>`_
+* :doc:`Groups <groups/index>`
+* :doc:`Monoids <monoids/index>`
+* :doc:`Matrices and Spaces of Matrices <matrices/index>`
+* :doc:`Modules <modules/index>`
 
 Geometry and Topology
 ---------------------
 
-   * `Combinatorial Geometry <geometry/index.html>`_
-   * `Cell Complexes and their Homology <homology/index.html>`_
-   * `Differential Forms <tensor/index.html>`_
+* :doc:`Combinatorial Geometry <geometry/index>`
+* :doc:`Cell Complexes and their Homology <homology/index>`
+* :doc:`Differential Forms <tensor/index>`
 
 Number Theory, Algebraic Geometry
 ---------------------------------
 
-   * `Quadratic Forms <quadratic_forms/index.html>`_
-   * `L-Functions <lfunctions/index.html>`_
-   * `Schemes <schemes/index.html>`_
-   * `Elliptic, Plane, and Hyperelliptic Curves <plane_curves/index.html>`_
-   * `Arithmetic Subgroups of SL_2(Z) <arithgroup/index.html>`_
-   * `General Hecke Algebras and Hecke Modules <hecke/index.html>`_
-   * `Modular Symbols <modsym/index.html>`_
-   * `Modular Forms <modfrm/index.html>`_
-   * `Modular Abelian Varieties <modabvar/index.html>`_
-   * `Miscellaneous Modular-Form-Related Modules <modmisc/index.html>`_
+* :doc:`Quadratic Forms <quadratic_forms/index>`
+* :doc:`L-Functions <lfunctions/index>`
+* :doc:`Schemes <schemes/index>`
+* :doc:`Elliptic, Plane, and Hyperelliptic Curves <plane_curves/index>`
+* :doc:`Arithmetic Subgroups of SL_2(Z) <arithgroup/index>`
+* :doc:`General Hecke Algebras and Hecke Modules <hecke/index>`
+* :doc:`Modular Symbols <modsym/index>`
+* :doc:`Modular Forms <modfrm/index>`
+* :doc:`Modular Abelian Varieties <modabvar/index>`
+* :doc:`Miscellaneous Modular-Form-Related Modules <modmisc/index>`
 
 Miscellaneous Mathematics
 -------------------------
 
-   * `Games <games/index.html>`_
-   * `Symbolic Logic <logic/index.html>`_
-   * `Cryptography <cryptography/index.html>`_
-   * `Numerical Optimization <numerical/index.html>`_
-   * `Probability <probability/index.html>`_
-   * `Statistics <stats/index.html>`_
-   * `Quantitative Finance <finance/index.html>`_
-   * `Coding Theory <coding/index.html>`_
+* :doc:`Games <games/index>`
+* :doc:`Symbolic Logic <logic/index>`
+* :doc:`Cryptography <cryptography/index>`
+* :doc:`Numerical Optimization <numerical/index>`
+* :doc:`Probability <probability/index>`
+* :doc:`Statistics <stats/index>`
+* :doc:`Quantitative Finance <finance/index>`
+* :doc:`Coding Theory <coding/index>`
 
 Interfaces, Databases, Miscellany
 ---------------------------------
 
-   * `Interpreter Interfaces <interfaces/index.html>`_
-   * `C/C++ Library Interfaces <libs/index.html>`_
-   * `Databases <databases/index.html>`_
-   * `Parallel Computing <parallel/index.html>`_
-   * `Miscellaneous <misc/index.html>`_
+* :doc:`Interpreter Interfaces <interfaces/index>`
+* :doc:`C/C++ Library Interfaces <libs/index>`
+* :doc:`Databases <databases/index>`
+* :doc:`Parallel Computing <parallel/index>`
+* :doc:`Miscellaneous <misc/index>`
 
 Other
 -----
 
-   * `Sage's To Do List <todolist/index.html>`_
-   * `History and License <history_and_license/index.html>`_
+.. toctree::
+   :maxdepth: 2
 
-.. include:: footer.txt
+   todolist
+
+* :doc:`History and License <history_and_license/index>`
+
+Indices and Tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/doc/en/reference/todolist.rst b/doc/en/reference/todolist.rst
new file mode 100644
--- /dev/null
+++ b/doc/en/reference/todolist.rst
@@ -0,0 +1,18 @@
+SAGE's To Do list
+=================
+
+There is still some work to do :-) :
+
+.. warning::
+
+    This list is currently very incomplete as most doctests do not use the
+    ``.. todo::`` markup.
+
+    .. todo::
+
+        Rewrite the hand-written TODOs by using the correct ``.. todo::``
+        markup.
+
+The combined to do list is only available in the html version of the reference manual.
+
+.. todolist::
diff --git a/doc/en/reference/todolist/conf.py b/doc/en/reference/todolist/conf.py
deleted file mode 100644
--- a/doc/en/reference/todolist/conf.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# -*- coding: utf-8 -*-
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# The contents of this file are pickled, so don't put values in the
-# namespace that aren't pickleable (module imports are okay, they're
-# removed automatically).
-#
-# All configuration values have a default; values that are commented
-# out serve to show the default.
-
-# See the parent directory's conf_sub module for details.
-import sys
-sys.path.append('..')
-from conf_sub import *
diff --git a/doc/en/reference/todolist/index.rst b/doc/en/reference/todolist/index.rst
deleted file mode 100644
--- a/doc/en/reference/todolist/index.rst
+++ /dev/null
@@ -1,18 +0,0 @@
-SAGE's To Do list
-=================
-
-There is still some work to do :-) :
-
-.. warning::
-
-    This list is currently very incomplete as most doctests do not use the
-    ``.. todo::`` markup.
-
-    .. todo::
-
-        Rewrite the hand-written TODOs by using the correct ``.. todo::``
-        markup.
-
-The combined to do list is only available in the html version of the reference manual.
-
-.. todolist::
