Ticket #6255: paral.patch

File paral.patch, 2.0 KB (added by hivert, 8 years ago)

Experimental parallel doc output patch

  • builders/__init__.py

    # HG changeset patch
    # User Florent Hivert <Florent.Hivert@univ-rouen.fr>
    # Date 1335023898 -7200
    # Node ID 8902c2655ee72286e003730170a1fbd6eba5e421
    # Parent  5e10ad3f16da15a33dff7ea01cb2bb365930438e
    imported patch paral.patch
    
    diff --git a/builders/__init__.py b/builders/__init__.py
    a b class Builder(object): 
    291291        # write target files
    292292        warnings = []
    293293        self.env.set_warnfunc(lambda *args: warnings.append(args))
     294        # for docname in self.status_iterator(
     295        #     sorted(docnames), 'writing output... ', darkgreen, len(docnames)):
     296        #     doctree = self.env.get_and_resolve_doctree(docname, self)
     297        #     self.write_doc(docname, doctree)
     298        from sage.parallel.decorate import parallel
     299        import itertools
     300        worker = parallel('fork')(self.write_doc_parallel_worker)
     301        sdocnames = sorted(docnames)
     302        docnameslices = []
     303        slicesize = 20 # max(min(20, len(docname)/20), 5)
     304        for i in range(0, len(sdocnames), slicesize):
     305            docnameslices.append(sdocnames[i:i+slicesize])
     306        pariter = itertools.imap(lambda x:x[1], worker(sorted(docnameslices)))
    294307        for docname in self.status_iterator(
    295             sorted(docnames), 'writing output... ', darkgreen, len(docnames)):
    296             doctree = self.env.get_and_resolve_doctree(docname, self)
    297             self.write_doc(docname, doctree)
     308            pariter, 'writing output... ', darkgreen, len(docnameslices)):
     309            # done in the iterator !!!
     310            pass
    298311        for warning in warnings:
    299312            self.warn(*warning)
    300313        self.env.set_warnfunc(self.warn)
    301314
     315    def write_doc_parallel_worker(self, docnames):
     316        for docname in docnames:
     317            # print docname
     318            doctree = self.env.get_and_resolve_doctree(docname, self)
     319            self.write_doc(docname, doctree)
     320        return docname
     321
    302322    def prepare_writing(self, docnames):
    303323        raise NotImplementedError
    304324