Ticket #9631: trac_9631-fork_decorator.5.patch

File trac_9631-fork_decorator.5.patch, 14.4 KB (added by leif, 10 years ago)

Some more docstring fixes.

  • sage/parallel/all.py

    # HG changeset patch
    # User Leif Leonhardy <not.really@online.de>
    # Date 1313395172 -7200
    # Node ID 59a811865bb9a0a7451fa5ca838787c7a80f5a6d
    # Parent  310879d07e4781a655ea93fb349ef9d87e877f93
    #9631: Make a @fork decorator
    
    diff -r 310879d07e47 -r 59a811865bb9 sage/parallel/all.py
    a b  
    1 from decorate import parallel
     1from decorate import parallel, fork
    22
  • sage/parallel/decorate.py

    diff -r 310879d07e47 -r 59a811865bb9 sage/parallel/decorate.py
    a b  
    11r"""
    2 Decorate interface for parallel computation
     2Decorate interface for parallel computation.
    33"""
    44
    55import os
     
    1313
    1414def normalize_input(a):
    1515    r"""
    16     Convert a to a pair (args, kwds) using some rules:
     16    Convert ``a`` to a pair ``(args, kwds)`` using some rules:
    1717
    18         * if already of that form, leave that way.
    19         * if a is a tuple make (a,{})
    20         * if a is a dict make (tuple([]),a)
    21         * otherwise make ((a,),{})
     18    - if already of that form, leave that way.
     19    - if ``a`` is a tuple make ``(a,{})``
     20    - if ``a`` is a dict make ``(tuple([]),a)``
     21    - otherwise make ``((a,),{})``
    2222
    2323    INPUT:
    2424
     
    2626
    2727    OUTPUT:
    2828
    29     - ``args`` -- tuple
    30     - ``kwds`` -- dictionary
     29     - ``args`` -- tuple
     30     - ``kwds`` -- dictionary
    3131
    3232    EXAMPLES::
    3333
     
    5252
    5353class Parallel:
    5454    r"""
    55     Create parallel decorated function.
     55    Create a ``parallel``-decorated function.
     56    """
     57    def __init__(self, p_iter='fork', ncpus=None, **kwds):
     58        """
     59        EXAMPLES::
    5660
    57     """
    58     def __init__(self, p_iter = 'fork', ncpus=None, **kwds):
    59         # The default p_iter is currently the reference implementation.
    60         # This may change.
     61            sage: P = sage.parallel.decorate.Parallel(); P
     62            <sage.parallel.decorate.Parallel instance at 0x...>
     63        """
     64        # The default p_iter is currently the 'fork' implementation.
     65        # This has changed.
     66
    6167        self.p_iter = None
    6268
    6369        if isinstance(p_iter, (int, long, Integer)):
     
    8187
    8288    def __call__(self, f):
    8389        r"""
    84         Create a function that wraps f and that when called with a
    85         list of inputs returns an iterator over pairs
    86              (x, f(x))
    87         in possibly random order. Here x is replaced by its
    88         normalized form (args, kwds) using normalize_inputs.
     90        Create a function that wraps ``f()`` and that when called with a
     91        list of inputs returns an iterator over pairs ``(x, f(x))``
     92        in possibly random order.  Here ``x`` is replaced by its
     93        normalized form ``(args, kwds)`` using ``normalize_inputs()``.
    8994
    9095        INPUT:
    9196
     
    9398
    9499        OUTPUT:
    95100
    96         decorated version of f
     101         - Decorated version of ``f``
    97102
    98103        EXAMPLES::
    99104
     
    105110            sage: p(f)(x=5)
    106111            24
    107112
     113            sage: P = sage.parallel.decorate.Parallel()
     114            sage: def g(n,m): return n+m
     115            sage: h = P(g)          # indirect doctest
     116            sage: list(h([(2,3)]))
     117            [(((2, 3), {}), 5)]
    108118        """
    109119        # Construct the wrapper parallel version of the function we're wrapping.
    110120        # We may rework this so g is a class instance, which has the plus that
     
    116126                return f(*args, **kwds)
    117127        return g
    118128
    119 def parallel(p_iter = 'fork', ncpus=None, **kwds):
     129def parallel(p_iter='fork', ncpus=None, **kwds):
    120130    r"""
    121131    This is a decorator that gives a function a parallel interface,
    122132    allowing it to be called with a list of inputs, whose values will
    123133    be computed in parallel.
    124134
     135    .. warning::
     136
     137         The parallel subprocesses will not have access to data
     138         created in pexpect interfaces.  This behavior with respect to
     139         pexpect interfaces is very important to keep in mind when
     140         setting up certain computations.  It's the one big limitation
     141         of this decorator.
     142
    125143    INPUT:
    126144
    127145     - ``p_iter`` -- parallel iterator function or string:
    128             - 'fork'            -- (default) use a new fork for each input
    129             - 'multiprocessing' -- use multiprocessing library
    130             - 'reference'       -- use a fake serial reference implementation
    131      - ``ncpus`` -- integer, number of cpus
    132      - ``timeout`` -- number of seconds until task is killed (only supported by 'fork')
     146            - ``'fork'``            -- (default) use a new forked subprocess for each input
     147            - ``'multiprocessing'`` -- use multiprocessing library
     148            - ``'reference'``       -- use a fake serial reference implementation
     149     - ``ncpus`` -- integer, maximal number of subprocesses to use at the same time
     150     - ``timeout`` -- number of seconds until each subprocess is killed (only supported
     151       by 'fork'; zero means not at all)
     152
     153    .. warning::
     154
     155         If you use anything but ``'fork'`` above, then a whole new
     156         subprocess is spawned, so none of your local state (variables,
     157         certain functions, etc.) is available.
     158
    133159
    134160    EXAMPLES:
    135161
    136     We create a simple decoration for a simple function. The number
    137     of cpus is automatically detected::
     162    We create a simple decoration for a simple function.  The number
     163    of cpus (or cores, or hardware threads) is automatically detected::
    138164
    139165        sage: @parallel
    140166        ... def f(n): return n*n
     
    143169        sage: sorted(list(f([1,2,3])))
    144170        [(((1,), {}), 1), (((2,), {}), 4), (((3,), {}), 9)]
    145171
    146     We use exactly 2 cpus::
     172    We use exactly two cpus::
    147173
    148174        sage: @parallel(2)
    149175        ... def f(n): return n*n
    150176
    151177
    152     We create a decorator that uses 3 processes, and times out
     178    We create a decorator that uses three subprocesses, and times out
    153179    individual processes after 10 seconds::
    154180
    155181        sage: @parallel(ncpus=3, timeout=10)
     
    174200    if isinstance(p_iter, types.FunctionType):
    175201        return Parallel()(p_iter)
    176202    return Parallel(p_iter, ncpus, **kwds)
     203
     204
     205
     206
     207###################################################################
     208# The @fork decorator -- evaluate a function with no side effects
     209# in memory, so the only side effects (if any) are on disk.
     210#
     211# We have both a function and a class below, so that the decorator
     212# can be used with or without options:
     213#
     214#   @fork
     215#   def f(...): ...
     216# and
     217#   @fork(...options...):
     218#   def f(...): ...
     219###################################################################
     220
     221class Fork:
     222    """
     223    A ``fork`` decorator class.
     224    """
     225    def __init__(self, timeout=0, verbose=False):
     226        """
     227        INPUT:
     228
     229         - ``timeout`` -- (default: 0) kill the subprocess after it has run this
     230           many seconds (wall time), or if ``timeout`` is zero, do not kill it.
     231         - ``verbose`` -- (default: ``False``) whether to print anything about
     232           what the decorator does (e.g., killing the subprocess)
     233
     234        EXAMPLES::
     235       
     236            sage: sage.parallel.decorate.Fork()
     237            <sage.parallel.decorate.Fork instance at 0x...>
     238            sage: sage.parallel.decorate.Fork(timeout=3)
     239            <sage.parallel.decorate.Fork instance at 0x...>
     240        """
     241        self.timeout = timeout
     242        self.verbose = verbose
     243       
     244    def __call__(self, f):
     245        """
     246        INPUT:
     247
     248         - ``f`` -- a function
     249
     250        OUTPUT:
     251
     252         - A decorated function.
     253
     254        EXAMPLES::
     255       
     256            sage: F = sage.parallel.decorate.Fork(timeout=3)
     257            sage: def g(n,m): return n+m
     258            sage: h = F(g)     # indirect doctest
     259            sage: h(2,3)
     260            5           
     261        """
     262        P = Parallel(p_iter='fork', ncpus=1, timeout=self.timeout,
     263                     verbose=self.verbose)
     264        g = P(f)
     265        def h(*args, **kwds):
     266            return list(g([(args, kwds)]))[0][1]
     267        return h
     268
     269def fork(f=None, timeout=0, verbose=False):
     270    """
     271    Decorate a function so that when called it runs in a forked
     272    subprocess.  This means that it won't have any in-memory
     273    side effects on the parent Sage process.  The pexpect interfaces
     274    are all reset.
     275   
     276    INPUT:
     277
     278      - ``f`` -- a function
     279      - ``timeout`` -- (default: 0) if positive, kill the subprocess after
     280        this many seconds (wall time)
     281      - ``verbose`` -- (default: ``False``) whether to print anything
     282        about what the decorator does (e.g., killing the subprocess)
     283
     284    .. warning::
     285
     286        The forked subprocess will not have access to data created
     287        in pexpect interfaces.  This behavior with respect to pexpect
     288        interfaces is very important to keep in mind when setting up
     289        certain computations.  It's the one big limitation of this
     290        decorator.
     291
     292    EXAMPLES:
     293
     294    We create a function and run it with the ``fork`` decorator.  Note
     295    that it does not have a side effect.  Despite trying to change
     296    the global variable ``a`` below in ``g``, the variable ``a`` does not
     297    get changed::
     298   
     299        sage: a = 5
     300        sage: @fork
     301        ... def g(n, m):
     302        ...       global a
     303        ...       a = 10
     304        ...       return factorial(n).ndigits() + m
     305        sage: g(5, m=5)
     306        8
     307        sage: a
     308        5
     309
     310    We use ``fork`` to make sure that the function terminates after one
     311    second, no matter what::
     312   
     313        sage: @fork(timeout=1, verbose=True)
     314        ... def g(n, m): return factorial(n).ndigits() + m
     315        sage: g(5, m=5)
     316        8
     317        sage: g(10^7, m=5)
     318        Killing subprocess ... with input ((10000000,), {'m': 5}) which took too long
     319        'NO DATA (timed out)'
     320
     321    We illustrate that the state of the pexpect interface is not altered by
     322    forked functions (they get their own new pexpect interfaces!)::
     323   
     324        sage: gp.eval('a = 5')
     325        '5'
     326        sage: @fork()
     327        ... def g():
     328        ...       gp.eval('a = 10')
     329        ...       return gp.eval('a')
     330        sage: g()
     331        '10'
     332        sage: gp.eval('a')
     333        '5'
     334
     335    We illustrate that the forked function has its own pexpect
     336    interface::
     337   
     338        sage: gp.eval('a = 15')
     339        '15'
     340        sage: @fork()
     341        ... def g(): return gp.eval('a')
     342        sage: g()
     343        'a'
     344
     345    We illustrate that segfaulting subprocesses are no trouble at all::
     346
     347        sage: cython('def f(): print <char*>0')
     348        sage: @fork
     349        ... def g(): f()
     350        sage: g()
     351        'NO DATA'
     352    """
     353    F = Fork(timeout=timeout, verbose=verbose)
     354    return F(f) if f else F
  • sage/parallel/use_fork.py

    diff -r 310879d07e47 -r 59a811865bb9 sage/parallel/use_fork.py
    a b  
    11"""
    2 Parallel Iterator built using the fork system call
     2Parallel iterator built using the ``fork()`` system call.
    33"""
    44
    55################################################################################
     
    1313
    1414class p_iter_fork:
    1515    """
    16     A parallel iterator implemented using fork.
     16    A parallel iterator implemented using ``fork()``.
    1717
    1818    EXAMPLES::
    1919
     
    2626        sage: X.verbose
    2727        False
    2828    """
    29     def __init__(self, ncpus, timeout=0, verbose=False):
     29    def __init__(self, ncpus, timeout=0, verbose=False, reset_interfaces=True):
    3030        """
    31         Create fork-based parallel iterator.
     31        Create a ``fork()``-based parallel iterator.
    3232       
    3333        INPUT:
    3434
    35             - ``ncpus`` -- the maximal number of simultaneous processes to spawn
    36             - ``timeout`` -- (float) time in seconds until a subprocess is automatically killed
    37             - ``verbose`` -- whether to print anything about what the iterator does (e.g., killing subprocesses)
     35            - ``ncpus`` -- the maximal number of simultaneous
     36              subprocesses to spawn
     37            - ``timeout`` -- (float, default: 0) wall time in seconds until
     38              a subprocess is automatically killed
     39            - ``verbose`` -- (default: False) whether to print
     40              anything about what the iterator does (e.g., killing
     41              subprocesses)
     42            - ``reset_interfaces`` -- (default: True) whether to reset
     43              all pexpect interfaces
    3844
    3945        EXAMPLES::
    4046
     
    5258            raise TypeError, "ncpus must be an integer"
    5359        self.timeout = float(timeout)  # require a float
    5460        self.verbose = verbose
     61        self.reset_interfaces = reset_interfaces
    5562
    5663    def __call__(self, f, inputs):
    5764        """
    58         Parallel iterator using fork.
     65        Parallel iterator using ``fork()``.
    5966
    6067        INPUT:
    6168
    62             - `f` -- a Python function that need not be picklable or anything else!
    63             - ``inputs`` -- a list of pickleable pairs (args, kwds), where
    64               args is a tuple and kwds is a dictionary.
     69            - ``f`` -- a Python function that need not be pickleable or anything else!
     70            - ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
     71              ``args`` is a tuple and ``kwds`` is a dictionary.
    6572
    6673        OUTPUT:
    6774
     
    171178           
    172179    def _subprocess(self, f, dir, x):
    173180        """
    174         Setup and run evaluation of f(*x[0], **x[1]), storing the
    175         result in the given directory.  This method is called by each
     181        Setup and run evaluation of ``f(*x[0], **x[1])``, storing the
     182        result in the given directory ``dir``.  This method is called by each
    176183        forked subprocess.
    177184
    178185        INPUT:
    179186
    180             - `f` -- a function
     187            - ``f`` -- a function
    181188            - ``dir`` -- name of a directory
    182             - `x` -- 2-tuple, with args and kwds
     189            - ``x`` -- 2-tuple, with args and kwds
    183190
    184         EXAMPLES::
     191        EXAMPLES:
    185192
    186193        We have only this indirect test, since a direct test would terminate the Sage session.
    187194       
     
    204211            import sage.misc.misc
    205212            reload(sage.misc.misc)
    206213
    207             # The expect interfaces (and objects defined in them) are
     214            # The pexpect interfaces (and objects defined in them) are
    208215            # not valid.
    209             sage.interfaces.quit.invalidate_all()
     216            if self.reset_interfaces:
     217                sage.interfaces.quit.invalidate_all()
    210218
    211219            # Now evaluate the function f.
    212220            value = f(*x[0], **x[1])