Ticket #9631: trac_9631-fork_decorator.3.patch

File trac_9631-fork_decorator.3.patch, 10.7 KB (added by vbraun, 10 years ago)

One more docfix

  • sage/parallel/all.py

    # HG changeset patch
    # User William Stein <wstein@gmail.com>
    # Date 1279238019 -7200
    # Node ID edf108f6206371421ae4f2fa27b40ffc8c544287
    # Parent  deec96694006a4a64290153462f827f44b3bcd57
    #9631: Make an @fork decorator
    
    diff -r deec96694006 -r edf108f62063 sage/parallel/all.py
    a b  
    1 from decorate import parallel
     1from decorate import parallel, fork
    22
  • sage/parallel/decorate.py

    diff -r deec96694006 -r edf108f62063 sage/parallel/decorate.py
    a b  
    1515    r"""
    1616    Convert a to a pair (args, kwds) using some rules:
    1717
    18         * if already of that form, leave that way.
    19         * if a is a tuple make (a,{})
    20         * if a is a dict make (tuple([]),a)
    21         * otherwise make ((a,),{})
     18    - if already of that form, leave that way.
     19    - if ``a`` is a tuple make ``(a,{})``
     20    - if ``a`` is a dict make ``(tuple([]),a)``
     21    - otherwise make ``((a,),{})``
    2222
    2323    INPUT:
    2424
     
    5353class Parallel:
    5454    r"""
    5555    Create parallel decorated function.
    56 
    5756    """
    5857    def __init__(self, p_iter = 'fork', ncpus=None, **kwds):
     58        """
     59        EXAMPLES::
     60
     61            sage: P = sage.parallel.decorate.Parallel(); P
     62            <sage.parallel.decorate.Parallel instance at 0x...>
     63        """
    5964        # The default p_iter is currently the reference implementation.
    6065        # This may change.
    6166        self.p_iter = None
     
    8186
    8287    def __call__(self, f):
    8388        r"""
    84         Create a function that wraps f and that when called with a
    85         list of inputs returns an iterator over pairs
    86              (x, f(x))
    87         in possibly random order. Here x is replaced by its
     89        Create a function that wraps ``f`` and that when called with a
     90        list of inputs returns an iterator over pairs ``(x, f(x))`` in
     91        possibly random order. Here ``x`` is replaced by its
    8892        normalized form (args, kwds) using normalize_inputs.
    8993
    9094        INPUT:
     
    9397
    9498        OUTPUT:
    9599
    96         decorated version of f
     100        Decorated version of ``f``
    97101
    98102        EXAMPLES::
    99103
     
    105109            sage: p(f)(x=5)
    106110            24
    107111
     112            sage: P = sage.parallel.decorate.Parallel()
     113            sage: def g(n,m): return n+m
     114            sage: h = P(g)          # indirect doctest
     115            sage: list(h([(2,3)]))
     116            [(((2, 3), {}), 5)]
    108117        """
    109118        # Construct the wrapper parallel version of the function we're wrapping.
    110119        # We may rework this so g is a class instance, which has the plus that
     
    122131    allowing it to be called with a list of inputs, whose values will
    123132    be computed in parallel.
    124133
     134    .. warning::
     135
     136         The parallel subprocesses will not have access to data
     137         created in pexpect interfaces.  This behavior with respect to
     138         pexpect interfaces is very important to keep in mind when
     139         setting up certain computations. It's the one big limitation
     140         of this decorator.
     141
    125142    INPUT:
    126143
    127144     - ``p_iter`` -- parallel iterator function or string:
    128             - 'fork'            -- (default) use a new fork for each input
    129             - 'multiprocessing' -- use multiprocessing library
    130             - 'reference'       -- use a fake serial reference implementation
     145            - ``fork``            -- (default) use a new fork for each input
     146            - ``multiprocessing`` -- use multiprocessing library
     147            - ``reference``       -- use a fake serial reference implementation
    131148     - ``ncpus`` -- integer, number of cpus
    132149     - ``timeout`` -- number of seconds until task is killed (only supported by 'fork')
    133150
     151    .. warning::
     152
     153         If you use anything but 'fork' above, then a whole new
     154         subprocess is spawned, so all your local state (variables,
     155         new functions, etc.,) is not available.
     156
    134157    EXAMPLES:
    135158
    136159    We create a simple decoration for a simple function. The number
     
    148171        sage: @parallel(2)
    149172        ... def f(n): return n*n
    150173
    151 
    152174    We create a decorator that uses 3 processes, and times out
    153175    individual processes after 10 seconds::
    154176
     
    174196    if isinstance(p_iter, types.FunctionType):
    175197        return Parallel()(p_iter)
    176198    return Parallel(p_iter, ncpus, **kwds)
     199
     200
     201
     202
     203###################################################################
     204# The @fork decorator -- run a function with no sideeffects in memory
     205# (so only side effects are on disk).
     206# We have both a function and a class below, so that the decorator
     207# can be used with or without options:
     208#   @fork
     209#   def f(...): ...
     210# and
     211#   @fork(...options...):
     212#   def f(...): ...
     213###################################################################
     214
     215class Fork:
     216    """
     217    A fork decorator class.
     218    """
     219    def __init__(self, timeout=0, verbose=False):
     220        """
     221        INPUT:
     222
     223        - ``timeout`` -- (default: 0) kills subrocess after this many
     224          seconds, or if timeout=0, do not kill the subprocess.
     225
     226        - ``verbose`` -- (default: ``False``) whether to print
     227          anything about what the decorator does (e.g., killing
     228          subprocesses)
     229
     230        EXAMPLES::
     231       
     232            sage: sage.parallel.decorate.Fork()
     233            <sage.parallel.decorate.Fork instance at 0x...>
     234            sage: sage.parallel.decorate.Fork(timeout=3)
     235            <sage.parallel.decorate.Fork instance at 0x...>
     236        """
     237        self.timeout = timeout
     238        self.verbose = verbose
     239       
     240    def __call__(self, f):
     241        """
     242        INPUT:
     243
     244        - ``f`` -- a function
     245
     246        OUTPUT:
     247           
     248        A decorated function.
     249
     250        EXAMPLES::
     251       
     252            sage: F = sage.parallel.decorate.Fork(timeout=3)
     253            sage: def g(n,m): return n+m
     254            sage: h = F(g)     # indirect doctest
     255            sage: h(2,3)
     256            5           
     257        """
     258        P = Parallel(p_iter='fork', ncpus=1, timeout=self.timeout,
     259                     verbose=self.verbose)
     260        g = P(f)
     261        def h(*args, **kwds):
     262            return list(g([(args, kwds)]))[0][1]
     263        return h
     264
     265def fork(f=None, timeout=0, verbose=False):
     266    """
     267    Decorate a function so that when called it runs in a forked
     268    subprocess.  This means that it won't have any in-memory
     269    side-effects on the master Sage process.  The pexpect interfaces
     270    are all reset.
     271   
     272    INPUT:
     273
     274    - ``f`` -- a function
     275   
     276    - ``timeout`` -- (default: 0) kills subrocess after this many
     277      seconds, or if timeout=0, do not kill the subprocess.
     278
     279    - ``verbose`` -- (default: ``False``) whether to print anything
     280      about what the decorator does (e.g., killing subprocesses)
     281
     282    .. warning::
     283
     284        The forked subprocesses will not have access to data created
     285        in pexpect interfaces.  This behavior with respect to pexpect
     286        interfaces is very important to keep in mind when setting up
     287        certain computations. It's the one big limitation of this
     288        decorator.
     289
     290    EXAMPLES:
     291
     292    We create a function and run it with the fork decorator.  Note
     293    that it does not have a side effect.  Despite trying to change
     294    that global variable "a" below, it does not get changed.::
     295   
     296        sage: a = 5
     297        sage: @fork
     298        ... def g(n, m):
     299        ...       global a
     300        ...       a = 10
     301        ...       return factorial(n).ndigits() + m
     302        sage: g(5, m=5)
     303        8
     304        sage: a
     305        5
     306
     307    We use fork to make sure that the function dies after 1 second no matter what::
     308   
     309        sage: @fork(timeout=1, verbose=True)
     310        ... def g(n, m): return factorial(n).ndigits() + m
     311        sage: g(5, m=5)
     312        8
     313        sage: g(10^7, m=5)
     314        Killing subprocess ... with input ((10000000,), {'m': 5}) which took too long
     315        'NO DATA (timed out)'
     316
     317    We illustrate that pexpect interface state is not effected by
     318    forked functions (they get their own new pexpect interfaces!)::
     319   
     320        sage: gp.eval('a = 5')
     321        '5'
     322        sage: @fork()
     323        ... def g():
     324        ...       gp.eval('a = 10')
     325        ...       return gp.eval('a')
     326        sage: g()
     327        '10'
     328        sage: gp.eval('a')
     329        '5'
     330
     331    We illustrate that the forked function has its own pexpect interface::
     332   
     333        sage: gp.eval('a = 15')
     334        '15'
     335        sage: @fork()
     336        ... def g(): return gp.eval('a')
     337        sage: g()
     338        'a'
     339
     340    We illustrate that segfaulting subprocesses are no trouble at all::
     341
     342        sage: cython('def f(): print <char*>0')
     343        sage: @fork
     344        ... def g(): f()
     345        sage: g()
     346        'NO DATA'
     347    """
     348    F = Fork(timeout=timeout, verbose=verbose)
     349    return F(f) if f else F
  • sage/parallel/use_fork.py

    diff -r deec96694006 -r edf108f62063 sage/parallel/use_fork.py
    a b  
    2626        sage: X.verbose
    2727        False
    2828    """
    29     def __init__(self, ncpus, timeout=0, verbose=False):
     29    def __init__(self, ncpus, timeout=0, verbose=False, reset_interfaces=True):
    3030        """
    3131        Create fork-based parallel iterator.
    3232       
    3333        INPUT:
    3434
    35             - ``ncpus`` -- the maximal number of simultaneous processes to spawn
    36             - ``timeout`` -- (float) time in seconds until a subprocess is automatically killed
    37             - ``verbose`` -- whether to print anything about what the iterator does (e.g., killing subprocesses)
     35            - ``ncpus`` -- the maximal number of simultaneous
     36              processes to spawn
     37            - ``timeout`` -- (float, default: 0) time in seconds until
     38              a subprocess is automatically killed
     39            - ``verbose`` -- (default: False) whether to print
     40              anything about what the iterator does (e.g., killing
     41              subprocesses)
     42            - ``reset_interfaces`` -- (default: True) whether to reset
     43              all expect interfaces
    3844
    3945        EXAMPLES::
    4046
     
    5258            raise TypeError, "ncpus must be an integer"
    5359        self.timeout = float(timeout)  # require a float
    5460        self.verbose = verbose
     61        self.reset_interfaces = reset_interfaces
    5562
    5663    def __call__(self, f, inputs):
    5764        """
     
    206213
    207214            # The expect interfaces (and objects defined in them) are
    208215            # not valid.
    209             sage.interfaces.quit.invalidate_all()
     216            if self.reset_interfaces:
     217                sage.interfaces.quit.invalidate_all()
    210218
    211219            # Now evaluate the function f.
    212220            value = f(*x[0], **x[1])