Ticket #9631: trac_9631-fork_decorator.patch

File trac_9631-fork_decorator.patch, 10.5 KB (added by mpatel, 11 years ago)

Version of #9501's v2, rebased for 4.6.rc0.

  • sage/parallel/all.py

    # HG changeset patch
    # User William Stein <wstein@gmail.com>
    # Date 1279238019 -7200
    # Node ID 04bf9ddfde1b18b586f26dde4d53e2644ba56886
    # Parent  f667e86a25fbe7138d4933ecf9b972ba8f40087d
    #9631: Make an @fork decorator
    
    diff --git a/sage/parallel/all.py b/sage/parallel/all.py
    a b  
    1 from decorate import parallel
     1from decorate import parallel, fork
    22
  • sage/parallel/decorate.py

    diff --git a/sage/parallel/decorate.py b/sage/parallel/decorate.py
    a b def normalize_input(a): 
    1515    """
    1616    Convert a to a pair (args, kwds) using some rules:
    1717
    18         * if already of that form, leave that way.
    19         * if a is a tuple make (a,{})
    20         * if a is a dict make (tuple([]),a)
    21         * otherwise make ((a,),{})
     18        - if already of that form, leave that way.
     19        - if a is a tuple make (a,{})
     20        - if a is a dict make (tuple([]),a)
     21        - otherwise make ((a,),{})
    2222   
    2323    INPUT:
    24         a -- object
     24        - ``a`` -- object
    2525    OUTPUT:
    26         args -- tuple
    27         kwds -- dictionary
     26        - ``args`` -- tuple
     27        - ``kwds`` -- dictionary
    2828
    29     EXAMPLES:
     29    EXAMPLES::
     30   
    3031        sage: sage.parallel.decorate.normalize_input( (2, {3:4}) )
    3132        ((2, {3: 4}), {})
    3233        sage: sage.parallel.decorate.normalize_input( (2,3) )
    def normalize_input(a): 
    4950class Parallel:
    5051    """
    5152    Create parallel decorated function.
    52 
    5353    """
    5454    def __init__(self, p_iter = 'fork', ncpus=None, **kwds):
     55        """
     56        EXAMPLES::
     57
     58            sage: P = sage.parallel.decorate.Parallel(); P
     59            <sage.parallel.decorate.Parallel instance at 0x...>
     60        """
    5561        # The default p_iter is currently the reference implementation.
    5662        # This may change.
    5763        self.p_iter = None
    class Parallel: 
    8490        normalized form (args, kwds) using normalize_inputs.
    8591
    8692        INPUT:
    87             f -- Python callable object or function
     93            - `f` -- Python callable object or function
     94           
    8895        OUTPUT:
    89             decorated version of f
     96            decorated version of `f`
    9097
    91         EXAMPLES:
     98        EXAMPLES::
    9299       
     100            sage: P = sage.parallel.decorate.Parallel()
     101            sage: def g(n,m): return n+m
     102            sage: h = P(g)          # indirect doctest
     103            sage: list(h([(2,3)]))
     104            [(((2, 3), {}), 5)]
    93105        """
    94106        # Construct the wrapper parallel version of the function we're wrapping.
    95107        # We may rework this so g is a class instance, which has the plus that
    def parallel(p_iter = 'fork', ncpus=None 
    106118    This is a decorator that gives a function a parallel interface,
    107119    allowing it to be called with a list of inputs, whose valuaes will
    108120    be computed in parallel.
    109    
     121
     122    .. warning::
     123
     124         The parallel subprocesses will not have access to data
     125         created in pexpect interfaces.  This behavior with respect to
     126         pexpect interfaces is very important to keep in mind when
     127         setting up certain computations. It's the one big limitation
     128         of this decorator.
     129
    110130    INPUT:
    111131   
    112132        - ``p_iter`` -- parallel iterator function or string:
    113             - 'fork'            -- (default) use a new fork for each input
    114             - 'multiprocessing' -- use multiprocessing library
    115             - 'reference'       -- use a fake serial reference implementation
     133            - ``fork``            -- (default) use a new fork for each input
     134            - ``multiprocessing`` -- use multiprocessing library
     135            - ``reference``       -- use a fake serial reference implementation
    116136        - ``ncpus`` -- integer, number of cpus
    117137        - ``timeout`` -- number of seconds until task is killed (only supported by 'fork')
    118138
     139    .. warning::
     140
     141         If you use anything but 'fork' above, then a whole new
     142         subprocess is spawned, so all your local state (variables,
     143         new functions, etc.,) is not available.
    119144
    120145    EXAMPLES::
    121146
    def parallel(p_iter = 'fork', ncpus=None 
    161186        return Parallel()(p_iter)
    162187    return Parallel(p_iter, ncpus, **kwds)
    163188   
     189
     190
     191###################################################################
     192# The @fork decorator -- run a function with no sideeffects in memory
     193# (so only side effects are on disk).
     194# We have both a function and a class below, so that the decorator
     195# can be used with or without options:
     196#   @fork
     197#   def f(...): ...
     198# and
     199#   @fork(...options...):
     200#   def f(...): ...
     201###################################################################
     202
     203class Fork:
     204    """
     205    A fork decorator class.
     206    """
     207    def __init__(self, timeout=0, verbose=False):
     208        """
     209        INPUT:
     210
     211          - ``timeout`` -- (default: 0) kills subrocess after this
     212            many seconds, or if timeout=0, do not kill the subprocess.
     213          - ``verbose`` -- (default: False) whether to print anything
     214            about what the decorator does (e.g., killing subprocesses)
     215
     216        EXAMPLES::
     217       
     218            sage: sage.parallel.decorate.Fork()
     219            <sage.parallel.decorate.Fork instance at 0x...>
     220            sage: sage.parallel.decorate.Fork(timeout=3)
     221            <sage.parallel.decorate.Fork instance at 0x...>
     222
     223        """
     224        self.timeout = timeout
     225        self.verbose = verbose
     226       
     227    def __call__(self, f):
     228        """
     229        INPUT:
     230            - f -- a function
     231        OUTPUT:
     232            - a decorated function
     233
     234        EXAMPLES::
     235       
     236            sage: F = sage.parallel.decorate.Fork(timeout=3)
     237            sage: def g(n,m): return n+m
     238            sage: h = F(g)     # indirect doctest
     239            sage: h(2,3)
     240            5           
     241        """
     242        P = Parallel(p_iter='fork', ncpus=1, timeout=self.timeout,
     243                     verbose=self.verbose)
     244        g = P(f)
     245        def h(*args, **kwds):
     246            return list(g([(args, kwds)]))[0][1]
     247        return h
     248
     249def fork(f=None, timeout=0, verbose=False):
     250    """
     251    Decorate a function so that when called it runs in a forked
     252    subprocess.  This means that it won't have any in-memory
     253    side-effects on the master Sage process.  The pexpect interfaces
     254    are all reset.
     255   
     256    INPUT:
     257
     258        - ``f`` -- a function
     259        - ``timeout`` -- (default: 0) kills subrocess after this many
     260          seconds, or if timeout=0, do not kill the subprocess.
     261        - ``verbose`` -- (default: False) whether to print anything
     262          about what the decorator does (e.g., killing subprocesses)
     263
     264    .. warning::
     265
     266        The forked subprocesses will not have access to data created
     267        in pexpect interfaces.  This behavior with respect to pexpect
     268        interfaces is very important to keep in mind when setting up
     269        certain computations. It's the one big limitation of this
     270        decorator.
     271
     272    EXAMPLES::
     273
     274    We create a function and run it with the fork decorator.  Note that
     275    it does not have a side effect.  Despite trying to change that global
     276    variable "a" below, it does not get changed.::
     277   
     278        sage: a = 5
     279        sage: @fork
     280        ... def g(n, m):
     281        ...       global a
     282        ...       a = 10
     283        ...       return factorial(n).ndigits() + m
     284        sage: g(5, m=5)
     285        8
     286        sage: a
     287        5
     288
     289    We use fork to make sure that the function dies after 1 second no matter what::
     290   
     291        sage: @fork(timeout=1, verbose=True)
     292        ... def g(n, m): return factorial(n).ndigits() + m
     293        sage: g(5, m=5)
     294        8
     295        sage: g(10^7, m=5)
     296        Killing subprocess ... with input ((10000000,), {'m': 5}) which took too long
     297        'NO DATA (timed out)'
     298
     299    We illustrate that pexpect interface state is not effected by
     300    forked functions (they get their own new pexpect interfaces!)::
     301   
     302        sage: gp.eval('a = 5')
     303        '5'
     304        sage: @fork()
     305        ... def g():
     306        ...       gp.eval('a = 10')
     307        ...       return gp.eval('a')
     308        sage: g()
     309        '10'
     310        sage: gp.eval('a')
     311        '5'
     312
     313    We illustrate that the forked function has its own pexpect interface.::
     314   
     315        sage: gp.eval('a = 15')
     316        '15'
     317        sage: @fork()
     318        ... def g(): return gp.eval('a')
     319        sage: g()
     320        'a'
     321
     322    We illustrate that segfaulting subprocesses are no trouble at all::
     323
     324        sage: cython('def f(): print <char*>0')
     325        sage: @fork
     326        ... def g(): f()
     327        sage: g()
     328        'NO DATA'
     329    """
     330    F = Fork(timeout=timeout, verbose=verbose)
     331    return F(f) if f else F
  • sage/parallel/use_fork.py

    diff --git a/sage/parallel/use_fork.py b/sage/parallel/use_fork.py
    a b class p_iter_fork: 
    1515    """
    1616    A parallel iterator implemented using fork.
    1717    """
    18     def __init__(self, ncpus, timeout=0, verbose=False):
     18    def __init__(self, ncpus, timeout=0, verbose=False, reset_interfaces=True):
    1919        """
    2020        Create fork-based parallel iterator.
    2121       
    2222        INPUT:
    2323
    24             - ``ncpus`` -- the maximal number of simultaneous processes to spawn
    25             - ``timeout`` -- (float) time in seconds until a subprocess is automatically killed
    26             - ``verbose`` -- whether to print anything about what the iterator does (e.g., killing subprocesses)
     24            - ``ncpus`` -- the maximal number of simultaneous
     25              processes to spawn
     26            - ``timeout`` -- (float, default: 0) time in seconds until
     27              a subprocess is automatically killed
     28            - ``verbose`` -- (default: False) whether to print
     29              anything about what the iterator does (e.g., killing
     30              subprocesses)
     31            - ``reset_interfaces`` -- (default: True) whether to reset
     32              all expect interfaces
    2733
    2834        EXAMPLES::
    2935
    class p_iter_fork: 
    4147            raise TypeError, "ncpus must be an integer"
    4248        self.timeout = float(timeout)  # require a float
    4349        self.verbose = verbose
     50        self.reset_interfaces = reset_interfaces
    4451
    4552    def __call__(self, f, inputs):
    4653        """
    class p_iter_fork: 
    195202
    196203            # The expect interfaces (and objects defined in them) are
    197204            # not valid.
    198             sage.interfaces.quit.invalidate_all()
     205            if self.reset_interfaces:
     206                sage.interfaces.quit.invalidate_all()
    199207
    200208            # Now evaluate the function f.
    201209            value = f(*x[0], **x[1])