Ticket #9501: trac_9501.patch

File trac_9501.patch, 9.9 KB (added by was, 11 years ago)
  • sage/parallel/all.py

    # HG changeset patch
    # User William Stein <wstein@gmail.com>
    # Date 1279238019 -7200
    # Node ID ced847706eb0d56ef1e478dd7b89b6f3ce58c529
    # Parent  f4e83f3f56e425d89d2b06e8b652464198267c54
    trac 9501: Make an @fork decorator
    
    diff -r f4e83f3f56e4 -r ced847706eb0 sage/parallel/all.py
    a b  
    1 from decorate import parallel
     1from decorate import parallel, fork
    22
  • sage/parallel/decorate.py

    diff -r f4e83f3f56e4 -r ced847706eb0 sage/parallel/decorate.py
    a b  
    1515    """
    1616    Convert a to a pair (args, kwds) using some rules:
    1717
    18         * if already of that form, leave that way.
    19         * if a is a tuple make (a,{})
    20         * if a is a dict make (tuple([]),a)
    21         * otherwise make ((a,),{})
     18        - if already of that form, leave that way.
     19        - if a is a tuple make (a,{})
     20        - if a is a dict make (tuple([]),a)
     21        - otherwise make ((a,),{})
    2222   
    2323    INPUT:
    24         a -- object
     24        - ``a`` -- object
    2525    OUTPUT:
    26         args -- tuple
    27         kwds -- dictionary
     26        - ``args`` -- tuple
     27        - ``kwds`` -- dictionary
    2828
    29     EXAMPLES:
     29    EXAMPLES::
     30   
    3031        sage: sage.parallel.decorate.normalize_input( (2, {3:4}) )
    3132        ((2, {3: 4}), {})
    3233        sage: sage.parallel.decorate.normalize_input( (2,3) )
     
    4950class Parallel:
    5051    """
    5152    Create parallel decorated function.
    52 
    5353    """
    5454    def __init__(self, p_iter = 'fork', ncpus=None, **kwds):
     55        """
     56        EXAMPLES::
     57
     58            sage: P = sage.parallel.decorate.Parallel(); P
     59            <sage.parallel.decorate.Parallel instance at 0x...>
     60        """
    5561        # The default p_iter is currently the reference implementation.
    5662        # This may change.
    5763        self.p_iter = None
     
    8490        normalized form (args, kwds) using normalize_inputs.
    8591
    8692        INPUT:
    87             f -- Python callable object or function
     93            - `f` -- Python callable object or function
     94           
    8895        OUTPUT:
    89             decorated version of f
     96            decorated version of `f`
    9097
    91         EXAMPLES:
     98        EXAMPLES::
    9299       
     100            sage: P = sage.parallel.decorate.Parallel()
     101            sage: def g(n,m): return n+m
     102            sage: h = P(g)          # indirect doctest
     103            sage: list(h([(2,3)]))
     104            [(((2, 3), {}), 5)]
    93105        """
    94106        # Construct the wrapper parallel version of the function we're wrapping.
    95107        # We may rework this so g is a class instance, which has the plus that
     
    106118    This is a decorator that gives a function a parallel interface,
    107119    allowing it to be called with a list of inputs, whose valuaes will
    108120    be computed in parallel.
    109    
     121
     122    WARNING: The parallel subprocesses will not have access to data
     123    created in pexpect interfaces.  This behavior with respect to
     124    pexpect interfaces is very important to keep in mind when setting
     125    up certain computations. It's the one big limitation of this
     126    decorator.
     127
    110128    INPUT:
    111129   
    112130        - ``p_iter`` -- parallel iterator function or string:
    113             - 'fork'            -- (default) use a new fork for each input
    114             - 'multiprocessing' -- use multiprocessing library
    115             - 'reference'       -- use a fake serial reference implementation
     131            - ``fork``            -- (default) use a new fork for each input
     132            - ``multiprocessing`` -- use multiprocessing library
     133            - ``reference``       -- use a fake serial reference implementation
    116134        - ``ncpus`` -- integer, number of cpus
    117135        - ``timeout`` -- number of seconds until task is killed (only supported by 'fork')
    118136
     137    WARNING: If you use anything but 'fork' above, then a whole new
     138    subprocess is spawned, so all your local state (variables, new
     139    functions, etc.,) is not available.
    119140
    120141    EXAMPLES::
    121142
     
    161182        return Parallel()(p_iter)
    162183    return Parallel(p_iter, ncpus, **kwds)
    163184   
     185
     186
     187###################################################################
     188# The @fork decorator -- run a function with no sideeffects in memory
     189# (so only side effects are on disk).
     190# We have both a function and a class below, so that the decorator
     191# can be used with or without options:
     192#   @fork
     193#   def f(...): ...
     194# and
     195#   @fork(...options...):
     196#   def f(...): ...
     197###################################################################
     198
     199class Fork:
     200    """
     201    A fork decorator class.
     202    """
     203    def __init__(self, timeout=0):
     204        """
     205        INPUT:
     206            - ``timeout`` -- (default: 0) kills subrocess after this many
     207              seconds, or if timeout=0, do not kill the subprocess.
     208
     209        EXAMPLES::
     210       
     211            sage: sage.parallel.decorate.Fork()
     212            <sage.parallel.decorate.Fork instance at 0x...>
     213            sage: sage.parallel.decorate.Fork(timeout=3)
     214            <sage.parallel.decorate.Fork instance at 0x...>
     215
     216        """
     217        self.timeout = timeout
     218       
     219    def __call__(self, f):
     220        """
     221        INPUT:
     222            - f -- a function
     223        OUTPUT:
     224            - a decorated function
     225
     226        EXAMPLES::
     227       
     228            sage: F = sage.parallel.decorate.Fork(timeout=3)
     229            sage: def g(n,m): return n+m
     230            sage: h = F(g)     # indirect doctest
     231            sage: h(2,3)
     232            5           
     233        """
     234        P = Parallel(p_iter='fork', ncpus=1, timeout=self.timeout)
     235        g = P(f)
     236        def h(*args, **kwds):
     237            return list(g([(args, kwds)]))[0][1]
     238        return h
     239
     240def fork(f=None, timeout=0):
     241    """
     242    Decorate a function so that when called it runs in a forked
     243    subprocess.  This means that it won't have any in-memory
     244    side-effects on the master Sage process.  The pexpect interfaces
     245    are all reset.
     246   
     247    INPUT:
     248        - ``f`` -- a function
     249        - ``timeout`` -- (default: 0) kills subrocess after this many
     250          seconds, or if timeout=0, do not kill the subprocess.
     251
     252    WARNING: The forked subprocesses will not have access to data
     253    created in pexpect interfaces.  This behavior with respect to
     254    pexpect interfaces is very important to keep in mind when setting
     255    up certain computations. It's the one big limitation of this
     256    decorator.
     257
     258    EXAMPLES::
     259
     260    We create a function and run it with the fork decorator.  Note that
     261    it does not have a side effect.  Despite trying to change that global
     262    variable "a" below, it does not get changed.::
     263   
     264        sage: a = 5
     265        sage: @fork
     266        ... def g(n, m):
     267        ...       global a
     268        ...       a = 10
     269        ...       return factorial(n).ndigits() + m
     270        sage: g(5, m=5)
     271        8
     272        sage: a
     273        5
     274
     275    We use fork to make sure that the function dies after 1 second no matter what::
     276   
     277        sage: @fork(timeout=1)
     278        ... def g(n, m): return factorial(n).ndigits() + m
     279        sage: g(5, m=5)
     280        8
     281        sage: g(10^7, m=5)
     282        Killing subprocess ... with input ((10000000,), {'m': 5}) which took too long
     283        'NO DATA (timed out)'
     284
     285    We illustrate that pexpect interface state is not effected by
     286    forked functions (they get their own new pexpect interfaces!)::
     287   
     288        sage: gp.eval('a = 5')
     289        '5'
     290        sage: @fork()
     291        ... def g():
     292        ...       gp.eval('a = 10')
     293        ...       return gp.eval('a')
     294        sage: g()
     295        '10'
     296        sage: gp.eval('a')
     297        '5'
     298
     299    We illustrate that the forked function has its own pexpect interface.::
     300   
     301        sage: gp.eval('a = 15')
     302        '15'
     303        sage: @fork()
     304        ... def g(): return gp.eval('a')
     305        sage: g()
     306        'a'
     307
     308    We illustrate that segfaulting subprocesses are no trouble at all::
     309
     310        sage: cython('def f(): print <char*>0')
     311        sage: @fork
     312        ... def g(): f()
     313        sage: g()
     314        'NO DATA'
     315    """
     316    F = Fork(timeout=timeout)
     317    return F(f) if f else F
  • sage/parallel/use_fork.py

    diff -r f4e83f3f56e4 -r ced847706eb0 sage/parallel/use_fork.py
    a b  
    1515    """
    1616    A parallel iterator implemented using fork.
    1717    """
    18     def __init__(self, ncpus, timeout=0, verbose=True):
     18    def __init__(self, ncpus, timeout=0, verbose=True, reset_interfaces=True):
    1919        """
    2020        Create fork-based parallel iterator.
    2121       
    2222        INPUT:
    2323
    2424            - ``ncpus`` -- the maximal number of simultaneous processes to spawn
    25             - ``timeout`` -- (float) time in seconds until a subprocess is automatically killed
    26             - ``verbose`` -- whether to print anything about what the iterator does (e.g., killing subprocesses)
     25            - ``timeout`` -- (float) time in seconds until a
     26              subprocess is automatically killed
     27            - ``verbose`` -- whether to print anything about what the
     28              iterator does (e.g., killing subprocesses)
     29            - ``reset_interface`` -- if True (the default), all the
     30              pexpect interfaces are reset in the forked off
     31              subprocesses.  You definitely want this, since not doing
     32              this can lead to weird issues. 
    2733
    2834        EXAMPLES::
    2935
     
    4147            raise TypeError, "ncpus must be an integer"
    4248        self.timeout = float(timeout)  # require a float
    4349        self.verbose = verbose
     50        self.reset_interfaces = reset_interfaces
    4451
    4552    def __call__(self, f, inputs):
    4653        """
     
    191198
    192199            # The expect interfaces (and objects defined in them) are
    193200            # not valid.
    194             sage.interfaces.quit.invalidate_all()
     201            if self.reset_interfaces:
     202                sage.interfaces.quit.invalidate_all()
    195203
    196204            # Now evaluate the function f.
    197205            value = f(*x[0], **x[1])