Ticket #9631: trac_9631-fork_decorator.4.patch

File trac_9631-fork_decorator.4.patch, 10.7 KB (added by was, 10 years ago)

polished docstrings fixing some typos.

  • sage/parallel/all.py

    # HG changeset patch
    # User William Stein <wstein@gmail.com>
    # Date 1279238019 -7200
    # Node ID 5b36a670c858c406cd730f862193ae5c7066d074
    # Parent  f8ced256ec92766abfb1f5c0b8418166554741b9
    #9631: Make an @fork decorator
    
    diff --git a/sage/parallel/all.py b/sage/parallel/all.py
    a b  
    1 from decorate import parallel
     1from decorate import parallel, fork
    22
  • sage/parallel/decorate.py

    diff --git a/sage/parallel/decorate.py b/sage/parallel/decorate.py
    a b  
    1515    r"""
    1616    Convert a to a pair (args, kwds) using some rules:
    1717
    18         * if already of that form, leave that way.
    19         * if a is a tuple make (a,{})
    20         * if a is a dict make (tuple([]),a)
    21         * otherwise make ((a,),{})
     18    - if already of that form, leave that way.
     19    - if ``a`` is a tuple make ``(a,{})``
     20    - if ``a`` is a dict make ``(tuple([]),a)``
     21    - otherwise make ``((a,),{})``
    2222
    2323    INPUT:
    24 
    2524     - ``a`` -- object
    2625
    2726    OUTPUT:
    28 
    29     - ``args`` -- tuple
    30     - ``kwds`` -- dictionary
     27     - ``args`` -- tuple
     28     - ``kwds`` -- dictionary
    3129
    3230    EXAMPLES::
    3331
     
    5351class Parallel:
    5452    r"""
    5553    Create parallel decorated function.
    56 
    5754    """
    5855    def __init__(self, p_iter = 'fork', ncpus=None, **kwds):
     56        """
     57        EXAMPLES::
     58
     59            sage: P = sage.parallel.decorate.Parallel(); P
     60            <sage.parallel.decorate.Parallel instance at 0x...>
     61        """
    5962        # The default p_iter is currently the reference implementation.
    6063        # This may change.
    6164        self.p_iter = None
     
    8184
    8285    def __call__(self, f):
    8386        r"""
    84         Create a function that wraps f and that when called with a
    85         list of inputs returns an iterator over pairs
    86              (x, f(x))
    87         in possibly random order. Here x is replaced by its
     87        Create a function that wraps ``f`` and that when called with a
     88        list of inputs returns an iterator over pairs ``(x, f(x))`` in
     89        possibly random order. Here ``x`` is replaced by its
    8890        normalized form (args, kwds) using normalize_inputs.
    8991
    9092        INPUT:
    91 
    9293         - ``f`` -- Python callable object or function
    9394
    9495        OUTPUT:
    95 
    96         decorated version of f
     96         - Decorated version of ``f``
    9797
    9898        EXAMPLES::
    9999
     
    105105            sage: p(f)(x=5)
    106106            24
    107107
     108            sage: P = sage.parallel.decorate.Parallel()
     109            sage: def g(n,m): return n+m
     110            sage: h = P(g)          # indirect doctest
     111            sage: list(h([(2,3)]))
     112            [(((2, 3), {}), 5)]
    108113        """
    109114        # Construct the wrapper parallel version of the function we're wrapping.
    110115        # We may rework this so g is a class instance, which has the plus that
     
    122127    allowing it to be called with a list of inputs, whose values will
    123128    be computed in parallel.
    124129
     130    .. warning::
     131
     132         The parallel subprocesses will not have access to data
     133         created in pexpect interfaces.  This behavior with respect to
     134         pexpect interfaces is very important to keep in mind when
     135         setting up certain computations. It's the one big limitation
     136         of this decorator.
     137
    125138    INPUT:
    126 
    127139     - ``p_iter`` -- parallel iterator function or string:
    128             - 'fork'            -- (default) use a new fork for each input
    129             - 'multiprocessing' -- use multiprocessing library
    130             - 'reference'       -- use a fake serial reference implementation
     140            - ``fork``            -- (default) use a new forked process for each input
     141            - ``multiprocessing`` -- use multiprocessing library
     142            - ``reference``       -- use a fake serial reference implementation
    131143     - ``ncpus`` -- integer, number of cpus
    132144     - ``timeout`` -- number of seconds until task is killed (only supported by 'fork')
    133145
     146    .. warning::
     147
     148         If you use anything but 'fork' above, then a whole new
     149         subprocess is spawned, so none of your local state (variables,
     150         certain functions, etc.) is available.
     151
    134152    EXAMPLES:
    135153
    136154    We create a simple decoration for a simple function. The number
     
    148166        sage: @parallel(2)
    149167        ... def f(n): return n*n
    150168
    151 
    152169    We create a decorator that uses 3 processes, and times out
    153170    individual processes after 10 seconds::
    154171
     
    174191    if isinstance(p_iter, types.FunctionType):
    175192        return Parallel()(p_iter)
    176193    return Parallel(p_iter, ncpus, **kwds)
     194
     195
     196
     197
     198###################################################################
     199# The @fork decorator -- evaluate a function with no side effects
     200# in memory, so the only side effects are on disk.
     201#
     202# We have both a function and a class below, so that the decorator
     203# can be used with or without options:
     204#
     205#   @fork
     206#   def f(...): ...
     207# and
     208#   @fork(...options...):
     209#   def f(...): ...
     210###################################################################
     211
     212class Fork:
     213    """
     214    A fork decorator class.
     215    """
     216    def __init__(self, timeout=0, verbose=False):
     217        """
     218        INPUT:
     219         - ``timeout`` -- (default: 0) kills subrocess after this many
     220           seconds, or if timeout=0, do not kill the subprocess.
     221         - ``verbose`` -- (default: ``False``) whether to print
     222           anything about what the decorator does (e.g., killing
     223           subprocesses)
     224
     225        EXAMPLES::
     226       
     227            sage: sage.parallel.decorate.Fork()
     228            <sage.parallel.decorate.Fork instance at 0x...>
     229            sage: sage.parallel.decorate.Fork(timeout=3)
     230            <sage.parallel.decorate.Fork instance at 0x...>
     231        """
     232        self.timeout = timeout
     233        self.verbose = verbose
     234       
     235    def __call__(self, f):
     236        """
     237        INPUT:
     238         - ``f`` -- a function
     239
     240        OUTPUT:
     241         - A decorated function.
     242
     243        EXAMPLES::
     244       
     245            sage: F = sage.parallel.decorate.Fork(timeout=3)
     246            sage: def g(n,m): return n+m
     247            sage: h = F(g)     # indirect doctest
     248            sage: h(2,3)
     249            5           
     250        """
     251        P = Parallel(p_iter='fork', ncpus=1, timeout=self.timeout,
     252                     verbose=self.verbose)
     253        g = P(f)
     254        def h(*args, **kwds):
     255            return list(g([(args, kwds)]))[0][1]
     256        return h
     257
     258def fork(f=None, timeout=0, verbose=False):
     259    """
     260    Decorate a function so that when called it runs in a forked
     261    subprocess.  This means that it won't have any in-memory
     262    side-effects on the parent Sage process.  The pexpect interfaces
     263    are all reset.
     264   
     265    INPUT:
     266      - ``f`` -- a function
     267      - ``timeout`` -- (default: 0) if positive, kills subrocess after
     268        this many seconds
     269      - ``verbose`` -- (default: ``False``) whether to print anything
     270        about what the decorator does (e.g., killing subprocesses)
     271
     272    .. warning::
     273
     274        The forked subprocesses will not have access to data created
     275        in pexpect interfaces.  This behavior with respect to pexpect
     276        interfaces is very important to keep in mind when setting up
     277        certain computations. It's the one big limitation of this
     278        decorator.
     279
     280    EXAMPLES:
     281
     282    We create a function and run it with the fork decorator.  Note
     283    that it does not have a side effect.  Despite trying to change
     284    the global variable "a" below in g, the variable a does not get
     285    changed.::
     286   
     287        sage: a = 5
     288        sage: @fork
     289        ... def g(n, m):
     290        ...       global a
     291        ...       a = 10
     292        ...       return factorial(n).ndigits() + m
     293        sage: g(5, m=5)
     294        8
     295        sage: a
     296        5
     297
     298    We use fork to make sure that the function terminates after 1
     299    second, no matter what::
     300   
     301        sage: @fork(timeout=1, verbose=True)
     302        ... def g(n, m): return factorial(n).ndigits() + m
     303        sage: g(5, m=5)
     304        8
     305        sage: g(10^7, m=5)
     306        Killing subprocess ... with input ((10000000,), {'m': 5}) which took too long
     307        'NO DATA (timed out)'
     308
     309    We illustrate that pexpect interface state is not affected by
     310    forked functions (they get their own new pexpect interfaces!)::
     311   
     312        sage: gp.eval('a = 5')
     313        '5'
     314        sage: @fork()
     315        ... def g():
     316        ...       gp.eval('a = 10')
     317        ...       return gp.eval('a')
     318        sage: g()
     319        '10'
     320        sage: gp.eval('a')
     321        '5'
     322
     323    We illustrate that the forked function has its own pexpect
     324    interface::
     325   
     326        sage: gp.eval('a = 15')
     327        '15'
     328        sage: @fork()
     329        ... def g(): return gp.eval('a')
     330        sage: g()
     331        'a'
     332
     333    We illustrate that segfaulting subprocesses are no trouble at all::
     334
     335        sage: cython('def f(): print <char*>0')
     336        sage: @fork
     337        ... def g(): f()
     338        sage: g()
     339        'NO DATA'
     340    """
     341    F = Fork(timeout=timeout, verbose=verbose)
     342    return F(f) if f else F
  • sage/parallel/use_fork.py

    diff --git a/sage/parallel/use_fork.py b/sage/parallel/use_fork.py
    a b  
    2626        sage: X.verbose
    2727        False
    2828    """
    29     def __init__(self, ncpus, timeout=0, verbose=False):
     29    def __init__(self, ncpus, timeout=0, verbose=False, reset_interfaces=True):
    3030        """
    3131        Create fork-based parallel iterator.
    3232       
    3333        INPUT:
    3434
    35             - ``ncpus`` -- the maximal number of simultaneous processes to spawn
    36             - ``timeout`` -- (float) time in seconds until a subprocess is automatically killed
    37             - ``verbose`` -- whether to print anything about what the iterator does (e.g., killing subprocesses)
     35            - ``ncpus`` -- the maximal number of simultaneous
     36              processes to spawn
     37            - ``timeout`` -- (float, default: 0) time in seconds until
     38              a subprocess is automatically killed
     39            - ``verbose`` -- (default: False) whether to print
     40              anything about what the iterator does (e.g., killing
     41              subprocesses)
     42            - ``reset_interfaces`` -- (default: True) whether to reset
     43              all expect interfaces
    3844
    3945        EXAMPLES::
    4046
     
    5258            raise TypeError, "ncpus must be an integer"
    5359        self.timeout = float(timeout)  # require a float
    5460        self.verbose = verbose
     61        self.reset_interfaces = reset_interfaces
    5562
    5663    def __call__(self, f, inputs):
    5764        """
     
    206213
    207214            # The expect interfaces (and objects defined in them) are
    208215            # not valid.
    209             sage.interfaces.quit.invalidate_all()
     216            if self.reset_interfaces:
     217                sage.interfaces.quit.invalidate_all()
    210218
    211219            # Now evaluate the function f.
    212220            value = f(*x[0], **x[1])