Ticket #10018: 10018_sig_retry.patch

File 10018_sig_retry.patch, 20.7 KB (added by jdemeyer, 11 years ago)

Apply on top of #9898

  • c_lib/include/interrupt.h

    # HG changeset patch
    # User Jeroen Demeyer <jdemeyer@cage.ugent.be>
    # Date 1285586635 -7200
    # Node ID 01b9380ee9909b2ef12f19dff9a06e7ee277c71e
    # Parent  932fe69b7f056e17e99c411344f2317087970ac6
    #9893: make PARI *not* catch signals
    #9893: clean up PariInstance.__init__()
    #10018: add _sig_retry macro for retrying computations in C code
    #10018: use _sig_retry for PARI error catching
    #10018: rewrite init_stack() in gen.pyx
    
    diff -r 932fe69b7f05 -r 01b9380ee990 c_lib/include/interrupt.h
    a b  
    170170 * See also @ref _sig_str
    171171 */
    172172
     173/* The function sigsetjmp() below can return the following:
     174 *  - zero: this happens in the actual _sig_on call and sets up the
     175 *    address for the Sage signal handler to jump to.  The program
     176 *    continues normally.
     177 *  - a signal number (e.g. 2 for SIGINT), assumed to be strictly
     178 *    positive: the Sage signal handler handled a signal and we return
     179 *    to the parent function of the function which called _sig_on.  At
     180 *    that point, the Exception (raised by sage_signal_handler) will be
     181 *    registered by Python.
     182 *  - a negative number: this is assumed to come from _sig_retry.  In
     183 *    this case, the program continues as if nothing happened between
     184 *    _sig_on and _sig_retry.
     185 */
    173186#define _sig_on if (_signals.mpio == 0) { _signals.mpio = 1+2; _signals.s = NULL;\
    174                  if (sigsetjmp(_signals.env,1)) { \
     187                 if (sigsetjmp(_signals.env,1) > 0) { \
    175188                  _signals.mpio = 0;   \
    176189                  return(0); \
    177190                } } // else { _signals.s = "Unbalanced _sig_on/_sig_off\n"; fprintf(stderr, _signals.s); sage_signal_handler(SIGABRT); }
     
    197210 
    198211#define _sig_str(mstring) if (_signals.mpio == 0) { _signals.mpio = 1+2; \
    199212                _signals.s = mstring; \
    200                 if (sigsetjmp(_signals.env,1)) { \
     213                if (sigsetjmp(_signals.env,1) > 0) { \
    201214                  _signals.mpio = 0; \
    202215                 return(0); \
    203216                } } //else { _signals.s = "Unbalanced _sig_str/_sig_off\n"; fprintf(stderr, _signals.s); sage_signal_handler(SIGABRT); }
     
    239252
    240253#define _sig_check _sig_on _sig_off
    241254
     255/* Retry a failed computation starting from the last _sig_on.  The
     256 * program will continue as if nothing happened between _sig_on and
     257 * _sig_retry. */
     258#define _sig_retry {_signals.mpio = 1+2; siglongjmp(_signals.env, -1);}
     259
    242260#endif /* C_LIB_INCLUDE_INTERRUPT_H */
    243261
    244262/*
  • sage/ext/cdefs.pxi

    diff -r 932fe69b7f05 -r 01b9380ee990 sage/ext/cdefs.pxi
    a b  
    1818    int fprintf(FILE *stream, char *format, ...)
    1919    int sprintf(char *str, char *format, ...)
    2020    FILE *fopen(char *path, char *mode)
    21     int fclose(FILE *strea)
     21    int fclose(FILE *stream)
     22    int fflush(FILE *stream)
    2223    cdef FILE *stdout
    2324    int scanf(char *format, ...)
    2425
  • sage/ext/interrupt.pxi

    diff -r 932fe69b7f05 -r 01b9380ee990 sage/ext/interrupt.pxi
    a b  
    55################################################################
    66
    77cdef extern from 'interrupt.h':
    8     int _sig_on, _sig_off, _sig_check
     8    int _sig_on, _sig_off, _sig_check, _sig_retry
    99    void _sig_str(char*)
  • sage/libs/pari/gen.pyx

    diff -r 932fe69b7f05 -r 01b9380ee990 sage/libs/pari/gen.pyx
    a b  
    174174
    175175# The unique running Pari instance.
    176176cdef PariInstance pari_instance, P
    177 pari_instance = PariInstance(16000000, 500000)
     177pari_instance = PariInstance()
    178178P = pari_instance   # shorthand notation
    179179
    180180# so Galois groups are represented in a sane way
    181181# See the polgalois section of the PARI users manual.
    182182new_galois_format = 1   
    183183
    184 cdef pari_sp mytop, initial_bot, initial_top
    185 
    186 # keep track of the stack
    187 cdef pari_sp stack_avma
     184cdef pari_sp mytop
    188185
    189186# real precision in decimal digits: see documentation for
    190187# get_real_precision() and set_real_precision().  This variable is used
     
    81778174
    81788175
    81798176cdef class PariInstance(sage.structure.parent_base.ParentWithBase):
    8180     def __init__(self, long size=16000000, unsigned long maxprime=500000):
     8177    def __init__(self, long size=10000000, unsigned long maxprime=500000):
    81818178        """
    81828179        Initialize the PARI system.
    81838180       
     
    82188215        if bot:
    82198216            return  # pari already initialized.
    82208217       
    8221         global initialized, num_primes, PARI_ZERO, PARI_ONE, PARI_TWO, avma, top, bot, \
    8222                initial_bot, initial_top, prec
    8223 
    8224 
    8225         #print "Initializing PARI (size=%s, maxprime=%s)"%(size,maxprime)
    8226         pari_init(1024, maxprime)
    8227         initial_bot = bot
    8228         initial_top = top
    8229         bot = 0
    8230 
    8231         _sig_on
     8218        global num_primes, avma, top, bot, prec
     8219
     8220        # The size here doesn't really matter, because we will allocate
     8221        # our own stack anyway. We ask PARI not to set up signal handlers.
     8222        pari_init_opts(10000, maxprime, INIT_JMPm | INIT_DFTm)
     8223        num_primes = maxprime
     8224
     8225        # NOTE: _sig_on can only come AFTER pari_init_opts()!
     8226        _sig_on
     8227
     8228        # Free the PARI stack and allocate our own (using Cython)
     8229        pari_free(<void*>bot); bot = 0
    82328230        init_stack(size)
    8233         _sig_off
    82348231
    82358232        GP_DATA.fmt.prettyp = 0
    82368233
     
    82458242        defaultOut.puts = sage_puts
    82468243        defaultOut.flush = sage_flush
    82478244
    8248         # Take control of SIGSEGV back from PARI.
    8249         import signal
    8250         signal.signal(signal.SIGSEGV, signal.SIG_DFL)
    8251 
    8252         # We do the following, since otherwise the IPython pager
    8253         # causes sage to crash when it is exited early.  This is
    8254         # because when PARI was initialized it set a trap for this
    8255         # signal.
    8256         import signal
    8257         signal.signal(signal.SIGPIPE, _my_sigpipe)
    8258         initialized = 1
    8259         stack_avma = avma
    8260         num_primes = maxprime
    8261         self.PARI_ZERO = self.new_gen(gen_0)
    8262         self.PARI_ONE = self.new_gen(gen_1)
    8263         self.PARI_TWO = self.new_gen(gen_2)
     8245        self.PARI_ZERO = self.new_gen_noclear(gen_0)
     8246        self.PARI_ONE = self.new_gen_noclear(gen_1)
     8247        self.PARI_TWO = self.new_gen_noclear(gen_2)
     8248        _sig_off
     8249
     8250    def debugstack(self):
     8251        r"""
     8252        Print the internal PARI variables ``top`` (top of stack), ``avma``
     8253        (available memory address, think of this as the stack pointer),
     8254        ``bot`` (bottom of stack).
     8255       
     8256        EXAMPLE::
     8257           
     8258            sage: pari.debugstack()  # not tested
     8259            top =  0x60b2c60
     8260            avma = 0x5875c38
     8261            bot =  0x57295e0
     8262            size = 10000000
     8263        """
     8264        # We deliberately use low-level functions to minimize the
     8265        # chances that something goes wrong here (for example, if we
     8266        # are out of memory).
     8267        global avma, top, bot
     8268        printf("top =  %p\navma = %p\nbot =  %p\nsize = %lu\n", top, avma, bot, <unsigned long>(top) - <unsigned long>(bot))
     8269        fflush(stdout)
    82648270
    82658271    def _unsafe_deallocate_pari_stack(self):
    82668272        if bot:
    82678273            sage_free(<void*>bot)
    8268         global top, bot
    8269         top = initial_top
    8270         bot = initial_bot
     8274        global top, bot, avma
     8275        bot = 0; avma = 0; top = 0
    82718276        pari_close()
    82728277
    82738278    def __repr__(self):
     
    88128817    # Initialization
    88138818    ############################################################
    88148819
     8820    def stacksize(self):
     8821        r"""
     8822        Returns the current size of the PARI stack, which is `10^7`
     8823        by default.  However, the stack size is automatically doubled
     8824        when needed.  It can also be set directly using
     8825        ``pari.allocatemem()``.
     8826       
     8827        EXAMPLES::
     8828           
     8829            sage: pari.stacksize()
     8830            10000000
     8831           
     8832        """
     8833        global top, bot
     8834        return int(<unsigned long>(top) - <unsigned long>(bot))
     8835
     8836    def _allocate_huge_mem(self):
     8837        r"""
     8838        This function allocates an impossibly large amount of PARI
     8839        stack in order to test ``init_stack()`` failing.
     8840       
     8841        TESTS::
     8842           
     8843            sage: pari.allocatemem(10^6, silent=True)
     8844            sage: pari._allocate_huge_mem()
     8845            Traceback (most recent call last):
     8846            ...
     8847            MemoryError: Unable to allocate ... (instead, allocated 1000000 bytes)
     8848       
     8849        Test that the PARI stack is sane after this failure::
     8850           
     8851            sage: a = pari('2^10000000')
     8852            sage: pari.allocatemem(10^6, silent=True)
     8853        """
     8854        # Since size_t is unsigned, this should wrap over to a very
     8855        # large positive number.
     8856        init_stack(<size_t>(-4096))
     8857
     8858    def _setup_failed_retry(self):
     8859        r"""
     8860        This function pretends that the PARI stack is larger than it
     8861        actually is such that allocatemem(0) will allocate much more
     8862        than double the current stack.  That allocation will then fail.
     8863        This function is meant to be used only in this doctest.  This
     8864        test is successful as long as we do not get any segmentation
     8865        faults.
     8866       
     8867        TESTS::
     8868           
     8869            sage: pari.allocatemem(4096, silent=True)
     8870            sage: pari._setup_failed_retry()
     8871            sage: try:  # Any result is fine as long as it's not a segfault
     8872            ...     a = pari('2^1000000')
     8873            ... except MemoryError:
     8874            ...     pass
     8875            sage: pari.allocatemem(10^6, silent=True)
     8876        """
     8877        global top
     8878        # Pretend top is at a very high address
     8879        top = <pari_sp>(<size_t>(-16))
     8880
    88158881    def allocatemem(self, s=0, silent=False):
    88168882        r"""
    8817         Double the *PARI* stack.
    8818         """
    8819         if s == 0 and not silent:
    8820             print "Doubling the PARI stack."
     8883        Change the PARI stack space to the given size (or double the
     8884        current size if ``s`` is `0`).
     8885       
     8886        The PARI stack is never automatically shrunk.  You can use the
     8887        command ``pari.allocatemem(10^7)`` to reset the size to `10^7`,
     8888        which is the default size at startup.  Note that the results of
     8889        computations using Sage's PARI interface are copied to the
     8890        Python heap, so they take up no space in the PARI stack.
     8891        The PARI stack is cleared after every computation.
     8892       
     8893        It does no real harm to set this to a small value as the PARI
     8894        stack will be automatically doubled when we run out of memory.
     8895        However, it could make some calculations slower (since they have
     8896        to be redone from the start after doubling the stack until the
     8897        stack is big enough).
     8898       
     8899        INPUT:
     8900       
     8901       
     8902        -  ``s`` - an integer (default: 0).  An argument of zero will be
     8903           interpreted as "double the current size".  Otherwise ``s``
     8904           should be the size in bytes of the new PARI stack.
     8905       
     8906       
     8907        EXAMPLES::
     8908       
     8909            sage: pari.allocatemem(10^7)
     8910            PARI stack size set to 10000000 bytes.
     8911            sage: pari.allocatemem()  # Double the current size
     8912            PARI stack size set to 20000000 bytes.
     8913            sage: pari.stacksize()
     8914            20000000
     8915            sage: pari.allocatemem(10^6)
     8916            PARI stack size set to 1000000 bytes.
     8917       
     8918        The following computation will automatically increase the PARI
     8919        stack size::
     8920           
     8921            sage: a = pari('2^100000000')
     8922       
     8923        ``a`` is now a Python variable on the Python heap and does not
     8924        take up any space on the PARI stack.  The PARI stack is still
     8925        large because of the computation of ``a``::
     8926           
     8927            sage: pari.stacksize()
     8928            32000000
     8929            sage: pari.allocatemem(10^7)
     8930            PARI stack size set to 10000000 bytes.
     8931            sage: pari.stacksize()
     8932            10000000
     8933       
     8934        TESTS:
     8935       
     8936        Do the same without using the string interface and starting
     8937        from a very small and odd stack size::
     8938           
     8939            sage: pari.allocatemem(1025)
     8940            PARI stack size set to 1040 bytes.
     8941            sage: a = pari(2)^100000000
     8942            sage: pari.stacksize()
     8943            34078720
     8944        """
    88218945        s = int(s)
    8822         cdef size_t a = s
    8823         if int(a) != s:
    8824             raise ValueError, "s must be nonnegative and not too big."
     8946        if s < 0:
     8947            raise ValueError, "s must be nonnegative."
    88258948        init_stack(s)
     8949        if not silent:
     8950            print "PARI stack size set to", self.stacksize(), "bytes."
    88268951
    88278952    def pari_version(self):
    88288953        return str(PARIVERSION)
     
    92379362
    92389363##############################################
    92399364
    9240 def init_pari_stack(size=8000000):
    9241     """
    9242     Change the PARI scratch stack space to the given size.
    9243    
    9244     The main application of this command is that you've done some
    9245     individual PARI computation that used a lot of stack space. As a
    9246     result the PARI stack may have doubled several times and is now
    9247     quite large. That object you computed is copied off to the heap,
    9248     but the PARI stack is never automatically shrunk back down. If you
    9249     call this function you can shrink it back down.
    9250    
    9251     If you set this too small then it will automatically be increased
    9252     if it is exceeded, which could make some calculations initially
    9253     slower (since they have to be redone until the stack is big
    9254     enough).
    9255    
    9256     INPUT:
    9257    
    9258    
    9259     -  ``size`` - an integer (default: 8000000)
    9260    
    9261    
    9262     EXAMPLES::
    9263    
    9264         sage: get_memory_usage()                       # random output
    9265         '122M+'
    9266         sage: a = pari('2^100000000')
    9267         sage: get_memory_usage()                       # random output
    9268         '157M+'
    9269         sage: del a
    9270         sage: get_memory_usage()                       # random output
    9271         '145M+'
    9272    
    9273     Hey, I want my memory back!
    9274    
    9275     ::
    9276    
    9277         sage: sage.libs.pari.gen.init_pari_stack()
    9278         sage: get_memory_usage()                       # random output
    9279         '114M+'
    9280    
    9281     Ahh, that's better.
    9282     """
    9283     init_stack(size)
    9284 
    9285 cdef int init_stack(size_t size) except -1:
     9365cdef int init_stack(size_t new_size) except -1:
     9366    r"""
     9367    Low-level Cython function to allocate the PARI stack.  This
     9368    function should not be called directly, use ``pari.allocatemem()``
     9369    instead.
     9370    """
     9371    global top, bot, avma, mytop
     9372    #printf("Stack before init_stack():\ntop =  %p\navma = %p\nbot =  %p\nsize = %lu\n", avma, top, bot, <unsigned long>(top) - <unsigned long>(bot)); fflush(stdout);
     9373
     9374    cdef size_t old_size = <size_t>(top) - <size_t>(bot)
     9375
     9376    # s is the number of bytes which will actually be allocated
    92869377    cdef size_t s
    9287     cdef pari_sp cur_stack_size
    9288 
    9289     global top, bot, avma, stack_avma, mytop
    9290 
    9291 
    9292     err = False    # whether or not a memory allocation error occurred.
    9293 
    9294    
    9295     # delete this if get core dumps and change the 2* to a 1* below.
     9378    if (new_size == 0):
     9379        if old_size >= (<size_t>(1) << (8*sizeof(void*)-1)):
     9380            # We cannot possibly double since we already use over half
     9381            # the addressable memory
     9382            raise MemoryError, "Unable to allocate %s bytes for the PARI stack (instead, allocated %s bytes)"%(2*long(old_size), old_size)
     9383        s = 2 * old_size
     9384    else:
     9385        s = new_size
     9386
     9387    # Align size to 16 bytes and take 1024 bytes as a minimum
     9388    s = ((s + 15)/16)*16
     9389    if (s < 1024):
     9390        s = 1024
     9391   
     9392    # Free the current stack
    92969393    if bot:
    92979394        sage_free(<void*>bot)
    9298 
    9299     prev_stack_size = top - bot
    9300     if size == 0:
    9301         size = 2*(top-bot)
    9302 
    9303     # Decide on size
    9304     s = fix_size(size)
     9395        bot = 0; top = 0; avma = 0; mytop = 0
    93059396
    93069397    # Allocate memory for new stack using Python's memory allocator.
    93079398    # As explained in the python/C API reference, using this instead
    93089399    # of malloc is much better (and more platform independent, etc.)
    93099400    bot = <pari_sp> sage_malloc(s)
    93109401
    9311     while not bot:
    9312         err = True
    9313         s = fix_size(prev_stack_size)
     9402    # If doubling failed, instead add 25% to the current stack size.
     9403    if (bot == 0) and (new_size == 0):
     9404        s = old_size + (old_size/64)*16
    93149405        bot = <pari_sp> sage_malloc(s)
    9315         if not bot:
    9316             prev_stack_size /= 2
    9317    
    9318     #endif
     9406   
     9407    # If this is non-zero, the size we failed to allocate
     9408    cdef size_t failed_size = 0
     9409    if not bot:
     9410        failed_size = s
     9411        # We lost our PARI stack and are not able to allocate the
     9412        # requested size.  If we just raise an exception now, we end up
     9413        # *without* a PARI stack which is not good.  We will raise an
     9414        # exception later, after allocating *some* PARI stack.
     9415        s = old_size
     9416        while True:
     9417            bot = <pari_sp> sage_malloc(s)
     9418            if bot: break
     9419            s = (s/32)*16
     9420            assert(s >= 1024)  # We are in BIG trouble if this fails!
     9421   
    93199422    top = bot + s
    93209423    mytop = top
    93219424    avma = top
    9322     stack_avma = avma
    9323 
    9324     if err:
    9325         raise MemoryError, "Unable to allocate %s bytes memory for PARI."%size       
    9326 
    9327 
    9328 def _my_sigpipe(signum, frame):
    9329     # If I do anything, it messes up IPython's pager.
    9330     pass
    9331 
    9332 cdef size_t fix_size(size_t a):
    9333     cdef size_t b
    9334     b = a - (a & (sizeof(long)-1))     # sizeof(long) | b <= a 
    9335     if b < 1024:
    9336         b = 1024
    9337     return b
     9425    #printf("Stack after init_stack():\ntop =  %p\navma = %p\nbot =  %p\nsize = %lu\n", avma, top, bot, <unsigned long>(top) - <unsigned long>(bot)); fflush(stdout);
     9426   
     9427    if failed_size:
     9428        raise MemoryError, "Unable to allocate %s bytes for the PARI stack (instead, allocated %s bytes)"%(failed_size, s)
     9429   
     9430    return 0
    93389431
    93399432cdef GEN deepcopy_to_python_heap(GEN x, pari_sp* address):
    93409433    cdef size_t s
     
    94809573
    94819574
    94829575# We expose a trap function to C.
    9483 # If this function returns without raising an exception,
    9484 # the code is retried.
    9485 # This is a proof-of-concept example.
    94869576# THE TRY CODE IS NOT REENTRANT -- NO CALLS TO PARI FROM HERE !!!
    94879577#              - Gonzalo Tornario
    9488 
    9489 cdef void _pari_trap "_pari_trap" (long errno, long retries) except *:
    9490     """
    9491     TESTS::
    9492    
     9578cdef void _pari_trap "_pari_trap" (long errnum) except *:
     9579    """
     9580    This function is called by _pari_catch in pari_err.h when a PARI
     9581    error occured.  If this function returns without raising an
     9582    exception, the code is retried starting from the _sig_on.
    94939583    """
    94949584    _sig_off
    9495     if retries > 100:
    9496         raise RuntimeError, "_pari_trap recursion too deep"
    9497     if errno == errpile:
    9498         P.allocatemem(silent=True)
    9499        
    9500         #raise RuntimeError, "The PARI stack overflowed.  It has automatically been doubled using pari.allocatemem().  Please retry your computation, possibly after you manually call pari.allocatemem() a few times."
    9501         #print "Stack overflow! (%d retries so far)"%retries
    9502         #print " enlarge the stack."
    9503        
    9504     elif errno == user:
    9505        
     9585    if errnum == errpile:
     9586        # PARI is out of memory.  We double the size of the PARI stack
     9587        # and retry the computation.
     9588        init_stack(0)
     9589    elif errnum == user:
    95069590        raise Exception, "PARI user exception"
    9507    
    95089591    else:
    9509        
    9510         raise PariError, errno
    9511 
     9592        raise PariError, errnum
    95129593
    95139594
    95149595def vecsmall_to_intlist(gen v):
  • sage/libs/pari/pari_err.h

    diff -r 932fe69b7f05 -r 01b9380ee990 sage/libs/pari/pari_err.h
    a b  
    1414
    1515#define _pari_endcatch { err_leave(__catcherr); }
    1616
    17 /* Careful with pari_retries !
    18  * It should not be in a register, we flag it as "volatile".
    19  */
    2017#define _pari_catch { \
    2118        long pari_errno; \
    22         long volatile pari_retries = 0; \
    2319        jmp_buf __env; \
    2420        __catcherr = 0; \
    2521        if ((pari_errno=setjmp(__env))) { \
    26             _pari_trap(pari_errno, pari_retries); \
    27             if(PyErr_Occurred()) { \
    28                 _pari_endcatch; \
    29                 return NULL; \
    30             } \
    31             pari_retries++; \
     22            _pari_trap(pari_errno); /* Implicitly calls _pari_endcatch */ \
     23            if(PyErr_Occurred()) return NULL; \
     24            _sig_retry \
    3225        } \
    3326        __catcherr = err_catch(CATCH_ALL, &__env); \
    3427    }
  • sage/libs/pari/pari_err.pxi

    diff -r 932fe69b7f05 -r 01b9380ee990 sage/libs/pari/pari_err.pxi
    a b  
    66    int _sig_on "_pari_sig_on"
    77    int _sig_off "_pari_sig_off"
    88    void _sig_str "_pari_sig_str" (char *)
     9    int _sig_retry   # from c_lib/include/interrupt.h