Ticket #10018: 10018_pari_init_stack.patch

File 10018_pari_init_stack.patch, 18.0 KB (added by jdemeyer, 8 years ago)
  • sage/libs/pari/gen.pyx

    # HG changeset patch
    # User Jeroen Demeyer <jdemeyer@cage.ugent.be>
    # Date 1287342612 -7200
    # Node ID 0bc0c7ba5107d49ee45cd5acd29ab9f1cef5831d
    # Parent  711435d5825529a2f525344e04a1f1acb18221d9
    Improve PARI's stack allocation
    
    diff --git a/sage/libs/pari/gen.pyx b/sage/libs/pari/gen.pyx
    a b  
    182182include 'pari_err.pxi'
    183183include 'sage/ext/stdsage.pxi'
    184184include 'sage/ext/python.pxi'
     185include 'sage/ext/interrupt.pxi'
     186
     187cimport libc.stdlib
    185188
    186189cdef extern from "misc.h":
    187190    int     factorint_withproof_sage(GEN* ans, GEN x, GEN cutoff)
     
    364367
    365368# The unique running Pari instance.
    366369cdef PariInstance pari_instance, P
    367 pari_instance = PariInstance(16000000, 500000)
     370pari_instance = PariInstance()
    368371P = pari_instance   # shorthand notation
    369372
    370373# PariInstance.__init__ must not create gen objects because their parent is not constructed yet
     
    86548657       
    86558658        INPUT:
    86568659       
    8657        
    8658         -  ``limit`` - (default: -1) is optional and can be set
     8660        -  ``limit`` -- (default: -1) is optional and can be set
    86598661           whenever x is of (possibly recursive) rational type. If limit is
    86608662           set return partial factorization, using primes up to limit (up to
    86618663           primelimit if limit=0).
    86628664       
    8663        
    8664         proof - (default: True) optional. If False (not the default),
    8665         returned factors `<10^{15}` may only be pseudoprimes.
     8665        - ``proof`` -- (default: True) optional. If False (not the default),
     8666          returned factors larger than `2^{64}` may only be pseudoprimes.
    86668667       
    86678668        .. note::
    86688669
     
    86928693            PariError: sorry, factor for general polynomials is not yet implemented
    86938694        """
    86948695        cdef int r
     8696        cdef GEN cutoff
    86958697        if limit == -1 and typ(self.g) == t_INT and proof:
    86968698            pari_catch_sig_on()
    8697             r = factorint_withproof_sage(&t0, self.g, ten_to_15)
     8699            # cutoff for checking true primality: 2^64 according to the
     8700            # PARI documentation ??ispseudoprime.
     8701            cutoff = mkintn(3, 1, 0, 0)  # expansion of 2^64 in base 2^32: (1,0,0)
     8702            r = factorint_withproof_sage(&t0, self.g, cutoff)
    86988703            z = P.new_gen(t0)
    86998704            if not r:
    87008705                return z
     
    92739278
    92749279
    92759280cdef class PariInstance(sage.structure.parent_base.ParentWithBase):
    9276     def __init__(self, long size=16000000, unsigned long maxprime=500000):
     9281    def __init__(self, long size=1000000, unsigned long maxprime=500000):
    92779282        """
    92789283        Initialize the PARI system.
    92799284       
    92809285        INPUT:
    92819286       
    92829287       
    9283         -  ``size`` - long, the number of bytes for the initial
     9288        -  ``size`` -- long, the number of bytes for the initial
    92849289           PARI stack (see note below)
    92859290       
    9286         -  ``maxprime`` - unsigned long, upper limit on a
     9291        -  ``maxprime`` -- unsigned long, upper limit on a
    92879292           precomputed prime number table (default: 500000)
    92889293       
    92899294       
    92909295        .. note::
    92919296       
    9292            In py_pari, the PARI stack is different than in gp or the
    9293            PARI C library. In Python, instead of the PARI stack
     9297           In Sage, the PARI stack is different than in GP or the
     9298           PARI C library. In Sage, instead of the PARI stack
    92949299           holding the results of all computations, it *only* holds
    92959300           the results of an individual computation. Each time a new
    92969301           Python/PARI object is computed, it it copied to its own
     
    93509355        pariErr.puts = sage_pariErr_puts
    93519356        pariErr.flush = sage_pariErr_flush
    93529357
     9358    def debugstack(self):
     9359        r"""
     9360        Print the internal PARI variables ``top`` (top of stack), ``avma``
     9361        (available memory address, think of this as the stack pointer),
     9362        ``bot`` (bottom of stack).
     9363       
     9364        EXAMPLE::
     9365           
     9366            sage: pari.debugstack()  # random
     9367            top =  0x60b2c60
     9368            avma = 0x5875c38
     9369            bot =  0x57295e0
     9370            size = 1000000
     9371        """
     9372        # We deliberately use low-level functions to minimize the
     9373        # chances that something goes wrong here (for example, if we
     9374        # are out of memory).
     9375        global avma, top, bot
     9376        printf("top =  %p\navma = %p\nbot =  %p\nsize = %lu\n", top, avma, bot, <unsigned long>(top) - <unsigned long>(bot))
     9377        fflush(stdout)
     9378
    93539379    def __dealloc__(self):
    93549380        """
    93559381        Deallocation of the Pari instance.
     
    93679393        """
    93689394        if bot:
    93699395            sage_free(<void*>bot)
    9370         global top, bot
     9396        global top, bot, avma
    93719397        top = 0
    93729398        bot = 0
     9399        avma = 0
    93739400        pari_close()
    93749401
    93759402    def __repr__(self):
     
    99289955    # Initialization
    99299956    ############################################################
    99309957
     9958    def stacksize(self):
     9959        r"""
     9960        Returns the current size of the PARI stack, which is `10^6`
     9961        by default.  However, the stack size is automatically doubled
     9962        when needed.  It can also be set directly using
     9963        ``pari.allocatemem()``.
     9964       
     9965        EXAMPLES::
     9966           
     9967            sage: pari.stacksize()
     9968            1000000
     9969           
     9970        """
     9971        global top, bot
     9972        return (<size_t>(top) - <size_t>(bot))
     9973
     9974    def _allocate_huge_mem(self):
     9975        r"""
     9976        This function tries to allocate an impossibly large amount of
     9977        PARI stack in order to test ``init_stack()`` failing.
     9978       
     9979        TESTS::
     9980           
     9981            sage: pari.allocatemem(10^6, silent=True)
     9982            sage: pari._allocate_huge_mem()
     9983            Traceback (most recent call last):
     9984            ...
     9985            MemoryError: Unable to allocate ... (instead, allocated 1000000 bytes)
     9986       
     9987        Test that the PARI stack is sane after this failure::
     9988           
     9989            sage: a = pari('2^10000000')
     9990            sage: pari.allocatemem(10^6, silent=True)
     9991        """
     9992        # Since size_t is unsigned, this should wrap over to a very
     9993        # large positive number.
     9994        init_stack(<size_t>(-4096))
     9995
     9996    def _setup_failed_retry(self):
     9997        r"""
     9998        This function pretends that the PARI stack is larger than it
     9999        actually is such that allocatemem(0) will allocate much more
     10000        than double the current stack.  That allocation will then fail.
     10001        This function is meant to be used only in this doctest.
     10002       
     10003        TESTS::
     10004           
     10005            sage: pari.allocatemem(4096, silent=True)
     10006            sage: pari._setup_failed_retry()
     10007            sage: try:  # Any result is fine as long as it's not a segfault
     10008            ....:     a = pari('2^1000000')
     10009            ....: except MemoryError:
     10010            ....:     pass
     10011            sage: pari.allocatemem(10^6, silent=True)
     10012        """
     10013        global top
     10014        # Pretend top is at a very high address
     10015        top = <pari_sp>(<size_t>(-16))
     10016
    993110017    def allocatemem(self, s=0, silent=False):
    993210018        r"""
    9933         Double the *PARI* stack.
    9934         """
    9935         if s == 0 and not silent:
    9936             print "Doubling the PARI stack."
    9937         s = int(s)
    9938         cdef size_t a = s
    9939         if int(a) != s:
    9940             raise ValueError, "s must be nonnegative and not too big."
     10019        Change the PARI stack space to the given size (or double the
     10020        current size if ``s`` is `0`).
     10021
     10022        If `s = 0` and insufficient memory is avaible to double, the
     10023        PARI stack will be enlarged by a smaller amount.  In any case,
     10024        a ``MemoryError`` will be raised if the requested memory cannot
     10025        be allocated.
     10026       
     10027        The PARI stack is never automatically shrunk.  You can use the
     10028        command ``pari.allocatemem(10^6)`` to reset the size to `10^6`,
     10029        which is the default size at startup.  Note that the results of
     10030        computations using Sage's PARI interface are copied to the
     10031        Python heap, so they take up no space in the PARI stack.
     10032        The PARI stack is cleared after every computation.
     10033       
     10034        It does no real harm to set this to a small value as the PARI
     10035        stack will be automatically doubled when we run out of memory.
     10036        However, it could make some calculations slower (since they have
     10037        to be redone from the start after doubling the stack until the
     10038        stack is big enough).
     10039       
     10040        INPUT:
     10041       
     10042        - ``s`` - an integer (default: 0).  A non-zero argument should
     10043          be the size in bytes of the new PARI stack.  If `s` is zero,
     10044          try to double the current stack size.
     10045       
     10046        EXAMPLES::
     10047       
     10048            sage: pari.allocatemem(10^7)
     10049            PARI stack size set to 10000000 bytes
     10050            sage: pari.allocatemem()  # Double the current size
     10051            PARI stack size set to 20000000 bytes
     10052            sage: pari.stacksize()
     10053            20000000
     10054            sage: pari.allocatemem(10^6)
     10055            PARI stack size set to 1000000 bytes
     10056       
     10057        The following computation will automatically increase the PARI
     10058        stack size::
     10059           
     10060            sage: a = pari('2^100000000')
     10061       
     10062        ``a`` is now a Python variable on the Python heap and does not
     10063        take up any space on the PARI stack.  The PARI stack is still
     10064        large because of the computation of ``a``::
     10065           
     10066            sage: pari.stacksize()
     10067            16000000
     10068            sage: pari.allocatemem(10^6)
     10069            PARI stack size set to 1000000 bytes
     10070            sage: pari.stacksize()
     10071            1000000
     10072       
     10073        TESTS:
     10074       
     10075        Do the same without using the string interface and starting
     10076        from a very small stack size::
     10077           
     10078            sage: pari.allocatemem(1)
     10079            PARI stack size set to 1024 bytes
     10080            sage: a = pari(2)^100000000
     10081            sage: pari.stacksize()
     10082            16777216
     10083        """
     10084        s = long(s)
     10085        if s < 0:
     10086            raise ValueError("Stack size must be nonnegative")
    994110087        init_stack(s)
     10088        if not silent:
     10089            print "PARI stack size set to", self.stacksize(), "bytes"
    994210090
    994310091    def pari_version(self):
    994410092        return str(PARIVERSION)
     
    1036010508        cdef gen x
    1036110509       
    1036210510        pari_catch_sig_on()
    10363          # The gtomat is very important!!  Without sage/PARI will segfault.
    10364          # I do not know why. -- William Stein
    10365         A = self.new_gen(gtomat(zeromat(m,n)))
     10511        A = self.new_gen(zeromatcopy(m,n))
    1036610512        if entries is not None:
    1036710513            if len(entries) != m*n:
    1036810514                raise IndexError, "len of entries (=%s) must be %s*%s=%s"%(len(entries),m,n,m*n)
     
    1037610522        return A
    1037710523
    1037810524
    10379 ##############################################
    10380 # Used in integer factorization -- must be done
    10381 # after the pari_instance creation above:
    10382 
    10383 cdef gen _tmp = P.new_gen_noclear(gp_read_str('1000000000000000'))
    10384 cdef GEN ten_to_15 = _tmp.g
    10385 
    10386 ##############################################
    10387 
    10388 def init_pari_stack(size=8000000):
     10525def init_pari_stack(s=8000000):
    1038910526    """
    10390     Change the PARI scratch stack space to the given size.
    10391    
    10392     The main application of this command is that you've done some
    10393     individual PARI computation that used a lot of stack space. As a
    10394     result the PARI stack may have doubled several times and is now
    10395     quite large. That object you computed is copied off to the heap,
    10396     but the PARI stack is never automatically shrunk back down. If you
    10397     call this function you can shrink it back down.
    10398    
    10399     If you set this too small then it will automatically be increased
    10400     if it is exceeded, which could make some calculations initially
    10401     slower (since they have to be redone until the stack is big
    10402     enough).
    10403    
    10404     INPUT:
    10405    
    10406    
    10407     -  ``size`` - an integer (default: 8000000)
    10408    
    10409    
     10527    Deprecated, use ``pari.allocatemem()`` instead.
     10528
    1041010529    EXAMPLES::
    10411    
    10412         sage: get_memory_usage()                       # random output
    10413         '122M+'
    10414         sage: a = pari('2^100000000')
    10415         sage: get_memory_usage()                       # random output
    10416         '157M+'
    10417         sage: del a
    10418         sage: get_memory_usage()                       # random output
    10419         '145M+'
    10420    
    10421     Hey, I want my memory back!
    10422    
    10423     ::
    10424    
    10425         sage: sage.libs.pari.gen.init_pari_stack()
    10426         sage: get_memory_usage()                       # random output
    10427         '114M+'
    10428    
    10429     Ahh, that's better.
     10530
     10531        sage: from sage.libs.pari.gen import init_pari_stack
     10532        sage: init_pari_stack()
     10533        doctest:...: DeprecationWarning: init_pari_stack() is deprecated; use pari.allocatemem() instead.
     10534        See http://trac.sagemath.org/10018 for details.
     10535        sage: pari.stacksize()
     10536        8000000
    1043010537    """
    10431     init_stack(size)
    10432 
    10433 cdef int init_stack(size_t size) except -1:
    10434     cdef size_t s
    10435     cdef pari_sp cur_stack_size
    10436 
     10538    from sage.misc.superseded import deprecation
     10539    deprecation(10018, 'init_pari_stack() is deprecated; use pari.allocatemem() instead.')
     10540    P.allocatemem(s, silent=True)
     10541
     10542
     10543cdef int init_stack(size_t requested_size) except -1:
     10544    r"""
     10545    Low-level Cython function to allocate the PARI stack.  This
     10546    function should not be called directly, use ``pari.allocatemem()``
     10547    instead.
     10548    """
    1043710549    global top, bot, avma, mytop
    1043810550
    10439 
    10440     err = False    # whether or not a memory allocation error occurred.
    10441 
    10442    
    10443     # delete this if get core dumps and change the 2* to a 1* below.
    10444     if bot:
    10445         sage_free(<void*>bot)
    10446 
    10447     prev_stack_size = top - bot
    10448     if size == 0:
    10449         size = 2 * prev_stack_size
    10450 
    10451     # Decide on size
    10452     s = fix_size(size)
    10453 
    10454     # Allocate memory for new stack using Python's memory allocator.
    10455     # As explained in the python/C API reference, using this instead
    10456     # of malloc is much better (and more platform independent, etc.)
    10457     bot = <pari_sp> sage_malloc(s)
    10458 
    10459     while not bot:
    10460         err = True
    10461         s = fix_size(prev_stack_size)
    10462         bot = <pari_sp> sage_malloc(s)
     10551    cdef size_t old_size = <size_t>(top) - <size_t>(bot)
     10552
     10553    cdef size_t new_size
     10554    cdef size_t max_size = <size_t>(-1)
     10555    if (requested_size == 0):
     10556        if old_size < max_size/2:
     10557            # Double the stack
     10558            new_size = 2*old_size
     10559        elif old_size < 4*(max_size/5):
     10560            # We cannot possibly double since we already use over half
     10561            # the addressable memory: take the average of current and
     10562            # maximum size
     10563            new_size = max_size/2 + old_size/2
     10564        else:
     10565            # We already use 80% of the addressable memory => give up
     10566            raise MemoryError("Unable to enlarge PARI stack (instead, kept the stack at %s bytes)"%(old_size))
     10567    else:
     10568        new_size = requested_size
     10569
     10570    # Align size to 16 bytes and take 1024 bytes as a minimum
     10571    new_size = (new_size/16)*16
     10572    if (new_size < 1024):
     10573        new_size = 1024
     10574   
     10575    # Disable interrupts
     10576    sig_on()
     10577    sig_block()
     10578
     10579    # If this is non-zero, the size we failed to allocate
     10580    cdef size_t failed_size = 0
     10581
     10582    try:
     10583        # Free the current stack
     10584        if bot:
     10585            libc.stdlib.free(<void*>bot)
     10586   
     10587        # Allocate memory for new stack.
     10588        bot = <pari_sp> libc.stdlib.malloc(new_size)
     10589   
     10590        # If doubling failed, instead add 25% to the current stack size.
     10591        # We already checked that we use less than 80% of the maximum value
     10592        # for s, so this will not overflow.
     10593        if (bot == 0) and (requested_size == 0):
     10594            new_size = (old_size/64)*80
     10595            bot = <pari_sp> libc.stdlib.malloc(new_size)
     10596       
    1046310597        if not bot:
    10464             prev_stack_size /= 2
    10465    
    10466     top = bot + s
    10467     mytop = top
    10468     avma = top
    10469 
    10470     if err:
    10471         raise MemoryError, "Unable to allocate %s bytes memory for PARI."%size
    10472 
    10473 
    10474 cdef size_t fix_size(size_t a):
    10475     cdef size_t b
    10476     b = a - (a & (sizeof(long)-1))     # sizeof(long) | b <= a 
    10477     if b < 1024:
    10478         b = 1024
    10479     return b
     10598            failed_size = new_size
     10599            # We lost our PARI stack and are not able to allocate the
     10600            # requested size.  If we just raise an exception now, we end up
     10601            # *without* a PARI stack which is not good.  We will raise an
     10602            # exception later, after allocating *some* PARI stack.
     10603            new_size = old_size
     10604            while new_size >= 1024:  # hope this never fails!
     10605                bot = <pari_sp> libc.stdlib.malloc(new_size)
     10606                if bot: break
     10607                new_size = (new_size/32)*16
     10608       
     10609        if not bot:
     10610            top = 0
     10611            avma = 0
     10612            raise SystemError("Unable to allocate PARI stack, all subsequent PARI computations will crash")
     10613
     10614        top = bot + new_size
     10615        mytop = top
     10616        avma = top
     10617
     10618        if failed_size:
     10619            raise MemoryError("Unable to allocate %s bytes for the PARI stack (instead, allocated %s bytes)"%(failed_size, new_size))
     10620   
     10621        return 0
     10622    finally:
     10623        sig_unblock()
     10624        sig_off()
     10625   
    1048010626
    1048110627cdef GEN deepcopy_to_python_heap(GEN x, pari_sp* address):
    1048210628    cdef size_t s = <size_t> gsizebyte(x)
     
    1048710633    address[0] = tmp_bot
    1048810634    return gcopy_avma(x, &tmp_top)
    1048910635
    10490 cdef gen _new_gen (GEN x):
     10636cdef gen _new_gen(GEN x):
    1049110637    cdef GEN h
    1049210638    cdef pari_sp address
    1049310639    cdef gen y
  • sage/libs/pari/handle_error.pyx

    diff --git a/sage/libs/pari/handle_error.pyx b/sage/libs/pari/handle_error.pyx
    a b  
    7878    """
    7979    from sage.libs.pari.gen import pari, PariError
    8080    if err == errpile:
     81        # PARI is out of memory.  We double the size of the PARI stack
     82        # and retry the computation.
    8183        pari.allocatemem(silent=True)
    8284        return 0
    8385
     
    9799    Perform a computation that requires doubling the default stack
    98100    several times::
    99101
    100         sage: from sage.libs.pari.gen import init_pari_stack
    101         sage: init_pari_stack(2^12)
     102        sage: pari.allocatemem(2^12)
     103        PARI stack size set to 4096 bytes
    102104        sage: x = pari('2^(2^26)')
    103105        sage: x == 2^(2^26)
    104106        True