Ticket #10296: trac10296_singular_overhead.patch

File trac10296_singular_overhead.patch, 19.5 KB (added by SimonKing, 8 years ago)

Modify garbage collection in Singular interface, which reduces the overhead. Synchronization of Gap interface.

  • sage/interfaces/expect.py

    # HG changeset patch
    # User Simon King <simon.king@uni-jena.de>
    # Date 1290512557 -3600
    # Node ID 7b33c4033cd13a869e4251a5bfd00e0b80c95b2c
    # Parent  b431a86793475e2ed1b4b9baafc7a509ffeead97
    #10296: Reduce overhead in Singular interface by better garbage collection. Synchronisation of GAP.
    
    diff --git a/sage/interfaces/expect.py b/sage/interfaces/expect.py
    a b  
    1919  which is important for forking.
    2020
    2121- Jean-Pierre Flori (2010,2011): Split non Pexpect stuff into a parent class.
     22
     23- Simon King (2010-11-23): Ensure that the interface is started again
     24  after a crash, when a command is executed in _eval_line. Allow
     25  synchronisation of the GAP interface.
     26
    2227"""
    2328
    2429#*****************************************************************************
     
    315320        pass
    316321
    317322    def pid(self):
     323        """
     324    Return the PID of the underlying sub-process.
     325
     326        REMARK:
     327
     328        If the interface terminates unexpectedly, the original
     329        PID will still be used. But if it was terminated using
     330        :meth:`quit`, a new sub-process with a new PID is
     331        automatically started.
     332
     333        EXAMPLE::
     334
     335            sage: pid = gap.pid()
     336            sage: gap.eval('quit;')
     337            ''
     338            sage: pid == gap.pid()
     339            True
     340            sage: gap.quit()
     341            sage: pid == gap.pid()
     342            False
     343
     344        """
    318345        if self._expect is None:
    319346            self._start()
    320347        return self._expect.pid
     
    643670    def _read_in_file_command(self, filename):
    644671        raise NotImplementedError
    645672
    646     def _eval_line_using_file(self, line):
     673    def _eval_line_using_file(self, line, first_call=True):
     674        """
     675        Evaluate a line of commands, using a temporary file.
     676
     677        REMARK:
     678
     679        By default, this is called when a long command is
     680        evaluated in :meth:`eval`.
     681
     682        If the command can not be evaluated since the interface
     683        has crashed, it is automatically restarted and tried
     684        again *once*.
     685
     686        INPUT:
     687
     688        - ``line`` -- (string) a command.
     689        - ``first_call`` - (optional bool, default ``True``) --
     690          If it is ``True``, the command evaluation is evaluated
     691          a second time after restarting the interface, if an
     692          EOFError occured.
     693
     694        TESTS::
     695
     696            sage: singular._eval_line_using_file('def a=3;')
     697            '< "...";'
     698            sage: singular('a')
     699            3
     700            sage: singular.eval('quit;')
     701            ''
     702            sage: singular._eval_line_using_file('def a=3;')
     703            Singular crashed -- automatically restarting.
     704            '< "...";'
     705            sage: singular('a')
     706            3
     707            sage: singular.eval('quit;')
     708            ''
     709            sage: singular._eval_line_using_file('def a=3;', first_call=False)
     710            Traceback (most recent call last):
     711            ...
     712            RuntimeError: Singular terminated unexpectedly while reading in a large line
     713
     714        We end by triggering a re-start of Singular, since otherwise
     715        the doc test of another method would fail by a side effect.
     716        ::
     717
     718            sage: singular(3)
     719            Singular crashed -- automatically restarting.
     720            3
     721
     722        """
    647723        F = open(self._local_tmpfile(), 'w')
    648724        F.write(line+'\n')
    649725        F.close()
     
    657733            if self._quit_string() in line:
    658734                # we expect to get an EOF if we're quitting.
    659735                return ''
     736            elif first_call==True: # the subprocess might have crashed
     737                try:
     738                    self._synchronize()
     739                    return self._eval_line_using_file(line, first_call=False)
     740                except (TypeError, RuntimeError):
     741                    pass
    660742            raise RuntimeError, '%s terminated unexpectedly while reading in a large line'%self
    661743        return self._post_process_from_file(s)
    662744
    663745    def _post_process_from_file(self, s):
    664746        return s
    665747
    666     def _eval_line(self, line, allow_use_file=True, wait_for_prompt=True):
    667         if allow_use_file and self._eval_using_file_cutoff and len(line) > self._eval_using_file_cutoff:
     748    def _eval_line(self, line, allow_use_file=True, wait_for_prompt=True, first_call=True):
     749        """
     750        Evaluate a line of commands.
     751
     752        REMARK:
     753
     754        By default, a long command (length exceeding ``self._eval_using_file_cutoff``)
     755        is evaluated using :meth:`_eval_line_using_file`.
     756
     757        If the command can not be evaluated since the interface
     758        has crashed, it is automatically restarted and tried
     759        again *once*.
     760
     761        If the optional ``wait_for_prompt`` is ``False`` then even a very
     762        long line will not be evaluated by :meth:`_eval_line_using_file`,
     763        since this does not support the ``wait_for_prompt`` option.
     764
     765        INPUT:
     766
     767        - ``line`` -- (string) a command.
     768        - ``allow_use_file`` (optional bool, default ``True``) --
     769          allow to evaluate long commands using :meth:`_eval_line_using_file`.
     770        - ``wait_for_prompt`` (optional bool, default ``True``) --
     771          wait until the prompt appears in the sub-process' output.
     772        - ``first_call`` (optional bool, default ``True``) --
     773          If it is ``True``, the command evaluation is evaluated
     774          a second time after restarting the interface, if an
     775          EOFError occured.
     776
     777        TESTS::
     778
     779            sage: singular._eval_line('def a=3;')
     780            'def a=3;'
     781            sage: singular('a')
     782            3
     783            sage: singular.eval('quit;')
     784            ''
     785            sage: singular._eval_line('def a=3;')
     786            Singular crashed -- automatically restarting.
     787            'def a=3;'
     788            sage: singular('a')
     789            3
     790            sage: singular.eval('kill a')
     791            'kill a;'
     792
     793        We are now sending a command that would run forever. But since
     794        we declare that we are not waiting for a prompt, we can interrupt
     795        it without a KeyboardInterrupt. At the same time, we test that
     796        the line is not forwarded to :meth:`_eval_line_using_file`, since
     797        that method would not support the ``wait_for_prompt`` option::
     798
     799            sage: cutoff = singular._eval_using_file_cutoff
     800            sage: singular._eval_using_file_cutoff = 4
     801            sage: singular._eval_line('for(int i=1;i<=3;i++){i=1;};', wait_for_prompt=False)
     802            ''
     803            sage: singular.interrupt(timeout=1)
     804            False
     805            sage: singular._eval_using_file_cutoff = cutoff
     806
     807        Last, we demonstrate that by default the execution of a command
     808        is tried twice if it fails the first time due to a crashed
     809        interface::
     810
     811            sage: singular.eval('quit;')
     812            ''
     813            sage: singular._eval_line_using_file('def a=3;', first_call=False)
     814            Traceback (most recent call last):
     815            ...
     816            RuntimeError: Singular terminated unexpectedly while reading in a large line
     817
     818        Since the doc test of the next method would fail, we re-start
     819        Singular now.
     820        ::
     821
     822            sage: singular(2+3)
     823            Singular crashed -- automatically restarting.
     824            5
     825
     826        """
     827        if allow_use_file and wait_for_prompt and self._eval_using_file_cutoff and len(line) > self._eval_using_file_cutoff:
    668828            return self._eval_line_using_file(line)
    669829        try:
    670830            if self._expect is None:
     
    699859                    if self._quit_string() in line:
    700860                        # we expect to get an EOF if we're quitting.
    701861                        return ''
     862                    elif first_call==True: # the subprocess might have crashed
     863                        try:
     864                            self._synchronize()
     865                            return self._eval_line(line,allow_use_file=allow_use_file, wait_for_prompt=wait_for_prompt, first_call=False)
     866                        except (TypeError, RuntimeError):
     867                            pass
    702868                    raise RuntimeError, "%s\n%s crashed executing %s"%(msg,self, line)
    703869                out = E.before
    704870            else:
     
    8491015            Control-C pressed.  Interrupting PARI/GP interpreter. Please wait a few seconds...
    8501016            ...
    8511017            KeyboardInterrupt: computation timed out because alarm was set for 1 seconds
     1018
     1019        Here is a doc test related with trac ticket #10296::
     1020
     1021            sage: gap._synchronize()
     1022
    8521023        """
    8531024        if expr is None:
    854             expr = self._prompt_wait
     1025            # the following works around gap._prompt_wait not being defined
     1026            expr = (hasattr(self,'_prompt_wait') and self._prompt_wait) or self._prompt
    8551027        if self._expect is None:
    8561028            self._start()
    8571029        try:
     
    11931365       
    11941366            sage: from sage.interfaces.expect import StdOutContext
    11951367            sage: with StdOutContext(singular):
    1196             ...       singular('1+1')
     1368            ...       singular.eval('1+1')
    11971369            ...
    1198             1+...
     1370            1+1;
     1371            ...
    11991372        """
    12001373        if self.silent:
    12011374            return
  • sage/interfaces/gap.py

    diff --git a/sage/interfaces/gap.py b/sage/interfaces/gap.py
    a b  
    379379
    380380
    381381    def _execute_line(self, line, wait_for_prompt=True, expect_eof=False):
     382        if self._expect is None: # interface is down
     383            self._start()
    382384        E = self._expect
     385       
    383386        try:
    384387            if len(line) > 4095:
    385388                raise RuntimeError("Passing commands this long to gap would hang")
     
    463466        self.quit()
    464467        raise KeyboardInterrupt, "Ctrl-c pressed while running %s"%self
    465468
    466     def _eval_line(self, line, allow_use_file=True, wait_for_prompt=True):
     469    def _eval_line(self, line, allow_use_file=True, wait_for_prompt=True, first_call=True):
    467470        """
    468         EXAMPLES::
     471        Evaluate a line of commands.
     472
     473        REMARK:
     474
     475        By default, a long command (length exceeding ``self._eval_using_file_cutoff``)
     476        is evaluated using :meth:`_eval_line_using_file`.
     477
     478        If the command can not be evaluated since the interface
     479        has crashed, it is automatically restarted and tried
     480        again *once*.
     481
     482        If the optional ``wait_for_prompt`` is ``False`` then even a very long line
     483        will not be evaluated by :meth:`_eval_line_using_file`, since this does not
     484        support the ``wait_for_prompt`` option.
     485
     486        INPUT:
     487
     488        - ``line`` -- (string) a command.
     489        - ``allow_use_file`` (optional bool, default ``True``) --
     490          allow to evaluate long commands using :meth:`_eval_line_using_file`.
     491        - ``wait_for_prompt`` (optional bool, default ``True``) --
     492          wait until the prompt appears in the sub-process' output.
     493        - ``first_call`` (optional bool, default ``True``) --
     494          If it is ``True``, the command evaluation is evaluated
     495          a second time after restarting the interface, if an
     496          EOFError occured.
     497
     498        TESTS::
    469499       
    470500            sage: gap._eval_line('2+2;')
    471501            '4'
     502
     503        We test the ``wait_for_prompt`` option by sending a command that
     504        creates an infinite loop in the GAP sub-process. But if we don't
     505        wait for the prompt to appear in the output, we can interrupt
     506        the loop without raising a KeyboardInterrupt. At the same time,
     507        we test that the line is not forwarded to :meth:`_eval_line_using_file`,
     508        since that method would not support the ``wait_for_prompt`` option::
     509
     510            sage: cutoff = gap._eval_using_file_cutoff
     511            sage: gap._eval_using_file_cutoff = 4
     512            sage: gap._eval_line('while(1=1) do i:=1;; od;', wait_for_prompt=False)
     513            ''
     514            sage: gap.interrupt(timeout=1) is not None
     515            True
     516            sage: gap._eval_using_file_cutoff = cutoff
     517
     518        The following tests against a bug fixed at trac ticket #10296:
     519
     520            sage: a = gap(3)
     521            sage: gap.eval('quit;')
     522            ''
     523            sage: a = gap(3)
     524            ** Gap crashed or quit executing '$sage...:=3;;' **
     525            Restarting Gap and trying again
     526            sage: a
     527            3
     528
    472529        """
    473530        #if line.find('\n') != -1:
    474531        #    raise ValueError, "line must not contain any newlines"
     
    477534                self._start()
    478535            E = self._expect
    479536            #import pdb; pdb.set_trace()
    480             if allow_use_file and len(line) > self._eval_using_file_cutoff:
     537            if allow_use_file and wait_for_prompt and len(line) > self._eval_using_file_cutoff:
    481538                return self._eval_line_using_file(line)
    482             try:
    483                 (normal, error) = self._execute_line(line, wait_for_prompt=wait_for_prompt,
     539            (normal, error) = self._execute_line(line, wait_for_prompt=wait_for_prompt,
    484540                                                 expect_eof= (self._quit_string() in line))
    485541
    486                 if len(error)> 0:
    487                     if 'Error, Rebuild completion files!' in error:
    488                         error += "\nRunning gap_reset_workspace()..."
    489                         self.quit()
    490                         gap_reset_workspace()
    491                     error = error.replace('\r','')
    492                     raise RuntimeError, "%s produced error output\n%s\n   executing %s"%(self, error,line)
    493                 if len(normal) == 0:
     542            if len(error)> 0:
     543                if 'Error, Rebuild completion files!' in error:
     544                    error += "\nRunning gap_reset_workspace()..."
     545                    self.quit()
     546                    gap_reset_workspace()
     547                error = error.replace('\r','')
     548                raise RuntimeError, "%s produced error output\n%s\n   executing %s"%(self, error,line)
     549            if len(normal) == 0:
     550                return ''
     551
     552            if isinstance(wait_for_prompt, str) and normal.ends_with(wait_for_prompt):
     553                n = len(wait_for_prompt)
     554            elif normal.endswith(self._prompt):
     555                n = len(self._prompt)
     556            elif normal.endswith(self._continuation_prompt()):
     557                n = len(self._continuation_prompt())
     558            else:
     559                n = 0
     560            out = normal[:-n]
     561            if len(out) > 0 and out[-1] == "\n":
     562                out = out[:-1]
     563            return out
     564
     565        except (RuntimeError,TypeError),message:
     566            if 'EOF' in message[0]:
     567                print "** %s crashed or quit executing '%s' **"%(self, line)
     568                print "Restarting %s and trying again"%self
     569                self._start()
     570                if line != '':
     571                    return self._eval_line(line, allow_use_file=allow_use_file)
     572                else:
    494573                    return ''
    495 
    496                 if isinstance(wait_for_prompt, str) and normal.ends_with(wait_for_prompt):
    497                     n = len(wait_for_prompt)
    498                 elif normal.endswith(self._prompt):
    499                     n = len(self._prompt)
    500                 elif normal.endswith(self._continuation_prompt()):
    501                     n = len(self._continuation_prompt())
    502                 else:
    503                     n = 0
    504                 out = normal[:-n]
    505                 if len(out) > 0 and out[-1] == "\n":
    506                     out = out[:-1]
    507                 return out
    508 
    509             except (RuntimeError,),message:
    510                 if 'EOF' in message:
    511                     print "** %s crashed or quit executing '%s' **"%(self, line)
    512                     print "Restarting %s and trying again"%self
    513                     self._start()
    514                     if line != '':
    515                         return self._eval_line(line, allow_use_file=allow_use_file)
    516                     else:
    517                         return ''
    518                 else:
    519                     raise RuntimeError, message
     574            else:
     575                raise RuntimeError, message
    520576
    521577        except KeyboardInterrupt:
    522578            self._keyboard_interrupt()
  • sage/interfaces/singular.py

    diff --git a/sage/interfaces/singular.py b/sage/interfaces/singular.py
    a b  
    1919
    2020- Martin Albrecht (2006-05-18): added sage_poly.
    2121
     22- Simon King (2010-11-23): Reduce the overhead caused by waiting for
     23  the Singular prompt by doing garbage collection differently.
     24
    2225Introduction
    2326------------
    2427
     
    525528            sage: set_verbose(0)
    526529            sage: o = s.hilb()
    527530        """
    528         # Synchronize the interface and clear any variables that are queued up to
    529         # be cleared.
    530         self._synchronize()
    531         if len(self.__to_clear) > 0:
    532             for var in self.__to_clear:
    533                 self._eval_line('if(defined(%s)>0){kill %s;};'%(var,var), wait_for_prompt=True)
    534             self.__to_clear = []
     531        # Simon King:
     532        # In previous versions, the interface was first synchronised and then
     533        # unused variables were killed. This created a considerable overhead.
     534        # By trac ticket #10296, killing unused variables is now done inside
     535        # singular.set(). Moreover, it is not done by calling a separate _eval_line.
     536        # In that way, the time spent by waiting for the singular prompt is reduced.
     537
     538        # Since nesting of _eval_line can not occur during garbage collection,
     539        # synchronisation is not needed anymore, saving even more waiting time
     540        # for the prompt.
    535541       
    536542        # Uncomment the print statements below for low-level debugging of
    537543        # code that involves the singular interfaces.  Everything goes
     
    562568        """
    563569        Set the variable with given name to the given value.
    564570       
     571        REMARK:
     572
     573        If a variable in the Singular interface was previously marked for
     574        deletion, the actual deletion is done here, before the new variable
     575        is created in Singular.
     576
    565577        EXAMPLES::
    566578       
    567579            sage: singular.set('int', 'x', '2')
    568580            sage: singular.get('x')
    569581            '2'
     582
     583        We test that an unused variable is only actually deleted if this method
     584        is called::
     585
     586            sage: a = singular(3)
     587            sage: n = a.name()
     588            sage: del a
     589            sage: singular.eval(n)
     590            '3'
     591            sage: singular.set('int', 'y', '5')
     592            sage: singular.eval('defined(%s)'%n)
     593            '0'
     594
    570595        """
    571         cmd = '%s %s=%s;'%(type, name, value)
     596        cmd = ''.join('if(defined(%s)){kill %s;};'%(v,v) for v in self.__to_clear) + '%s %s=%s;'%(type, name, value)
     597        self.__to_clear = []
    572598        try:
    573599            out = self.eval(cmd)
    574600        except RuntimeError, msg:
     
    588614       
    589615    def clear(self, var):
    590616        """
    591         Clear the variable named var.
     617        Clear the variable named ``var``.
    592618       
    593619        EXAMPLES::
    594620       
     
    596622            sage: singular.get('x')
    597623            '2'
    598624            sage: singular.clear('x')
     625
     626        "Clearing the variable" means to allow to free the memory
     627        that it uses in the Singular sub-process. However, the
     628        actual deletion of the variable is only committed when
     629        the next element in the Singular interface is created::
     630
     631            sage: singular.get('x')
     632            '2'
     633            sage: a = singular(3)
    599634            sage: singular.get('x')
    600635            '`x`'
     636
    601637        """
    602638        # We add the variable to the list of vars to clear when we do an eval.
    603639        # We queue up all the clears and do them at once to avoid synchronizing