Ticket #14713: new_transformers.patch

File new_transformers.patch, 16.1 KB (added by jason, 6 years ago)
  • sage/misc/interpreter.py

    # HG changeset patch
    # Parent 1077314f416653b28e199c382667a1f11e444bdd
    Implement the Sage preparsing transformations using the new IPython transformation framework.
    
    diff --git a/sage/misc/interpreter.py b/sage/misc/interpreter.py
    a b  
    234234###################################################################
    235235# Transformers used in the SageInputSplitter
    236236###################################################################
    237 # These used to be part of the older PrefilterTransformer framework,
    238 # but that is not the modern way of doing things. For more details, see
    239 # http://mail.scipy.org/pipermail/ipython-dev/2011-March/007334.html
     237from IPython.core.inputtransformer import (CoroutineInputTransformer,
     238                                           StatelessInputTransformer,
     239                                           _strip_prompts)
    240240
    241 class SagePreparseTransformer():
     241@CoroutineInputTransformer.wrap
     242def sage_preparse_transformer():
    242243    """
    243     Preparse the line of code before it get evaluated by IPython.
     244    Implement Sage-specific preparsing (see sage.misc.preparser.preparse)
     245
     246    EXAMPLES::
     247
     248        sage: from sage.misc.interpreter import sage_preparse_transformer
     249        sage: spt = sage_preparse_transformer()
     250        sage: spt.push('1+1r+2.3^2.3r')
     251        "Integer(1)+1+RealNumber('2.3')**2.3"
     252        sage: preparser(False)
     253        sage: spt.push('2.3^2')
     254        '2.3^2'
    244255    """
    245     def __call__(self, line, line_number):
    246         """
    247         Transform ``line``.
     256    line = (yield '')
     257    while True:
     258        if line is None:
     259            rline = preparse('', reset=True)
     260        elif do_preparse and not line.startswith('%'):
     261            rline = preparse(line)
     262        else:
     263            rline = line
     264        line = (yield rline)
    248265
    249         INPUT:
    250 
    251         - ``line`` -- string. The line to be transformed.
    252 
    253         OUTPUT:
    254 
    255         A string, the transformed line.
    256 
    257         EXAMPLES::
    258 
    259             sage: from sage.misc.interpreter import SagePreparseTransformer
    260             sage: spt = SagePreparseTransformer()
    261             sage: spt('2', 0)
    262             'Integer(2)'
    263             sage: preparser(False)
    264             sage: spt('2', 0)
    265             '2'
    266             sage: preparser(True)
    267         """
    268         if do_preparse and not line.startswith('%'):
    269             # we use preparse_file instead of just preparse because preparse_file
    270             # automatically prepends attached files
    271             return preparse(line, reset=(line_number==0))
    272         else:
    273             return line
    274 
    275 class MagicTransformer():
     266_magic_deprecations = {'load': '%runfile',
     267                       'attach': '%attach',
     268                       'time': '%time'}
     269@StatelessInputTransformer.wrap
     270def magic_transformer(line):
    276271    r"""
    277272    Handle input lines that start out like ``load ...`` or ``attach
    278273    ...``.
     
    281276    ``attach``, IPython's automagic will not transform these lines
    282277    into ``%load ...`` and ``%attach ...``, respectively.  Thus, we
    283278    have to do it manually.
    284     """
    285     deprecations = {'load': '%runfile',
    286                     'attach': '%attach',
    287                     'time': '%time'}
    288     def __call__(self, line, line_number):
    289         """
    290         Transform ``line``.
    291 
    292         INPUT:
    293 
    294         - ``line`` -- string. The line to be transformed.
    295 
    296         OUTPUT:
    297 
    298         A string, the transformed line.
    299279
    300280        EXAMPLES::
    301281
    302             sage: from sage.misc.interpreter import get_test_shell, MagicTransformer
    303             sage: mt = MagicTransformer()
    304             sage: mt('load /path/to/file', 0)
    305             doctest:1: DeprecationWarning: Use %runfile instead of load.
     282            sage: from sage.misc.interpreter import get_test_shell, magic_transformer
     283            sage: mt = magic_transformer()
     284            sage: mt.push('load /path/to/file')
     285            doctest:...: DeprecationWarning: Use %runfile instead of load.
    306286            See http://trac.sagemath.org/12719 for details.
    307287            '%runfile /path/to/file'
    308             sage: mt('attach /path/to/file', 0)
    309             doctest:1: DeprecationWarning: Use %attach instead of attach.
     288            sage: mt.push('attach /path/to/file')
     289            doctest:...: DeprecationWarning: Use %attach instead of attach.
    310290            See http://trac.sagemath.org/12719 for details.
    311291            '%attach /path/to/file'
    312             sage: mt('time 1+2', 0)
    313             doctest:1: DeprecationWarning: Use %time instead of time.
     292            sage: mt.push('time 1+2')
     293            doctest:...: DeprecationWarning: Use %time instead of time.
    314294            See http://trac.sagemath.org/12719 for details.
    315295            '%time 1+2'
    316         """
    317         for old,new in self.deprecations.items():
    318             if line.startswith(old+' '):
    319                 from sage.misc.superseded import deprecation
    320                 deprecation(12719, 'Use %s instead of %s.'%(new,old))
    321                 return new+line[len(old):]
    322         return line
     296    """
     297    global _magic_deprecations
     298    for old,new in _magic_deprecations.items():
     299        if line.startswith(old+' '):
     300            from sage.misc.superseded import deprecation
     301            deprecation(12719, 'Use %s instead of %s.'%(new,old))
     302            return new+line[len(old):]
     303    return line
    323304
    324 class SagePromptTransformer():
     305
     306@CoroutineInputTransformer.wrap
     307def sage_prompt_transformer():
    325308    """
    326     Remove Sage prompts from the input line.
     309    Strip the sage:/... prompts of Sage.
     310
     311    EXAMPLES::
     312   
     313        sage: from sage.misc.interpreter import sage_prompt_transformer
     314        sage: spt = sage_prompt_transformer()
     315        sage: spt.push("sage: sage: 2 + 2")
     316        '2 + 2'
     317        sage: spt.push('')
     318        ''
     319        sage: spt.push("sage: 2+2")
     320        '2+2'
     321        sage: spt.push("... .... ....: ...: 2+2")
     322        '2+2'
    327323    """
    328     _sage_prompt_re = re.compile(r'(^[ \t]*sage: |^[ \t]*\.+:? )+')
    329 
    330     def __call__(self, line, line_number):
    331         """
    332         Transform ``line``.
    333 
    334         INPUT:
    335 
    336         - ``line`` -- string. The line to be transformed.
    337 
    338         OUTPUT:
    339 
    340         A string, the transformed line.
    341 
    342         EXAMPLES::
    343 
    344             sage: from sage.misc.interpreter import SagePromptTransformer
    345             sage: spt = SagePromptTransformer()
    346             sage: spt("sage: sage: 2 + 2", 0)
    347             '2 + 2'
    348             sage: spt('', 0)
    349             ''
    350             sage: spt("      sage: 2+2", 0)
    351             '2+2'
    352             sage: spt("      ... 2+2", 0)
    353             '2+2'
    354         """
    355         if not line or line.isspace() or line.strip() == '...':
    356             # This allows us to recognize multiple input prompts separated by
    357             # blank lines and pasted in a single chunk, very common when
    358             # pasting doctests or long tutorial passages.
    359             return ''
    360         while True:
    361             m = self._sage_prompt_re.match(line)
    362             if m:
    363                 line = line[len(m.group(0)):]
    364             else:
    365                 break
    366         return line
    367 
    368 class SagePromptDedenter():
    369     """
    370     Remove leading spaces from the input line.
    371     """
    372     def __call__(self, line, line_number):
    373         """
    374         Transform ``line``.
    375 
    376         INPUT:
    377 
    378         - ``line`` -- string. The line to be transformed.
    379 
    380         - ``line_number`` -- integer. The line number. For a single-line input, this is always zero.
    381        
    382         OUTPUT:
    383 
    384         A string, the transformed line.
    385 
    386         EXAMPLES::
    387 
    388             sage: from sage.misc.interpreter import SagePromptDedenter
    389             sage: spd = SagePromptDedenter()
    390             sage: spd('  1 + \\', 0)
    391             '1 + \\'
    392             sage: spd('  2', 1)
    393             '2'
    394             sage: spd('3', 2)   # try our best with incorrect indentation
    395             '3'
    396         """
    397         if line_number == 0:
    398             dedent_line = line.lstrip()
    399             self._dedent = len(line) - len(dedent_line)
    400             return dedent_line
    401         else:
    402             dedent = min(len(line)-len(line.lstrip()), self._dedent)
    403             return line[dedent:]
     324    _sage_prompt_re = re.compile(r'^(sage: |\.\.\.\.?:? )+')
     325    return _strip_prompts(_sage_prompt_re)
    404326
    405327###################
    406328# Interface shell #
  • sage/misc/sage_extension.py

    diff --git a/sage/misc/sage_extension.py b/sage/misc/sage_extension.py
    a b  
    226226            sage: shell.display_formatter.formatters['text/plain']
    227227            <...sage_extension.SagePlainTextFormatter object at 0x...>
    228228            sage: shell.displayhook.compute_format_data(2)
    229             {u'text/plain': '2'}
     229            ({u'text/plain': '2'}, {})
    230230            sage: a = identity_matrix(ZZ, 2)
    231231            sage: shell.displayhook.compute_format_data([a,a])
    232             {u'text/plain': '[\n[1 0]  [1 0]\n[0 1], [0 1]\n]'}
     232            ({u'text/plain': '[\n[1 0]  [1 0]\n[0 1], [0 1]\n]'}, {})
    233233        """
    234234        import sage
    235235        from sage.misc.displayhook import format_obj
     
    238238            s = super(SagePlainTextFormatter, self).__call__(obj)
    239239        return s
    240240
    241 
    242 # SageInputSplitter:
    243 #  Hopefully most or all of this code can go away when
    244 #  https://github.com/ipython/ipython/issues/2293 is resolved
    245 #  apparently we will have stateful transformations then.
    246 #  see also https://github.com/ipython/ipython/pull/2402
    247 from IPython.core.inputsplitter import (transform_ipy_prompt, transform_classic_prompt,
    248                                         transform_help_end, transform_escaped,
    249                                         transform_assign_system, transform_assign_magic,
    250                                         cast_unicode,
    251                                         IPythonInputSplitter)
    252 
    253 def first_arg(f):
    254     def tm(arg1, arg2):
    255         return f(arg1)
    256     return tm
    257 
    258 class SageInputSplitter(IPythonInputSplitter):
    259     """
    260     We override the input splitter for two reasons:
    261 
    262     1. to make the list of transforms a class attribute that can be modified
    263 
    264     2. to pass the line number to transforms (we strip the line number off for IPython transforms)
    265     """
    266 
    267     # List of input transforms to apply
    268     transforms = map(first_arg, [transform_ipy_prompt, transform_classic_prompt,
    269                                  transform_help_end, transform_escaped,
    270                                  transform_assign_system, transform_assign_magic])
    271 
    272     # a direct copy of the IPython splitter, except that the
    273     # transforms are called with the line numbers, and the transforms come from the class attribute
    274     # and except that we add a .startswith('@') in the test to see if we should transform a line
    275     # (see http://mail.scipy.org/pipermail/ipython-dev/2012-September/010329.html; the current behavior
    276     # doesn't run the preparser on a 'def' line following an decorator.
    277     def push(self, lines):
    278         """Push one or more lines of IPython input.
    279 
    280         This stores the given lines and returns a status code indicating
    281         whether the code forms a complete Python block or not, after processing
    282         all input lines for special IPython syntax.
    283 
    284         Any exceptions generated in compilation are swallowed, but if an
    285         exception was produced, the method returns True.
    286 
    287         Parameters
    288         ----------
    289         lines : string
    290           One or more lines of Python input.
    291 
    292         Returns
    293         -------
    294         is_complete : boolean
    295           True if the current input source (the result of the current input
    296         plus prior inputs) forms a complete Python execution block.  Note that
    297         this value is also stored as a private attribute (_is_complete), so it
    298         can be queried at any time.
    299         """
    300         if not lines:
    301             return super(IPythonInputSplitter, self).push(lines)
    302 
    303         # We must ensure all input is pure unicode
    304         lines = cast_unicode(lines, self.encoding)
    305 
    306         # If the entire input block is a cell magic, return after handling it
    307         # as the rest of the transformation logic should be skipped.
    308         if lines.startswith('%%') and not \
    309           (len(lines.splitlines()) == 1 and lines.strip().endswith('?')):
    310             return self._handle_cell_magic(lines)
    311 
    312         # In line mode, a cell magic can arrive in separate pieces
    313         if self.input_mode == 'line' and self.processing_cell_magic:
    314             return self._line_mode_cell_append(lines)
    315 
    316         # The rest of the processing is for 'normal' content, i.e. IPython
    317         # source that we process through our transformations pipeline.
    318         lines_list = lines.splitlines()
    319 
    320         # Transform logic
    321         #
    322         # We only apply the line transformers to the input if we have either no
    323         # input yet, or complete input, or if the last line of the buffer ends
    324         # with ':' (opening an indented block).  This prevents the accidental
    325         # transformation of escapes inside multiline expressions like
    326         # triple-quoted strings or parenthesized expressions.
    327         #
    328         # The last heuristic, while ugly, ensures that the first line of an
    329         # indented block is correctly transformed.
    330         #
    331         # FIXME: try to find a cleaner approach for this last bit.
    332 
    333         # If we were in 'block' mode, since we're going to pump the parent
    334         # class by hand line by line, we need to temporarily switch out to
    335         # 'line' mode, do a single manual reset and then feed the lines one
    336         # by one.  Note that this only matters if the input has more than one
    337         # line.
    338         changed_input_mode = False
    339 
    340         if self.input_mode == 'cell':
    341             self.reset()
    342             changed_input_mode = True
    343             saved_input_mode = 'cell'
    344             self.input_mode = 'line'
    345 
    346         # Store raw source before applying any transformations to it.  Note
    347         # that this must be done *after* the reset() call that would otherwise
    348         # flush the buffer.
    349         self._store(lines, self._buffer_raw, 'source_raw')
    350 
    351         try:
    352             push = super(IPythonInputSplitter, self).push
    353             buf = self._buffer
    354             for line in lines_list:
    355                 line_number = len(buf)
    356                 if (self._is_complete or not buf or
    357                     buf[-1].rstrip().endswith((':', ',')) or buf[-1].lstrip().startswith('@')):
    358                     for f in self.transforms:
    359                         line = f(line, line_number)
    360                 else:
    361                     for f in self.always_transform:
    362                         line = f(line, line_number)
    363                 out = push(line)
    364         finally:
    365             if changed_input_mode:
    366                 self.input_mode = saved_input_mode
    367         return out
    368 
    369 # END SageIPythonInputSplitter
    370 #
    371 #
    372 
    373 
    374241class SageCustomizations(object):
    375242    startup_code = """from sage.all_cmdline import *
    376243from sage.misc.interpreter import sage_prompt
     
    471338        """
    472339        Set up transforms (like the preparser).
    473340        """
    474         self.shell.input_splitter = SageInputSplitter()
    475341        import sage
    476342        import sage.all
    477         from sage.misc.interpreter import (SagePromptDedenter, SagePromptTransformer,
    478                                            MagicTransformer, SagePreparseTransformer)
    479 
    480         p = SagePreparseTransformer()
    481         self.shell.input_splitter.transforms = [SagePromptDedenter(),
    482                                                 SagePromptTransformer(),
    483                                                 MagicTransformer(),
    484                                                 p] + self.shell.input_splitter.transforms
    485         self.shell.input_splitter.always_transform = [p]
    486 
     343        from interpreter import (sage_preparse_transformer,
     344                                 sage_prompt_transformer,
     345                                 magic_transformer)
     346        for s in (self.shell.input_splitter, self.shell.input_transformer_manager):
     347            s.physical_line_transforms.extend([sage_prompt_transformer()])
     348            s.logical_line_transforms.insert(0, magic_transformer())
     349            s.python_line_transforms.extend([sage_preparse_transformer()])
    487350        preparser(True)
    488351
    489352    def deprecated(self):