Ticket #14713: trac_14713_new_transformers.patch

File trac_14713_new_transformers.patch, 18.8 KB (added by vbraun, 6 years ago)

Updated patch

  • sage/misc/displayhook.py

    # HG changeset patch
    # User Volker Braun <vbraun.name@gmail.com>
    # Date 1376061594 -3600
    #      Fri Aug 09 16:19:54 2013 +0100
    # Node ID 781c0789c38583494c6dcdbe4c40bb6890e94535
    # Parent  5adce6f5b34bf456beda4de1d2c949ab0f44a581
    Implement the Sage preparsing transformations using the new IPython transformation framework.
    
    diff --git a/sage/misc/displayhook.py b/sage/misc/displayhook.py
    a b  
    297297            sage: shell.display_formatter.formatters['text/plain']
    298298            <...displayhook.SagePlainTextFormatter object at 0x...>
    299299            sage: shell.displayhook.compute_format_data(2)
    300             {u'text/plain': '2'}
     300            ({u'text/plain': '2'}, {})
    301301            sage: a = identity_matrix(ZZ, 2)
    302302            sage: shell.displayhook.compute_format_data([a,a])
    303             {u'text/plain': '[\n[1 0]  [1 0]\n[0 1], [0 1]\n]'}
     303            ({u'text/plain': '[\n[1 0]  [1 0]\n[0 1], [0 1]\n]'}, {})
    304304            sage: from sage.misc.displayhook import SPTextFormatter
    305305            sage: SPTextFormatter.set_display("ascii_art")
    306306            sage: i = var('i')
    307307            sage: shell.displayhook.compute_format_data(sum(i*x^i, i, 0, 10))
    308             {u'text/plain':     10      9      8      7      6      5      4      3      2   
    309                             10*x   + 9*x  + 8*x  + 7*x  + 6*x  + 5*x  + 4*x  + 3*x  + 2*x  + x}
     308            ({u'text/plain':     10      9      8      7      6      5      4      3      2   
     309            10*x   + 9*x  + 8*x  + 7*x  + 6*x  + 5*x  + 4*x  + 3*x  + 2*x  + x}, {})
    310310        """
    311311        if hasattr(obj, '_graphics_') and not isinstance(obj, type):
    312312            if obj._graphics_():
  • sage/misc/interpreter.py

    diff --git a/sage/misc/interpreter.py b/sage/misc/interpreter.py
    a b  
    155155####################
    156156# InteractiveShell #
    157157####################
    158 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
     158from IPython.terminal.interactiveshell import TerminalInteractiveShell
    159159class SageInteractiveShell(TerminalInteractiveShell):
    160160
    161161    def system_raw(self, cmd):
     
    229229###################################################################
    230230# Transformers used in the SageInputSplitter
    231231###################################################################
    232 # These used to be part of the older PrefilterTransformer framework,
    233 # but that is not the modern way of doing things. For more details, see
    234 # http://mail.scipy.org/pipermail/ipython-dev/2011-March/007334.html
     232from IPython.core.inputtransformer import (CoroutineInputTransformer,
     233                                           StatelessInputTransformer,
     234                                           _strip_prompts)
    235235
    236 class SagePreparseTransformer():
     236@CoroutineInputTransformer.wrap
     237def sage_preparse_transformer():
    237238    """
    238     Preparse the line of code before it get evaluated by IPython.
     239    Implement Sage-specific preparsing (see sage.misc.preparser.preparse)
     240
     241    EXAMPLES::
     242
     243        sage: from sage.misc.interpreter import sage_preparse_transformer
     244        sage: spt = sage_preparse_transformer()
     245        sage: spt.push('1+1r+2.3^2.3r')
     246        "Integer(1)+1+RealNumber('2.3')**2.3"
     247        sage: preparser(False)
     248        sage: spt.push('2.3^2')
     249        '2.3^2'
    239250    """
    240     def __call__(self, line, line_number):
    241         """
    242         Transform ``line``.
     251    line = (yield '')
     252    while True:
     253        if line is None:
     254            rline = preparse('', reset=True)
     255        elif do_preparse and not line.startswith('%'):
     256            rline = preparse(line)
     257        else:
     258            rline = line
     259        line = (yield rline)
    243260
    244         INPUT:
    245 
    246         - ``line`` -- string. The line to be transformed.
    247 
    248         OUTPUT:
    249 
    250         A string, the transformed line.
    251 
    252         EXAMPLES::
    253 
    254             sage: from sage.misc.interpreter import SagePreparseTransformer
    255             sage: spt = SagePreparseTransformer()
    256             sage: spt('2', 0)
    257             'Integer(2)'
    258             sage: preparser(False)
    259             sage: spt('2', 0)
    260             '2'
    261             sage: preparser(True)
    262         """
    263         if do_preparse and not line.startswith('%'):
    264             # we use preparse_file instead of just preparse because preparse_file
    265             # automatically prepends attached files
    266             return preparse(line, reset=(line_number==0))
    267         else:
    268             return line
    269 
    270 class MagicTransformer():
     261_magic_deprecations = {'load': '%runfile',
     262                       'attach': '%attach',
     263                       'time': '%time'}
     264@StatelessInputTransformer.wrap
     265def magic_transformer(line):
    271266    r"""
    272267    Handle input lines that start out like ``load ...`` or ``attach
    273268    ...``.
     
    276271    ``attach``, IPython's automagic will not transform these lines
    277272    into ``%load ...`` and ``%attach ...``, respectively.  Thus, we
    278273    have to do it manually.
    279     """
    280     deprecations = {'load': '%runfile',
    281                     'attach': '%attach',
    282                     'time': '%time'}
    283     def __call__(self, line, line_number):
    284         """
    285         Transform ``line``.
    286 
    287         INPUT:
    288 
    289         - ``line`` -- string. The line to be transformed.
    290 
    291         OUTPUT:
    292 
    293         A string, the transformed line.
    294274
    295275        EXAMPLES::
    296276
    297             sage: from sage.misc.interpreter import get_test_shell, MagicTransformer
    298             sage: mt = MagicTransformer()
    299             sage: mt('load /path/to/file', 0)
    300             doctest:1: DeprecationWarning: Use %runfile instead of load.
     277            sage: from sage.misc.interpreter import get_test_shell, magic_transformer
     278            sage: mt = magic_transformer()
     279            sage: mt.push('load /path/to/file')
     280            doctest:...: DeprecationWarning: Use %runfile instead of load.
    301281            See http://trac.sagemath.org/12719 for details.
    302282            '%runfile /path/to/file'
    303             sage: mt('attach /path/to/file', 0)
    304             doctest:1: DeprecationWarning: Use %attach instead of attach.
     283            sage: mt.push('attach /path/to/file')
     284            doctest:...: DeprecationWarning: Use %attach instead of attach.
    305285            See http://trac.sagemath.org/12719 for details.
    306286            '%attach /path/to/file'
    307             sage: mt('time 1+2', 0)
    308             doctest:1: DeprecationWarning: Use %time instead of time.
     287            sage: mt.push('time 1+2')
     288            doctest:...: DeprecationWarning: Use %time instead of time.
    309289            See http://trac.sagemath.org/12719 for details.
    310290            '%time 1+2'
    311         """
    312         for old,new in self.deprecations.items():
    313             if line.startswith(old+' '):
    314                 from sage.misc.superseded import deprecation
    315                 deprecation(12719, 'Use %s instead of %s.'%(new,old))
    316                 return new+line[len(old):]
    317         return line
     291    """
     292    global _magic_deprecations
     293    for old,new in _magic_deprecations.items():
     294        if line.startswith(old+' '):
     295            from sage.misc.superseded import deprecation
     296            deprecation(12719, 'Use %s instead of %s.'%(new,old))
     297            return new+line[len(old):]
     298    return line
    318299
    319 class SagePromptTransformer():
     300
     301@CoroutineInputTransformer.wrap
     302def sage_prompt_transformer():
    320303    """
    321     Remove Sage prompts from the input line.
     304    Strip the sage:/... prompts of Sage.
     305
     306    EXAMPLES::
     307   
     308        sage: from sage.misc.interpreter import sage_prompt_transformer
     309        sage: spt = sage_prompt_transformer()
     310        sage: spt.push("sage: sage: 2 + 2")
     311        '2 + 2'
     312        sage: spt.push('')
     313        ''
     314        sage: spt.push("sage: 2+2")
     315        '2+2'
     316        sage: spt.push("... .... ....: ...: 2+2")
     317        '2+2'
    322318    """
    323     _sage_prompt_re = re.compile(r'(^[ \t]*sage: |^[ \t]*\.+:? )+')
    324 
    325     def __call__(self, line, line_number):
    326         """
    327         Transform ``line``.
    328 
    329         INPUT:
    330 
    331         - ``line`` -- string. The line to be transformed.
    332 
    333         OUTPUT:
    334 
    335         A string, the transformed line.
    336 
    337         EXAMPLES::
    338 
    339             sage: from sage.misc.interpreter import SagePromptTransformer
    340             sage: spt = SagePromptTransformer()
    341             sage: spt("sage: sage: 2 + 2", 0)
    342             '2 + 2'
    343             sage: spt('', 0)
    344             ''
    345             sage: spt("      sage: 2+2", 0)
    346             '2+2'
    347             sage: spt("      ... 2+2", 0)
    348             '2+2'
    349         """
    350         if not line or line.isspace() or line.strip() == '...':
    351             # This allows us to recognize multiple input prompts separated by
    352             # blank lines and pasted in a single chunk, very common when
    353             # pasting doctests or long tutorial passages.
    354             return ''
    355         while True:
    356             m = self._sage_prompt_re.match(line)
    357             if m:
    358                 line = line[len(m.group(0)):]
    359             else:
    360                 break
    361         return line
    362 
    363 class SagePromptDedenter():
    364     """
    365     Remove leading spaces from the input line.
    366     """
    367     def __call__(self, line, line_number):
    368         """
    369         Transform ``line``.
    370 
    371         INPUT:
    372 
    373         - ``line`` -- string. The line to be transformed.
    374 
    375         - ``line_number`` -- integer. The line number. For a single-line input, this is always zero.
    376        
    377         OUTPUT:
    378 
    379         A string, the transformed line.
    380 
    381         EXAMPLES::
    382 
    383             sage: from sage.misc.interpreter import SagePromptDedenter
    384             sage: spd = SagePromptDedenter()
    385             sage: spd('  1 + \\', 0)
    386             '1 + \\'
    387             sage: spd('  2', 1)
    388             '2'
    389             sage: spd('3', 2)   # try our best with incorrect indentation
    390             '3'
    391         """
    392         if line_number == 0:
    393             dedent_line = line.lstrip()
    394             self._dedent = len(line) - len(dedent_line)
    395             return dedent_line
    396         else:
    397             dedent = min(len(line)-len(line.lstrip()), self._dedent)
    398             return line[dedent:]
     319    _sage_prompt_re = re.compile(r'^(sage: |\.\.\.\.?:? )+')
     320    return _strip_prompts(_sage_prompt_re)
    399321
    400322###################
    401323# Interface shell #
    402324###################
    403325from IPython.core.prefilter import PrefilterTransformer
    404 from IPython.frontend.terminal.embed import InteractiveShellEmbed
     326from IPython.terminal.embed import InteractiveShellEmbed
    405327from IPython import Config
    406328
    407329class InterfaceShellTransformer(PrefilterTransformer):
     
    619541#######################
    620542# IPython TerminalApp #
    621543#######################
    622 from IPython.frontend.terminal.ipapp import TerminalIPythonApp, IPAppCrashHandler
     544from IPython.terminal.ipapp import TerminalIPythonApp, IPAppCrashHandler
    623545from IPython.core.crashhandler import CrashHandler
    624546from IPython import Config
    625547
     
    679601
    680602
    681603    def load_config_file(self, *args, **kwds):
    682         from IPython.frontend.terminal.ipapp import default_config_file_name
    683604        from IPython.config.loader import PyFileConfigLoader, ConfigFileNotFound
    684605        from IPython.core.profiledir import ProfileDir
    685606        from IPython.utils.path import get_ipython_dir
     
    689610        conf._merge(self.command_line_config)
    690611
    691612        # Get user config.
    692         sage_profile_dir = ProfileDir.find_profile_dir_by_name(
     613        sage_config_dir = ProfileDir.find_profile_dir_by_name(
    693614            get_ipython_dir(), 'sage').location
     615        sage_config_file = os.path.join(sage_config_dir, 'ipython_config.py')
    694616        try:
    695             cl = PyFileConfigLoader(default_config_file_name, sage_profile_dir)
     617            cl = PyFileConfigLoader(sage_config_file)
    696618            conf._merge(cl.load_config())
    697619        except ConfigFileNotFound:
    698620            pass
  • sage/misc/sage_extension.py

    diff --git a/sage/misc/sage_extension.py b/sage/misc/sage_extension.py
    a b  
    251251            print mode, args_split
    252252            raise AttributeError("First argument must be `simple` or `ascii_art` or the method must be call without argument")
    253253
    254 # SageInputSplitter:
    255 #  Hopefully most or all of this code can go away when
    256 #  https://github.com/ipython/ipython/issues/2293 is resolved
    257 #  apparently we will have stateful transformations then.
    258 #  see also https://github.com/ipython/ipython/pull/2402
    259 from IPython.core.inputsplitter import (transform_ipy_prompt, transform_classic_prompt,
    260                                         transform_help_end, transform_escaped,
    261                                         transform_assign_system, transform_assign_magic,
    262                                         cast_unicode,
    263                                         IPythonInputSplitter)
    264 
    265 def first_arg(f):
    266     def tm(arg1, arg2):
    267         return f(arg1)
    268     return tm
    269 
    270 class SageInputSplitter(IPythonInputSplitter):
    271     """
    272     We override the input splitter for two reasons:
    273 
    274     1. to make the list of transforms a class attribute that can be modified
    275 
    276     2. to pass the line number to transforms (we strip the line number off for IPython transforms)
    277     """
    278 
    279     # List of input transforms to apply
    280     transforms = map(first_arg, [transform_ipy_prompt, transform_classic_prompt,
    281                                  transform_help_end, transform_escaped,
    282                                  transform_assign_system, transform_assign_magic])
    283 
    284     # a direct copy of the IPython splitter, except that the
    285     # transforms are called with the line numbers, and the transforms come from the class attribute
    286     # and except that we add a .startswith('@') in the test to see if we should transform a line
    287     # (see http://mail.scipy.org/pipermail/ipython-dev/2012-September/010329.html; the current behavior
    288     # doesn't run the preparser on a 'def' line following an decorator.
    289     def push(self, lines):
    290         """Push one or more lines of IPython input.
    291 
    292         This stores the given lines and returns a status code indicating
    293         whether the code forms a complete Python block or not, after processing
    294         all input lines for special IPython syntax.
    295 
    296         Any exceptions generated in compilation are swallowed, but if an
    297         exception was produced, the method returns True.
    298 
    299         Parameters
    300         ----------
    301         lines : string
    302           One or more lines of Python input.
    303 
    304         Returns
    305         -------
    306         is_complete : boolean
    307           True if the current input source (the result of the current input
    308         plus prior inputs) forms a complete Python execution block.  Note that
    309         this value is also stored as a private attribute (_is_complete), so it
    310         can be queried at any time.
    311         """
    312         if not lines:
    313             return super(IPythonInputSplitter, self).push(lines)
    314 
    315         # We must ensure all input is pure unicode
    316         lines = cast_unicode(lines, self.encoding)
    317 
    318         # If the entire input block is a cell magic, return after handling it
    319         # as the rest of the transformation logic should be skipped.
    320         if lines.startswith('%%') and not \
    321           (len(lines.splitlines()) == 1 and lines.strip().endswith('?')):
    322             return self._handle_cell_magic(lines)
    323 
    324         # In line mode, a cell magic can arrive in separate pieces
    325         if self.input_mode == 'line' and self.processing_cell_magic:
    326             return self._line_mode_cell_append(lines)
    327 
    328         # The rest of the processing is for 'normal' content, i.e. IPython
    329         # source that we process through our transformations pipeline.
    330         lines_list = lines.splitlines()
    331 
    332         # Transform logic
    333         #
    334         # We only apply the line transformers to the input if we have either no
    335         # input yet, or complete input, or if the last line of the buffer ends
    336         # with ':' (opening an indented block).  This prevents the accidental
    337         # transformation of escapes inside multiline expressions like
    338         # triple-quoted strings or parenthesized expressions.
    339         #
    340         # The last heuristic, while ugly, ensures that the first line of an
    341         # indented block is correctly transformed.
    342         #
    343         # FIXME: try to find a cleaner approach for this last bit.
    344 
    345         # If we were in 'block' mode, since we're going to pump the parent
    346         # class by hand line by line, we need to temporarily switch out to
    347         # 'line' mode, do a single manual reset and then feed the lines one
    348         # by one.  Note that this only matters if the input has more than one
    349         # line.
    350         changed_input_mode = False
    351 
    352         if self.input_mode == 'cell':
    353             self.reset()
    354             changed_input_mode = True
    355             saved_input_mode = 'cell'
    356             self.input_mode = 'line'
    357 
    358         # Store raw source before applying any transformations to it.  Note
    359         # that this must be done *after* the reset() call that would otherwise
    360         # flush the buffer.
    361         self._store(lines, self._buffer_raw, 'source_raw')
    362 
    363         try:
    364             push = super(IPythonInputSplitter, self).push
    365             buf = self._buffer
    366             for line in lines_list:
    367                 line_number = len(buf)
    368                 if (self._is_complete or not buf or
    369                     buf[-1].rstrip().endswith((':', ',')) or buf[-1].lstrip().startswith('@')):
    370                     for f in self.transforms:
    371                         line = f(line, line_number)
    372                 else:
    373                     for f in self.always_transform:
    374                         line = f(line, line_number)
    375                 out = push(line)
    376         finally:
    377             if changed_input_mode:
    378                 self.input_mode = saved_input_mode
    379         return out
    380 
    381 # END SageIPythonInputSplitter
    382 #
    383 #
    384254
    385255import displayhook
    386256class SageCustomizations(object):
     
    482352        """
    483353        Set up transforms (like the preparser).
    484354        """
    485         self.shell.input_splitter = SageInputSplitter()
    486355        import sage
    487356        import sage.all
    488         from sage.misc.interpreter import (SagePromptDedenter, SagePromptTransformer,
    489                                            MagicTransformer, SagePreparseTransformer)
    490 
    491         p = SagePreparseTransformer()
    492         self.shell.input_splitter.transforms = [SagePromptDedenter(),
    493                                                 SagePromptTransformer(),
    494                                                 MagicTransformer(),
    495                                                 p] + self.shell.input_splitter.transforms
    496         self.shell.input_splitter.always_transform = [p]
    497 
     357        from interpreter import (sage_preparse_transformer,
     358                                 sage_prompt_transformer,
     359                                 magic_transformer)
     360        for s in (self.shell.input_splitter, self.shell.input_transformer_manager):
     361            s.physical_line_transforms.extend([sage_prompt_transformer()])
     362            s.logical_line_transforms.insert(0, magic_transformer())
     363            s.python_line_transforms.extend([sage_preparse_transformer()])
    498364        preparser(True)
    499365
    500366