Ticket #14713: new_transformers.patch
File new_transformers.patch, 16.1 KB (added by , 6 years ago) 


sage/misc/interpreter.py
# HG changeset patch # Parent 1077314f416653b28e199c382667a1f11e444bdd Implement the Sage preparsing transformations using the new IPython transformation framework. diff git a/sage/misc/interpreter.py b/sage/misc/interpreter.py
a b 234 234 ################################################################### 235 235 # Transformers used in the SageInputSplitter 236 236 ################################################################### 237 # These used to be part of the older PrefilterTransformer framework,238 # but that is not the modern way of doing things. For more details, see 239 # http://mail.scipy.org/pipermail/ipythondev/2011March/007334.html 237 from IPython.core.inputtransformer import (CoroutineInputTransformer, 238 StatelessInputTransformer, 239 _strip_prompts) 240 240 241 class SagePreparseTransformer(): 241 @CoroutineInputTransformer.wrap 242 def sage_preparse_transformer(): 242 243 """ 243 Preparse the line of code before it get evaluated by IPython. 244 Implement Sagespecific preparsing (see sage.misc.preparser.preparse) 245 246 EXAMPLES:: 247 248 sage: from sage.misc.interpreter import sage_preparse_transformer 249 sage: spt = sage_preparse_transformer() 250 sage: spt.push('1+1r+2.3^2.3r') 251 "Integer(1)+1+RealNumber('2.3')**2.3" 252 sage: preparser(False) 253 sage: spt.push('2.3^2') 254 '2.3^2' 244 255 """ 245 def __call__(self, line, line_number): 246 """ 247 Transform ``line``. 256 line = (yield '') 257 while True: 258 if line is None: 259 rline = preparse('', reset=True) 260 elif do_preparse and not line.startswith('%'): 261 rline = preparse(line) 262 else: 263 rline = line 264 line = (yield rline) 248 265 249 INPUT: 250 251  ``line``  string. The line to be transformed. 252 253 OUTPUT: 254 255 A string, the transformed line. 256 257 EXAMPLES:: 258 259 sage: from sage.misc.interpreter import SagePreparseTransformer 260 sage: spt = SagePreparseTransformer() 261 sage: spt('2', 0) 262 'Integer(2)' 263 sage: preparser(False) 264 sage: spt('2', 0) 265 '2' 266 sage: preparser(True) 267 """ 268 if do_preparse and not line.startswith('%'): 269 # we use preparse_file instead of just preparse because preparse_file 270 # automatically prepends attached files 271 return preparse(line, reset=(line_number==0)) 272 else: 273 return line 274 275 class MagicTransformer(): 266 _magic_deprecations = {'load': '%runfile', 267 'attach': '%attach', 268 'time': '%time'} 269 @StatelessInputTransformer.wrap 270 def magic_transformer(line): 276 271 r""" 277 272 Handle input lines that start out like ``load ...`` or ``attach 278 273 ...``. … … 281 276 ``attach``, IPython's automagic will not transform these lines 282 277 into ``%load ...`` and ``%attach ...``, respectively. Thus, we 283 278 have to do it manually. 284 """285 deprecations = {'load': '%runfile',286 'attach': '%attach',287 'time': '%time'}288 def __call__(self, line, line_number):289 """290 Transform ``line``.291 292 INPUT:293 294  ``line``  string. The line to be transformed.295 296 OUTPUT:297 298 A string, the transformed line.299 279 300 280 EXAMPLES:: 301 281 302 sage: from sage.misc.interpreter import get_test_shell, MagicTransformer303 sage: mt = MagicTransformer()304 sage: mt ('load /path/to/file', 0)305 doctest: 1: DeprecationWarning: Use %runfile instead of load.282 sage: from sage.misc.interpreter import get_test_shell, magic_transformer 283 sage: mt = magic_transformer() 284 sage: mt.push('load /path/to/file') 285 doctest:...: DeprecationWarning: Use %runfile instead of load. 306 286 See http://trac.sagemath.org/12719 for details. 307 287 '%runfile /path/to/file' 308 sage: mt ('attach /path/to/file', 0)309 doctest: 1: DeprecationWarning: Use %attach instead of attach.288 sage: mt.push('attach /path/to/file') 289 doctest:...: DeprecationWarning: Use %attach instead of attach. 310 290 See http://trac.sagemath.org/12719 for details. 311 291 '%attach /path/to/file' 312 sage: mt ('time 1+2', 0)313 doctest: 1: DeprecationWarning: Use %time instead of time.292 sage: mt.push('time 1+2') 293 doctest:...: DeprecationWarning: Use %time instead of time. 314 294 See http://trac.sagemath.org/12719 for details. 315 295 '%time 1+2' 316 """ 317 for old,new in self.deprecations.items(): 318 if line.startswith(old+' '): 319 from sage.misc.superseded import deprecation 320 deprecation(12719, 'Use %s instead of %s.'%(new,old)) 321 return new+line[len(old):] 322 return line 296 """ 297 global _magic_deprecations 298 for old,new in _magic_deprecations.items(): 299 if line.startswith(old+' '): 300 from sage.misc.superseded import deprecation 301 deprecation(12719, 'Use %s instead of %s.'%(new,old)) 302 return new+line[len(old):] 303 return line 323 304 324 class SagePromptTransformer(): 305 306 @CoroutineInputTransformer.wrap 307 def sage_prompt_transformer(): 325 308 """ 326 Remove Sage prompts from the input line. 309 Strip the sage:/... prompts of Sage. 310 311 EXAMPLES:: 312 313 sage: from sage.misc.interpreter import sage_prompt_transformer 314 sage: spt = sage_prompt_transformer() 315 sage: spt.push("sage: sage: 2 + 2") 316 '2 + 2' 317 sage: spt.push('') 318 '' 319 sage: spt.push("sage: 2+2") 320 '2+2' 321 sage: spt.push("... .... ....: ...: 2+2") 322 '2+2' 327 323 """ 328 _sage_prompt_re = re.compile(r'(^[ \t]*sage: ^[ \t]*\.+:? )+') 329 330 def __call__(self, line, line_number): 331 """ 332 Transform ``line``. 333 334 INPUT: 335 336  ``line``  string. The line to be transformed. 337 338 OUTPUT: 339 340 A string, the transformed line. 341 342 EXAMPLES:: 343 344 sage: from sage.misc.interpreter import SagePromptTransformer 345 sage: spt = SagePromptTransformer() 346 sage: spt("sage: sage: 2 + 2", 0) 347 '2 + 2' 348 sage: spt('', 0) 349 '' 350 sage: spt(" sage: 2+2", 0) 351 '2+2' 352 sage: spt(" ... 2+2", 0) 353 '2+2' 354 """ 355 if not line or line.isspace() or line.strip() == '...': 356 # This allows us to recognize multiple input prompts separated by 357 # blank lines and pasted in a single chunk, very common when 358 # pasting doctests or long tutorial passages. 359 return '' 360 while True: 361 m = self._sage_prompt_re.match(line) 362 if m: 363 line = line[len(m.group(0)):] 364 else: 365 break 366 return line 367 368 class SagePromptDedenter(): 369 """ 370 Remove leading spaces from the input line. 371 """ 372 def __call__(self, line, line_number): 373 """ 374 Transform ``line``. 375 376 INPUT: 377 378  ``line``  string. The line to be transformed. 379 380  ``line_number``  integer. The line number. For a singleline input, this is always zero. 381 382 OUTPUT: 383 384 A string, the transformed line. 385 386 EXAMPLES:: 387 388 sage: from sage.misc.interpreter import SagePromptDedenter 389 sage: spd = SagePromptDedenter() 390 sage: spd(' 1 + \\', 0) 391 '1 + \\' 392 sage: spd(' 2', 1) 393 '2' 394 sage: spd('3', 2) # try our best with incorrect indentation 395 '3' 396 """ 397 if line_number == 0: 398 dedent_line = line.lstrip() 399 self._dedent = len(line)  len(dedent_line) 400 return dedent_line 401 else: 402 dedent = min(len(line)len(line.lstrip()), self._dedent) 403 return line[dedent:] 324 _sage_prompt_re = re.compile(r'^(sage: \.\.\.\.?:? )+') 325 return _strip_prompts(_sage_prompt_re) 404 326 405 327 ################### 406 328 # Interface shell # 
sage/misc/sage_extension.py
diff git a/sage/misc/sage_extension.py b/sage/misc/sage_extension.py
a b 226 226 sage: shell.display_formatter.formatters['text/plain'] 227 227 <...sage_extension.SagePlainTextFormatter object at 0x...> 228 228 sage: shell.displayhook.compute_format_data(2) 229 {u'text/plain': '2'}229 ({u'text/plain': '2'}, {}) 230 230 sage: a = identity_matrix(ZZ, 2) 231 231 sage: shell.displayhook.compute_format_data([a,a]) 232 {u'text/plain': '[\n[1 0] [1 0]\n[0 1], [0 1]\n]'}232 ({u'text/plain': '[\n[1 0] [1 0]\n[0 1], [0 1]\n]'}, {}) 233 233 """ 234 234 import sage 235 235 from sage.misc.displayhook import format_obj … … 238 238 s = super(SagePlainTextFormatter, self).__call__(obj) 239 239 return s 240 240 241 242 # SageInputSplitter:243 # Hopefully most or all of this code can go away when244 # https://github.com/ipython/ipython/issues/2293 is resolved245 # apparently we will have stateful transformations then.246 # see also https://github.com/ipython/ipython/pull/2402247 from IPython.core.inputsplitter import (transform_ipy_prompt, transform_classic_prompt,248 transform_help_end, transform_escaped,249 transform_assign_system, transform_assign_magic,250 cast_unicode,251 IPythonInputSplitter)252 253 def first_arg(f):254 def tm(arg1, arg2):255 return f(arg1)256 return tm257 258 class SageInputSplitter(IPythonInputSplitter):259 """260 We override the input splitter for two reasons:261 262 1. to make the list of transforms a class attribute that can be modified263 264 2. to pass the line number to transforms (we strip the line number off for IPython transforms)265 """266 267 # List of input transforms to apply268 transforms = map(first_arg, [transform_ipy_prompt, transform_classic_prompt,269 transform_help_end, transform_escaped,270 transform_assign_system, transform_assign_magic])271 272 # a direct copy of the IPython splitter, except that the273 # transforms are called with the line numbers, and the transforms come from the class attribute274 # and except that we add a .startswith('@') in the test to see if we should transform a line275 # (see http://mail.scipy.org/pipermail/ipythondev/2012September/010329.html; the current behavior276 # doesn't run the preparser on a 'def' line following an decorator.277 def push(self, lines):278 """Push one or more lines of IPython input.279 280 This stores the given lines and returns a status code indicating281 whether the code forms a complete Python block or not, after processing282 all input lines for special IPython syntax.283 284 Any exceptions generated in compilation are swallowed, but if an285 exception was produced, the method returns True.286 287 Parameters288 289 lines : string290 One or more lines of Python input.291 292 Returns293 294 is_complete : boolean295 True if the current input source (the result of the current input296 plus prior inputs) forms a complete Python execution block. Note that297 this value is also stored as a private attribute (_is_complete), so it298 can be queried at any time.299 """300 if not lines:301 return super(IPythonInputSplitter, self).push(lines)302 303 # We must ensure all input is pure unicode304 lines = cast_unicode(lines, self.encoding)305 306 # If the entire input block is a cell magic, return after handling it307 # as the rest of the transformation logic should be skipped.308 if lines.startswith('%%') and not \309 (len(lines.splitlines()) == 1 and lines.strip().endswith('?')):310 return self._handle_cell_magic(lines)311 312 # In line mode, a cell magic can arrive in separate pieces313 if self.input_mode == 'line' and self.processing_cell_magic:314 return self._line_mode_cell_append(lines)315 316 # The rest of the processing is for 'normal' content, i.e. IPython317 # source that we process through our transformations pipeline.318 lines_list = lines.splitlines()319 320 # Transform logic321 #322 # We only apply the line transformers to the input if we have either no323 # input yet, or complete input, or if the last line of the buffer ends324 # with ':' (opening an indented block). This prevents the accidental325 # transformation of escapes inside multiline expressions like326 # triplequoted strings or parenthesized expressions.327 #328 # The last heuristic, while ugly, ensures that the first line of an329 # indented block is correctly transformed.330 #331 # FIXME: try to find a cleaner approach for this last bit.332 333 # If we were in 'block' mode, since we're going to pump the parent334 # class by hand line by line, we need to temporarily switch out to335 # 'line' mode, do a single manual reset and then feed the lines one336 # by one. Note that this only matters if the input has more than one337 # line.338 changed_input_mode = False339 340 if self.input_mode == 'cell':341 self.reset()342 changed_input_mode = True343 saved_input_mode = 'cell'344 self.input_mode = 'line'345 346 # Store raw source before applying any transformations to it. Note347 # that this must be done *after* the reset() call that would otherwise348 # flush the buffer.349 self._store(lines, self._buffer_raw, 'source_raw')350 351 try:352 push = super(IPythonInputSplitter, self).push353 buf = self._buffer354 for line in lines_list:355 line_number = len(buf)356 if (self._is_complete or not buf or357 buf[1].rstrip().endswith((':', ',')) or buf[1].lstrip().startswith('@')):358 for f in self.transforms:359 line = f(line, line_number)360 else:361 for f in self.always_transform:362 line = f(line, line_number)363 out = push(line)364 finally:365 if changed_input_mode:366 self.input_mode = saved_input_mode367 return out368 369 # END SageIPythonInputSplitter370 #371 #372 373 374 241 class SageCustomizations(object): 375 242 startup_code = """from sage.all_cmdline import * 376 243 from sage.misc.interpreter import sage_prompt … … 471 338 """ 472 339 Set up transforms (like the preparser). 473 340 """ 474 self.shell.input_splitter = SageInputSplitter()475 341 import sage 476 342 import sage.all 477 from sage.misc.interpreter import (SagePromptDedenter, SagePromptTransformer, 478 MagicTransformer, SagePreparseTransformer) 479 480 p = SagePreparseTransformer() 481 self.shell.input_splitter.transforms = [SagePromptDedenter(), 482 SagePromptTransformer(), 483 MagicTransformer(), 484 p] + self.shell.input_splitter.transforms 485 self.shell.input_splitter.always_transform = [p] 486 343 from interpreter import (sage_preparse_transformer, 344 sage_prompt_transformer, 345 magic_transformer) 346 for s in (self.shell.input_splitter, self.shell.input_transformer_manager): 347 s.physical_line_transforms.extend([sage_prompt_transformer()]) 348 s.logical_line_transforms.insert(0, magic_transformer()) 349 s.python_line_transforms.extend([sage_preparse_transformer()]) 487 350 preparser(True) 488 351 489 352 def deprecated(self):