1 # encoding: utf-8 2 """ 3 Prefiltering components. 4 5 Prefilters transform user input before it is exec'd by Python. These 6 transforms are used to implement additional syntax such as !ls and %magic. 7 """ 8 9 # Copyright (c) IPython Development Team. 10 # Distributed under the terms of the Modified BSD License. 11 12 from keyword import iskeyword 13 import re 14 15 from .autocall import IPyAutocall 16 from traitlets.config.configurable import Configurable 17 from .inputtransformer2 import ( 18 ESC_MAGIC, 19 ESC_QUOTE, 20 ESC_QUOTE2, 21 ESC_PAREN, 22 ) 23 from .macro import Macro 24 from .splitinput import LineInfo 25 26 from traitlets import ( 27 List, Integer, Unicode, Bool, Instance, CRegExp 28 ) 29 30 #----------------------------------------------------------------------------- 31 # Global utilities, errors and constants 32 #----------------------------------------------------------------------------- 33 34 35 class PrefilterError(Exception): 36 pass 37 38 39 # RegExp to identify potential function names 40 re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$') 41 42 # RegExp to exclude strings with this start from autocalling. In 43 # particular, all binary operators should be excluded, so that if foo is 44 # callable, foo OP bar doesn't become foo(OP bar), which is invalid. The 45 # characters '!=()' don't need to be checked for, as the checkPythonChars 46 # routine explicitly does so, to catch direct calls and rebindings of 47 # existing names. 48 49 # Warning: the '-' HAS TO BE AT THE END of the first group, otherwise 50 # it affects the rest of the group in square brackets. 51 re_exclude_auto = re.compile(r'^[,&^\|\*/\+-]' 52 r'|^is |^not |^in |^and |^or ') 53 54 # try to catch also methods for stuff in lists/tuples/dicts: off 55 # (experimental). For this to work, the line_split regexp would need 56 # to be modified so it wouldn't break things at '['. That line is 57 # nasty enough that I shouldn't change it until I can test it _well_. 58 #self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$') 59 60 61 # Handler Check Utilities 62 def is_shadowed(identifier, ip): 63 """Is the given identifier defined in one of the namespaces which shadow 64 the alias and magic namespaces? Note that an identifier is different 65 than ifun, because it can not contain a '.' character.""" 66 # This is much safer than calling ofind, which can change state 67 return (identifier in ip.user_ns \ 68 or identifier in ip.user_global_ns \ 69 or identifier in ip.ns_table['builtin']\ 70 or iskeyword(identifier)) 71 72 73 #----------------------------------------------------------------------------- 74 # Main Prefilter manager 75 #----------------------------------------------------------------------------- 76 77 78 class PrefilterManager(Configurable): 79 """Main prefilter component. 80 81 The IPython prefilter is run on all user input before it is run. The 82 prefilter consumes lines of input and produces transformed lines of 83 input. 84 85 The implementation consists of two phases: 86 87 1. Transformers 88 2. Checkers and handlers 89 90 Over time, we plan on deprecating the checkers and handlers and doing 91 everything in the transformers. 92 93 The transformers are instances of :class:`PrefilterTransformer` and have 94 a single method :meth:`transform` that takes a line and returns a 95 transformed line. The transformation can be accomplished using any 96 tool, but our current ones use regular expressions for speed. 97 98 After all the transformers have been run, the line is fed to the checkers, 99 which are instances of :class:`PrefilterChecker`. The line is passed to 100 the :meth:`check` method, which either returns `None` or a 101 :class:`PrefilterHandler` instance. If `None` is returned, the other 102 checkers are tried. If an :class:`PrefilterHandler` instance is returned, 103 the line is passed to the :meth:`handle` method of the returned 104 handler and no further checkers are tried. 105 106 Both transformers and checkers have a `priority` attribute, that determines 107 the order in which they are called. Smaller priorities are tried first. 108 109 Both transformers and checkers also have `enabled` attribute, which is 110 a boolean that determines if the instance is used. 111 112 Users or developers can change the priority or enabled attribute of 113 transformers or checkers, but they must call the :meth:`sort_checkers` 114 or :meth:`sort_transformers` method after changing the priority. 115 """ 116 117 multi_line_specials = Bool(True).tag(config=True) 118 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) 119 120 def __init__(self, shell=None, **kwargs): 121 super(PrefilterManager, self).__init__(shell=shell, **kwargs) 122 self.shell = shell 123 self._transformers = [] 124 self.init_handlers() 125 self.init_checkers() 126 127 #------------------------------------------------------------------------- 128 # API for managing transformers 129 #------------------------------------------------------------------------- 130 131 def sort_transformers(self): 132 """Sort the transformers by priority. 133 134 This must be called after the priority of a transformer is changed. 135 The :meth:`register_transformer` method calls this automatically. 136 """ 137 self._transformers.sort(key=lambda x: x.priority) 138 139 @property 140 def transformers(self): 141 """Return a list of checkers, sorted by priority.""" 142 return self._transformers 143 144 def register_transformer(self, transformer): 145 """Register a transformer instance.""" 146 if transformer not in self._transformers: 147 self._transformers.append(transformer) 148 self.sort_transformers() 149 150 def unregister_transformer(self, transformer): 151 """Unregister a transformer instance.""" 152 if transformer in self._transformers: 153 self._transformers.remove(transformer) 154 155 #------------------------------------------------------------------------- 156 # API for managing checkers 157 #------------------------------------------------------------------------- 158 159 def init_checkers(self): 160 """Create the default checkers.""" 161 self._checkers = [] 162 for checker in _default_checkers: 163 checker( 164 shell=self.shell, prefilter_manager=self, parent=self 165 ) 166 167 def sort_checkers(self): 168 """Sort the checkers by priority. 169 170 This must be called after the priority of a checker is changed. 171 The :meth:`register_checker` method calls this automatically. 172 """ 173 self._checkers.sort(key=lambda x: x.priority) 174 175 @property 176 def checkers(self): 177 """Return a list of checkers, sorted by priority.""" 178 return self._checkers 179 180 def register_checker(self, checker): 181 """Register a checker instance.""" 182 if checker not in self._checkers: 183 self._checkers.append(checker) 184 self.sort_checkers() 185 186 def unregister_checker(self, checker): 187 """Unregister a checker instance.""" 188 if checker in self._checkers: 189 self._checkers.remove(checker) 190 191 #------------------------------------------------------------------------- 192 # API for managing handlers 193 #------------------------------------------------------------------------- 194 195 def init_handlers(self): 196 """Create the default handlers.""" 197 self._handlers = {} 198 self._esc_handlers = {} 199 for handler in _default_handlers: 200 handler( 201 shell=self.shell, prefilter_manager=self, parent=self 202 ) 203 204 @property 205 def handlers(self): 206 """Return a dict of all the handlers.""" 207 return self._handlers 208 209 def register_handler(self, name, handler, esc_strings): 210 """Register a handler instance by name with esc_strings.""" 211 self._handlers[name] = handler 212 for esc_str in esc_strings: 213 self._esc_handlers[esc_str] = handler 214 215 def unregister_handler(self, name, handler, esc_strings): 216 """Unregister a handler instance by name with esc_strings.""" 217 try: 218 del self._handlers[name] 219 except KeyError: 220 pass 221 for esc_str in esc_strings: 222 h = self._esc_handlers.get(esc_str) 223 if h is handler: 224 del self._esc_handlers[esc_str] 225 226 def get_handler_by_name(self, name): 227 """Get a handler by its name.""" 228 return self._handlers.get(name) 229 230 def get_handler_by_esc(self, esc_str): 231 """Get a handler by its escape string.""" 232 return self._esc_handlers.get(esc_str) 233 234 #------------------------------------------------------------------------- 235 # Main prefiltering API 236 #------------------------------------------------------------------------- 237 238 def prefilter_line_info(self, line_info): 239 """Prefilter a line that has been converted to a LineInfo object. 240 241 This implements the checker/handler part of the prefilter pipe. 242 """ 243 # print("prefilter_line_info: ", line_info) 244 handler = self.find_handler(line_info) 245 return handler.handle(line_info) 246 247 def find_handler(self, line_info): 248 """Find a handler for the line_info by trying checkers.""" 249 for checker in self.checkers: 250 if checker.enabled: 251 handler = checker.check(line_info) 252 if handler: 253 return handler 254 return self.get_handler_by_name('normal') 255 256 def transform_line(self, line, continue_prompt): 257 """Calls the enabled transformers in order of increasing priority.""" 258 for transformer in self.transformers: 259 if transformer.enabled: 260 line = transformer.transform(line, continue_prompt) 261 return line 262 263 def prefilter_line(self, line, continue_prompt=False): 264 """Prefilter a single input line as text. 265 266 This method prefilters a single line of text by calling the 267 transformers and then the checkers/handlers. 268 """ 269 270 # print("prefilter_line: ", line, continue_prompt) 271 # All handlers *must* return a value, even if it's blank (''). 272 273 # save the line away in case we crash, so the post-mortem handler can 274 # record it 275 self.shell._last_input_line = line 276 277 if not line: 278 # Return immediately on purely empty lines, so that if the user 279 # previously typed some whitespace that started a continuation 280 # prompt, he can break out of that loop with just an empty line. 281 # This is how the default python prompt works. 282 return '' 283 284 # At this point, we invoke our transformers. 285 if not continue_prompt or (continue_prompt and self.multi_line_specials): 286 line = self.transform_line(line, continue_prompt) 287 288 # Now we compute line_info for the checkers and handlers 289 line_info = LineInfo(line, continue_prompt) 290 291 # the input history needs to track even empty lines 292 stripped = line.strip() 293 294 normal_handler = self.get_handler_by_name('normal') 295 if not stripped: 296 return normal_handler.handle(line_info) 297 298 # special handlers are only allowed for single line statements 299 if continue_prompt and not self.multi_line_specials: 300 return normal_handler.handle(line_info) 301 302 prefiltered = self.prefilter_line_info(line_info) 303 # print("prefiltered line: %r" % prefiltered) 304 return prefiltered 305 306 def prefilter_lines(self, lines, continue_prompt=False): 307 """Prefilter multiple input lines of text. 308 309 This is the main entry point for prefiltering multiple lines of 310 input. This simply calls :meth:`prefilter_line` for each line of 311 input. 312 313 This covers cases where there are multiple lines in the user entry, 314 which is the case when the user goes back to a multiline history 315 entry and presses enter. 316 """ 317 llines = lines.rstrip('\n').split('\n') 318 # We can get multiple lines in one shot, where multiline input 'blends' 319 # into one line, in cases like recalling from the readline history 320 # buffer. We need to make sure that in such cases, we correctly 321 # communicate downstream which line is first and which are continuation 322 # ones. 323 if len(llines) > 1: 324 out = '\n'.join([self.prefilter_line(line, lnum>0) 325 for lnum, line in enumerate(llines) ]) 326 else: 327 out = self.prefilter_line(llines[0], continue_prompt) 328 329 return out 330 331 #----------------------------------------------------------------------------- 332 # Prefilter transformers 333 #----------------------------------------------------------------------------- 334 335 336 class PrefilterTransformer(Configurable): 337 """Transform a line of user input.""" 338 339 priority = Integer(100).tag(config=True) 340 # Transformers don't currently use shell or prefilter_manager, but as we 341 # move away from checkers and handlers, they will need them. 342 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) 343 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True) 344 enabled = Bool(True).tag(config=True) 345 346 def __init__(self, shell=None, prefilter_manager=None, **kwargs): 347 super(PrefilterTransformer, self).__init__( 348 shell=shell, prefilter_manager=prefilter_manager, **kwargs 349 ) 350 self.prefilter_manager.register_transformer(self) 351 352 def transform(self, line, continue_prompt): 353 """Transform a line, returning the new one.""" 354 return None 355 356 def __repr__(self): 357 return "<%s(priority=%r, enabled=%r)>" % ( 358 self.__class__.__name__, self.priority, self.enabled) 359 360 361 #----------------------------------------------------------------------------- 362 # Prefilter checkers 363 #----------------------------------------------------------------------------- 364 365 366 class PrefilterChecker(Configurable): 367 """Inspect an input line and return a handler for that line.""" 368 369 priority = Integer(100).tag(config=True) 370 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) 371 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True) 372 enabled = Bool(True).tag(config=True) 373 374 def __init__(self, shell=None, prefilter_manager=None, **kwargs): 375 super(PrefilterChecker, self).__init__( 376 shell=shell, prefilter_manager=prefilter_manager, **kwargs 377 ) 378 self.prefilter_manager.register_checker(self) 379 380 def check(self, line_info): 381 """Inspect line_info and return a handler instance or None.""" 382 return None 383 384 def __repr__(self): 385 return "<%s(priority=%r, enabled=%r)>" % ( 386 self.__class__.__name__, self.priority, self.enabled) 387 388 389 class EmacsChecker(PrefilterChecker): 390 391 priority = Integer(100).tag(config=True) 392 enabled = Bool(False).tag(config=True) 393 394 def check(self, line_info): 395 "Emacs ipython-mode tags certain input lines." 396 if line_info.line.endswith('# PYTHON-MODE'): 397 return self.prefilter_manager.get_handler_by_name('emacs') 398 else: 399 return None 400 401 402 class MacroChecker(PrefilterChecker): 403 404 priority = Integer(250).tag(config=True) 405 406 def check(self, line_info): 407 obj = self.shell.user_ns.get(line_info.ifun) 408 if isinstance(obj, Macro): 409 return self.prefilter_manager.get_handler_by_name('macro') 410 else: 411 return None 412 413 414 class IPyAutocallChecker(PrefilterChecker): 415 416 priority = Integer(300).tag(config=True) 417 418 def check(self, line_info): 419 "Instances of IPyAutocall in user_ns get autocalled immediately" 420 obj = self.shell.user_ns.get(line_info.ifun, None) 421 if isinstance(obj, IPyAutocall): 422 obj.set_ip(self.shell) 423 return self.prefilter_manager.get_handler_by_name('auto') 424 else: 425 return None 426 427 428 class AssignmentChecker(PrefilterChecker): 429 430 priority = Integer(600).tag(config=True) 431 432 def check(self, line_info): 433 """Check to see if user is assigning to a var for the first time, in 434 which case we want to avoid any sort of automagic / autocall games. 435 436 This allows users to assign to either alias or magic names true python 437 variables (the magic/alias systems always take second seat to true 438 python code). E.g. ls='hi', or ls,that=1,2""" 439 if line_info.the_rest: 440 if line_info.the_rest[0] in '=,': 441 return self.prefilter_manager.get_handler_by_name('normal') 442 else: 443 return None 444 445 446 class AutoMagicChecker(PrefilterChecker): 447 448 priority = Integer(700).tag(config=True) 449 450 def check(self, line_info): 451 """If the ifun is magic, and automagic is on, run it. Note: normal, 452 non-auto magic would already have been triggered via '%' in 453 check_esc_chars. This just checks for automagic. Also, before 454 triggering the magic handler, make sure that there is nothing in the 455 user namespace which could shadow it.""" 456 if not self.shell.automagic or not self.shell.find_magic(line_info.ifun): 457 return None 458 459 # We have a likely magic method. Make sure we should actually call it. 460 if line_info.continue_prompt and not self.prefilter_manager.multi_line_specials: 461 return None 462 463 head = line_info.ifun.split('.',1)[0] 464 if is_shadowed(head, self.shell): 465 return None 466 467 return self.prefilter_manager.get_handler_by_name('magic') 468 469 470 class PythonOpsChecker(PrefilterChecker): 471 472 priority = Integer(900).tag(config=True) 473 474 def check(self, line_info): 475 """If the 'rest' of the line begins with a function call or pretty much 476 any python operator, we should simply execute the line (regardless of 477 whether or not there's a possible autocall expansion). This avoids 478 spurious (and very confusing) geattr() accesses.""" 479 if line_info.the_rest and line_info.the_rest[0] in "!=()<>,+*/%^&|": 480 return self.prefilter_manager.get_handler_by_name("normal") 481 else: 482 return None 483 484 485 class AutocallChecker(PrefilterChecker): 486 487 priority = Integer(1000).tag(config=True) 488 489 function_name_regexp = CRegExp(re_fun_name, 490 help="RegExp to identify potential function names." 491 ).tag(config=True) 492 exclude_regexp = CRegExp(re_exclude_auto, 493 help="RegExp to exclude strings with this start from autocalling." 494 ).tag(config=True) 495 496 def check(self, line_info): 497 "Check if the initial word/function is callable and autocall is on." 498 if not self.shell.autocall: 499 return None 500 501 oinfo = line_info.ofind(self.shell) # This can mutate state via getattr 502 if not oinfo.found: 503 return None 504 505 ignored_funs = ['b', 'f', 'r', 'u', 'br', 'rb', 'fr', 'rf'] 506 ifun = line_info.ifun 507 line = line_info.line 508 if ifun.lower() in ignored_funs and (line.startswith(ifun + "'") or line.startswith(ifun + '"')): 509 return None 510 511 if ( 512 callable(oinfo.obj) 513 and (not self.exclude_regexp.match(line_info.the_rest)) 514 and self.function_name_regexp.match(line_info.ifun) 515 and ( 516 line_info.raw_the_rest.startswith(" ") 517 or not line_info.raw_the_rest.strip() 518 ) 519 ): 520 return self.prefilter_manager.get_handler_by_name("auto") 521 else: 522 return None 523 524 525 #----------------------------------------------------------------------------- 526 # Prefilter handlers 527 #----------------------------------------------------------------------------- 528 529 530 class PrefilterHandler(Configurable): 531 handler_name = Unicode("normal") 532 esc_strings: List = List([]) 533 shell = Instance( 534 "IPython.core.interactiveshell.InteractiveShellABC", allow_none=True 535 ) 536 prefilter_manager = Instance( 537 "IPython.core.prefilter.PrefilterManager", allow_none=True 538 ) 539 540 def __init__(self, shell=None, prefilter_manager=None, **kwargs): 541 super(PrefilterHandler, self).__init__( 542 shell=shell, prefilter_manager=prefilter_manager, **kwargs 543 ) 544 self.prefilter_manager.register_handler( 545 self.handler_name, 546 self, 547 self.esc_strings 548 ) 549 550 def handle(self, line_info): 551 # print("normal: ", line_info) 552 """Handle normal input lines. Use as a template for handlers.""" 553 554 # With autoindent on, we need some way to exit the input loop, and I 555 # don't want to force the user to have to backspace all the way to 556 # clear the line. The rule will be in this case, that either two 557 # lines of pure whitespace in a row, or a line of pure whitespace but 558 # of a size different to the indent level, will exit the input loop. 559 line = line_info.line 560 continue_prompt = line_info.continue_prompt 561 562 if (continue_prompt and 563 self.shell.autoindent and 564 line.isspace() and 565 0 < abs(len(line) - self.shell.indent_current_nsp) <= 2): 566 line = '' 567 568 return line 569 570 def __str__(self): 571 return "<%s(name=%s)>" % (self.__class__.__name__, self.handler_name) 572 573 574 class MacroHandler(PrefilterHandler): 575 handler_name = Unicode("macro") 576 577 def handle(self, line_info): 578 obj = self.shell.user_ns.get(line_info.ifun) 579 pre_space = line_info.pre_whitespace 580 line_sep = "\n" + pre_space 581 return pre_space + line_sep.join(obj.value.splitlines()) 582 583 584 class MagicHandler(PrefilterHandler): 585 586 handler_name = Unicode('magic') 587 esc_strings = List([ESC_MAGIC]) 588 589 def handle(self, line_info): 590 """Execute magic functions.""" 591 ifun = line_info.ifun 592 the_rest = line_info.the_rest 593 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) 594 t_arg_s = ifun + " " + the_rest 595 t_magic_name, _, t_magic_arg_s = t_arg_s.partition(' ') 596 t_magic_name = t_magic_name.lstrip(ESC_MAGIC) 597 cmd = '%sget_ipython().run_line_magic(%r, %r)' % (line_info.pre_whitespace, t_magic_name, t_magic_arg_s) 598 return cmd 599 600 601 class AutoHandler(PrefilterHandler): 602 603 handler_name = Unicode('auto') 604 esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2]) 605 606 def handle(self, line_info): 607 """Handle lines which can be auto-executed, quoting if requested.""" 608 line = line_info.line 609 ifun = line_info.ifun 610 the_rest = line_info.the_rest 611 esc = line_info.esc 612 continue_prompt = line_info.continue_prompt 613 obj = line_info.ofind(self.shell).obj 614 615 # This should only be active for single-line input! 616 if continue_prompt: 617 return line 618 619 force_auto = isinstance(obj, IPyAutocall) 620 621 # User objects sometimes raise exceptions on attribute access other 622 # than AttributeError (we've seen it in the past), so it's safest to be 623 # ultra-conservative here and catch all. 624 try: 625 auto_rewrite = obj.rewrite 626 except Exception: 627 auto_rewrite = True 628 629 if esc == ESC_QUOTE: 630 # Auto-quote splitting on whitespace 631 newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) ) 632 elif esc == ESC_QUOTE2: 633 # Auto-quote whole string 634 newcmd = '%s("%s")' % (ifun,the_rest) 635 elif esc == ESC_PAREN: 636 newcmd = '%s(%s)' % (ifun,",".join(the_rest.split())) 637 else: 638 # Auto-paren. 639 if force_auto: 640 # Don't rewrite if it is already a call. 641 do_rewrite = not the_rest.startswith('(') 642 else: 643 if not the_rest: 644 # We only apply it to argument-less calls if the autocall 645 # parameter is set to 2. 646 do_rewrite = (self.shell.autocall >= 2) 647 elif the_rest.startswith('[') and hasattr(obj, '__getitem__'): 648 # Don't autocall in this case: item access for an object 649 # which is BOTH callable and implements __getitem__. 650 do_rewrite = False 651 else: 652 do_rewrite = True 653 654 # Figure out the rewritten command 655 if do_rewrite: 656 if the_rest.endswith(';'): 657 newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1]) 658 else: 659 newcmd = '%s(%s)' % (ifun.rstrip(), the_rest) 660 else: 661 normal_handler = self.prefilter_manager.get_handler_by_name('normal') 662 return normal_handler.handle(line_info) 663 664 # Display the rewritten call 665 if auto_rewrite: 666 self.shell.auto_rewrite_input(newcmd) 667 668 return newcmd 669 670 671 class EmacsHandler(PrefilterHandler): 672 673 handler_name = Unicode('emacs') 674 esc_strings = List([]) 675 676 def handle(self, line_info): 677 """Handle input lines marked by python-mode.""" 678 679 # Currently, nothing is done. Later more functionality can be added 680 # here if needed. 681 682 # The input cache shouldn't be updated 683 return line_info.line 684 685 686 #----------------------------------------------------------------------------- 687 # Defaults 688 #----------------------------------------------------------------------------- 689 690 691 _default_checkers = [ 692 EmacsChecker, 693 MacroChecker, 694 IPyAutocallChecker, 695 AssignmentChecker, 696 AutoMagicChecker, 697 PythonOpsChecker, 698 AutocallChecker 699 ] 700 701 _default_handlers = [ 702 PrefilterHandler, 703 MacroHandler, 704 MagicHandler, 705 AutoHandler, 706 EmacsHandler 707 ] 708