1 """DEPRECATED: Input transformer classes to support IPython special syntax. 2 3 This module was deprecated in IPython 7.0, in favour of inputtransformer2. 4 5 This includes the machinery to recognise and transform ``%magic`` commands, 6 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth. 7 """ 8 import abc 9 import functools 10 import re 11 import tokenize 12 import warnings 13 from tokenize import untokenize, TokenError 14 from io import StringIO 15 16 from IPython.core.splitinput import LineInfo 17 from IPython.utils import tokenutil 18 19 #----------------------------------------------------------------------------- 20 # Globals 21 #----------------------------------------------------------------------------- 22 23 # The escape sequences that define the syntax transformations IPython will 24 # apply to user input. These can NOT be just changed here: many regular 25 # expressions and other parts of the code may use their hardcoded values, and 26 # for all intents and purposes they constitute the 'IPython syntax', so they 27 # should be considered fixed. 28 29 ESC_SHELL = '!' # Send line to underlying system shell 30 ESC_SH_CAP = '!!' # Send line to system shell and capture output 31 ESC_HELP = '?' # Find information about object 32 ESC_HELP2 = '??' # Find extra-detailed information about object 33 ESC_MAGIC = '%' # Call magic function 34 ESC_MAGIC2 = '%%' # Call cell-magic function 35 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call 36 ESC_QUOTE2 = ';' # Quote all args as a single string, call 37 ESC_PAREN = '/' # Call first argument with rest of line as arguments 38 39 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\ 40 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\ 41 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ] 42 43 44 class InputTransformer(metaclass=abc.ABCMeta): 45 """Abstract base class for line-based input transformers.""" 46 47 def __init__(self): 48 warnings.warn( 49 "`InputTransformer` has been deprecated since IPython 7.0" 50 " and emit a warnig since IPython 8.31, it" 51 " will be removed in the future", 52 DeprecationWarning, 53 stacklevel=2, 54 ) 55 56 @abc.abstractmethod 57 def push(self, line): 58 """Send a line of input to the transformer, returning the transformed 59 input or None if the transformer is waiting for more input. 60 61 Must be overridden by subclasses. 62 63 Implementations may raise ``SyntaxError`` if the input is invalid. No 64 other exceptions may be raised. 65 """ 66 pass 67 68 @abc.abstractmethod 69 def reset(self): 70 """Return, transformed any lines that the transformer has accumulated, 71 and reset its internal state. 72 73 Must be overridden by subclasses. 74 """ 75 pass 76 77 @classmethod 78 def wrap(cls, func): 79 """Can be used by subclasses as a decorator, to return a factory that 80 will allow instantiation with the decorated object. 81 """ 82 @functools.wraps(func) 83 def transformer_factory(**kwargs): 84 return cls(func, **kwargs) # type: ignore [call-arg] 85 86 return transformer_factory 87 88 class StatelessInputTransformer(InputTransformer): 89 """Wrapper for a stateless input transformer implemented as a function.""" 90 def __init__(self, func): 91 super().__init__() 92 warnings.warn( 93 "`StatelessInputTransformer` has been deprecated since IPython 7.0" 94 " and emit a warnig since IPython 8.31, it" 95 " will be removed in the future", 96 DeprecationWarning, 97 stacklevel=2, 98 ) 99 self.func = func 100 101 def __repr__(self): 102 return "StatelessInputTransformer(func={0!r})".format(self.func) 103 104 def push(self, line): 105 """Send a line of input to the transformer, returning the 106 transformed input.""" 107 return self.func(line) 108 109 def reset(self): 110 """No-op - exists for compatibility.""" 111 pass 112 113 class CoroutineInputTransformer(InputTransformer): 114 """Wrapper for an input transformer implemented as a coroutine.""" 115 def __init__(self, coro, **kwargs): 116 # Prime it 117 super().__init__() 118 warnings.warn( 119 "`CoroutineInputTransformer` has been deprecated since IPython 7.0" 120 " and emit a warnig since IPython 8.31, it" 121 " will be removed in the future", 122 DeprecationWarning, 123 stacklevel=2, 124 ) 125 self.coro = coro(**kwargs) 126 next(self.coro) 127 128 def __repr__(self): 129 return "CoroutineInputTransformer(coro={0!r})".format(self.coro) 130 131 def push(self, line): 132 """Send a line of input to the transformer, returning the 133 transformed input or None if the transformer is waiting for more 134 input. 135 """ 136 return self.coro.send(line) 137 138 def reset(self): 139 """Return, transformed any lines that the transformer has 140 accumulated, and reset its internal state. 141 """ 142 return self.coro.send(None) 143 144 class TokenInputTransformer(InputTransformer): 145 """Wrapper for a token-based input transformer. 146 147 func should accept a list of tokens (5-tuples, see tokenize docs), and 148 return an iterable which can be passed to tokenize.untokenize(). 149 """ 150 def __init__(self, func): 151 warnings.warn( 152 "`CoroutineInputTransformer` has been deprecated since IPython 7.0" 153 " and emit a warnig since IPython 8.31, it" 154 " will be removed in the future", 155 DeprecationWarning, 156 stacklevel=2, 157 ) 158 self.func = func 159 self.buf = [] 160 self.reset_tokenizer() 161 162 def reset_tokenizer(self): 163 it = iter(self.buf) 164 self.tokenizer = tokenutil.generate_tokens_catch_errors(it.__next__) 165 166 def push(self, line): 167 self.buf.append(line + '\n') 168 if all(l.isspace() for l in self.buf): 169 return self.reset() 170 171 tokens = [] 172 stop_at_NL = False 173 try: 174 for intok in self.tokenizer: 175 tokens.append(intok) 176 t = intok[0] 177 if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL): 178 # Stop before we try to pull a line we don't have yet 179 break 180 elif t == tokenize.ERRORTOKEN: 181 stop_at_NL = True 182 except TokenError: 183 # Multi-line statement - stop and try again with the next line 184 self.reset_tokenizer() 185 return None 186 187 return self.output(tokens) 188 189 def output(self, tokens): 190 self.buf.clear() 191 self.reset_tokenizer() 192 return untokenize(self.func(tokens)).rstrip('\n') 193 194 def reset(self): 195 l = ''.join(self.buf) 196 self.buf.clear() 197 self.reset_tokenizer() 198 if l: 199 return l.rstrip('\n') 200 201 class assemble_python_lines(TokenInputTransformer): 202 def __init__(self): 203 super().__init__(None) 204 205 def output(self, tokens): 206 return self.reset() 207 208 @CoroutineInputTransformer.wrap 209 def assemble_logical_lines(): 210 r"""Join lines following explicit line continuations (\)""" 211 line = '' 212 while True: 213 line = (yield line) 214 if not line or line.isspace(): 215 continue 216 217 parts = [] 218 while line is not None: 219 if line.endswith('\\') and (not has_comment(line)): 220 parts.append(line[:-1]) 221 line = (yield None) # Get another line 222 else: 223 parts.append(line) 224 break 225 226 # Output 227 line = ''.join(parts) 228 229 # Utilities 230 def _make_help_call(target: str, esc: str, lspace: str) -> str: 231 """Prepares a pinfo(2)/psearch call from a target name and the escape 232 (i.e. ? or ??)""" 233 method = 'pinfo2' if esc == '??' \ 234 else 'psearch' if '*' in target \ 235 else 'pinfo' 236 arg = " ".join([method, target]) 237 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) 238 t_magic_name, _, t_magic_arg_s = arg.partition(' ') 239 t_magic_name = t_magic_name.lstrip(ESC_MAGIC) 240 return "%sget_ipython().run_line_magic(%r, %r)" % ( 241 lspace, 242 t_magic_name, 243 t_magic_arg_s, 244 ) 245 246 247 # These define the transformations for the different escape characters. 248 def _tr_system(line_info: LineInfo): 249 "Translate lines escaped with: !" 250 cmd = line_info.line.lstrip().lstrip(ESC_SHELL) 251 return '%sget_ipython().system(%r)' % (line_info.pre, cmd) 252 253 254 def _tr_system2(line_info: LineInfo): 255 "Translate lines escaped with: !!" 256 cmd = line_info.line.lstrip()[2:] 257 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd) 258 259 260 def _tr_help(line_info: LineInfo): 261 "Translate lines escaped with: ?/??" 262 # A naked help line should just fire the intro help screen 263 if not line_info.line[1:]: 264 return 'get_ipython().show_usage()' 265 266 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre) 267 268 269 def _tr_magic(line_info: LineInfo): 270 "Translate lines escaped with: %" 271 tpl = '%sget_ipython().run_line_magic(%r, %r)' 272 if line_info.line.startswith(ESC_MAGIC2): 273 return line_info.line 274 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip() 275 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) 276 t_magic_name, _, t_magic_arg_s = cmd.partition(' ') 277 t_magic_name = t_magic_name.lstrip(ESC_MAGIC) 278 return tpl % (line_info.pre, t_magic_name, t_magic_arg_s) 279 280 281 def _tr_quote(line_info: LineInfo): 282 "Translate lines escaped with: ," 283 return '%s%s("%s")' % (line_info.pre, line_info.ifun, 284 '", "'.join(line_info.the_rest.split()) ) 285 286 287 def _tr_quote2(line_info: LineInfo): 288 "Translate lines escaped with: ;" 289 return '%s%s("%s")' % (line_info.pre, line_info.ifun, 290 line_info.the_rest) 291 292 293 def _tr_paren(line_info: LineInfo): 294 "Translate lines escaped with: /" 295 return '%s%s(%s)' % (line_info.pre, line_info.ifun, 296 ", ".join(line_info.the_rest.split())) 297 298 tr = { ESC_SHELL : _tr_system, 299 ESC_SH_CAP : _tr_system2, 300 ESC_HELP : _tr_help, 301 ESC_HELP2 : _tr_help, 302 ESC_MAGIC : _tr_magic, 303 ESC_QUOTE : _tr_quote, 304 ESC_QUOTE2 : _tr_quote2, 305 ESC_PAREN : _tr_paren } 306 307 @StatelessInputTransformer.wrap 308 def escaped_commands(line: str): 309 """Transform escaped commands - %magic, !system, ?help + various autocalls.""" 310 if not line or line.isspace(): 311 return line 312 lineinf = LineInfo(line) 313 if lineinf.esc not in tr: 314 return line 315 316 return tr[lineinf.esc](lineinf) 317 318 _initial_space_re = re.compile(r'\s*') 319 320 _help_end_re = re.compile(r"""(%{0,2} 321 (?!\d)[\w*]+ # Variable name 322 (\.(?!\d)[\w*]+)* # .etc.etc 323 ) 324 (\?\??)$ # ? or ?? 325 """, 326 re.VERBOSE) 327 328 # Extra pseudotokens for multiline strings and data structures 329 _MULTILINE_STRING = object() 330 _MULTILINE_STRUCTURE = object() 331 332 def _line_tokens(line): 333 """Helper for has_comment and ends_in_comment_or_string.""" 334 readline = StringIO(line).readline 335 toktypes = set() 336 try: 337 for t in tokenutil.generate_tokens_catch_errors(readline): 338 toktypes.add(t[0]) 339 except TokenError as e: 340 # There are only two cases where a TokenError is raised. 341 if 'multi-line string' in e.args[0]: 342 toktypes.add(_MULTILINE_STRING) 343 else: 344 toktypes.add(_MULTILINE_STRUCTURE) 345 return toktypes 346 347 def has_comment(src): 348 """Indicate whether an input line has (i.e. ends in, or is) a comment. 349 350 This uses tokenize, so it can distinguish comments from # inside strings. 351 352 Parameters 353 ---------- 354 src : string 355 A single line input string. 356 357 Returns 358 ------- 359 comment : bool 360 True if source has a comment. 361 """ 362 return (tokenize.COMMENT in _line_tokens(src)) 363 364 def ends_in_comment_or_string(src): 365 """Indicates whether or not an input line ends in a comment or within 366 a multiline string. 367 368 Parameters 369 ---------- 370 src : string 371 A single line input string. 372 373 Returns 374 ------- 375 comment : bool 376 True if source ends in a comment or multiline string. 377 """ 378 toktypes = _line_tokens(src) 379 return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes) 380 381 382 @StatelessInputTransformer.wrap 383 def help_end(line: str): 384 """Translate lines with ?/?? at the end""" 385 m = _help_end_re.search(line) 386 if m is None or ends_in_comment_or_string(line): 387 return line 388 target = m.group(1) 389 esc = m.group(3) 390 match = _initial_space_re.match(line) 391 assert match is not None 392 lspace = match.group(0) 393 394 return _make_help_call(target, esc, lspace) 395 396 397 @CoroutineInputTransformer.wrap 398 def cellmagic(end_on_blank_line: bool = False): 399 """Captures & transforms cell magics. 400 401 After a cell magic is started, this stores up any lines it gets until it is 402 reset (sent None). 403 """ 404 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)' 405 cellmagic_help_re = re.compile(r'%%\w+\?') 406 line = '' 407 while True: 408 line = (yield line) 409 # consume leading empty lines 410 while not line: 411 line = (yield line) 412 413 if not line.startswith(ESC_MAGIC2): 414 # This isn't a cell magic, idle waiting for reset then start over 415 while line is not None: 416 line = (yield line) 417 continue 418 419 if cellmagic_help_re.match(line): 420 # This case will be handled by help_end 421 continue 422 423 first = line 424 body = [] 425 line = (yield None) 426 while (line is not None) and \ 427 ((line.strip() != '') or not end_on_blank_line): 428 body.append(line) 429 line = (yield None) 430 431 # Output 432 magic_name, _, first = first.partition(' ') 433 magic_name = magic_name.lstrip(ESC_MAGIC2) 434 line = tpl % (magic_name, first, u'\n'.join(body)) 435 436 437 def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None): 438 """Remove matching input prompts from a block of input. 439 440 Parameters 441 ---------- 442 prompt_re : regular expression 443 A regular expression matching any input prompt (including continuation) 444 initial_re : regular expression, optional 445 A regular expression matching only the initial prompt, but not continuation. 446 If no initial expression is given, prompt_re will be used everywhere. 447 Used mainly for plain Python prompts, where the continuation prompt 448 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped. 449 450 Notes 451 ----- 452 If `initial_re` and `prompt_re differ`, 453 only `initial_re` will be tested against the first line. 454 If any prompt is found on the first two lines, 455 prompts will be stripped from the rest of the block. 456 """ 457 if initial_re is None: 458 initial_re = prompt_re 459 line = '' 460 while True: 461 line = (yield line) 462 463 # First line of cell 464 if line is None: 465 continue 466 out, n1 = initial_re.subn('', line, count=1) 467 if turnoff_re and not n1: 468 if turnoff_re.match(line): 469 # We're in e.g. a cell magic; disable this transformer for 470 # the rest of the cell. 471 while line is not None: 472 line = (yield line) 473 continue 474 475 line = (yield out) 476 477 if line is None: 478 continue 479 # check for any prompt on the second line of the cell, 480 # because people often copy from just after the first prompt, 481 # so we might not see it in the first line. 482 out, n2 = prompt_re.subn('', line, count=1) 483 line = (yield out) 484 485 if n1 or n2: 486 # Found a prompt in the first two lines - check for it in 487 # the rest of the cell as well. 488 while line is not None: 489 line = (yield prompt_re.sub('', line, count=1)) 490 491 else: 492 # Prompts not in input - wait for reset 493 while line is not None: 494 line = (yield line) 495 496 @CoroutineInputTransformer.wrap 497 def classic_prompt(): 498 """Strip the >>>/... prompts of the Python interactive shell.""" 499 # FIXME: non-capturing version (?:...) usable? 500 prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)') 501 initial_re = re.compile(r'^>>>( |$)') 502 # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts 503 turnoff_re = re.compile(r'^[%!]') 504 return _strip_prompts(prompt_re, initial_re, turnoff_re) 505 506 @CoroutineInputTransformer.wrap 507 def ipy_prompt(): 508 """Strip IPython's In [1]:/...: prompts.""" 509 # FIXME: non-capturing version (?:...) usable? 510 prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)') 511 # Disable prompt stripping inside cell magics 512 turnoff_re = re.compile(r'^%%') 513 return _strip_prompts(prompt_re, turnoff_re=turnoff_re) 514 515 516 @CoroutineInputTransformer.wrap 517 def leading_indent(): 518 """Remove leading indentation. 519 520 If the first line starts with a spaces or tabs, the same whitespace will be 521 removed from each following line until it is reset. 522 """ 523 space_re = re.compile(r'^[ \t]+') 524 line = '' 525 while True: 526 line = (yield line) 527 528 if line is None: 529 continue 530 531 m = space_re.match(line) 532 if m: 533 space = m.group(0) 534 while line is not None: 535 if line.startswith(space): 536 line = line[len(space):] 537 line = (yield line) 538 else: 539 # No leading spaces - wait for reset 540 while line is not None: 541 line = (yield line) 542 543 544 _assign_pat = \ 545 r'''(?P<lhs>(\s*) 546 ([\w\.]+) # Initial identifier 547 (\s*,\s* 548 \*?[\w\.]+)* # Further identifiers for unpacking 549 \s*?,? # Trailing comma 550 ) 551 \s*=\s* 552 ''' 553 554 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE) 555 assign_system_template = '%s = get_ipython().getoutput(%r)' 556 @StatelessInputTransformer.wrap 557 def assign_from_system(line): 558 """Transform assignment from system commands (e.g. files = !ls)""" 559 m = assign_system_re.match(line) 560 if m is None: 561 return line 562 563 return assign_system_template % m.group('lhs', 'cmd') 564 565 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE) 566 assign_magic_template = '%s = get_ipython().run_line_magic(%r, %r)' 567 @StatelessInputTransformer.wrap 568 def assign_from_magic(line): 569 """Transform assignment from magic commands (e.g. a = %who_ls)""" 570 m = assign_magic_re.match(line) 571 if m is None: 572 return line 573 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) 574 m_lhs, m_cmd = m.group('lhs', 'cmd') 575 t_magic_name, _, t_magic_arg_s = m_cmd.partition(' ') 576 t_magic_name = t_magic_name.lstrip(ESC_MAGIC) 577 return assign_magic_template % (m_lhs, t_magic_name, t_magic_arg_s) 578