1 # -*- coding: utf-8 -*- 2 """Tests for the inputsplitter module.""" 3 4 5 # Copyright (c) IPython Development Team. 6 # Distributed under the terms of the Modified BSD License. 7 8 import unittest 9 import pytest 10 import sys 11 12 with pytest.warns(DeprecationWarning, match="inputsplitter"): 13 from IPython.core import inputsplitter as isp 14 from IPython.core.inputtransformer import InputTransformer 15 from IPython.core.tests.test_inputtransformer import syntax, syntax_ml 16 from IPython.testing import tools as tt 17 18 #----------------------------------------------------------------------------- 19 # Semi-complete examples (also used as tests) 20 #----------------------------------------------------------------------------- 21 22 # Note: at the bottom, there's a slightly more complete version of this that 23 # can be useful during development of code here. 24 25 def mini_interactive_loop(input_func): 26 """Minimal example of the logic of an interactive interpreter loop. 27 28 This serves as an example, and it is used by the test system with a fake 29 raw_input that simulates interactive input.""" 30 31 from IPython.core.inputsplitter import InputSplitter 32 33 isp = InputSplitter() 34 # In practice, this input loop would be wrapped in an outside loop to read 35 # input indefinitely, until some exit/quit command was issued. Here we 36 # only illustrate the basic inner loop. 37 while isp.push_accepts_more(): 38 indent = ' '*isp.get_indent_spaces() 39 prompt = '>>> ' + indent 40 line = indent + input_func(prompt) 41 isp.push(line) 42 43 # Here we just return input so we can use it in a test suite, but a real 44 # interpreter would instead send it for execution somewhere. 45 src = isp.source_reset() 46 # print('Input source was:\n', src) # dbg 47 return src 48 49 #----------------------------------------------------------------------------- 50 # Test utilities, just for local use 51 #----------------------------------------------------------------------------- 52 53 54 def pseudo_input(lines): 55 """Return a function that acts like raw_input but feeds the input list.""" 56 ilines = iter(lines) 57 def raw_in(prompt): 58 try: 59 return next(ilines) 60 except StopIteration: 61 return '' 62 return raw_in 63 64 #----------------------------------------------------------------------------- 65 # Tests 66 #----------------------------------------------------------------------------- 67 def test_spaces(): 68 tests = [('', 0), 69 (' ', 1), 70 ('\n', 0), 71 (' \n', 1), 72 ('x', 0), 73 (' x', 1), 74 (' x',2), 75 (' x',4), 76 # Note: tabs are counted as a single whitespace! 77 ('\tx', 1), 78 ('\t x', 2), 79 ] 80 with pytest.warns(PendingDeprecationWarning): 81 tt.check_pairs(isp.num_ini_spaces, tests) 82 83 84 def test_remove_comments(): 85 tests = [('text', 'text'), 86 ('text # comment', 'text '), 87 ('text # comment\n', 'text \n'), 88 ('text # comment \n', 'text \n'), 89 ('line # c \nline\n','line \nline\n'), 90 ('line # c \nline#c2 \nline\nline #c\n\n', 91 'line \nline\nline\nline \n\n'), 92 ] 93 tt.check_pairs(isp.remove_comments, tests) 94 95 96 def test_get_input_encoding(): 97 encoding = isp.get_input_encoding() 98 assert isinstance(encoding, str) 99 # simple-minded check that at least encoding a simple string works with the 100 # encoding we got. 101 assert "test".encode(encoding) == b"test" 102 103 104 class NoInputEncodingTestCase(unittest.TestCase): 105 def setUp(self): 106 self.old_stdin = sys.stdin 107 class X: pass 108 fake_stdin = X() 109 sys.stdin = fake_stdin 110 111 def test(self): 112 # Verify that if sys.stdin has no 'encoding' attribute we do the right 113 # thing 114 enc = isp.get_input_encoding() 115 self.assertEqual(enc, 'ascii') 116 117 def tearDown(self): 118 sys.stdin = self.old_stdin 119 120 121 class InputSplitterTestCase(unittest.TestCase): 122 def setUp(self): 123 self.isp = isp.InputSplitter() 124 125 def test_reset(self): 126 isp = self.isp 127 isp.push('x=1') 128 isp.reset() 129 self.assertEqual(isp._buffer, []) 130 self.assertEqual(isp.get_indent_spaces(), 0) 131 self.assertEqual(isp.source, '') 132 self.assertEqual(isp.code, None) 133 self.assertEqual(isp._is_complete, False) 134 135 def test_source(self): 136 self.isp._store('1') 137 self.isp._store('2') 138 self.assertEqual(self.isp.source, '1\n2\n') 139 self.assertEqual(len(self.isp._buffer)>0, True) 140 self.assertEqual(self.isp.source_reset(), '1\n2\n') 141 self.assertEqual(self.isp._buffer, []) 142 self.assertEqual(self.isp.source, '') 143 144 def test_indent(self): 145 isp = self.isp # shorthand 146 isp.push('x=1') 147 self.assertEqual(isp.get_indent_spaces(), 0) 148 isp.push('if 1:\n x=1') 149 self.assertEqual(isp.get_indent_spaces(), 4) 150 isp.push('y=2\n') 151 self.assertEqual(isp.get_indent_spaces(), 0) 152 153 def test_indent2(self): 154 isp = self.isp 155 isp.push('if 1:') 156 self.assertEqual(isp.get_indent_spaces(), 4) 157 isp.push(' x=1') 158 self.assertEqual(isp.get_indent_spaces(), 4) 159 # Blank lines shouldn't change the indent level 160 isp.push(' '*2) 161 self.assertEqual(isp.get_indent_spaces(), 4) 162 163 def test_indent3(self): 164 isp = self.isp 165 # When a multiline statement contains parens or multiline strings, we 166 # shouldn't get confused. 167 isp.push("if 1:") 168 isp.push(" x = (1+\n 2)") 169 self.assertEqual(isp.get_indent_spaces(), 4) 170 171 def test_indent4(self): 172 isp = self.isp 173 # whitespace after ':' should not screw up indent level 174 isp.push('if 1: \n x=1') 175 self.assertEqual(isp.get_indent_spaces(), 4) 176 isp.push('y=2\n') 177 self.assertEqual(isp.get_indent_spaces(), 0) 178 isp.push('if 1:\t\n x=1') 179 self.assertEqual(isp.get_indent_spaces(), 4) 180 isp.push('y=2\n') 181 self.assertEqual(isp.get_indent_spaces(), 0) 182 183 def test_dedent_pass(self): 184 isp = self.isp # shorthand 185 # should NOT cause dedent 186 isp.push('if 1:\n passes = 5') 187 self.assertEqual(isp.get_indent_spaces(), 4) 188 isp.push('if 1:\n pass') 189 self.assertEqual(isp.get_indent_spaces(), 0) 190 isp.push('if 1:\n pass ') 191 self.assertEqual(isp.get_indent_spaces(), 0) 192 193 def test_dedent_break(self): 194 isp = self.isp # shorthand 195 # should NOT cause dedent 196 isp.push('while 1:\n breaks = 5') 197 self.assertEqual(isp.get_indent_spaces(), 4) 198 isp.push('while 1:\n break') 199 self.assertEqual(isp.get_indent_spaces(), 0) 200 isp.push('while 1:\n break ') 201 self.assertEqual(isp.get_indent_spaces(), 0) 202 203 def test_dedent_continue(self): 204 isp = self.isp # shorthand 205 # should NOT cause dedent 206 isp.push('while 1:\n continues = 5') 207 self.assertEqual(isp.get_indent_spaces(), 4) 208 isp.push('while 1:\n continue') 209 self.assertEqual(isp.get_indent_spaces(), 0) 210 isp.push('while 1:\n continue ') 211 self.assertEqual(isp.get_indent_spaces(), 0) 212 213 def test_dedent_raise(self): 214 isp = self.isp # shorthand 215 # should NOT cause dedent 216 isp.push('if 1:\n raised = 4') 217 self.assertEqual(isp.get_indent_spaces(), 4) 218 isp.push('if 1:\n raise TypeError()') 219 self.assertEqual(isp.get_indent_spaces(), 0) 220 isp.push('if 1:\n raise') 221 self.assertEqual(isp.get_indent_spaces(), 0) 222 isp.push('if 1:\n raise ') 223 self.assertEqual(isp.get_indent_spaces(), 0) 224 225 def test_dedent_return(self): 226 isp = self.isp # shorthand 227 # should NOT cause dedent 228 isp.push('if 1:\n returning = 4') 229 self.assertEqual(isp.get_indent_spaces(), 4) 230 isp.push('if 1:\n return 5 + 493') 231 self.assertEqual(isp.get_indent_spaces(), 0) 232 isp.push('if 1:\n return') 233 self.assertEqual(isp.get_indent_spaces(), 0) 234 isp.push('if 1:\n return ') 235 self.assertEqual(isp.get_indent_spaces(), 0) 236 isp.push('if 1:\n return(0)') 237 self.assertEqual(isp.get_indent_spaces(), 0) 238 239 def test_push(self): 240 isp = self.isp 241 self.assertEqual(isp.push('x=1'), True) 242 243 def test_push2(self): 244 isp = self.isp 245 self.assertEqual(isp.push('if 1:'), False) 246 for line in [' x=1', '# a comment', ' y=2']: 247 print(line) 248 self.assertEqual(isp.push(line), True) 249 250 def test_push3(self): 251 isp = self.isp 252 isp.push('if True:') 253 isp.push(' a = 1') 254 self.assertEqual(isp.push('b = [1,'), False) 255 256 def test_push_accepts_more(self): 257 isp = self.isp 258 isp.push('x=1') 259 self.assertEqual(isp.push_accepts_more(), False) 260 261 def test_push_accepts_more2(self): 262 isp = self.isp 263 isp.push('if 1:') 264 self.assertEqual(isp.push_accepts_more(), True) 265 isp.push(' x=1') 266 self.assertEqual(isp.push_accepts_more(), True) 267 isp.push('') 268 self.assertEqual(isp.push_accepts_more(), False) 269 270 def test_push_accepts_more3(self): 271 isp = self.isp 272 isp.push("x = (2+\n3)") 273 self.assertEqual(isp.push_accepts_more(), False) 274 275 def test_push_accepts_more4(self): 276 isp = self.isp 277 # When a multiline statement contains parens or multiline strings, we 278 # shouldn't get confused. 279 # FIXME: we should be able to better handle de-dents in statements like 280 # multiline strings and multiline expressions (continued with \ or 281 # parens). Right now we aren't handling the indentation tracking quite 282 # correctly with this, though in practice it may not be too much of a 283 # problem. We'll need to see. 284 isp.push("if 1:") 285 isp.push(" x = (2+") 286 isp.push(" 3)") 287 self.assertEqual(isp.push_accepts_more(), True) 288 isp.push(" y = 3") 289 self.assertEqual(isp.push_accepts_more(), True) 290 isp.push('') 291 self.assertEqual(isp.push_accepts_more(), False) 292 293 def test_push_accepts_more5(self): 294 isp = self.isp 295 isp.push('try:') 296 isp.push(' a = 5') 297 isp.push('except:') 298 isp.push(' raise') 299 # We want to be able to add an else: block at this point, so it should 300 # wait for a blank line. 301 self.assertEqual(isp.push_accepts_more(), True) 302 303 def test_continuation(self): 304 isp = self.isp 305 isp.push("import os, \\") 306 self.assertEqual(isp.push_accepts_more(), True) 307 isp.push("sys") 308 self.assertEqual(isp.push_accepts_more(), False) 309 310 def test_syntax_error(self): 311 isp = self.isp 312 # Syntax errors immediately produce a 'ready' block, so the invalid 313 # Python can be sent to the kernel for evaluation with possible ipython 314 # special-syntax conversion. 315 isp.push('run foo') 316 self.assertEqual(isp.push_accepts_more(), False) 317 318 def test_unicode(self): 319 self.isp.push(u"Pérez") 320 self.isp.push(u'\xc3\xa9') 321 self.isp.push(u"u'\xc3\xa9'") 322 323 @pytest.mark.xfail( 324 reason="Bug in python 3.9.8 – bpo 45738", 325 condition=sys.version_info in [(3, 11, 0, "alpha", 2)], 326 raises=SystemError, 327 strict=True, 328 ) 329 def test_line_continuation(self): 330 """ Test issue #2108.""" 331 isp = self.isp 332 # A blank line after a line continuation should not accept more 333 isp.push("1 \\\n\n") 334 self.assertEqual(isp.push_accepts_more(), False) 335 # Whitespace after a \ is a SyntaxError. The only way to test that 336 # here is to test that push doesn't accept more (as with 337 # test_syntax_error() above). 338 isp.push(r"1 \ ") 339 self.assertEqual(isp.push_accepts_more(), False) 340 # Even if the line is continuable (c.f. the regular Python 341 # interpreter) 342 isp.push(r"(1 \ ") 343 self.assertEqual(isp.push_accepts_more(), False) 344 345 def test_check_complete(self): 346 isp = self.isp 347 self.assertEqual(isp.check_complete("a = 1"), ('complete', None)) 348 self.assertEqual(isp.check_complete("for a in range(5):"), ('incomplete', 4)) 349 self.assertEqual(isp.check_complete("raise = 2"), ('invalid', None)) 350 self.assertEqual(isp.check_complete("a = [1,\n2,"), ('incomplete', 0)) 351 self.assertEqual(isp.check_complete("def a():\n x=1\n global x"), ('invalid', None)) 352 353 class InteractiveLoopTestCase(unittest.TestCase): 354 """Tests for an interactive loop like a python shell. 355 """ 356 def check_ns(self, lines, ns): 357 """Validate that the given input lines produce the resulting namespace. 358 359 Note: the input lines are given exactly as they would be typed in an 360 auto-indenting environment, as mini_interactive_loop above already does 361 auto-indenting and prepends spaces to the input. 362 """ 363 src = mini_interactive_loop(pseudo_input(lines)) 364 test_ns = {} 365 exec(src, test_ns) 366 # We can't check that the provided ns is identical to the test_ns, 367 # because Python fills test_ns with extra keys (copyright, etc). But 368 # we can check that the given dict is *contained* in test_ns 369 for k,v in ns.items(): 370 self.assertEqual(test_ns[k], v) 371 372 def test_simple(self): 373 self.check_ns(['x=1'], dict(x=1)) 374 375 def test_simple2(self): 376 self.check_ns(['if 1:', 'x=2'], dict(x=2)) 377 378 def test_xy(self): 379 self.check_ns(['x=1; y=2'], dict(x=1, y=2)) 380 381 def test_abc(self): 382 self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3)) 383 384 def test_multi(self): 385 self.check_ns(['x =(1+','1+','2)'], dict(x=4)) 386 387 388 class IPythonInputTestCase(InputSplitterTestCase): 389 """By just creating a new class whose .isp is a different instance, we 390 re-run the same test battery on the new input splitter. 391 392 In addition, this runs the tests over the syntax and syntax_ml dicts that 393 were tested by individual functions, as part of the OO interface. 394 395 It also makes some checks on the raw buffer storage. 396 """ 397 398 def setUp(self): 399 self.isp = isp.IPythonInputSplitter() 400 401 def test_syntax(self): 402 """Call all single-line syntax tests from the main object""" 403 isp = self.isp 404 for example in syntax.values(): 405 for raw, out_t in example: 406 if raw.startswith(' '): 407 continue 408 409 isp.push(raw+'\n') 410 out_raw = isp.source_raw 411 out = isp.source_reset() 412 self.assertEqual(out.rstrip(), out_t, 413 tt.pair_fail_msg.format("inputsplitter",raw, out_t, out)) 414 self.assertEqual(out_raw.rstrip(), raw.rstrip()) 415 416 def test_syntax_multiline(self): 417 isp = self.isp 418 for example in syntax_ml.values(): 419 for line_pairs in example: 420 out_t_parts = [] 421 raw_parts = [] 422 for lraw, out_t_part in line_pairs: 423 if out_t_part is not None: 424 out_t_parts.append(out_t_part) 425 426 if lraw is not None: 427 isp.push(lraw) 428 raw_parts.append(lraw) 429 430 out_raw = isp.source_raw 431 out = isp.source_reset() 432 out_t = '\n'.join(out_t_parts).rstrip() 433 raw = '\n'.join(raw_parts).rstrip() 434 self.assertEqual(out.rstrip(), out_t) 435 self.assertEqual(out_raw.rstrip(), raw) 436 437 def test_syntax_multiline_cell(self): 438 isp = self.isp 439 for example in syntax_ml.values(): 440 441 out_t_parts = [] 442 for line_pairs in example: 443 raw = '\n'.join(r for r, _ in line_pairs if r is not None) 444 out_t = '\n'.join(t for _,t in line_pairs if t is not None) 445 out = isp.transform_cell(raw) 446 # Match ignoring trailing whitespace 447 self.assertEqual(out.rstrip(), out_t.rstrip()) 448 449 def test_cellmagic_preempt(self): 450 isp = self.isp 451 for raw, name, line, cell in [ 452 ("%%cellm a\nIn[1]:", u'cellm', u'a', u'In[1]:'), 453 ("%%cellm \nline\n>>> hi", u'cellm', u'', u'line\n>>> hi'), 454 (">>> %%cellm \nline\n>>> hi", u'cellm', u'', u'line\nhi'), 455 ("%%cellm \n>>> hi", u'cellm', u'', u'>>> hi'), 456 ("%%cellm \nline1\nline2", u'cellm', u'', u'line1\nline2'), 457 ("%%cellm \nline1\\\\\nline2", u'cellm', u'', u'line1\\\\\nline2'), 458 ]: 459 expected = "get_ipython().run_cell_magic(%r, %r, %r)" % ( 460 name, line, cell 461 ) 462 out = isp.transform_cell(raw) 463 self.assertEqual(out.rstrip(), expected.rstrip()) 464 465 def test_multiline_passthrough(self): 466 isp = self.isp 467 class CommentTransformer(InputTransformer): 468 def __init__(self): 469 self._lines = [] 470 471 def push(self, line): 472 self._lines.append(line + '#') 473 474 def reset(self): 475 text = '\n'.join(self._lines) 476 self._lines = [] 477 return text 478 479 isp.physical_line_transforms.insert(0, CommentTransformer()) 480 481 for raw, expected in [ 482 ("a=5", "a=5#"), 483 ("%ls foo", "get_ipython().run_line_magic(%r, %r)" % (u'ls', u'foo#')), 484 ("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().run_line_magic(%r, %r)" % ( 485 u'ls foo#', u'ls', u'bar#' 486 )), 487 ("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().run_line_magic(%r, %r)\n4#\n5#" % (u'ls', u'foo#')), 488 ]: 489 out = isp.transform_cell(raw) 490 self.assertEqual(out.rstrip(), expected.rstrip()) 491 492 #----------------------------------------------------------------------------- 493 # Main - use as a script, mostly for developer experiments 494 #----------------------------------------------------------------------------- 495 496 if __name__ == '__main__': 497 # A simple demo for interactive experimentation. This code will not get 498 # picked up by any test suite. 499 from IPython.core.inputsplitter import IPythonInputSplitter 500 501 # configure here the syntax to use, prompt and whether to autoindent 502 #isp, start_prompt = InputSplitter(), '>>> ' 503 isp, start_prompt = IPythonInputSplitter(), 'In> ' 504 505 autoindent = True 506 #autoindent = False 507 508 try: 509 while True: 510 prompt = start_prompt 511 while isp.push_accepts_more(): 512 indent = ' '*isp.get_indent_spaces() 513 if autoindent: 514 line = indent + input(prompt+indent) 515 else: 516 line = input(prompt) 517 isp.push(line) 518 prompt = '... ' 519 520 # Here we just return input so we can use it in a test suite, but a 521 # real interpreter would instead send it for execution somewhere. 522 #src = isp.source; raise EOFError # dbg 523 raw = isp.source_raw 524 src = isp.source_reset() 525 print('Input source was:\n', src) 526 print('Raw source was:\n', raw) 527 except EOFError: 528 print('Bye') 529 530 # Tests for cell magics support 531 532 def test_last_blank(): 533 assert isp.last_blank("") is False 534 assert isp.last_blank("abc") is False 535 assert isp.last_blank("abc\n") is False 536 assert isp.last_blank("abc\na") is False 537 538 assert isp.last_blank("\n") is True 539 assert isp.last_blank("\n ") is True 540 assert isp.last_blank("abc\n ") is True 541 assert isp.last_blank("abc\n\n") is True 542 assert isp.last_blank("abc\nd\n\n") is True 543 assert isp.last_blank("abc\nd\ne\n\n") is True 544 assert isp.last_blank("abc \n \n \n\n") is True 545 546 547 def test_last_two_blanks(): 548 assert isp.last_two_blanks("") is False 549 assert isp.last_two_blanks("abc") is False 550 assert isp.last_two_blanks("abc\n") is False 551 assert isp.last_two_blanks("abc\n\na") is False 552 assert isp.last_two_blanks("abc\n \n") is False 553 assert isp.last_two_blanks("abc\n\n") is False 554 555 assert isp.last_two_blanks("\n\n") is True 556 assert isp.last_two_blanks("\n\n ") is True 557 assert isp.last_two_blanks("\n \n") is True 558 assert isp.last_two_blanks("abc\n\n ") is True 559 assert isp.last_two_blanks("abc\n\n\n") is True 560 assert isp.last_two_blanks("abc\n\n \n") is True 561 assert isp.last_two_blanks("abc\n\n \n ") is True 562 assert isp.last_two_blanks("abc\n\n \n \n") is True 563 assert isp.last_two_blanks("abc\nd\n\n\n") is True 564 assert isp.last_two_blanks("abc\nd\ne\nf\n\n\n") is True 565 566 567 class CellMagicsCommon(object): 568 569 def test_whole_cell(self): 570 src = "%%cellm line\nbody\n" 571 out = self.sp.transform_cell(src) 572 ref = "get_ipython().run_cell_magic('cellm', 'line', 'body')\n" 573 assert out == ref 574 575 def test_cellmagic_help(self): 576 self.sp.push('%%cellm?') 577 assert self.sp.push_accepts_more() is False 578 579 def tearDown(self): 580 self.sp.reset() 581 582 583 class CellModeCellMagics(CellMagicsCommon, unittest.TestCase): 584 sp = isp.IPythonInputSplitter(line_input_checker=False) 585 586 def test_incremental(self): 587 sp = self.sp 588 sp.push("%%cellm firstline\n") 589 assert sp.push_accepts_more() is True # 1 590 sp.push("line2\n") 591 assert sp.push_accepts_more() is True # 2 592 sp.push("\n") 593 # This should accept a blank line and carry on until the cell is reset 594 assert sp.push_accepts_more() is True # 3 595 596 def test_no_strip_coding(self): 597 src = '\n'.join([ 598 '%%writefile foo.py', 599 '# coding: utf-8', 600 'print(u"üñîçø∂é")', 601 ]) 602 out = self.sp.transform_cell(src) 603 assert "# coding: utf-8" in out 604 605 606 class LineModeCellMagics(CellMagicsCommon, unittest.TestCase): 607 sp = isp.IPythonInputSplitter(line_input_checker=True) 608 609 def test_incremental(self): 610 sp = self.sp 611 sp.push("%%cellm line2\n") 612 assert sp.push_accepts_more() is True # 1 613 sp.push("\n") 614 # In this case, a blank line should end the cell magic 615 assert sp.push_accepts_more() is False # 2 616 617 618 indentation_samples = [ 619 ('a = 1', 0), 620 ('for a in b:', 4), 621 ('def f():', 4), 622 ('def f(): #comment', 4), 623 ('a = ":#not a comment"', 0), 624 ('def f():\n a = 1', 4), 625 ('def f():\n return 1', 0), 626 ('for a in b:\n' 627 ' if a < 0:' 628 ' continue', 3), 629 ('a = {', 4), 630 ('a = {\n' 631 ' 1,', 5), 632 ('b = """123', 0), 633 ('', 0), 634 ('def f():\n pass', 0), 635 ('class Bar:\n def f():\n pass', 4), 636 ('class Bar:\n def f():\n raise', 4), 637 ] 638 639 def test_find_next_indent(): 640 for code, exp in indentation_samples: 641 res = isp.find_next_indent(code) 642 msg = "{!r} != {!r} (expected)\n Code: {!r}".format(res, exp, code) 643 assert res == exp, msg 644