1  import tokenize
   2  
   3  from IPython.testing import tools as tt
   4  
   5  from IPython.core import inputtransformer as ipt
   6  
   7  def transform_and_reset(transformer):
   8      transformer = transformer()
   9      def transform(inp):
  10          try:
  11              return transformer.push(inp)
  12          finally:
  13              transformer.reset()
  14      
  15      return transform
  16  
  17  # Transformer tests
  18  def transform_checker(tests, transformer, **kwargs):
  19      """Utility to loop over test inputs"""
  20      transformer = transformer(**kwargs)
  21      try:
  22          for inp, tr in tests:
  23              if inp is None:
  24                  out = transformer.reset()
  25              else:
  26                  out = transformer.push(inp)
  27              assert out == tr
  28      finally:
  29          transformer.reset()
  30  
  31  # Data for all the syntax tests in the form of lists of pairs of
  32  # raw/transformed input.  We store it here as a global dict so that we can use
  33  # it both within single-function tests and also to validate the behavior of the
  34  # larger objects
  35  
  36  syntax = \
  37    dict(assign_system =
  38         [('a =! ls', "a = get_ipython().getoutput('ls')"),
  39          ('b = !ls', "b = get_ipython().getoutput('ls')"),
  40          ('c= !ls', "c = get_ipython().getoutput('ls')"),
  41          ('d == !ls', 'd == !ls'), # Invalid syntax, but we leave == alone.
  42          ('x=1', 'x=1'), # normal input is unmodified
  43          ('    ','    '),  # blank lines are kept intact
  44          # Tuple unpacking
  45          ("a, b = !echo 'a\\nb'", "a, b = get_ipython().getoutput(\"echo 'a\\\\nb'\")"),
  46          ("a,= !echo 'a'", "a, = get_ipython().getoutput(\"echo 'a'\")"),
  47          ("a, *bc = !echo 'a\\nb\\nc'", "a, *bc = get_ipython().getoutput(\"echo 'a\\\\nb\\\\nc'\")"),
  48          # Tuple unpacking with regular Python expressions, not our syntax.
  49          ("a, b = range(2)", "a, b = range(2)"),
  50          ("a, = range(1)", "a, = range(1)"),
  51          ("a, *bc = range(3)", "a, *bc = range(3)"),
  52          ],
  53  
  54         assign_magic =
  55         [('a =% who', "a = get_ipython().run_line_magic('who', '')"),
  56          ('b = %who', "b = get_ipython().run_line_magic('who', '')"),
  57          ('c= %ls', "c = get_ipython().run_line_magic('ls', '')"),
  58          ('d == %ls', 'd == %ls'), # Invalid syntax, but we leave == alone.
  59          ('x=1', 'x=1'), # normal input is unmodified
  60          ('    ','    '),  # blank lines are kept intact
  61          ("a, b = %foo", "a, b = get_ipython().run_line_magic('foo', '')"),
  62      ],
  63      classic_prompt=[
  64          (">>> x=1", "x=1"),
  65          ("x=1", "x=1"),  # normal input is unmodified
  66          ("    ", "    "),  # blank lines are kept intact
  67      ],
  68      ipy_prompt=[
  69          ("In [1]: x=1", "x=1"),
  70          ("x=1", "x=1"),  # normal input is unmodified
  71          ("    ", "    "),  # blank lines are kept intact
  72      ],
  73      # Tests for the escape transformer to leave normal code alone
  74      escaped_noesc=[
  75          ("    ", "    "),
  76          ("x=1", "x=1"),
  77      ],
  78      # System calls
  79      escaped_shell=[
  80          ("!ls", "get_ipython().system('ls')"),
  81          # Double-escape shell, this means to capture the output of the
  82          # subprocess and return it
  83          ("!!ls", "get_ipython().getoutput('ls')"),
  84      ],
  85      # Help/object info
  86      escaped_help=[
  87          ("?", "get_ipython().show_usage()"),
  88          ("?x1", "get_ipython().run_line_magic('pinfo', 'x1')"),
  89          ("??x2", "get_ipython().run_line_magic('pinfo2', 'x2')"),
  90          ("?a.*s", "get_ipython().run_line_magic('psearch', 'a.*s')"),
  91          ("?%hist1", "get_ipython().run_line_magic('pinfo', '%hist1')"),
  92          ("?%%hist2", "get_ipython().run_line_magic('pinfo', '%%hist2')"),
  93          ("?abc = qwe", "get_ipython().run_line_magic('pinfo', 'abc')"),
  94      ],
  95      end_help=[
  96          ("x3?", "get_ipython().run_line_magic('pinfo', 'x3')"),
  97          ("x4??", "get_ipython().run_line_magic('pinfo2', 'x4')"),
  98          ("%hist1?", "get_ipython().run_line_magic('pinfo', '%hist1')"),
  99          ("%hist2??", "get_ipython().run_line_magic('pinfo2', '%hist2')"),
 100          ("%%hist3?", "get_ipython().run_line_magic('pinfo', '%%hist3')"),
 101          ("%%hist4??", "get_ipython().run_line_magic('pinfo2', '%%hist4')"),
 102          ("π.foo?", "get_ipython().run_line_magic('pinfo', 'π.foo')"),
 103          ("f*?", "get_ipython().run_line_magic('psearch', 'f*')"),
 104          ("ax.*aspe*?", "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"),
 105          ("a = abc?", "get_ipython().run_line_magic('pinfo', 'abc')"),
 106          ("a = abc.qe??", "get_ipython().run_line_magic('pinfo2', 'abc.qe')"),
 107          ("a = *.items?", "get_ipython().run_line_magic('psearch', '*.items')"),
 108          ("plot(a?", "get_ipython().run_line_magic('pinfo', 'a')"),
 109          ("a*2 #comment?", "a*2 #comment?"),
 110      ],
 111      # Explicit magic calls
 112      escaped_magic=[
 113          ("%cd", "get_ipython().run_line_magic('cd', '')"),
 114          ("%cd /home", "get_ipython().run_line_magic('cd', '/home')"),
 115          # Backslashes need to be escaped.
 116          ("%cd C:\\User", "get_ipython().run_line_magic('cd', 'C:\\\\User')"),
 117          ("    %magic", "    get_ipython().run_line_magic('magic', '')"),
 118      ],
 119      # Quoting with separate arguments
 120      escaped_quote=[
 121          (",f", 'f("")'),
 122          (",f x", 'f("x")'),
 123          ("  ,f y", '  f("y")'),
 124          (",f a b", 'f("a", "b")'),
 125      ],
 126      # Quoting with single argument
 127      escaped_quote2=[
 128          (";f", 'f("")'),
 129          (";f x", 'f("x")'),
 130          ("  ;f y", '  f("y")'),
 131          (";f a b", 'f("a b")'),
 132      ],
 133      # Simply apply parens
 134      escaped_paren=[
 135          ("/f", "f()"),
 136          ("/f x", "f(x)"),
 137          ("  /f y", "  f(y)"),
 138          ("/f a b", "f(a, b)"),
 139      ],
 140      # Check that we transform prompts before other transforms
 141      mixed=[
 142          ("In [1]: %lsmagic", "get_ipython().run_line_magic('lsmagic', '')"),
 143          (">>> %lsmagic", "get_ipython().run_line_magic('lsmagic', '')"),
 144          ("In [2]: !ls", "get_ipython().system('ls')"),
 145          ("In [3]: abs?", "get_ipython().run_line_magic('pinfo', 'abs')"),
 146          ("In [4]: b = %who", "b = get_ipython().run_line_magic('who', '')"),
 147      ],
 148  )
 149  
 150  # multiline syntax examples.  Each of these should be a list of lists, with
 151  # each entry itself having pairs of raw/transformed input.  The union (with
 152  # '\n'.join() of the transformed inputs is what the splitter should produce
 153  # when fed the raw lines one at a time via push.
 154  syntax_ml = \
 155    dict(classic_prompt =
 156         [ [('>>> for i in range(10):','for i in range(10):'),
 157            ('...     print i','    print i'),
 158            ('... ', ''),
 159            ],
 160           [('>>> a="""','a="""'),
 161            ('... 123"""','123"""'),
 162            ],
 163           [('a="""','a="""'),
 164            ('... 123','123'),
 165            ('... 456"""','456"""'),
 166            ],
 167           [('a="""','a="""'),
 168            ('>>> 123','123'),
 169            ('... 456"""','456"""'),
 170            ],
 171           [('a="""','a="""'),
 172            ('123','123'),
 173            ('... 456"""','... 456"""'),
 174            ],
 175           [('....__class__','....__class__'),
 176           ],
 177           [('a=5', 'a=5'),
 178            ('...', ''),
 179           ],
 180           [('>>> def f(x):', 'def f(x):'),
 181            ('...', ''),
 182            ('...     return x', '    return x'),
 183            ],
 184           [('board = """....', 'board = """....'),
 185            ('....', '....'),
 186            ('...."""', '...."""'),
 187            ],
 188          ],
 189  
 190         ipy_prompt =
 191         [ [('In [24]: for i in range(10):','for i in range(10):'),
 192            ('   ....:     print i','    print i'),
 193            ('   ....: ', ''),
 194            ],
 195           [('In [24]: for i in range(10):','for i in range(10):'),
 196            # Qt console prompts expand with spaces, not dots
 197            ('    ...:     print i','    print i'),
 198            ('    ...: ', ''),
 199            ],
 200           [('In [24]: for i in range(10):','for i in range(10):'),
 201            # Sometimes whitespace preceding '...' has been removed
 202            ('...:     print i','    print i'),
 203            ('...: ', ''),
 204            ],
 205           [('In [24]: for i in range(10):','for i in range(10):'),
 206            # Space after last continuation prompt has been removed (issue #6674)
 207            ('...:     print i','    print i'),
 208            ('...:', ''),
 209            ],
 210           [('In [2]: a="""','a="""'),
 211            ('   ...: 123"""','123"""'),
 212            ],
 213           [('a="""','a="""'),
 214            ('   ...: 123','123'),
 215            ('   ...: 456"""','456"""'),
 216            ],
 217           [('a="""','a="""'),
 218            ('In [1]: 123','123'),
 219            ('   ...: 456"""','456"""'),
 220            ],
 221           [('a="""','a="""'),
 222            ('123','123'),
 223            ('   ...: 456"""','   ...: 456"""'),
 224            ],
 225           ],
 226  
 227         multiline_datastructure_prompt =
 228         [ [('>>> a = [1,','a = [1,'),
 229            ('... 2]','2]'),
 230           ],
 231         ],
 232          
 233         multiline_datastructure =
 234         [ [('b = ("%s"', None),
 235            ('# comment', None),
 236            ('%foo )', 'b = ("%s"\n# comment\n%foo )'),
 237           ],
 238         ],
 239         
 240         multiline_string =
 241         [ [("'''foo?", None),
 242            ("bar'''", "'''foo?\nbar'''"),
 243           ],
 244         ],
 245         
 246         leading_indent =
 247         [ [('    print "hi"','print "hi"'),
 248            ],
 249           [('  for a in range(5):','for a in range(5):'),
 250            ('    a*2','  a*2'),
 251            ],
 252           [('    a="""','a="""'),
 253            ('    123"""','123"""'),
 254             ],
 255           [('a="""','a="""'),
 256            ('    123"""','    123"""'),
 257            ],
 258         ],
 259         
 260         cellmagic =
 261         [ [('%%foo a', None),
 262            (None, "get_ipython().run_cell_magic('foo', 'a', '')"),
 263            ],
 264           [('%%bar 123', None),
 265            ('hello', None),
 266            (None , "get_ipython().run_cell_magic('bar', '123', 'hello')"),
 267            ],
 268           [('a=5', 'a=5'),
 269            ('%%cellmagic', '%%cellmagic'),
 270            ],
 271         ],
 272         
 273         escaped =
 274         [ [('%abc def \\', None),
 275            ('ghi', "get_ipython().run_line_magic('abc', 'def ghi')"),
 276            ],
 277           [('%abc def \\', None),
 278            ('ghi\\', None),
 279            (None, "get_ipython().run_line_magic('abc', 'def ghi')"),
 280            ],
 281         ],
 282         
 283         assign_magic =
 284         [ [('a = %bc de \\', None),
 285            ('fg', "a = get_ipython().run_line_magic('bc', 'de fg')"),
 286            ],
 287           [('a = %bc de \\', None),
 288            ('fg\\', None),
 289            (None, "a = get_ipython().run_line_magic('bc', 'de fg')"),
 290            ],
 291         ],
 292         
 293         assign_system =
 294         [ [('a = !bc de \\', None),
 295            ('fg', "a = get_ipython().getoutput('bc de fg')"),
 296            ],
 297           [('a = !bc de \\', None),
 298            ('fg\\', None),
 299            (None, "a = get_ipython().getoutput('bc de fg')"),
 300            ],
 301         ],
 302         )
 303  
 304  
 305  def test_assign_system():
 306      tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system'])
 307  
 308  def test_assign_magic():
 309      tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic'])
 310  
 311  def test_classic_prompt():
 312      tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
 313      for example in syntax_ml['classic_prompt']:
 314          transform_checker(example, ipt.classic_prompt)
 315      for example in syntax_ml['multiline_datastructure_prompt']:
 316          transform_checker(example, ipt.classic_prompt)
 317  
 318      # Check that we don't transform the second line if the first is obviously
 319      # IPython syntax
 320      transform_checker([
 321          ('%foo', '%foo'),
 322          ('>>> bar', '>>> bar'),
 323      ], ipt.classic_prompt)
 324  
 325  
 326  def test_ipy_prompt():
 327      tt.check_pairs(transform_and_reset(ipt.ipy_prompt), syntax['ipy_prompt'])
 328      for example in syntax_ml['ipy_prompt']:
 329          transform_checker(example, ipt.ipy_prompt)
 330  
 331      # Check that we don't transform the second line if we're inside a cell magic
 332      transform_checker([
 333          ('%%foo', '%%foo'),
 334          ('In [1]: bar', 'In [1]: bar'),
 335      ], ipt.ipy_prompt)
 336  
 337  def test_assemble_logical_lines():
 338      tests = \
 339      [ [("a = \\", None),
 340         ("123", "a = 123"),
 341        ],
 342        [("a = \\", None),  # Test resetting when within a multi-line string
 343         ("12 *\\", None),
 344         (None, "a = 12 *"),
 345        ],
 346        [("# foo\\", "# foo\\"), # Comments can't be continued like this
 347        ],
 348      ]
 349      for example in tests:
 350          transform_checker(example, ipt.assemble_logical_lines)
 351  
 352  def test_assemble_python_lines():
 353      tests = \
 354      [ [("a = '''", None),
 355         ("abc'''", "a = '''\nabc'''"),
 356        ],
 357        [("a = '''", None),  # Test resetting when within a multi-line string
 358         ("def", None),
 359         (None, "a = '''\ndef"),
 360        ],
 361        [("a = [1,", None),
 362         ("2]", "a = [1,\n2]"),
 363        ],
 364        [("a = [1,", None),  # Test resetting when within a multi-line string
 365         ("2,", None),
 366         (None, "a = [1,\n2,"),
 367        ],
 368        [("a = '''", None),  # Test line continuation within a multi-line string
 369         ("abc\\", None),
 370         ("def", None),
 371         ("'''", "a = '''\nabc\\\ndef\n'''"),
 372        ],
 373      ] + syntax_ml['multiline_datastructure']
 374      for example in tests:
 375          transform_checker(example, ipt.assemble_python_lines)
 376  
 377  
 378  def test_help_end():
 379      tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help'])
 380  
 381  def test_escaped_noesc():
 382      tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc'])
 383  
 384  
 385  def test_escaped_shell():
 386      tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell'])
 387  
 388  
 389  def test_escaped_help():
 390      tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help'])
 391  
 392  
 393  def test_escaped_magic():
 394      tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic'])
 395  
 396  
 397  def test_escaped_quote():
 398      tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote'])
 399  
 400  
 401  def test_escaped_quote2():
 402      tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2'])
 403  
 404  
 405  def test_escaped_paren():
 406      tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren'])
 407  
 408  
 409  def test_cellmagic():
 410      for example in syntax_ml['cellmagic']:
 411          transform_checker(example, ipt.cellmagic)
 412      
 413      line_example = [('%%bar 123', None),
 414                      ('hello', None),
 415                      ('' , "get_ipython().run_cell_magic('bar', '123', 'hello')"),
 416                     ]
 417      transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True)
 418  
 419  def test_has_comment():
 420      tests = [('text', False),
 421               ('text #comment', True),
 422               ('text #comment\n', True),
 423               ('#comment', True),
 424               ('#comment\n', True),
 425               ('a = "#string"', False),
 426               ('a = "#string" # comment', True),
 427               ('a #comment not "string"', True),
 428               ]
 429      tt.check_pairs(ipt.has_comment, tests)
 430  
 431  @ipt.TokenInputTransformer.wrap
 432  def decistmt(tokens):
 433      """Substitute Decimals for floats in a string of statements.
 434  
 435      Based on an example from the tokenize module docs.
 436      """
 437      result = []
 438      for toknum, tokval, _, _, _  in tokens:
 439          if toknum == tokenize.NUMBER and '.' in tokval:  # replace NUMBER tokens
 440              yield from [
 441                  (tokenize.NAME, 'Decimal'),
 442                  (tokenize.OP, '('),
 443                  (tokenize.STRING, repr(tokval)),
 444                  (tokenize.OP, ')')
 445              ]
 446          else:
 447              yield (toknum, tokval)
 448  
 449  
 450  
 451  def test_token_input_transformer():
 452      tests = [('1.2', "Decimal ('1.2')"),
 453               ('"1.2"', '"1.2"'),
 454               ]
 455      tt.check_pairs(transform_and_reset(decistmt), tests)
 456      ml_tests = \
 457      [ [("a = 1.2; b = '''x", None),
 458         ("y'''", "a =Decimal ('1.2');b ='''x\ny'''"),
 459        ],
 460        [("a = [1.2,", None),
 461         ("3]", "a =[Decimal ('1.2'),\n3 ]"),
 462        ],
 463        [("a = '''foo", None),  # Test resetting when within a multi-line string
 464         ("bar", None),
 465         (None, "a = '''foo\nbar"),
 466        ],
 467      ]
 468      for example in ml_tests:
 469          transform_checker(example, decistmt)
 470