1  """Tests for the token-based transformers in IPython.core.inputtransformer2
   2  
   3  Line-based transformers are the simpler ones; token-based transformers are
   4  more complex. See test_inputtransformer2_line for tests for line-based
   5  transformations.
   6  """
   7  
   8  import platform
   9  import string
  10  import sys
  11  from textwrap import dedent
  12  
  13  import pytest
  14  
  15  from IPython.core import inputtransformer2 as ipt2
  16  from IPython.core.inputtransformer2 import _find_assign_op, make_tokens_by_line
  17  
  18  MULTILINE_MAGIC = (
  19      """\
  20  a = f()
  21  %foo \\
  22  bar
  23  g()
  24  """.splitlines(
  25          keepends=True
  26      ),
  27      (2, 0),
  28      """\
  29  a = f()
  30  get_ipython().run_line_magic('foo', ' bar')
  31  g()
  32  """.splitlines(
  33          keepends=True
  34      ),
  35  )
  36  
  37  INDENTED_MAGIC = (
  38      """\
  39  for a in range(5):
  40      %ls
  41  """.splitlines(
  42          keepends=True
  43      ),
  44      (2, 4),
  45      """\
  46  for a in range(5):
  47      get_ipython().run_line_magic('ls', '')
  48  """.splitlines(
  49          keepends=True
  50      ),
  51  )
  52  
  53  CRLF_MAGIC = (
  54      ["a = f()\n", "%ls\r\n", "g()\n"],
  55      (2, 0),
  56      ["a = f()\n", "get_ipython().run_line_magic('ls', '')\n", "g()\n"],
  57  )
  58  
  59  MULTILINE_MAGIC_ASSIGN = (
  60      """\
  61  a = f()
  62  b = %foo \\
  63    bar
  64  g()
  65  """.splitlines(
  66          keepends=True
  67      ),
  68      (2, 4),
  69      """\
  70  a = f()
  71  b = get_ipython().run_line_magic('foo', '   bar')
  72  g()
  73  """.splitlines(
  74          keepends=True
  75      ),
  76  )
  77  
  78  MULTILINE_SYSTEM_ASSIGN = ("""\
  79  a = f()
  80  b = !foo \\
  81    bar
  82  g()
  83  """.splitlines(keepends=True), (2, 4), """\
  84  a = f()
  85  b = get_ipython().getoutput('foo    bar')
  86  g()
  87  """.splitlines(keepends=True))
  88  
  89  #####
  90  
  91  MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = (
  92      """\
  93  def test():
  94    for i in range(1):
  95      print(i)
  96    res =! ls
  97  """.splitlines(
  98          keepends=True
  99      ),
 100      (4, 7),
 101      """\
 102  def test():
 103    for i in range(1):
 104      print(i)
 105    res =get_ipython().getoutput(\' ls\')
 106  """.splitlines(
 107          keepends=True
 108      ),
 109  )
 110  
 111  ######
 112  
 113  AUTOCALL_QUOTE = ([",f 1 2 3\n"], (1, 0), ['f("1", "2", "3")\n'])
 114  
 115  AUTOCALL_QUOTE2 = ([";f 1 2 3\n"], (1, 0), ['f("1 2 3")\n'])
 116  
 117  AUTOCALL_PAREN = (["/f 1 2 3\n"], (1, 0), ["f(1, 2, 3)\n"])
 118  
 119  SIMPLE_HELP = (["foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', 'foo')\n"])
 120  
 121  DETAILED_HELP = (
 122      ["foo??\n"],
 123      (1, 0),
 124      ["get_ipython().run_line_magic('pinfo2', 'foo')\n"],
 125  )
 126  
 127  MAGIC_HELP = (["%foo?\n"], (1, 0), ["get_ipython().run_line_magic('pinfo', '%foo')\n"])
 128  
 129  HELP_IN_EXPR = (
 130      ["a = b + c?\n"],
 131      (1, 0),
 132      ["get_ipython().run_line_magic('pinfo', 'c')\n"],
 133  )
 134  
 135  HELP_CONTINUED_LINE = (
 136      """\
 137  a = \\
 138  zip?
 139  """.splitlines(
 140          keepends=True
 141      ),
 142      (1, 0),
 143      [r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
 144  )
 145  
 146  HELP_MULTILINE = (
 147      """\
 148  (a,
 149  b) = zip?
 150  """.splitlines(
 151          keepends=True
 152      ),
 153      (1, 0),
 154      [r"get_ipython().run_line_magic('pinfo', 'zip')" + "\n"],
 155  )
 156  
 157  HELP_UNICODE = (
 158      ["π.foo?\n"],
 159      (1, 0),
 160      ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"],
 161  )
 162  
 163  
 164  def null_cleanup_transformer(lines):
 165      """
 166      A cleanup transform that returns an empty list.
 167      """
 168      return []
 169  
 170  
 171  def test_check_make_token_by_line_never_ends_empty():
 172      """
 173      Check that not sequence of single or double characters ends up leading to en empty list of tokens
 174      """
 175      from string import printable
 176  
 177      for c in printable:
 178          assert make_tokens_by_line(c)[-1] != []
 179          for k in printable:
 180              assert make_tokens_by_line(c + k)[-1] != []
 181  
 182  
 183  def check_find(transformer, case, match=True):
 184      sample, expected_start, _ = case
 185      tbl = make_tokens_by_line(sample)
 186      res = transformer.find(tbl)
 187      if match:
 188          # start_line is stored 0-indexed, expected values are 1-indexed
 189          assert (res.start_line + 1, res.start_col) == expected_start
 190          return res
 191      else:
 192          assert res is None
 193  
 194  
 195  def check_transform(transformer_cls, case):
 196      lines, start, expected = case
 197      transformer = transformer_cls(start)
 198      assert transformer.transform(lines) == expected
 199  
 200  
 201  def test_continued_line():
 202      lines = MULTILINE_MAGIC_ASSIGN[0]
 203      assert ipt2.find_end_of_continued_line(lines, 1) == 2
 204  
 205      assert ipt2.assemble_continued_line(lines, (1, 5), 2) == "foo    bar"
 206  
 207  
 208  def test_find_assign_magic():
 209      check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
 210      check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
 211      check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)
 212  
 213  
 214  def test_transform_assign_magic():
 215      check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
 216  
 217  
 218  def test_find_assign_system():
 219      check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
 220      check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
 221      check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
 222      check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
 223      check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
 224  
 225  
 226  def test_transform_assign_system():
 227      check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
 228      check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
 229  
 230  
 231  def test_find_magic_escape():
 232      check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
 233      check_find(ipt2.EscapedCommand, INDENTED_MAGIC)
 234      check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False)
 235  
 236  
 237  def test_transform_magic_escape():
 238      check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC)
 239      check_transform(ipt2.EscapedCommand, INDENTED_MAGIC)
 240      check_transform(ipt2.EscapedCommand, CRLF_MAGIC)
 241  
 242  
 243  def test_find_autocalls():
 244      for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
 245          print("Testing %r" % case[0])
 246          check_find(ipt2.EscapedCommand, case)
 247  
 248  
 249  def test_transform_autocall():
 250      for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]:
 251          print("Testing %r" % case[0])
 252          check_transform(ipt2.EscapedCommand, case)
 253  
 254  
 255  def test_find_help():
 256      for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]:
 257          check_find(ipt2.HelpEnd, case)
 258  
 259      tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE)
 260      assert tf.q_line == 1
 261      assert tf.q_col == 3
 262  
 263      tf = check_find(ipt2.HelpEnd, HELP_MULTILINE)
 264      assert tf.q_line == 1
 265      assert tf.q_col == 8
 266  
 267      # ? in a comment does not trigger help
 268      check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False)
 269      # Nor in a string
 270      check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False)
 271  
 272  
 273  def test_transform_help():
 274      tf = ipt2.HelpEnd((1, 0), (1, 9))
 275      assert tf.transform(HELP_IN_EXPR[0]) == HELP_IN_EXPR[2]
 276  
 277      tf = ipt2.HelpEnd((1, 0), (2, 3))
 278      assert tf.transform(HELP_CONTINUED_LINE[0]) == HELP_CONTINUED_LINE[2]
 279  
 280      tf = ipt2.HelpEnd((1, 0), (2, 8))
 281      assert tf.transform(HELP_MULTILINE[0]) == HELP_MULTILINE[2]
 282  
 283      tf = ipt2.HelpEnd((1, 0), (1, 0))
 284      assert tf.transform(HELP_UNICODE[0]) == HELP_UNICODE[2]
 285  
 286  
 287  def test_find_assign_op_dedent():
 288      """
 289      be careful that empty token like dedent are not counted as parens
 290      """
 291  
 292      class Tk:
 293          def __init__(self, s):
 294              self.string = s
 295  
 296      assert _find_assign_op([Tk(s) for s in ("", "a", "=", "b")]) == 2
 297      assert (
 298          _find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6
 299      )
 300  
 301  
 302  extra_closing_paren_param = (
 303      pytest.param("(\n))", "invalid", None)
 304      if sys.version_info >= (3, 12)
 305      else pytest.param("(\n))", "incomplete", 0)
 306  )
 307  examples = [
 308      pytest.param("a = 1", "complete", None),
 309      pytest.param("for a in range(5):", "incomplete", 4),
 310      pytest.param("for a in range(5):\n    if a > 0:", "incomplete", 8),
 311      pytest.param("raise = 2", "invalid", None),
 312      pytest.param("a = [1,\n2,", "incomplete", 0),
 313      extra_closing_paren_param,
 314      pytest.param("\\\r\n", "incomplete", 0),
 315      pytest.param("a = '''\n   hi", "incomplete", 3),
 316      pytest.param("def a():\n x=1\n global x", "invalid", None),
 317      pytest.param(
 318          "a \\ ",
 319          "invalid",
 320          None,
 321          marks=pytest.mark.xfail(
 322              reason="Bug in python 3.9.8 – bpo 45738",
 323              condition=sys.version_info in [(3, 11, 0, "alpha", 2)],
 324              raises=SystemError,
 325              strict=True,
 326          ),
 327      ),  # Nothing allowed after backslash,
 328      pytest.param("1\\\n+2", "complete", None),
 329  ]
 330  
 331  
 332  @pytest.mark.parametrize("code, expected, number", examples)
 333  def test_check_complete_param(code, expected, number):
 334      cc = ipt2.TransformerManager().check_complete
 335      assert cc(code) == (expected, number)
 336  
 337  
 338  @pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
 339  @pytest.mark.xfail(
 340      reason="Bug in python 3.9.8 – bpo 45738",
 341      condition=sys.version_info in [(3, 11, 0, "alpha", 2)],
 342      raises=SystemError,
 343      strict=True,
 344  )
 345  def test_check_complete():
 346      cc = ipt2.TransformerManager().check_complete
 347  
 348      example = dedent(
 349          """
 350          if True:
 351              a=1"""
 352      )
 353  
 354      assert cc(example) == ("incomplete", 4)
 355      assert cc(example + "\n") == ("complete", None)
 356      assert cc(example + "\n    ") == ("complete", None)
 357  
 358      # no need to loop on all the letters/numbers.
 359      short = "12abAB" + string.printable[62:]
 360      for c in short:
 361          # test does not raise:
 362          cc(c)
 363          for k in short:
 364              cc(c + k)
 365  
 366      assert cc("def f():\n  x=0\n  \\\n  ") == ("incomplete", 2)
 367  
 368  
 369  @pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
 370  @pytest.mark.parametrize(
 371      "value, expected",
 372      [
 373          ('''def foo():\n    """''', ("incomplete", 4)),
 374          ("""async with example:\n    pass""", ("incomplete", 4)),
 375          ("""async with example:\n    pass\n    """, ("complete", None)),
 376      ],
 377  )
 378  def test_check_complete_II(value, expected):
 379      """
 380      Test that multiple line strings are properly handled.
 381  
 382      Separate test function for convenience
 383  
 384      """
 385      cc = ipt2.TransformerManager().check_complete
 386      assert cc(value) == expected
 387  
 388  
 389  @pytest.mark.parametrize(
 390      "value, expected",
 391      [
 392          (")", ("invalid", None)),
 393          ("]", ("invalid", None)),
 394          ("}", ("invalid", None)),
 395          (")(", ("invalid", None)),
 396          ("][", ("invalid", None)),
 397          ("}{", ("invalid", None)),
 398          ("]()(", ("invalid", None)),
 399          ("())(", ("invalid", None)),
 400          (")[](", ("invalid", None)),
 401          ("()](", ("invalid", None)),
 402      ],
 403  )
 404  def test_check_complete_invalidates_sunken_brackets(value, expected):
 405      """
 406      Test that a single line with more closing brackets than the opening ones is
 407      interpreted as invalid
 408      """
 409      cc = ipt2.TransformerManager().check_complete
 410      assert cc(value) == expected
 411  
 412  
 413  def test_null_cleanup_transformer():
 414      manager = ipt2.TransformerManager()
 415      manager.cleanup_transforms.insert(0, null_cleanup_transformer)
 416      assert manager.transform_cell("") == ""
 417  
 418  
 419  def test_side_effects_I():
 420      count = 0
 421  
 422      def counter(lines):
 423          nonlocal count
 424          count += 1
 425          return lines
 426  
 427      counter.has_side_effects = True
 428  
 429      manager = ipt2.TransformerManager()
 430      manager.cleanup_transforms.insert(0, counter)
 431      assert manager.check_complete("a=1\n") == ("complete", None)
 432      assert count == 0
 433  
 434  
 435  def test_side_effects_II():
 436      count = 0
 437  
 438      def counter(lines):
 439          nonlocal count
 440          count += 1
 441          return lines
 442  
 443      counter.has_side_effects = True
 444  
 445      manager = ipt2.TransformerManager()
 446      manager.line_transforms.insert(0, counter)
 447      assert manager.check_complete("b=1\n") == ("complete", None)
 448      assert count == 0
 449