1  """Tests for tokenutil"""
   2  # Copyright (c) IPython Development Team.
   3  # Distributed under the terms of the Modified BSD License.
   4  
   5  import pytest
   6  
   7  from IPython.utils.tokenutil import token_at_cursor, line_at_cursor
   8  
   9  def expect_token(expected, cell, cursor_pos):
  10      token = token_at_cursor(cell, cursor_pos)
  11      offset = 0
  12      for line in cell.splitlines():
  13          if offset + len(line) >= cursor_pos:
  14              break
  15          else:
  16              offset += len(line)+1
  17      column = cursor_pos - offset
  18      line_with_cursor = "%s|%s" % (line[:column], line[column:])
  19      assert token == expected, "Expected %r, got %r in: %r (pos %i)" % (
  20          expected,
  21          token,
  22          line_with_cursor,
  23          cursor_pos,
  24      )
  25  
  26  
  27  def test_simple():
  28      cell = "foo"
  29      for i in range(len(cell)):
  30          expect_token("foo", cell, i)
  31  
  32  def test_function():
  33      cell = "foo(a=5, b='10')"
  34      expected = 'foo'
  35      # up to `foo(|a=`
  36      for i in range(cell.find('a=') + 1):
  37          expect_token("foo", cell, i)
  38      # find foo after `=`
  39      for i in [cell.find('=') + 1, cell.rfind('=') + 1]:
  40          expect_token("foo", cell, i)
  41      # in between `5,|` and `|b=`
  42      for i in range(cell.find(','), cell.find('b=')):
  43          expect_token("foo", cell, i)
  44  
  45  def test_multiline():
  46      cell = '\n'.join([
  47          'a = 5',
  48          'b = hello("string", there)'
  49      ])
  50      expected = 'hello'
  51      start = cell.index(expected) + 1
  52      for i in range(start, start + len(expected)):
  53          expect_token(expected, cell, i)
  54      expected = 'hello'
  55      start = cell.index(expected) + 1
  56      for i in range(start, start + len(expected)):
  57          expect_token(expected, cell, i)
  58  
  59  def test_multiline_token():
  60      cell = '\n'.join([
  61          '"""\n\nxxxxxxxxxx\n\n"""',
  62          '5, """',
  63          'docstring',
  64          'multiline token',
  65          '""", [',
  66          '2, 3, "complicated"]',
  67          'b = hello("string", there)'
  68      ])
  69      expected = 'hello'
  70      start = cell.index(expected) + 1
  71      for i in range(start, start + len(expected)):
  72          expect_token(expected, cell, i)
  73      expected = 'hello'
  74      start = cell.index(expected) + 1
  75      for i in range(start, start + len(expected)):
  76          expect_token(expected, cell, i)
  77  
  78  def test_nested_call():
  79      cell = "foo(bar(a=5), b=10)"
  80      expected = 'foo'
  81      start = cell.index('bar') + 1
  82      for i in range(start, start + 3):
  83          expect_token(expected, cell, i)
  84      expected = 'bar'
  85      start = cell.index('a=')
  86      for i in range(start, start + 3):
  87          expect_token(expected, cell, i)
  88      expected = 'foo'
  89      start = cell.index(')') + 1
  90      for i in range(start, len(cell)-1):
  91          expect_token(expected, cell, i)
  92  
  93  def test_attrs():
  94      cell = "a = obj.attr.subattr"
  95      expected = 'obj'
  96      idx = cell.find('obj') + 1
  97      for i in range(idx, idx + 3):
  98          expect_token(expected, cell, i)
  99      idx = cell.find('.attr') + 2
 100      expected = 'obj.attr'
 101      for i in range(idx, idx + 4):
 102          expect_token(expected, cell, i)
 103      idx = cell.find('.subattr') + 2
 104      expected = 'obj.attr.subattr'
 105      for i in range(idx, len(cell)):
 106          expect_token(expected, cell, i)
 107  
 108  def test_line_at_cursor():
 109      cell = ""
 110      (line, offset) = line_at_cursor(cell, cursor_pos=11)
 111      assert line == ""
 112      assert offset == 0
 113  
 114      # The position after a newline should be the start of the following line.
 115      cell = "One\nTwo\n"
 116      (line, offset) = line_at_cursor(cell, cursor_pos=4)
 117      assert line == "Two\n"
 118      assert offset == 4
 119  
 120      # The end of a cell should be on the last line
 121      cell = "pri\npri"
 122      (line, offset) = line_at_cursor(cell, cursor_pos=7)
 123      assert line == "pri"
 124      assert offset == 4
 125  
 126  
 127  @pytest.mark.parametrize(
 128      "c, token",
 129      zip(
 130          list(range(16, 22)) + list(range(22, 28)),
 131          ["int"] * (22 - 16) + ["map"] * (28 - 22),
 132      ),
 133  )
 134  def test_multiline_statement(c, token):
 135      cell = """a = (1,
 136      3)
 137  
 138  int()
 139  map()
 140  """
 141      expect_token(token, cell, c)
 142