1  """Token-related utilities"""
   2  
   3  # Copyright (c) IPython Development Team.
   4  # Distributed under the terms of the Modified BSD License.
   5  
   6  from collections import namedtuple
   7  from io import StringIO
   8  from keyword import iskeyword
   9  
  10  import tokenize
  11  from tokenize import TokenInfo
  12  from typing import List, Optional
  13  
  14  
  15  Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
  16  
  17  def generate_tokens(readline):
  18      """wrap generate_tkens to catch EOF errors"""
  19      try:
  20          for token in tokenize.generate_tokens(readline):
  21              yield token
  22      except tokenize.TokenError:
  23          # catch EOF error
  24          return
  25  
  26  
  27  def generate_tokens_catch_errors(
  28      readline, extra_errors_to_catch: Optional[List[str]] = None
  29  ):
  30      default_errors_to_catch = [
  31          "unterminated string literal",
  32          "invalid non-printable character",
  33          "after line continuation character",
  34      ]
  35      assert extra_errors_to_catch is None or isinstance(extra_errors_to_catch, list)
  36      errors_to_catch = default_errors_to_catch + (extra_errors_to_catch or [])
  37  
  38      tokens: List[TokenInfo] = []
  39      try:
  40          for token in tokenize.generate_tokens(readline):
  41              tokens.append(token)
  42              yield token
  43      except tokenize.TokenError as exc:
  44          if any(error in exc.args[0] for error in errors_to_catch):
  45              if tokens:
  46                  start = tokens[-1].start[0], tokens[-1].end[0]
  47                  end = start
  48                  line = tokens[-1].line
  49              else:
  50                  start = end = (1, 0)
  51                  line = ""
  52              yield tokenize.TokenInfo(tokenize.ERRORTOKEN, "", start, end, line)
  53          else:
  54              # Catch EOF
  55              raise
  56  
  57  
  58  def line_at_cursor(cell, cursor_pos=0):
  59      """Return the line in a cell at a given cursor position
  60  
  61      Used for calling line-based APIs that don't support multi-line input, yet.
  62  
  63      Parameters
  64      ----------
  65      cell : str
  66          multiline block of text
  67      cursor_pos : integer
  68          the cursor position
  69  
  70      Returns
  71      -------
  72      (line, offset): (string, integer)
  73          The line with the current cursor, and the character offset of the start of the line.
  74      """
  75      offset = 0
  76      lines = cell.splitlines(True)
  77      for line in lines:
  78          next_offset = offset + len(line)
  79          if not line.endswith('\n'):
  80              # If the last line doesn't have a trailing newline, treat it as if
  81              # it does so that the cursor at the end of the line still counts
  82              # as being on that line.
  83              next_offset += 1
  84          if next_offset > cursor_pos:
  85              break
  86          offset = next_offset
  87      else:
  88          line = ""
  89      return (line, offset)
  90  
  91  
  92  def token_at_cursor(cell: str, cursor_pos: int = 0):
  93      """Get the token at a given cursor
  94  
  95      Used for introspection.
  96  
  97      Function calls are prioritized, so the token for the callable will be returned
  98      if the cursor is anywhere inside the call.
  99  
 100      Parameters
 101      ----------
 102      cell : str
 103          A block of Python code
 104      cursor_pos : int
 105          The location of the cursor in the block where the token should be found
 106      """
 107      names: List[str] = []
 108      tokens: List[Token] = []
 109      call_names = []
 110      
 111      offsets = {1: 0} # lines start at 1
 112      for tup in generate_tokens(StringIO(cell).readline):
 113          
 114          tok = Token(*tup)
 115          
 116          # token, text, start, end, line = tup
 117          start_line, start_col = tok.start
 118          end_line, end_col = tok.end
 119          if end_line + 1 not in offsets:
 120              # keep track of offsets for each line
 121              lines = tok.line.splitlines(True)
 122              for lineno, line in enumerate(lines, start_line + 1):
 123                  if lineno not in offsets:
 124                      offsets[lineno] = offsets[lineno-1] + len(line)
 125          
 126          offset = offsets[start_line]
 127          # allow '|foo' to find 'foo' at the beginning of a line
 128          boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
 129          if offset + start_col >= boundary:
 130              # current token starts after the cursor,
 131              # don't consume it
 132              break
 133          
 134          if tok.token == tokenize.NAME and not iskeyword(tok.text):
 135              if names and tokens and tokens[-1].token == tokenize.OP and tokens[-1].text == '.':
 136                  names[-1] = "%s.%s" % (names[-1], tok.text)
 137              else:
 138                  names.append(tok.text)
 139          elif tok.token == tokenize.OP:
 140              if tok.text == '=' and names:
 141                  # don't inspect the lhs of an assignment
 142                  names.pop(-1)
 143              if tok.text == '(' and names:
 144                  # if we are inside a function call, inspect the function
 145                  call_names.append(names[-1])
 146              elif tok.text == ')' and call_names:
 147                  call_names.pop(-1)
 148          
 149          tokens.append(tok)
 150          
 151          if offsets[end_line] + end_col > cursor_pos:
 152              # we found the cursor, stop reading
 153              break
 154          
 155      if call_names:
 156          return call_names[-1]
 157      elif names:
 158          return names[-1]
 159      else:
 160          return ''
 161