1  """
   2  Helpers for the API
   3  """
   4  import re
   5  from collections import namedtuple
   6  from textwrap import dedent
   7  from itertools import chain
   8  from functools import wraps
   9  from inspect import Parameter
  10  
  11  from parso.python.parser import Parser
  12  from parso.python import tree
  13  
  14  from jedi.inference.base_value import NO_VALUES
  15  from jedi.inference.syntax_tree import infer_atom
  16  from jedi.inference.helpers import infer_call_of_leaf
  17  from jedi.inference.compiled import get_string_value_set
  18  from jedi.cache import signature_time_cache, memoize_method
  19  from jedi.parser_utils import get_parent_scope
  20  
  21  
  22  CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
  23  
  24  
  25  def _start_match(string, like_name):
  26      return string.startswith(like_name)
  27  
  28  
  29  def _fuzzy_match(string, like_name):
  30      if len(like_name) <= 1:
  31          return like_name in string
  32      pos = string.find(like_name[0])
  33      if pos >= 0:
  34          return _fuzzy_match(string[pos + 1:], like_name[1:])
  35      return False
  36  
  37  
  38  def match(string, like_name, fuzzy=False):
  39      if fuzzy:
  40          return _fuzzy_match(string, like_name)
  41      else:
  42          return _start_match(string, like_name)
  43  
  44  
  45  def sorted_definitions(defs):
  46      # Note: `or ''` below is required because `module_path` could be
  47      return sorted(defs, key=lambda x: (str(x.module_path or ''),
  48                                         x.line or 0,
  49                                         x.column or 0,
  50                                         x.name))
  51  
  52  
  53  def get_on_completion_name(module_node, lines, position):
  54      leaf = module_node.get_leaf_for_position(position)
  55      if leaf is None or leaf.type in ('string', 'error_leaf'):
  56          # Completions inside strings are a bit special, we need to parse the
  57          # string. The same is true for comments and error_leafs.
  58          line = lines[position[0] - 1]
  59          # The first step of completions is to get the name
  60          return re.search(r'(?!\d)\w+$|$', line[:position[1]]).group(0)
  61      elif leaf.type not in ('name', 'keyword'):
  62          return ''
  63  
  64      return leaf.value[:position[1] - leaf.start_pos[1]]
  65  
  66  
  67  def _get_code(code_lines, start_pos, end_pos):
  68      # Get relevant lines.
  69      lines = code_lines[start_pos[0] - 1:end_pos[0]]
  70      # Remove the parts at the end of the line.
  71      lines[-1] = lines[-1][:end_pos[1]]
  72      # Remove first line indentation.
  73      lines[0] = lines[0][start_pos[1]:]
  74      return ''.join(lines)
  75  
  76  
  77  class OnErrorLeaf(Exception):
  78      @property
  79      def error_leaf(self):
  80          return self.args[0]
  81  
  82  
  83  def _get_code_for_stack(code_lines, leaf, position):
  84      # It might happen that we're on whitespace or on a comment. This means
  85      # that we would not get the right leaf.
  86      if leaf.start_pos >= position:
  87          # If we're not on a comment simply get the previous leaf and proceed.
  88          leaf = leaf.get_previous_leaf()
  89          if leaf is None:
  90              return ''  # At the beginning of the file.
  91  
  92      is_after_newline = leaf.type == 'newline'
  93      while leaf.type == 'newline':
  94          leaf = leaf.get_previous_leaf()
  95          if leaf is None:
  96              return ''
  97  
  98      if leaf.type == 'error_leaf' or leaf.type == 'string':
  99          if leaf.start_pos[0] < position[0]:
 100              # On a different line, we just begin anew.
 101              return ''
 102  
 103          # Error leafs cannot be parsed, completion in strings is also
 104          # impossible.
 105          raise OnErrorLeaf(leaf)
 106      else:
 107          user_stmt = leaf
 108          while True:
 109              if user_stmt.parent.type in ('file_input', 'suite', 'simple_stmt'):
 110                  break
 111              user_stmt = user_stmt.parent
 112  
 113          if is_after_newline:
 114              if user_stmt.start_pos[1] > position[1]:
 115                  # This means that it's actually a dedent and that means that we
 116                  # start without value (part of a suite).
 117                  return ''
 118  
 119          # This is basically getting the relevant lines.
 120          return _get_code(code_lines, user_stmt.get_start_pos_of_prefix(), position)
 121  
 122  
 123  def get_stack_at_position(grammar, code_lines, leaf, pos):
 124      """
 125      Returns the possible node names (e.g. import_from, xor_test or yield_stmt).
 126      """
 127      class EndMarkerReached(Exception):
 128          pass
 129  
 130      def tokenize_without_endmarker(code):
 131          # TODO This is for now not an official parso API that exists purely
 132          #   for Jedi.
 133          tokens = grammar._tokenize(code)
 134          for token in tokens:
 135              if token.string == safeword:
 136                  raise EndMarkerReached()
 137              elif token.prefix.endswith(safeword):
 138                  # This happens with comments.
 139                  raise EndMarkerReached()
 140              elif token.string.endswith(safeword):
 141                  yield token  # Probably an f-string literal that was not finished.
 142                  raise EndMarkerReached()
 143              else:
 144                  yield token
 145  
 146      # The code might be indedented, just remove it.
 147      code = dedent(_get_code_for_stack(code_lines, leaf, pos))
 148      # We use a word to tell Jedi when we have reached the start of the
 149      # completion.
 150      # Use Z as a prefix because it's not part of a number suffix.
 151      safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI'
 152      code = code + ' ' + safeword
 153  
 154      p = Parser(grammar._pgen_grammar, error_recovery=True)
 155      try:
 156          p.parse(tokens=tokenize_without_endmarker(code))
 157      except EndMarkerReached:
 158          return p.stack
 159      raise SystemError(
 160          "This really shouldn't happen. There's a bug in Jedi:\n%s"
 161          % list(tokenize_without_endmarker(code))
 162      )
 163  
 164  
 165  def infer(inference_state, context, leaf):
 166      if leaf.type == 'name':
 167          return inference_state.infer(context, leaf)
 168  
 169      parent = leaf.parent
 170      definitions = NO_VALUES
 171      if parent.type == 'atom':
 172          # e.g. `(a + b)`
 173          definitions = context.infer_node(leaf.parent)
 174      elif parent.type == 'trailer':
 175          # e.g. `a()`
 176          definitions = infer_call_of_leaf(context, leaf)
 177      elif isinstance(leaf, tree.Literal):
 178          # e.g. `"foo"` or `1.0`
 179          return infer_atom(context, leaf)
 180      elif leaf.type in ('fstring_string', 'fstring_start', 'fstring_end'):
 181          return get_string_value_set(inference_state)
 182      return definitions
 183  
 184  
 185  def filter_follow_imports(names, follow_builtin_imports=False):
 186      for name in names:
 187          if name.is_import():
 188              new_names = list(filter_follow_imports(
 189                  name.goto(),
 190                  follow_builtin_imports=follow_builtin_imports,
 191              ))
 192              found_builtin = False
 193              if follow_builtin_imports:
 194                  for new_name in new_names:
 195                      if new_name.start_pos is None:
 196                          found_builtin = True
 197  
 198              if found_builtin:
 199                  yield name
 200              else:
 201                  yield from new_names
 202          else:
 203              yield name
 204  
 205  
 206  class CallDetails:
 207      def __init__(self, bracket_leaf, children, position):
 208          self.bracket_leaf = bracket_leaf
 209          self._children = children
 210          self._position = position
 211  
 212      @property
 213      def index(self):
 214          return _get_index_and_key(self._children, self._position)[0]
 215  
 216      @property
 217      def keyword_name_str(self):
 218          return _get_index_and_key(self._children, self._position)[1]
 219  
 220      @memoize_method
 221      def _list_arguments(self):
 222          return list(_iter_arguments(self._children, self._position))
 223  
 224      def calculate_index(self, param_names):
 225          positional_count = 0
 226          used_names = set()
 227          star_count = -1
 228          args = self._list_arguments()
 229          if not args:
 230              if param_names:
 231                  return 0
 232              else:
 233                  return None
 234  
 235          is_kwarg = False
 236          for i, (star_count, key_start, had_equal) in enumerate(args):
 237              is_kwarg |= had_equal | (star_count == 2)
 238              if star_count:
 239                  pass  # For now do nothing, we don't know what's in there here.
 240              else:
 241                  if i + 1 != len(args):  # Not last
 242                      if had_equal:
 243                          used_names.add(key_start)
 244                      else:
 245                          positional_count += 1
 246  
 247          for i, param_name in enumerate(param_names):
 248              kind = param_name.get_kind()
 249  
 250              if not is_kwarg:
 251                  if kind == Parameter.VAR_POSITIONAL:
 252                      return i
 253                  if kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.POSITIONAL_ONLY):
 254                      if i == positional_count:
 255                          return i
 256  
 257              if key_start is not None and not star_count == 1 or star_count == 2:
 258                  if param_name.string_name not in used_names \
 259                          and (kind == Parameter.KEYWORD_ONLY
 260                               or kind == Parameter.POSITIONAL_OR_KEYWORD
 261                               and positional_count <= i):
 262                      if star_count:
 263                          return i
 264                      if had_equal:
 265                          if param_name.string_name == key_start:
 266                              return i
 267                      else:
 268                          if param_name.string_name.startswith(key_start):
 269                              return i
 270  
 271                  if kind == Parameter.VAR_KEYWORD:
 272                      return i
 273          return None
 274  
 275      def iter_used_keyword_arguments(self):
 276          for star_count, key_start, had_equal in list(self._list_arguments()):
 277              if had_equal and key_start:
 278                  yield key_start
 279  
 280      def count_positional_arguments(self):
 281          count = 0
 282          for star_count, key_start, had_equal in self._list_arguments()[:-1]:
 283              if star_count or key_start:
 284                  break
 285              count += 1
 286          return count
 287  
 288  
 289  def _iter_arguments(nodes, position):
 290      def remove_after_pos(name):
 291          if name.type != 'name':
 292              return None
 293          return name.value[:position[1] - name.start_pos[1]]
 294  
 295      # Returns Generator[Tuple[star_count, Optional[key_start: str], had_equal]]
 296      nodes_before = [c for c in nodes if c.start_pos < position]
 297      if nodes_before[-1].type == 'arglist':
 298          yield from _iter_arguments(nodes_before[-1].children, position)
 299          return
 300  
 301      previous_node_yielded = False
 302      stars_seen = 0
 303      for i, node in enumerate(nodes_before):
 304          if node.type == 'argument':
 305              previous_node_yielded = True
 306              first = node.children[0]
 307              second = node.children[1]
 308              if second == '=':
 309                  if second.start_pos < position and first.type == 'name':
 310                      yield 0, first.value, True
 311                  else:
 312                      yield 0, remove_after_pos(first), False
 313              elif first in ('*', '**'):
 314                  yield len(first.value), remove_after_pos(second), False
 315              else:
 316                  # Must be a Comprehension
 317                  first_leaf = node.get_first_leaf()
 318                  if first_leaf.type == 'name' and first_leaf.start_pos >= position:
 319                      yield 0, remove_after_pos(first_leaf), False
 320                  else:
 321                      yield 0, None, False
 322              stars_seen = 0
 323          elif node.type == 'testlist_star_expr':
 324              for n in node.children[::2]:
 325                  if n.type == 'star_expr':
 326                      stars_seen = 1
 327                      n = n.children[1]
 328                  yield stars_seen, remove_after_pos(n), False
 329                  stars_seen = 0
 330              # The count of children is even if there's a comma at the end.
 331              previous_node_yielded = bool(len(node.children) % 2)
 332          elif isinstance(node, tree.PythonLeaf) and node.value == ',':
 333              if not previous_node_yielded:
 334                  yield stars_seen, '', False
 335                  stars_seen = 0
 336              previous_node_yielded = False
 337          elif isinstance(node, tree.PythonLeaf) and node.value in ('*', '**'):
 338              stars_seen = len(node.value)
 339          elif node == '=' and nodes_before[-1]:
 340              previous_node_yielded = True
 341              before = nodes_before[i - 1]
 342              if before.type == 'name':
 343                  yield 0, before.value, True
 344              else:
 345                  yield 0, None, False
 346              # Just ignore the star that is probably a syntax error.
 347              stars_seen = 0
 348  
 349      if not previous_node_yielded:
 350          if nodes_before[-1].type == 'name':
 351              yield stars_seen, remove_after_pos(nodes_before[-1]), False
 352          else:
 353              yield stars_seen, '', False
 354  
 355  
 356  def _get_index_and_key(nodes, position):
 357      """
 358      Returns the amount of commas and the keyword argument string.
 359      """
 360      nodes_before = [c for c in nodes if c.start_pos < position]
 361      if nodes_before[-1].type == 'arglist':
 362          return _get_index_and_key(nodes_before[-1].children, position)
 363  
 364      key_str = None
 365  
 366      last = nodes_before[-1]
 367      if last.type == 'argument' and last.children[1] == '=' \
 368              and last.children[1].end_pos <= position:
 369          # Checked if the argument
 370          key_str = last.children[0].value
 371      elif last == '=':
 372          key_str = nodes_before[-2].value
 373  
 374      return nodes_before.count(','), key_str
 375  
 376  
 377  def _get_signature_details_from_error_node(node, additional_children, position):
 378      for index, element in reversed(list(enumerate(node.children))):
 379          # `index > 0` means that it's a trailer and not an atom.
 380          if element == '(' and element.end_pos <= position and index > 0:
 381              # It's an error node, we don't want to match too much, just
 382              # until the parentheses is enough.
 383              children = node.children[index:]
 384              name = element.get_previous_leaf()
 385              if name is None:
 386                  continue
 387              if name.type == 'name' or name.parent.type in ('trailer', 'atom'):
 388                  return CallDetails(element, children + additional_children, position)
 389  
 390  
 391  def get_signature_details(module, position):
 392      leaf = module.get_leaf_for_position(position, include_prefixes=True)
 393      # It's easier to deal with the previous token than the next one in this
 394      # case.
 395      if leaf.start_pos >= position:
 396          # Whitespace / comments after the leaf count towards the previous leaf.
 397          leaf = leaf.get_previous_leaf()
 398          if leaf is None:
 399              return None
 400  
 401      # Now that we know where we are in the syntax tree, we start to look at
 402      # parents for possible function definitions.
 403      node = leaf.parent
 404      while node is not None:
 405          if node.type in ('funcdef', 'classdef', 'decorated', 'async_stmt'):
 406              # Don't show signatures if there's stuff before it that just
 407              # makes it feel strange to have a signature.
 408              return None
 409  
 410          additional_children = []
 411          for n in reversed(node.children):
 412              if n.start_pos < position:
 413                  if n.type == 'error_node':
 414                      result = _get_signature_details_from_error_node(
 415                          n, additional_children, position
 416                      )
 417                      if result is not None:
 418                          return result
 419  
 420                      additional_children[0:0] = n.children
 421                      continue
 422                  additional_children.insert(0, n)
 423  
 424          # Find a valid trailer
 425          if node.type == 'trailer' and node.children[0] == '(' \
 426                  or node.type == 'decorator' and node.children[2] == '(':
 427              # Additionally we have to check that an ending parenthesis isn't
 428              # interpreted wrong. There are two cases:
 429              # 1. Cursor before paren -> The current signature is good
 430              # 2. Cursor after paren -> We need to skip the current signature
 431              if not (leaf is node.children[-1] and position >= leaf.end_pos):
 432                  leaf = node.get_previous_leaf()
 433                  if leaf is None:
 434                      return None
 435                  return CallDetails(
 436                      node.children[0] if node.type == 'trailer' else node.children[2],
 437                      node.children,
 438                      position
 439                  )
 440  
 441          node = node.parent
 442  
 443      return None
 444  
 445  
 446  @signature_time_cache("call_signatures_validity")
 447  def cache_signatures(inference_state, context, bracket_leaf, code_lines, user_pos):
 448      """This function calculates the cache key."""
 449      line_index = user_pos[0] - 1
 450  
 451      before_cursor = code_lines[line_index][:user_pos[1]]
 452      other_lines = code_lines[bracket_leaf.start_pos[0]:line_index]
 453      whole = ''.join(other_lines + [before_cursor])
 454      before_bracket = re.match(r'.*\(', whole, re.DOTALL)
 455  
 456      module_path = context.get_root_context().py__file__()
 457      if module_path is None:
 458          yield None  # Don't cache!
 459      else:
 460          yield (module_path, before_bracket, bracket_leaf.start_pos)
 461      yield infer(
 462          inference_state,
 463          context,
 464          bracket_leaf.get_previous_leaf(),
 465      )
 466  
 467  
 468  def validate_line_column(func):
 469      @wraps(func)
 470      def wrapper(self, line=None, column=None, *args, **kwargs):
 471          line = max(len(self._code_lines), 1) if line is None else line
 472          if not (0 < line <= len(self._code_lines)):
 473              raise ValueError('`line` parameter is not in a valid range.')
 474  
 475          line_string = self._code_lines[line - 1]
 476          line_len = len(line_string)
 477          if line_string.endswith('\r\n'):
 478              line_len -= 2
 479          elif line_string.endswith('\n'):
 480              line_len -= 1
 481  
 482          column = line_len if column is None else column
 483          if not (0 <= column <= line_len):
 484              raise ValueError('`column` parameter (%d) is not in a valid range '
 485                               '(0-%d) for line %d (%r).' % (
 486                                   column, line_len, line, line_string))
 487          return func(self, line, column, *args, **kwargs)
 488      return wrapper
 489  
 490  
 491  def get_module_names(module, all_scopes, definitions=True, references=False):
 492      """
 493      Returns a dictionary with name parts as keys and their call paths as
 494      values.
 495      """
 496      def def_ref_filter(name):
 497          is_def = name.is_definition()
 498          return definitions and is_def or references and not is_def
 499  
 500      names = list(chain.from_iterable(module.get_used_names().values()))
 501      if not all_scopes:
 502          # We have to filter all the names that don't have the module as a
 503          # parent_scope. There's None as a parent, because nodes in the module
 504          # node have the parent module and not suite as all the others.
 505          # Therefore it's important to catch that case.
 506  
 507          def is_module_scope_name(name):
 508              parent_scope = get_parent_scope(name)
 509              # async functions have an extra wrapper. Strip it.
 510              if parent_scope and parent_scope.type == 'async_stmt':
 511                  parent_scope = parent_scope.parent
 512              return parent_scope in (module, None)
 513  
 514          names = [n for n in names if is_module_scope_name(n)]
 515      return filter(def_ref_filter, names)
 516  
 517  
 518  def split_search_string(name):
 519      type, _, dotted_names = name.rpartition(' ')
 520      if type == 'def':
 521          type = 'function'
 522      return type, dotted_names.split('.')
 523