1 import re 2 from textwrap import dedent 3 from inspect import Parameter 4 5 from parso.python.token import PythonTokenTypes 6 from parso.python import tree 7 from parso.tree import search_ancestor, Leaf 8 from parso import split_lines 9 10 from jedi import debug 11 from jedi import settings 12 from jedi.api import classes 13 from jedi.api import helpers 14 from jedi.api import keywords 15 from jedi.api.strings import complete_dict 16 from jedi.api.file_name import complete_file_name 17 from jedi.inference import imports 18 from jedi.inference.base_value import ValueSet 19 from jedi.inference.helpers import infer_call_of_leaf, parse_dotted_names 20 from jedi.inference.context import get_global_filters 21 from jedi.inference.value import TreeInstance 22 from jedi.inference.docstring_utils import DocstringModule 23 from jedi.inference.names import ParamNameWrapper, SubModuleName 24 from jedi.inference.gradual.conversion import convert_values, convert_names 25 from jedi.parser_utils import cut_value_at_position 26 from jedi.plugins import plugin_manager 27 28 29 class ParamNameWithEquals(ParamNameWrapper): 30 def get_public_name(self): 31 return self.string_name + '=' 32 33 34 def _get_signature_param_names(signatures, positional_count, used_kwargs): 35 # Add named params 36 for call_sig in signatures: 37 for i, p in enumerate(call_sig.params): 38 kind = p.kind 39 if i < positional_count and kind == Parameter.POSITIONAL_OR_KEYWORD: 40 continue 41 if kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.KEYWORD_ONLY) \ 42 and p.name not in used_kwargs: 43 yield ParamNameWithEquals(p._name) 44 45 46 def _must_be_kwarg(signatures, positional_count, used_kwargs): 47 if used_kwargs: 48 return True 49 50 must_be_kwarg = True 51 for signature in signatures: 52 for i, p in enumerate(signature.params): 53 kind = p.kind 54 if kind is Parameter.VAR_POSITIONAL: 55 # In case there were not already kwargs, the next param can 56 # always be a normal argument. 57 return False 58 59 if i >= positional_count and kind in (Parameter.POSITIONAL_OR_KEYWORD, 60 Parameter.POSITIONAL_ONLY): 61 must_be_kwarg = False 62 break 63 if not must_be_kwarg: 64 break 65 return must_be_kwarg 66 67 68 def filter_names(inference_state, completion_names, stack, like_name, fuzzy, 69 imported_names, cached_name): 70 comp_dct = set() 71 if settings.case_insensitive_completion: 72 like_name = like_name.lower() 73 for name in completion_names: 74 string = name.string_name 75 if string in imported_names and string != like_name: 76 continue 77 if settings.case_insensitive_completion: 78 string = string.lower() 79 if helpers.match(string, like_name, fuzzy=fuzzy): 80 new = classes.Completion( 81 inference_state, 82 name, 83 stack, 84 len(like_name), 85 is_fuzzy=fuzzy, 86 cached_name=cached_name, 87 ) 88 k = (new.name, new.complete) # key 89 if k not in comp_dct: 90 comp_dct.add(k) 91 tree_name = name.tree_name 92 if tree_name is not None: 93 definition = tree_name.get_definition() 94 if definition is not None and definition.type == 'del_stmt': 95 continue 96 yield new 97 98 99 def _remove_duplicates(completions, other_completions): 100 names = {d.name for d in other_completions} 101 return [c for c in completions if c.name not in names] 102 103 104 def get_user_context(module_context, position): 105 """ 106 Returns the scope in which the user resides. This includes flows. 107 """ 108 leaf = module_context.tree_node.get_leaf_for_position(position, include_prefixes=True) 109 return module_context.create_context(leaf) 110 111 112 def get_flow_scope_node(module_node, position): 113 node = module_node.get_leaf_for_position(position, include_prefixes=True) 114 while not isinstance(node, (tree.Scope, tree.Flow)): 115 node = node.parent 116 117 return node 118 119 120 @plugin_manager.decorate() 121 def complete_param_names(context, function_name, decorator_nodes): 122 # Basically there's no way to do param completion. The plugins are 123 # responsible for this. 124 return [] 125 126 127 class Completion: 128 def __init__(self, inference_state, module_context, code_lines, position, 129 signatures_callback, fuzzy=False): 130 self._inference_state = inference_state 131 self._module_context = module_context 132 self._module_node = module_context.tree_node 133 self._code_lines = code_lines 134 135 # The first step of completions is to get the name 136 self._like_name = helpers.get_on_completion_name(self._module_node, code_lines, position) 137 # The actual cursor position is not what we need to calculate 138 # everything. We want the start of the name we're on. 139 self._original_position = position 140 self._signatures_callback = signatures_callback 141 142 self._fuzzy = fuzzy 143 144 # Return list of completions in this order: 145 # - Beginning with what user is typing 146 # - Public (alphabet) 147 # - Private ("_xxx") 148 # - Dunder ("__xxx") 149 def complete(self): 150 leaf = self._module_node.get_leaf_for_position( 151 self._original_position, 152 include_prefixes=True 153 ) 154 string, start_leaf, quote = _extract_string_while_in_string(leaf, self._original_position) 155 156 prefixed_completions = complete_dict( 157 self._module_context, 158 self._code_lines, 159 start_leaf or leaf, 160 self._original_position, 161 None if string is None else quote + string, 162 fuzzy=self._fuzzy, 163 ) 164 165 if string is not None and not prefixed_completions: 166 prefixed_completions = list(complete_file_name( 167 self._inference_state, self._module_context, start_leaf, quote, string, 168 self._like_name, self._signatures_callback, 169 self._code_lines, self._original_position, 170 self._fuzzy 171 )) 172 if string is not None: 173 if not prefixed_completions and '\n' in string: 174 # Complete only multi line strings 175 prefixed_completions = self._complete_in_string(start_leaf, string) 176 return prefixed_completions 177 178 cached_name, completion_names = self._complete_python(leaf) 179 180 imported_names = [] 181 if leaf.parent is not None and leaf.parent.type in ['import_as_names', 'dotted_as_names']: 182 imported_names.extend(extract_imported_names(leaf.parent)) 183 184 completions = list(filter_names(self._inference_state, completion_names, 185 self.stack, self._like_name, 186 self._fuzzy, imported_names, cached_name=cached_name)) 187 188 return ( 189 # Removing duplicates mostly to remove False/True/None duplicates. 190 _remove_duplicates(prefixed_completions, completions) 191 + sorted(completions, key=lambda x: (not x.name.startswith(self._like_name), 192 x.name.startswith('__'), 193 x.name.startswith('_'), 194 x.name.lower())) 195 ) 196 197 def _complete_python(self, leaf): 198 """ 199 Analyzes the current context of a completion and decides what to 200 return. 201 202 Technically this works by generating a parser stack and analysing the 203 current stack for possible grammar nodes. 204 205 Possible enhancements: 206 - global/nonlocal search global 207 - yield from / raise from <- could be only exceptions/generators 208 - In args: */**: no completion 209 - In params (also lambda): no completion before = 210 """ 211 grammar = self._inference_state.grammar 212 self.stack = stack = None 213 self._position = ( 214 self._original_position[0], 215 self._original_position[1] - len(self._like_name) 216 ) 217 cached_name = None 218 219 try: 220 self.stack = stack = helpers.get_stack_at_position( 221 grammar, self._code_lines, leaf, self._position 222 ) 223 except helpers.OnErrorLeaf as e: 224 value = e.error_leaf.value 225 if value == '.': 226 # After ErrorLeaf's that are dots, we will not do any 227 # completions since this probably just confuses the user. 228 return cached_name, [] 229 230 # If we don't have a value, just use global completion. 231 return cached_name, self._complete_global_scope() 232 233 allowed_transitions = \ 234 list(stack._allowed_transition_names_and_token_types()) 235 236 if 'if' in allowed_transitions: 237 leaf = self._module_node.get_leaf_for_position(self._position, include_prefixes=True) 238 previous_leaf = leaf.get_previous_leaf() 239 240 indent = self._position[1] 241 if not (leaf.start_pos <= self._position <= leaf.end_pos): 242 indent = leaf.start_pos[1] 243 244 if previous_leaf is not None: 245 stmt = previous_leaf 246 while True: 247 stmt = search_ancestor( 248 stmt, 'if_stmt', 'for_stmt', 'while_stmt', 'try_stmt', 249 'error_node', 250 ) 251 if stmt is None: 252 break 253 254 type_ = stmt.type 255 if type_ == 'error_node': 256 first = stmt.children[0] 257 if isinstance(first, Leaf): 258 type_ = first.value + '_stmt' 259 # Compare indents 260 if stmt.start_pos[1] == indent: 261 if type_ == 'if_stmt': 262 allowed_transitions += ['elif', 'else'] 263 elif type_ == 'try_stmt': 264 allowed_transitions += ['except', 'finally', 'else'] 265 elif type_ == 'for_stmt': 266 allowed_transitions.append('else') 267 268 completion_names = [] 269 270 kwargs_only = False 271 if any(t in allowed_transitions for t in (PythonTokenTypes.NAME, 272 PythonTokenTypes.INDENT)): 273 # This means that we actually have to do type inference. 274 275 nonterminals = [stack_node.nonterminal for stack_node in stack] 276 277 nodes = _gather_nodes(stack) 278 if nodes and nodes[-1] in ('as', 'def', 'class'): 279 # No completions for ``with x as foo`` and ``import x as foo``. 280 # Also true for defining names as a class or function. 281 return cached_name, list(self._complete_inherited(is_function=True)) 282 elif "import_stmt" in nonterminals: 283 level, names = parse_dotted_names(nodes, "import_from" in nonterminals) 284 285 only_modules = not ("import_from" in nonterminals and 'import' in nodes) 286 completion_names += self._get_importer_names( 287 names, 288 level, 289 only_modules=only_modules, 290 ) 291 elif nonterminals[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.': 292 dot = self._module_node.get_leaf_for_position(self._position) 293 if dot.type == "endmarker": 294 # This is a bit of a weird edge case, maybe we can somehow 295 # generalize this. 296 dot = leaf.get_previous_leaf() 297 cached_name, n = self._complete_trailer(dot.get_previous_leaf()) 298 completion_names += n 299 elif self._is_parameter_completion(): 300 completion_names += self._complete_params(leaf) 301 else: 302 # Apparently this looks like it's good enough to filter most cases 303 # so that signature completions don't randomly appear. 304 # To understand why this works, three things are important: 305 # 1. trailer with a `,` in it is either a subscript or an arglist. 306 # 2. If there's no `,`, it's at the start and only signatures start 307 # with `(`. Other trailers could start with `.` or `[`. 308 # 3. Decorators are very primitive and have an optional `(` with 309 # optional arglist in them. 310 if nodes[-1] in ['(', ','] \ 311 and nonterminals[-1] in ('trailer', 'arglist', 'decorator'): 312 signatures = self._signatures_callback(*self._position) 313 if signatures: 314 call_details = signatures[0]._call_details 315 used_kwargs = list(call_details.iter_used_keyword_arguments()) 316 positional_count = call_details.count_positional_arguments() 317 318 completion_names += _get_signature_param_names( 319 signatures, 320 positional_count, 321 used_kwargs, 322 ) 323 324 kwargs_only = _must_be_kwarg(signatures, positional_count, used_kwargs) 325 326 if not kwargs_only: 327 completion_names += self._complete_global_scope() 328 completion_names += self._complete_inherited(is_function=False) 329 330 if not kwargs_only: 331 current_line = self._code_lines[self._position[0] - 1][:self._position[1]] 332 completion_names += self._complete_keywords( 333 allowed_transitions, 334 only_values=not (not current_line or current_line[-1] in ' \t.;' 335 and current_line[-3:] != '...') 336 ) 337 338 return cached_name, completion_names 339 340 def _is_parameter_completion(self): 341 tos = self.stack[-1] 342 if tos.nonterminal == 'lambdef' and len(tos.nodes) == 1: 343 # We are at the position `lambda `, where basically the next node 344 # is a param. 345 return True 346 if tos.nonterminal in 'parameters': 347 # Basically we are at the position `foo(`, there's nothing there 348 # yet, so we have no `typedargslist`. 349 return True 350 # var args is for lambdas and typed args for normal functions 351 return tos.nonterminal in ('typedargslist', 'varargslist') and tos.nodes[-1] == ',' 352 353 def _complete_params(self, leaf): 354 stack_node = self.stack[-2] 355 if stack_node.nonterminal == 'parameters': 356 stack_node = self.stack[-3] 357 if stack_node.nonterminal == 'funcdef': 358 context = get_user_context(self._module_context, self._position) 359 node = search_ancestor(leaf, 'error_node', 'funcdef') 360 if node is not None: 361 if node.type == 'error_node': 362 n = node.children[0] 363 if n.type == 'decorators': 364 decorators = n.children 365 elif n.type == 'decorator': 366 decorators = [n] 367 else: 368 decorators = [] 369 else: 370 decorators = node.get_decorators() 371 function_name = stack_node.nodes[1] 372 373 return complete_param_names(context, function_name.value, decorators) 374 return [] 375 376 def _complete_keywords(self, allowed_transitions, only_values): 377 for k in allowed_transitions: 378 if isinstance(k, str) and k.isalpha(): 379 if not only_values or k in ('True', 'False', 'None'): 380 yield keywords.KeywordName(self._inference_state, k) 381 382 def _complete_global_scope(self): 383 context = get_user_context(self._module_context, self._position) 384 debug.dbg('global completion scope: %s', context) 385 flow_scope_node = get_flow_scope_node(self._module_node, self._position) 386 filters = get_global_filters( 387 context, 388 self._position, 389 flow_scope_node 390 ) 391 completion_names = [] 392 for filter in filters: 393 completion_names += filter.values() 394 return completion_names 395 396 def _complete_trailer(self, previous_leaf): 397 inferred_context = self._module_context.create_context(previous_leaf) 398 values = infer_call_of_leaf(inferred_context, previous_leaf) 399 debug.dbg('trailer completion values: %s', values, color='MAGENTA') 400 401 # The cached name simply exists to make speed optimizations for certain 402 # modules. 403 cached_name = None 404 if len(values) == 1: 405 v, = values 406 if v.is_module(): 407 if len(v.string_names) == 1: 408 module_name = v.string_names[0] 409 if module_name in ('numpy', 'tensorflow', 'matplotlib', 'pandas'): 410 cached_name = module_name 411 412 return cached_name, self._complete_trailer_for_values(values) 413 414 def _complete_trailer_for_values(self, values): 415 user_context = get_user_context(self._module_context, self._position) 416 417 return complete_trailer(user_context, values) 418 419 def _get_importer_names(self, names, level=0, only_modules=True): 420 names = [n.value for n in names] 421 i = imports.Importer(self._inference_state, names, self._module_context, level) 422 return i.completion_names(self._inference_state, only_modules=only_modules) 423 424 def _complete_inherited(self, is_function=True): 425 """ 426 Autocomplete inherited methods when overriding in child class. 427 """ 428 leaf = self._module_node.get_leaf_for_position(self._position, include_prefixes=True) 429 cls = tree.search_ancestor(leaf, 'classdef') 430 if cls is None: 431 return 432 433 # Complete the methods that are defined in the super classes. 434 class_value = self._module_context.create_value(cls) 435 436 if cls.start_pos[1] >= leaf.start_pos[1]: 437 return 438 439 filters = class_value.get_filters(is_instance=True) 440 # The first dict is the dictionary of class itself. 441 next(filters) 442 for filter in filters: 443 for name in filter.values(): 444 # TODO we should probably check here for properties 445 if (name.api_type == 'function') == is_function: 446 yield name 447 448 def _complete_in_string(self, start_leaf, string): 449 """ 450 To make it possible for people to have completions in doctests or 451 generally in "Python" code in docstrings, we use the following 452 heuristic: 453 454 - Having an indented block of code 455 - Having some doctest code that starts with `>>>` 456 - Having backticks that doesn't have whitespace inside it 457 """ 458 459 def iter_relevant_lines(lines): 460 include_next_line = False 461 for l in code_lines: 462 if include_next_line or l.startswith('>>>') or l.startswith(' '): 463 yield re.sub(r'^( *>>> ?| +)', '', l) 464 else: 465 yield None 466 467 include_next_line = bool(re.match(' *>>>', l)) 468 469 string = dedent(string) 470 code_lines = split_lines(string, keepends=True) 471 relevant_code_lines = list(iter_relevant_lines(code_lines)) 472 if relevant_code_lines[-1] is not None: 473 # Some code lines might be None, therefore get rid of that. 474 relevant_code_lines = ['\n' if c is None else c for c in relevant_code_lines] 475 return self._complete_code_lines(relevant_code_lines) 476 match = re.search(r'`([^`\s]+)', code_lines[-1]) 477 if match: 478 return self._complete_code_lines([match.group(1)]) 479 return [] 480 481 def _complete_code_lines(self, code_lines): 482 module_node = self._inference_state.grammar.parse(''.join(code_lines)) 483 module_value = DocstringModule( 484 in_module_context=self._module_context, 485 inference_state=self._inference_state, 486 module_node=module_node, 487 code_lines=code_lines, 488 ) 489 return Completion( 490 self._inference_state, 491 module_value.as_context(), 492 code_lines=code_lines, 493 position=module_node.end_pos, 494 signatures_callback=lambda *args, **kwargs: [], 495 fuzzy=self._fuzzy 496 ).complete() 497 498 499 def _gather_nodes(stack): 500 nodes = [] 501 for stack_node in stack: 502 if stack_node.dfa.from_rule == 'small_stmt': 503 nodes = [] 504 else: 505 nodes += stack_node.nodes 506 return nodes 507 508 509 _string_start = re.compile(r'^\w*(\'{3}|"{3}|\'|")') 510 511 512 def _extract_string_while_in_string(leaf, position): 513 def return_part_of_leaf(leaf): 514 kwargs = {} 515 if leaf.line == position[0]: 516 kwargs['endpos'] = position[1] - leaf.column 517 match = _string_start.match(leaf.value, **kwargs) 518 if not match: 519 return None, None, None 520 start = match.group(0) 521 if leaf.line == position[0] and position[1] < leaf.column + match.end(): 522 return None, None, None 523 return cut_value_at_position(leaf, position)[match.end():], leaf, start 524 525 if position < leaf.start_pos: 526 return None, None, None 527 528 if leaf.type == 'string': 529 return return_part_of_leaf(leaf) 530 531 leaves = [] 532 while leaf is not None: 533 if leaf.type == 'error_leaf' and ('"' in leaf.value or "'" in leaf.value): 534 if len(leaf.value) > 1: 535 return return_part_of_leaf(leaf) 536 prefix_leaf = None 537 if not leaf.prefix: 538 prefix_leaf = leaf.get_previous_leaf() 539 if prefix_leaf is None or prefix_leaf.type != 'name' \ 540 or not all(c in 'rubf' for c in prefix_leaf.value.lower()): 541 prefix_leaf = None 542 543 return ( 544 ''.join(cut_value_at_position(l, position) for l in leaves), 545 prefix_leaf or leaf, 546 ('' if prefix_leaf is None else prefix_leaf.value) 547 + cut_value_at_position(leaf, position), 548 ) 549 if leaf.line != position[0]: 550 # Multi line strings are always simple error leaves and contain the 551 # whole string, single line error leaves are atherefore important 552 # now and since the line is different, it's not really a single 553 # line string anymore. 554 break 555 leaves.insert(0, leaf) 556 leaf = leaf.get_previous_leaf() 557 return None, None, None 558 559 560 def complete_trailer(user_context, values): 561 completion_names = [] 562 for value in values: 563 for filter in value.get_filters(origin_scope=user_context.tree_node): 564 completion_names += filter.values() 565 566 if not value.is_stub() and isinstance(value, TreeInstance): 567 completion_names += _complete_getattr(user_context, value) 568 569 python_values = convert_values(values) 570 for c in python_values: 571 if c not in values: 572 for filter in c.get_filters(origin_scope=user_context.tree_node): 573 completion_names += filter.values() 574 return completion_names 575 576 577 def _complete_getattr(user_context, instance): 578 """ 579 A heuristic to make completion for proxy objects work. This is not 580 intended to work in all cases. It works exactly in this case: 581 582 def __getattr__(self, name): 583 ... 584 return getattr(any_object, name) 585 586 It is important that the return contains getattr directly, otherwise it 587 won't work anymore. It's really just a stupid heuristic. It will not 588 work if you write e.g. `return (getatr(o, name))`, because of the 589 additional parentheses. It will also not work if you move the getattr 590 to some other place that is not the return statement itself. 591 592 It is intentional that it doesn't work in all cases. Generally it's 593 really hard to do even this case (as you can see below). Most people 594 will write it like this anyway and the other ones, well they are just 595 out of luck I guess :) ~dave. 596 """ 597 names = (instance.get_function_slot_names('__getattr__') 598 or instance.get_function_slot_names('__getattribute__')) 599 functions = ValueSet.from_sets( 600 name.infer() 601 for name in names 602 ) 603 for func in functions: 604 tree_node = func.tree_node 605 if tree_node is None or tree_node.type != 'funcdef': 606 continue 607 608 for return_stmt in tree_node.iter_return_stmts(): 609 # Basically until the next comment we just try to find out if a 610 # return statement looks exactly like `return getattr(x, name)`. 611 if return_stmt.type != 'return_stmt': 612 continue 613 atom_expr = return_stmt.children[1] 614 if atom_expr.type != 'atom_expr': 615 continue 616 atom = atom_expr.children[0] 617 trailer = atom_expr.children[1] 618 if len(atom_expr.children) != 2 or atom.type != 'name' \ 619 or atom.value != 'getattr': 620 continue 621 arglist = trailer.children[1] 622 if arglist.type != 'arglist' or len(arglist.children) < 3: 623 continue 624 context = func.as_context() 625 object_node = arglist.children[0] 626 627 # Make sure it's a param: foo in __getattr__(self, foo) 628 name_node = arglist.children[2] 629 name_list = context.goto(name_node, name_node.start_pos) 630 if not any(n.api_type == 'param' for n in name_list): 631 continue 632 633 # Now that we know that these are most probably completion 634 # objects, we just infer the object and return them as 635 # completions. 636 objects = context.infer_node(object_node) 637 return complete_trailer(user_context, objects) 638 return [] 639 640 641 def search_in_module(inference_state, module_context, names, wanted_names, 642 wanted_type, complete=False, fuzzy=False, 643 ignore_imports=False, convert=False): 644 for s in wanted_names[:-1]: 645 new_names = [] 646 for n in names: 647 if s == n.string_name: 648 if n.tree_name is not None and n.api_type in ('module', 'namespace') \ 649 and ignore_imports: 650 continue 651 new_names += complete_trailer( 652 module_context, 653 n.infer() 654 ) 655 debug.dbg('dot lookup on search %s from %s', new_names, names[:10]) 656 names = new_names 657 658 last_name = wanted_names[-1].lower() 659 for n in names: 660 string = n.string_name.lower() 661 if complete and helpers.match(string, last_name, fuzzy=fuzzy) \ 662 or not complete and string == last_name: 663 if isinstance(n, SubModuleName): 664 names = [v.name for v in n.infer()] 665 else: 666 names = [n] 667 if convert: 668 names = convert_names(names) 669 for n2 in names: 670 if complete: 671 def_ = classes.Completion( 672 inference_state, n2, 673 stack=None, 674 like_name_length=len(last_name), 675 is_fuzzy=fuzzy, 676 ) 677 else: 678 def_ = classes.Name(inference_state, n2) 679 if not wanted_type or wanted_type == def_.type: 680 yield def_ 681 682 683 def extract_imported_names(node): 684 imported_names = [] 685 686 if node.type in ['import_as_names', 'dotted_as_names', 'import_as_name']: 687 for index, child in enumerate(node.children): 688 if child.type == 'name': 689 if (index > 0 and node.children[index - 1].type == "keyword" 690 and node.children[index - 1].value == "as"): 691 continue 692 imported_names.append(child.value) 693 elif child.type == 'import_as_name': 694 imported_names.extend(extract_imported_names(child)) 695 696 return imported_names 697