1  """
   2  Projects are a way to handle Python projects within Jedi. For simpler plugins
   3  you might not want to deal with projects, but if you want to give the user more
   4  flexibility to define sys paths and Python interpreters for a project,
   5  :class:`.Project` is the perfect way to allow for that.
   6  
   7  Projects can be saved to disk and loaded again, to allow project definitions to
   8  be used across repositories.
   9  """
  10  import json
  11  from pathlib import Path
  12  from itertools import chain
  13  
  14  from jedi import debug
  15  from jedi.api.environment import get_cached_default_environment, create_environment
  16  from jedi.api.exceptions import WrongVersion
  17  from jedi.api.completion import search_in_module
  18  from jedi.api.helpers import split_search_string, get_module_names
  19  from jedi.inference.imports import load_module_from_path, \
  20      load_namespace_from_path, iter_module_names
  21  from jedi.inference.sys_path import discover_buildout_paths
  22  from jedi.inference.cache import inference_state_as_method_param_cache
  23  from jedi.inference.references import recurse_find_python_folders_and_files, search_in_file_ios
  24  from jedi.file_io import FolderIO
  25  
  26  _CONFIG_FOLDER = '.jedi'
  27  _CONTAINS_POTENTIAL_PROJECT = \
  28      'setup.py', '.git', '.hg', 'requirements.txt', 'MANIFEST.in', 'pyproject.toml'
  29  
  30  _SERIALIZER_VERSION = 1
  31  
  32  
  33  def _try_to_skip_duplicates(func):
  34      def wrapper(*args, **kwargs):
  35          found_tree_nodes = []
  36          found_modules = []
  37          for definition in func(*args, **kwargs):
  38              tree_node = definition._name.tree_name
  39              if tree_node is not None and tree_node in found_tree_nodes:
  40                  continue
  41              if definition.type == 'module' and definition.module_path is not None:
  42                  if definition.module_path in found_modules:
  43                      continue
  44                  found_modules.append(definition.module_path)
  45              yield definition
  46              found_tree_nodes.append(tree_node)
  47      return wrapper
  48  
  49  
  50  def _remove_duplicates_from_path(path):
  51      used = set()
  52      for p in path:
  53          if p in used:
  54              continue
  55          used.add(p)
  56          yield p
  57  
  58  
  59  class Project:
  60      """
  61      Projects are a simple way to manage Python folders and define how Jedi does
  62      import resolution. It is mostly used as a parameter to :class:`.Script`.
  63      Additionally there are functions to search a whole project.
  64      """
  65      _environment = None
  66  
  67      @staticmethod
  68      def _get_config_folder_path(base_path):
  69          return base_path.joinpath(_CONFIG_FOLDER)
  70  
  71      @staticmethod
  72      def _get_json_path(base_path):
  73          return Project._get_config_folder_path(base_path).joinpath('project.json')
  74  
  75      @classmethod
  76      def load(cls, path):
  77          """
  78          Loads a project from a specific path. You should not provide the path
  79          to ``.jedi/project.json``, but rather the path to the project folder.
  80  
  81          :param path: The path of the directory you want to use as a project.
  82          """
  83          if isinstance(path, str):
  84              path = Path(path)
  85          with open(cls._get_json_path(path)) as f:
  86              version, data = json.load(f)
  87  
  88          if version == 1:
  89              return cls(**data)
  90          else:
  91              raise WrongVersion(
  92                  "The Jedi version of this project seems newer than what we can handle."
  93              )
  94  
  95      def save(self):
  96          """
  97          Saves the project configuration in the project in ``.jedi/project.json``.
  98          """
  99          data = dict(self.__dict__)
 100          data.pop('_environment', None)
 101          data.pop('_django', None)  # TODO make django setting public?
 102          data = {k.lstrip('_'): v for k, v in data.items()}
 103          data['path'] = str(data['path'])
 104  
 105          self._get_config_folder_path(self._path).mkdir(parents=True, exist_ok=True)
 106          with open(self._get_json_path(self._path), 'w') as f:
 107              return json.dump((_SERIALIZER_VERSION, data), f)
 108  
 109      def __init__(
 110          self,
 111          path,
 112          *,
 113          environment_path=None,
 114          load_unsafe_extensions=False,
 115          sys_path=None,
 116          added_sys_path=(),
 117          smart_sys_path=True,
 118      ) -> None:
 119          """
 120          :param path: The base path for this project.
 121          :param environment_path: The Python executable path, typically the path
 122              of a virtual environment.
 123          :param load_unsafe_extensions: Default False, Loads extensions that are not in the
 124              sys path and in the local directories. With this option enabled,
 125              this is potentially unsafe if you clone a git repository and
 126              analyze it's code, because those compiled extensions will be
 127              important and therefore have execution privileges.
 128          :param sys_path: list of str. You can override the sys path if you
 129              want. By default the ``sys.path.`` is generated by the
 130              environment (virtualenvs, etc).
 131          :param added_sys_path: list of str. Adds these paths at the end of the
 132              sys path.
 133          :param smart_sys_path: If this is enabled (default), adds paths from
 134              local directories. Otherwise you will have to rely on your packages
 135              being properly configured on the ``sys.path``.
 136          """
 137  
 138          if isinstance(path, str):
 139              path = Path(path).absolute()
 140          self._path = path
 141  
 142          self._environment_path = environment_path
 143          if sys_path is not None:
 144              # Remap potential pathlib.Path entries
 145              sys_path = list(map(str, sys_path))
 146          self._sys_path = sys_path
 147          self._smart_sys_path = smart_sys_path
 148          self._load_unsafe_extensions = load_unsafe_extensions
 149          self._django = False
 150          # Remap potential pathlib.Path entries
 151          self.added_sys_path = list(map(str, added_sys_path))
 152          """The sys path that is going to be added at the end of the """
 153  
 154      @property
 155      def path(self):
 156          """
 157          The base path for this project.
 158          """
 159          return self._path
 160  
 161      @property
 162      def sys_path(self):
 163          """
 164          The sys path provided to this project. This can be None and in that
 165          case will be auto generated.
 166          """
 167          return self._sys_path
 168  
 169      @property
 170      def smart_sys_path(self):
 171          """
 172          If the sys path is going to be calculated in a smart way, where
 173          additional paths are added.
 174          """
 175          return self._smart_sys_path
 176  
 177      @property
 178      def load_unsafe_extensions(self):
 179          """
 180          Wheter the project loads unsafe extensions.
 181          """
 182          return self._load_unsafe_extensions
 183  
 184      @inference_state_as_method_param_cache()
 185      def _get_base_sys_path(self, inference_state):
 186          # The sys path has not been set explicitly.
 187          sys_path = list(inference_state.environment.get_sys_path())
 188          try:
 189              sys_path.remove('')
 190          except ValueError:
 191              pass
 192          return sys_path
 193  
 194      @inference_state_as_method_param_cache()
 195      def _get_sys_path(self, inference_state, add_parent_paths=True, add_init_paths=False):
 196          """
 197          Keep this method private for all users of jedi. However internally this
 198          one is used like a public method.
 199          """
 200          suffixed = list(self.added_sys_path)
 201          prefixed = []
 202  
 203          if self._sys_path is None:
 204              sys_path = list(self._get_base_sys_path(inference_state))
 205          else:
 206              sys_path = list(self._sys_path)
 207  
 208          if self._smart_sys_path:
 209              prefixed.append(str(self._path))
 210  
 211              if inference_state.script_path is not None:
 212                  suffixed += map(str, discover_buildout_paths(
 213                      inference_state,
 214                      inference_state.script_path
 215                  ))
 216  
 217                  if add_parent_paths:
 218                      # Collect directories in upward search by:
 219                      #   1. Skipping directories with __init__.py
 220                      #   2. Stopping immediately when above self._path
 221                      traversed = []
 222                      for parent_path in inference_state.script_path.parents:
 223                          if parent_path == self._path \
 224                                  or self._path not in parent_path.parents:
 225                              break
 226                          if not add_init_paths \
 227                                  and parent_path.joinpath("__init__.py").is_file():
 228                              continue
 229                          traversed.append(str(parent_path))
 230  
 231                      # AFAIK some libraries have imports like `foo.foo.bar`, which
 232                      # leads to the conclusion to by default prefer longer paths
 233                      # rather than shorter ones by default.
 234                      suffixed += reversed(traversed)
 235  
 236          if self._django:
 237              prefixed.append(str(self._path))
 238  
 239          path = prefixed + sys_path + suffixed
 240          return list(_remove_duplicates_from_path(path))
 241  
 242      def get_environment(self):
 243          if self._environment is None:
 244              if self._environment_path is not None:
 245                  self._environment = create_environment(self._environment_path, safe=False)
 246              else:
 247                  self._environment = get_cached_default_environment()
 248          return self._environment
 249  
 250      def search(self, string, *, all_scopes=False):
 251          """
 252          Searches a name in the whole project. If the project is very big,
 253          at some point Jedi will stop searching. However it's also very much
 254          recommended to not exhaust the generator. Just display the first ten
 255          results to the user.
 256  
 257          There are currently three different search patterns:
 258  
 259          - ``foo`` to search for a definition foo in any file or a file called
 260            ``foo.py`` or ``foo.pyi``.
 261          - ``foo.bar`` to search for the ``foo`` and then an attribute ``bar``
 262            in it.
 263          - ``class foo.bar.Bar`` or ``def foo.bar.baz`` to search for a specific
 264            API type.
 265  
 266          :param bool all_scopes: Default False; searches not only for
 267              definitions on the top level of a module level, but also in
 268              functions and classes.
 269          :yields: :class:`.Name`
 270          """
 271          return self._search_func(string, all_scopes=all_scopes)
 272  
 273      def complete_search(self, string, **kwargs):
 274          """
 275          Like :meth:`.Script.search`, but completes that string. An empty string
 276          lists all definitions in a project, so be careful with that.
 277  
 278          :param bool all_scopes: Default False; searches not only for
 279              definitions on the top level of a module level, but also in
 280              functions and classes.
 281          :yields: :class:`.Completion`
 282          """
 283          return self._search_func(string, complete=True, **kwargs)
 284  
 285      @_try_to_skip_duplicates
 286      def _search_func(self, string, complete=False, all_scopes=False):
 287          # Using a Script is they easiest way to get an empty module context.
 288          from jedi import Script
 289          s = Script('', project=self)
 290          inference_state = s._inference_state
 291          empty_module_context = s._get_module_context()
 292  
 293          debug.dbg('Search for string %s, complete=%s', string, complete)
 294          wanted_type, wanted_names = split_search_string(string)
 295          name = wanted_names[0]
 296          stub_folder_name = name + '-stubs'
 297  
 298          ios = recurse_find_python_folders_and_files(FolderIO(str(self._path)))
 299          file_ios = []
 300  
 301          # 1. Search for modules in the current project
 302          for folder_io, file_io in ios:
 303              if file_io is None:
 304                  file_name = folder_io.get_base_name()
 305                  if file_name == name or file_name == stub_folder_name:
 306                      f = folder_io.get_file_io('__init__.py')
 307                      try:
 308                          m = load_module_from_path(inference_state, f).as_context()
 309                      except FileNotFoundError:
 310                          f = folder_io.get_file_io('__init__.pyi')
 311                          try:
 312                              m = load_module_from_path(inference_state, f).as_context()
 313                          except FileNotFoundError:
 314                              m = load_namespace_from_path(inference_state, folder_io).as_context()
 315                  else:
 316                      continue
 317              else:
 318                  file_ios.append(file_io)
 319                  if Path(file_io.path).name in (name + '.py', name + '.pyi'):
 320                      m = load_module_from_path(inference_state, file_io).as_context()
 321                  else:
 322                      continue
 323  
 324              debug.dbg('Search of a specific module %s', m)
 325              yield from search_in_module(
 326                  inference_state,
 327                  m,
 328                  names=[m.name],
 329                  wanted_type=wanted_type,
 330                  wanted_names=wanted_names,
 331                  complete=complete,
 332                  convert=True,
 333                  ignore_imports=True,
 334              )
 335  
 336          # 2. Search for identifiers in the project.
 337          for module_context in search_in_file_ios(inference_state, file_ios,
 338                                                   name, complete=complete):
 339              names = get_module_names(module_context.tree_node, all_scopes=all_scopes)
 340              names = [module_context.create_name(n) for n in names]
 341              names = _remove_imports(names)
 342              yield from search_in_module(
 343                  inference_state,
 344                  module_context,
 345                  names=names,
 346                  wanted_type=wanted_type,
 347                  wanted_names=wanted_names,
 348                  complete=complete,
 349                  ignore_imports=True,
 350              )
 351  
 352          # 3. Search for modules on sys.path
 353          sys_path = [
 354              p for p in self._get_sys_path(inference_state)
 355              # Exclude the current folder which is handled by recursing the folders.
 356              if p != self._path
 357          ]
 358          names = list(iter_module_names(inference_state, empty_module_context, sys_path))
 359          yield from search_in_module(
 360              inference_state,
 361              empty_module_context,
 362              names=names,
 363              wanted_type=wanted_type,
 364              wanted_names=wanted_names,
 365              complete=complete,
 366              convert=True,
 367          )
 368  
 369      def __repr__(self):
 370          return '<%s: %s>' % (self.__class__.__name__, self._path)
 371  
 372  
 373  def _is_potential_project(path):
 374      for name in _CONTAINS_POTENTIAL_PROJECT:
 375          try:
 376              if path.joinpath(name).exists():
 377                  return True
 378          except OSError:
 379              continue
 380      return False
 381  
 382  
 383  def _is_django_path(directory):
 384      """ Detects the path of the very well known Django library (if used) """
 385      try:
 386          with open(directory.joinpath('manage.py'), 'rb') as f:
 387              return b"DJANGO_SETTINGS_MODULE" in f.read()
 388      except (FileNotFoundError, IsADirectoryError, PermissionError):
 389          return False
 390  
 391  
 392  def get_default_project(path=None):
 393      """
 394      If a project is not defined by the user, Jedi tries to define a project by
 395      itself as well as possible. Jedi traverses folders until it finds one of
 396      the following:
 397  
 398      1. A ``.jedi/config.json``
 399      2. One of the following files: ``setup.py``, ``.git``, ``.hg``,
 400         ``requirements.txt`` and ``MANIFEST.in``.
 401      """
 402      if path is None:
 403          path = Path.cwd()
 404      elif isinstance(path, str):
 405          path = Path(path)
 406  
 407      check = path.absolute()
 408      probable_path = None
 409      first_no_init_file = None
 410      for dir in chain([check], check.parents):
 411          try:
 412              return Project.load(dir)
 413          except (FileNotFoundError, IsADirectoryError, PermissionError):
 414              pass
 415          except NotADirectoryError:
 416              continue
 417  
 418          if first_no_init_file is None:
 419              if dir.joinpath('__init__.py').exists():
 420                  # In the case that a __init__.py exists, it's in 99% just a
 421                  # Python package and the project sits at least one level above.
 422                  continue
 423              elif not dir.is_file():
 424                  first_no_init_file = dir
 425  
 426          if _is_django_path(dir):
 427              project = Project(dir)
 428              project._django = True
 429              return project
 430  
 431          if probable_path is None and _is_potential_project(dir):
 432              probable_path = dir
 433  
 434      if probable_path is not None:
 435          return Project(probable_path)
 436  
 437      if first_no_init_file is not None:
 438          return Project(first_no_init_file)
 439  
 440      curdir = path if path.is_dir() else path.parent
 441      return Project(curdir)
 442  
 443  
 444  def _remove_imports(names):
 445      return [
 446          n for n in names
 447          if n.tree_name is None or n.api_type not in ('module', 'namespace')
 448      ]
 449