1 """ 2 Environments are a way to activate different Python versions or Virtualenvs for 3 static analysis. The Python binary in that environment is going to be executed. 4 """ 5 import os 6 import sys 7 import hashlib 8 import filecmp 9 from collections import namedtuple 10 from shutil import which 11 from typing import TYPE_CHECKING 12 13 from jedi.cache import memoize_method, time_cache 14 from jedi.inference.compiled.subprocess import CompiledSubprocess, \ 15 InferenceStateSameProcess, InferenceStateSubprocess 16 17 import parso 18 19 if TYPE_CHECKING: 20 from jedi.inference import InferenceState 21 22 23 _VersionInfo = namedtuple('VersionInfo', 'major minor micro') # type: ignore[name-match] 24 25 _SUPPORTED_PYTHONS = ['3.13', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7', '3.6'] 26 _SAFE_PATHS = ['/usr/bin', '/usr/local/bin'] 27 _CONDA_VAR = 'CONDA_PREFIX' 28 _CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor) 29 30 31 class InvalidPythonEnvironment(Exception): 32 """ 33 If you see this exception, the Python executable or Virtualenv you have 34 been trying to use is probably not a correct Python version. 35 """ 36 37 38 class _BaseEnvironment: 39 @memoize_method 40 def get_grammar(self): 41 version_string = '%s.%s' % (self.version_info.major, self.version_info.minor) 42 return parso.load_grammar(version=version_string) 43 44 @property 45 def _sha256(self): 46 try: 47 return self._hash 48 except AttributeError: 49 self._hash = _calculate_sha256_for_file(self.executable) 50 return self._hash 51 52 53 def _get_info(): 54 return ( 55 sys.executable, 56 sys.prefix, 57 sys.version_info[:3], 58 ) 59 60 61 class Environment(_BaseEnvironment): 62 """ 63 This class is supposed to be created by internal Jedi architecture. You 64 should not create it directly. Please use create_environment or the other 65 functions instead. It is then returned by that function. 66 """ 67 _subprocess = None 68 69 def __init__(self, executable, env_vars=None): 70 self._start_executable = executable 71 self._env_vars = env_vars 72 # Initialize the environment 73 self._get_subprocess() 74 75 def _get_subprocess(self): 76 if self._subprocess is not None and not self._subprocess.is_crashed: 77 return self._subprocess 78 79 try: 80 self._subprocess = CompiledSubprocess(self._start_executable, 81 env_vars=self._env_vars) 82 info = self._subprocess._send(None, _get_info) 83 except Exception as exc: 84 raise InvalidPythonEnvironment( 85 "Could not get version information for %r: %r" % ( 86 self._start_executable, 87 exc)) 88 89 # Since it could change and might not be the same(?) as the one given, 90 # set it here. 91 self.executable = info[0] 92 """ 93 The Python executable, matches ``sys.executable``. 94 """ 95 self.path = info[1] 96 """ 97 The path to an environment, matches ``sys.prefix``. 98 """ 99 self.version_info = _VersionInfo(*info[2]) 100 """ 101 Like :data:`sys.version_info`: a tuple to show the current 102 Environment's Python version. 103 """ 104 return self._subprocess 105 106 def __repr__(self): 107 version = '.'.join(str(i) for i in self.version_info) 108 return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path) 109 110 def get_inference_state_subprocess( 111 self, 112 inference_state: 'InferenceState', 113 ) -> InferenceStateSubprocess: 114 return InferenceStateSubprocess(inference_state, self._get_subprocess()) 115 116 @memoize_method 117 def get_sys_path(self): 118 """ 119 The sys path for this environment. Does not include potential 120 modifications from e.g. appending to :data:`sys.path`. 121 122 :returns: list of str 123 """ 124 # It's pretty much impossible to generate the sys path without actually 125 # executing Python. The sys path (when starting with -S) itself depends 126 # on how the Python version was compiled (ENV variables). 127 # If you omit -S when starting Python (normal case), additionally 128 # site.py gets executed. 129 return self._get_subprocess().get_sys_path() 130 131 132 class _SameEnvironmentMixin: 133 def __init__(self): 134 self._start_executable = self.executable = sys.executable 135 self.path = sys.prefix 136 self.version_info = _VersionInfo(*sys.version_info[:3]) 137 self._env_vars = None 138 139 140 class SameEnvironment(_SameEnvironmentMixin, Environment): 141 pass 142 143 144 class InterpreterEnvironment(_SameEnvironmentMixin, _BaseEnvironment): 145 def get_inference_state_subprocess( 146 self, 147 inference_state: 'InferenceState', 148 ) -> InferenceStateSameProcess: 149 return InferenceStateSameProcess(inference_state) 150 151 def get_sys_path(self): 152 return sys.path 153 154 155 def _get_virtual_env_from_var(env_var='VIRTUAL_ENV'): 156 """Get virtualenv environment from VIRTUAL_ENV environment variable. 157 158 It uses `safe=False` with ``create_environment``, because the environment 159 variable is considered to be safe / controlled by the user solely. 160 """ 161 var = os.environ.get(env_var) 162 if var: 163 # Under macOS in some cases - notably when using Pipenv - the 164 # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of 165 # /path/to/env so we need to fully resolve the paths in order to 166 # compare them. 167 if os.path.realpath(var) == os.path.realpath(sys.prefix): 168 return _try_get_same_env() 169 170 try: 171 return create_environment(var, safe=False) 172 except InvalidPythonEnvironment: 173 pass 174 175 176 def _calculate_sha256_for_file(path): 177 sha256 = hashlib.sha256() 178 with open(path, 'rb') as f: 179 for block in iter(lambda: f.read(filecmp.BUFSIZE), b''): 180 sha256.update(block) 181 return sha256.hexdigest() 182 183 184 def get_default_environment(): 185 """ 186 Tries to return an active Virtualenv or conda environment. 187 If there is no VIRTUAL_ENV variable or no CONDA_PREFIX variable set 188 set it will return the latest Python version installed on the system. This 189 makes it possible to use as many new Python features as possible when using 190 autocompletion and other functionality. 191 192 :returns: :class:`.Environment` 193 """ 194 virtual_env = _get_virtual_env_from_var() 195 if virtual_env is not None: 196 return virtual_env 197 198 conda_env = _get_virtual_env_from_var(_CONDA_VAR) 199 if conda_env is not None: 200 return conda_env 201 202 return _try_get_same_env() 203 204 205 def _try_get_same_env(): 206 env = SameEnvironment() 207 if not os.path.basename(env.executable).lower().startswith('python'): 208 # This tries to counter issues with embedding. In some cases (e.g. 209 # VIM's Python Mac/Windows, sys.executable is /foo/bar/vim. This 210 # happens, because for Mac a function called `_NSGetExecutablePath` is 211 # used and for Windows `GetModuleFileNameW`. These are both platform 212 # specific functions. For all other systems sys.executable should be 213 # alright. However here we try to generalize: 214 # 215 # 1. Check if the executable looks like python (heuristic) 216 # 2. In case it's not try to find the executable 217 # 3. In case we don't find it use an interpreter environment. 218 # 219 # The last option will always work, but leads to potential crashes of 220 # Jedi - which is ok, because it happens very rarely and even less, 221 # because the code below should work for most cases. 222 if os.name == 'nt': 223 # The first case would be a virtualenv and the second a normal 224 # Python installation. 225 checks = (r'Scripts\python.exe', 'python.exe') 226 else: 227 # For unix it looks like Python is always in a bin folder. 228 checks = ( 229 'bin/python%s.%s' % (sys.version_info[0], sys.version[1]), 230 'bin/python%s' % (sys.version_info[0]), 231 'bin/python', 232 ) 233 for check in checks: 234 guess = os.path.join(sys.exec_prefix, check) 235 if os.path.isfile(guess): 236 # Bingo - We think we have our Python. 237 return Environment(guess) 238 # It looks like there is no reasonable Python to be found. 239 return InterpreterEnvironment() 240 # If no virtualenv is found, use the environment we're already 241 # using. 242 return env 243 244 245 def get_cached_default_environment(): 246 var = os.environ.get('VIRTUAL_ENV') or os.environ.get(_CONDA_VAR) 247 environment = _get_cached_default_environment() 248 249 # Under macOS in some cases - notably when using Pipenv - the 250 # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of 251 # /path/to/env so we need to fully resolve the paths in order to 252 # compare them. 253 if var and os.path.realpath(var) != os.path.realpath(environment.path): 254 _get_cached_default_environment.clear_cache() 255 return _get_cached_default_environment() 256 return environment 257 258 259 @time_cache(seconds=10 * 60) # 10 Minutes 260 def _get_cached_default_environment(): 261 try: 262 return get_default_environment() 263 except InvalidPythonEnvironment: 264 # It's possible that `sys.executable` is wrong. Typically happens 265 # when Jedi is used in an executable that embeds Python. For further 266 # information, have a look at: 267 # https://github.com/davidhalter/jedi/issues/1531 268 return InterpreterEnvironment() 269 270 271 def find_virtualenvs(paths=None, *, safe=True, use_environment_vars=True): 272 """ 273 :param paths: A list of paths in your file system to be scanned for 274 Virtualenvs. It will search in these paths and potentially execute the 275 Python binaries. 276 :param safe: Default True. In case this is False, it will allow this 277 function to execute potential `python` environments. An attacker might 278 be able to drop an executable in a path this function is searching by 279 default. If the executable has not been installed by root, it will not 280 be executed. 281 :param use_environment_vars: Default True. If True, the VIRTUAL_ENV 282 variable will be checked if it contains a valid VirtualEnv. 283 CONDA_PREFIX will be checked to see if it contains a valid conda 284 environment. 285 286 :yields: :class:`.Environment` 287 """ 288 if paths is None: 289 paths = [] 290 291 _used_paths = set() 292 293 if use_environment_vars: 294 # Using this variable should be safe, because attackers might be 295 # able to drop files (via git) but not environment variables. 296 virtual_env = _get_virtual_env_from_var() 297 if virtual_env is not None: 298 yield virtual_env 299 _used_paths.add(virtual_env.path) 300 301 conda_env = _get_virtual_env_from_var(_CONDA_VAR) 302 if conda_env is not None: 303 yield conda_env 304 _used_paths.add(conda_env.path) 305 306 for directory in paths: 307 if not os.path.isdir(directory): 308 continue 309 310 directory = os.path.abspath(directory) 311 for path in os.listdir(directory): 312 path = os.path.join(directory, path) 313 if path in _used_paths: 314 # A path shouldn't be inferred twice. 315 continue 316 _used_paths.add(path) 317 318 try: 319 executable = _get_executable_path(path, safe=safe) 320 yield Environment(executable) 321 except InvalidPythonEnvironment: 322 pass 323 324 325 def find_system_environments(*, env_vars=None): 326 """ 327 Ignores virtualenvs and returns the Python versions that were installed on 328 your system. This might return nothing, if you're running Python e.g. from 329 a portable version. 330 331 The environments are sorted from latest to oldest Python version. 332 333 :yields: :class:`.Environment` 334 """ 335 for version_string in _SUPPORTED_PYTHONS: 336 try: 337 yield get_system_environment(version_string, env_vars=env_vars) 338 except InvalidPythonEnvironment: 339 pass 340 341 342 # TODO: this function should probably return a list of environments since 343 # multiple Python installations can be found on a system for the same version. 344 def get_system_environment(version, *, env_vars=None): 345 """ 346 Return the first Python environment found for a string of the form 'X.Y' 347 where X and Y are the major and minor versions of Python. 348 349 :raises: :exc:`.InvalidPythonEnvironment` 350 :returns: :class:`.Environment` 351 """ 352 exe = which('python' + version) 353 if exe: 354 if exe == sys.executable: 355 return SameEnvironment() 356 return Environment(exe) 357 358 if os.name == 'nt': 359 for exe in _get_executables_from_windows_registry(version): 360 try: 361 return Environment(exe, env_vars=env_vars) 362 except InvalidPythonEnvironment: 363 pass 364 raise InvalidPythonEnvironment("Cannot find executable python%s." % version) 365 366 367 def create_environment(path, *, safe=True, env_vars=None): 368 """ 369 Make it possible to manually create an Environment object by specifying a 370 Virtualenv path or an executable path and optional environment variables. 371 372 :raises: :exc:`.InvalidPythonEnvironment` 373 :returns: :class:`.Environment` 374 """ 375 if os.path.isfile(path): 376 _assert_safe(path, safe) 377 return Environment(path, env_vars=env_vars) 378 return Environment(_get_executable_path(path, safe=safe), env_vars=env_vars) 379 380 381 def _get_executable_path(path, safe=True): 382 """ 383 Returns None if it's not actually a virtual env. 384 """ 385 386 if os.name == 'nt': 387 pythons = [os.path.join(path, 'Scripts', 'python.exe'), os.path.join(path, 'python.exe')] 388 else: 389 pythons = [os.path.join(path, 'bin', 'python')] 390 for python in pythons: 391 if os.path.exists(python): 392 break 393 else: 394 raise InvalidPythonEnvironment("%s seems to be missing." % python) 395 396 _assert_safe(python, safe) 397 return python 398 399 400 def _get_executables_from_windows_registry(version): 401 import winreg 402 403 # TODO: support Python Anaconda. 404 sub_keys = [ 405 r'SOFTWARE\Python\PythonCore\{version}\InstallPath', 406 r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}\InstallPath', 407 r'SOFTWARE\Python\PythonCore\{version}-32\InstallPath', 408 r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}-32\InstallPath' 409 ] 410 for root_key in [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE]: 411 for sub_key in sub_keys: 412 sub_key = sub_key.format(version=version) 413 try: 414 with winreg.OpenKey(root_key, sub_key) as key: 415 prefix = winreg.QueryValueEx(key, '')[0] 416 exe = os.path.join(prefix, 'python.exe') 417 if os.path.isfile(exe): 418 yield exe 419 except WindowsError: 420 pass 421 422 423 def _assert_safe(executable_path, safe): 424 if safe and not _is_safe(executable_path): 425 raise InvalidPythonEnvironment( 426 "The python binary is potentially unsafe.") 427 428 429 def _is_safe(executable_path): 430 # Resolve sym links. A venv typically is a symlink to a known Python 431 # binary. Only virtualenvs copy symlinks around. 432 real_path = os.path.realpath(executable_path) 433 434 if _is_unix_safe_simple(real_path): 435 return True 436 437 # Just check the list of known Python versions. If it's not in there, 438 # it's likely an attacker or some Python that was not properly 439 # installed in the system. 440 for environment in find_system_environments(): 441 if environment.executable == real_path: 442 return True 443 444 # If the versions don't match, just compare the binary files. If we 445 # don't do that, only venvs will be working and not virtualenvs. 446 # venvs are symlinks while virtualenvs are actual copies of the 447 # Python files. 448 # This still means that if the system Python is updated and the 449 # virtualenv's Python is not (which is probably never going to get 450 # upgraded), it will not work with Jedi. IMO that's fine, because 451 # people should just be using venv. ~ dave 452 if environment._sha256 == _calculate_sha256_for_file(real_path): 453 return True 454 return False 455 456 457 def _is_unix_safe_simple(real_path): 458 if _is_unix_admin(): 459 # In case we are root, just be conservative and 460 # only execute known paths. 461 return any(real_path.startswith(p) for p in _SAFE_PATHS) 462 463 uid = os.stat(real_path).st_uid 464 # The interpreter needs to be owned by root. This means that it wasn't 465 # written by a user and therefore attacking Jedi is not as simple. 466 # The attack could look like the following: 467 # 1. A user clones a repository. 468 # 2. The repository has an innocent looking folder called foobar. jedi 469 # searches for the folder and executes foobar/bin/python --version if 470 # there's also a foobar/bin/activate. 471 # 3. The attacker has gained code execution, since he controls 472 # foobar/bin/python. 473 return uid == 0 474 475 476 def _is_unix_admin(): 477 try: 478 return os.getuid() == 0 479 except AttributeError: 480 return False # Windows 481