1  # encoding: utf-8
   2  """Implementations for various useful completers.
   3  
   4  These are all loaded by default by IPython.
   5  """
   6  #-----------------------------------------------------------------------------
   7  #  Copyright (C) 2010-2011 The IPython Development Team.
   8  #
   9  #  Distributed under the terms of the BSD License.
  10  #
  11  #  The full license is in the file COPYING.txt, distributed with this software.
  12  #-----------------------------------------------------------------------------
  13  
  14  #-----------------------------------------------------------------------------
  15  # Imports
  16  #-----------------------------------------------------------------------------
  17  
  18  # Stdlib imports
  19  import glob
  20  import inspect
  21  import os
  22  import re
  23  import sys
  24  from importlib import import_module
  25  from importlib.machinery import all_suffixes
  26  
  27  
  28  # Third-party imports
  29  from time import time
  30  from zipimport import zipimporter
  31  
  32  # Our own imports
  33  from .completer import expand_user, compress_user
  34  from .error import TryNext
  35  from ..utils._process_common import arg_split
  36  
  37  # FIXME: this should be pulled in with the right call via the component system
  38  from IPython import get_ipython
  39  
  40  from typing import List
  41  
  42  #-----------------------------------------------------------------------------
  43  # Globals and constants
  44  #-----------------------------------------------------------------------------
  45  _suffixes = all_suffixes()
  46  
  47  # Time in seconds after which the rootmodules will be stored permanently in the
  48  # ipython ip.db database (kept in the user's .ipython dir).
  49  TIMEOUT_STORAGE = 2
  50  
  51  # Time in seconds after which we give up
  52  TIMEOUT_GIVEUP = 20
  53  
  54  # Regular expression for the python import statement
  55  import_re = re.compile(r'(?P<name>[^\W\d]\w*?)'
  56                         r'(?P<package>[/\\]__init__)?'
  57                         r'(?P<suffix>%s)$' %
  58                         r'|'.join(re.escape(s) for s in _suffixes))
  59  
  60  # RE for the ipython %run command (python + ipython scripts)
  61  magic_run_re = re.compile(r'.*(\.ipy|\.ipynb|\.py[w]?)$')
  62  
  63  #-----------------------------------------------------------------------------
  64  # Local utilities
  65  #-----------------------------------------------------------------------------
  66  
  67  
  68  def module_list(path: str) -> List[str]:
  69      """
  70      Return the list containing the names of the modules available in the given
  71      folder.
  72      """
  73      # sys.path has the cwd as an empty string, but isdir/listdir need it as '.'
  74      if path == '':
  75          path = '.'
  76  
  77      # A few local constants to be used in loops below
  78      pjoin = os.path.join
  79  
  80      if os.path.isdir(path):
  81          # Build a list of all files in the directory and all files
  82          # in its subdirectories. For performance reasons, do not
  83          # recurse more than one level into subdirectories.
  84          files: List[str] = []
  85          for root, dirs, nondirs in os.walk(path, followlinks=True):
  86              subdir = root[len(path)+1:]
  87              if subdir:
  88                  files.extend(pjoin(subdir, f) for f in nondirs)
  89                  dirs[:] = [] # Do not recurse into additional subdirectories.
  90              else:
  91                  files.extend(nondirs)
  92  
  93      else:
  94          try:
  95              files = list(zipimporter(path)._files.keys())  # type: ignore
  96          except Exception:
  97              files = []
  98  
  99      # Build a list of modules which match the import_re regex.
 100      modules = []
 101      for f in files:
 102          m = import_re.match(f)
 103          if m:
 104              modules.append(m.group('name'))
 105      return list(set(modules))
 106  
 107  
 108  def get_root_modules():
 109      """
 110      Returns a list containing the names of all the modules available in the
 111      folders of the pythonpath.
 112  
 113      ip.db['rootmodules_cache'] maps sys.path entries to list of modules.
 114      """
 115      ip = get_ipython()
 116      if ip is None:
 117          # No global shell instance to store cached list of modules.
 118          # Don't try to scan for modules every time.
 119          return list(sys.builtin_module_names)
 120  
 121      if getattr(ip.db, "_mock", False):
 122          rootmodules_cache = {}
 123      else:
 124          rootmodules_cache = ip.db.get("rootmodules_cache", {})
 125      rootmodules = list(sys.builtin_module_names)
 126      start_time = time()
 127      store = False
 128      for path in sys.path:
 129          try:
 130              modules = rootmodules_cache[path]
 131          except KeyError:
 132              modules = module_list(path)
 133              try:
 134                  modules.remove('__init__')
 135              except ValueError:
 136                  pass
 137              if path not in ('', '.'): # cwd modules should not be cached
 138                  rootmodules_cache[path] = modules
 139              if time() - start_time > TIMEOUT_STORAGE and not store:
 140                  store = True
 141                  print("\nCaching the list of root modules, please wait!")
 142                  print("(This will only be done once - type '%rehashx' to "
 143                        "reset cache!)\n")
 144                  sys.stdout.flush()
 145              if time() - start_time > TIMEOUT_GIVEUP:
 146                  print("This is taking too long, we give up.\n")
 147                  return []
 148          rootmodules.extend(modules)
 149      if store:
 150          ip.db['rootmodules_cache'] = rootmodules_cache
 151      rootmodules = list(set(rootmodules))
 152      return rootmodules
 153  
 154  
 155  def is_importable(module, attr: str, only_modules) -> bool:
 156      if only_modules:
 157          try:
 158              mod = getattr(module, attr)
 159          except ModuleNotFoundError:
 160              # See gh-14434
 161              return False
 162          return inspect.ismodule(mod)
 163      else:
 164          return not(attr[:2] == '__' and attr[-2:] == '__')
 165  
 166  def is_possible_submodule(module, attr):
 167      try:
 168          obj = getattr(module, attr)
 169      except AttributeError:
 170          # Is possibly an unimported submodule
 171          return True
 172      except TypeError:
 173          # https://github.com/ipython/ipython/issues/9678
 174          return False
 175      return inspect.ismodule(obj)
 176  
 177  
 178  def try_import(mod: str, only_modules=False) -> List[str]:
 179      """
 180      Try to import given module and return list of potential completions.
 181      """
 182      mod = mod.rstrip('.')
 183      try:
 184          m = import_module(mod)
 185      except:
 186          return []
 187  
 188      m_is_init = '__init__' in (getattr(m, '__file__', '') or '')
 189  
 190      completions = []
 191      if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init:
 192          completions.extend( [attr for attr in dir(m) if
 193                               is_importable(m, attr, only_modules)])
 194  
 195      m_all = getattr(m, "__all__", [])
 196      if only_modules:
 197          completions.extend(attr for attr in m_all if is_possible_submodule(m, attr))
 198      else:
 199          completions.extend(m_all)
 200  
 201      if m_is_init:
 202          file_ = m.__file__
 203          file_path = os.path.dirname(file_)  # type: ignore
 204          if file_path is not None:
 205              completions.extend(module_list(file_path))
 206      completions_set = {c for c in completions if isinstance(c, str)}
 207      completions_set.discard('__init__')
 208      return list(completions_set)
 209  
 210  
 211  #-----------------------------------------------------------------------------
 212  # Completion-related functions.
 213  #-----------------------------------------------------------------------------
 214  
 215  def quick_completer(cmd, completions):
 216      r""" Easily create a trivial completer for a command.
 217  
 218      Takes either a list of completions, or all completions in string (that will
 219      be split on whitespace).
 220  
 221      Example::
 222  
 223          [d:\ipython]|1> import ipy_completers
 224          [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])
 225          [d:\ipython]|3> foo b<TAB>
 226          bar baz
 227          [d:\ipython]|3> foo ba
 228      """
 229  
 230      if isinstance(completions, str):
 231          completions = completions.split()
 232  
 233      def do_complete(self, event):
 234          return completions
 235  
 236      get_ipython().set_hook('complete_command',do_complete, str_key = cmd)
 237  
 238  def module_completion(line):
 239      """
 240      Returns a list containing the completion possibilities for an import line.
 241  
 242      The line looks like this :
 243      'import xml.d'
 244      'from xml.dom import'
 245      """
 246  
 247      words = line.split(' ')
 248      nwords = len(words)
 249  
 250      # from whatever <tab> -> 'import '
 251      if nwords == 3 and words[0] == 'from':
 252          return ['import ']
 253  
 254      # 'from xy<tab>' or 'import xy<tab>'
 255      if nwords < 3 and (words[0] in {'%aimport', 'import', 'from'}) :
 256          if nwords == 1:
 257              return get_root_modules()
 258          mod = words[1].split('.')
 259          if len(mod) < 2:
 260              return get_root_modules()
 261          completion_list = try_import('.'.join(mod[:-1]), True)
 262          return ['.'.join(mod[:-1] + [el]) for el in completion_list]
 263  
 264      # 'from xyz import abc<tab>'
 265      if nwords >= 3 and words[0] == 'from':
 266          mod = words[1]
 267          return try_import(mod)
 268  
 269  #-----------------------------------------------------------------------------
 270  # Completers
 271  #-----------------------------------------------------------------------------
 272  # These all have the func(self, event) signature to be used as custom
 273  # completers
 274  
 275  def module_completer(self,event):
 276      """Give completions after user has typed 'import ...' or 'from ...'"""
 277  
 278      # This works in all versions of python.  While 2.5 has
 279      # pkgutil.walk_packages(), that particular routine is fairly dangerous,
 280      # since it imports *EVERYTHING* on sys.path.  That is: a) very slow b) full
 281      # of possibly problematic side effects.
 282      # This search the folders in the sys.path for available modules.
 283  
 284      return module_completion(event.line)
 285  
 286  # FIXME: there's a lot of logic common to the run, cd and builtin file
 287  # completers, that is currently reimplemented in each.
 288  
 289  def magic_run_completer(self, event):
 290      """Complete files that end in .py or .ipy or .ipynb for the %run command.
 291      """
 292      comps = arg_split(event.line, strict=False)
 293      # relpath should be the current token that we need to complete.
 294      if (len(comps) > 1) and (not event.line.endswith(' ')):
 295          relpath = comps[-1].strip("'\"")
 296      else:
 297          relpath = ''
 298  
 299      #print("\nev=", event)  # dbg
 300      #print("rp=", relpath)  # dbg
 301      #print('comps=', comps)  # dbg
 302  
 303      lglob = glob.glob
 304      isdir = os.path.isdir
 305      relpath, tilde_expand, tilde_val = expand_user(relpath)
 306  
 307      # Find if the user has already typed the first filename, after which we
 308      # should complete on all files, since after the first one other files may
 309      # be arguments to the input script.
 310  
 311      if any(magic_run_re.match(c) for c in comps):
 312          matches =  [f.replace('\\','/') + ('/' if isdir(f) else '')
 313                              for f in lglob(relpath+'*')]
 314      else:
 315          dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)]
 316          pys =  [f.replace('\\','/')
 317                  for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
 318                  lglob(relpath+'*.ipynb') + lglob(relpath + '*.pyw')]
 319  
 320          matches = dirs + pys
 321  
 322      #print('run comp:', dirs+pys) # dbg
 323      return [compress_user(p, tilde_expand, tilde_val) for p in matches]
 324  
 325  
 326  def cd_completer(self, event):
 327      """Completer function for cd, which only returns directories."""
 328      ip = get_ipython()
 329      relpath = event.symbol
 330  
 331      #print(event) # dbg
 332      if event.line.endswith('-b') or ' -b ' in event.line:
 333          # return only bookmark completions
 334          bkms = self.db.get('bookmarks', None)
 335          if bkms:
 336              return bkms.keys()
 337          else:
 338              return []
 339  
 340      if event.symbol == '-':
 341          width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
 342          # jump in directory history by number
 343          fmt = '-%0' + width_dh +'d [%s]'
 344          ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
 345          if len(ents) > 1:
 346              return ents
 347          return []
 348  
 349      if event.symbol.startswith('--'):
 350          return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']]
 351  
 352      # Expand ~ in path and normalize directory separators.
 353      relpath, tilde_expand, tilde_val = expand_user(relpath)
 354      relpath = relpath.replace('\\','/')
 355  
 356      found = []
 357      for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
 358                if os.path.isdir(f)]:
 359          if ' ' in d:
 360              # we don't want to deal with any of that, complex code
 361              # for this is elsewhere
 362              raise TryNext
 363  
 364          found.append(d)
 365  
 366      if not found:
 367          if os.path.isdir(relpath):
 368              return [compress_user(relpath, tilde_expand, tilde_val)]
 369  
 370          # if no completions so far, try bookmarks
 371          bks = self.db.get('bookmarks',{})
 372          bkmatches = [s for s in bks if s.startswith(event.symbol)]
 373          if bkmatches:
 374              return bkmatches
 375  
 376          raise TryNext
 377  
 378      return [compress_user(p, tilde_expand, tilde_val) for p in found]
 379  
 380  def reset_completer(self, event):
 381      "A completer for %reset magic"
 382      return '-f -s in out array dhist'.split()
 383