1  """Generic testing tools.
   2  
   3  Authors
   4  -------
   5  - Fernando Perez <Fernando.Perez@berkeley.edu>
   6  """
   7  
   8  
   9  # Copyright (c) IPython Development Team.
  10  # Distributed under the terms of the Modified BSD License.
  11  
  12  import os
  13  from pathlib import Path
  14  import re
  15  import sys
  16  import tempfile
  17  import unittest
  18  
  19  from contextlib import contextmanager
  20  from io import StringIO
  21  from subprocess import Popen, PIPE
  22  from unittest.mock import patch
  23  
  24  from traitlets.config.loader import Config
  25  from IPython.utils.process import get_output_error_code
  26  from IPython.utils.text import list_strings
  27  from IPython.utils.io import temp_pyfile, Tee
  28  from IPython.utils import py3compat
  29  
  30  from . import decorators as dec
  31  from . import skipdoctest
  32  
  33  
  34  # The docstring for full_path doctests differently on win32 (different path
  35  # separator) so just skip the doctest there.  The example remains informative.
  36  doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
  37  
  38  @doctest_deco
  39  def full_path(startPath: str, files: list[str]) -> list[str]:
  40      """Make full paths for all the listed files, based on startPath.
  41  
  42      Only the base part of startPath is kept, since this routine is typically
  43      used with a script's ``__file__`` variable as startPath. The base of startPath
  44      is then prepended to all the listed files, forming the output list.
  45  
  46      Parameters
  47      ----------
  48      startPath : string
  49        Initial path to use as the base for the results.  This path is split
  50        using os.path.split() and only its first component is kept.
  51  
  52      files : list
  53        One or more files.
  54  
  55      Examples
  56      --------
  57  
  58      >>> full_path('/foo/bar.py',['a.txt','b.txt'])
  59      ['/foo/a.txt', '/foo/b.txt']
  60  
  61      >>> full_path('/foo',['a.txt','b.txt'])
  62      ['/a.txt', '/b.txt']
  63  
  64      """
  65      assert isinstance(files, list)
  66      base = os.path.split(startPath)[0]
  67      return [ os.path.join(base,f) for f in files ]
  68  
  69  
  70  def parse_test_output(txt):
  71      """Parse the output of a test run and return errors, failures.
  72  
  73      Parameters
  74      ----------
  75      txt : str
  76        Text output of a test run, assumed to contain a line of one of the
  77        following forms::
  78  
  79          'FAILED (errors=1)'
  80          'FAILED (failures=1)'
  81          'FAILED (errors=1, failures=1)'
  82  
  83      Returns
  84      -------
  85      nerr, nfail
  86        number of errors and failures.
  87      """
  88  
  89      err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
  90      if err_m:
  91          nerr = int(err_m.group(1))
  92          nfail = 0
  93          return  nerr, nfail
  94  
  95      fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
  96      if fail_m:
  97          nerr = 0
  98          nfail = int(fail_m.group(1))
  99          return  nerr, nfail
 100  
 101      both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
 102                         re.MULTILINE)
 103      if both_m:
 104          nerr = int(both_m.group(1))
 105          nfail = int(both_m.group(2))
 106          return  nerr, nfail
 107  
 108      # If the input didn't match any of these forms, assume no error/failures
 109      return 0, 0
 110  
 111  
 112  # So nose doesn't think this is a test
 113  parse_test_output.__test__ = False
 114  
 115  
 116  def default_argv():
 117      """Return a valid default argv for creating testing instances of ipython"""
 118  
 119      return ['--quick', # so no config file is loaded
 120              # Other defaults to minimize side effects on stdout
 121              '--colors=NoColor', '--no-term-title','--no-banner',
 122              '--autocall=0']
 123  
 124  
 125  def default_config():
 126      """Return a config object with good defaults for testing."""
 127      config = Config()
 128      config.TerminalInteractiveShell.colors = 'NoColor'
 129      config.TerminalTerminalInteractiveShell.term_title = False,
 130      config.TerminalInteractiveShell.autocall = 0
 131      f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False)
 132      config.HistoryManager.hist_file = Path(f.name)
 133      f.close()
 134      config.HistoryManager.db_cache_size = 10000
 135      return config
 136  
 137  
 138  def get_ipython_cmd(as_string=False):
 139      """
 140      Return appropriate IPython command line name. By default, this will return
 141      a list that can be used with subprocess.Popen, for example, but passing
 142      `as_string=True` allows for returning the IPython command as a string.
 143  
 144      Parameters
 145      ----------
 146      as_string: bool
 147          Flag to allow to return the command as a string.
 148      """
 149      ipython_cmd = [sys.executable, "-m", "IPython"]
 150  
 151      if as_string:
 152          ipython_cmd = " ".join(ipython_cmd)
 153  
 154      return ipython_cmd
 155  
 156  def ipexec(fname, options=None, commands=()):
 157      """Utility to call 'ipython filename'.
 158  
 159      Starts IPython with a minimal and safe configuration to make startup as fast
 160      as possible.
 161  
 162      Note that this starts IPython in a subprocess!
 163  
 164      Parameters
 165      ----------
 166      fname : str, Path
 167        Name of file to be executed (should have .py or .ipy extension).
 168  
 169      options : optional, list
 170        Extra command-line flags to be passed to IPython.
 171  
 172      commands : optional, list
 173        Commands to send in on stdin
 174  
 175      Returns
 176      -------
 177      ``(stdout, stderr)`` of ipython subprocess.
 178      """
 179      __tracebackhide__ = True
 180  
 181      if options is None:
 182          options = []
 183  
 184      cmdargs = default_argv() + options
 185  
 186      test_dir = os.path.dirname(__file__)
 187  
 188      ipython_cmd = get_ipython_cmd()
 189      # Absolute path for filename
 190      full_fname = os.path.join(test_dir, fname)
 191      full_cmd = ipython_cmd + cmdargs + ['--', full_fname]
 192      env = os.environ.copy()
 193      # FIXME: ignore all warnings in ipexec while we have shims
 194      # should we keep suppressing warnings here, even after removing shims?
 195      env['PYTHONWARNINGS'] = 'ignore'
 196      # env.pop('PYTHONWARNINGS', None)  # Avoid extraneous warnings appearing on stderr
 197      # Prevent coloring under PyCharm ("\x1b[0m" at the end of the stdout)
 198      env.pop("PYCHARM_HOSTED", None)
 199      for k, v in env.items():
 200          # Debug a bizarre failure we've seen on Windows:
 201          # TypeError: environment can only contain strings
 202          if not isinstance(v, str):
 203              print(k, v)
 204      p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
 205      out, err = p.communicate(input=py3compat.encode('\n'.join(commands)) or None)
 206      out, err = py3compat.decode(out), py3compat.decode(err)
 207      # `import readline` causes 'ESC[?1034h' to be output sometimes,
 208      # so strip that out before doing comparisons
 209      if out:
 210          out = re.sub(r'\x1b\[[^h]+h', '', out)
 211      return out, err
 212  
 213  
 214  def ipexec_validate(fname, expected_out, expected_err='',
 215                      options=None, commands=()):
 216      """Utility to call 'ipython filename' and validate output/error.
 217  
 218      This function raises an AssertionError if the validation fails.
 219  
 220      Note that this starts IPython in a subprocess!
 221  
 222      Parameters
 223      ----------
 224      fname : str, Path
 225        Name of the file to be executed (should have .py or .ipy extension).
 226  
 227      expected_out : str
 228        Expected stdout of the process.
 229  
 230      expected_err : optional, str
 231        Expected stderr of the process.
 232  
 233      options : optional, list
 234        Extra command-line flags to be passed to IPython.
 235  
 236      Returns
 237      -------
 238      None
 239      """
 240      __tracebackhide__ = True
 241  
 242      out, err = ipexec(fname, options, commands)
 243      # print('OUT', out)  # dbg
 244      # print('ERR', err)  # dbg
 245      # If there are any errors, we must check those before stdout, as they may be
 246      # more informative than simply having an empty stdout.
 247      if err:
 248          if expected_err:
 249              assert "\n".join(err.strip().splitlines()) == "\n".join(
 250                  expected_err.strip().splitlines()
 251              )
 252          else:
 253              raise ValueError('Running file %r produced error: %r' %
 254                               (fname, err))
 255      # If no errors or output on stderr was expected, match stdout
 256      assert "\n".join(out.strip().splitlines()) == "\n".join(
 257          expected_out.strip().splitlines()
 258      )
 259  
 260  
 261  class TempFileMixin(unittest.TestCase):
 262      """Utility class to create temporary Python/IPython files.
 263  
 264      Meant as a mixin class for test cases."""
 265  
 266      def mktmp(self, src, ext='.py'):
 267          """Make a valid python temp file."""
 268          fname = temp_pyfile(src, ext)
 269          if not hasattr(self, 'tmps'):
 270              self.tmps=[]
 271          self.tmps.append(fname)
 272          self.fname = fname
 273  
 274      def tearDown(self):
 275          # If the tmpfile wasn't made because of skipped tests, like in
 276          # win32, there's nothing to cleanup.
 277          if hasattr(self, 'tmps'):
 278              for fname in self.tmps:
 279                  # If the tmpfile wasn't made because of skipped tests, like in
 280                  # win32, there's nothing to cleanup.
 281                  try:
 282                      os.unlink(fname)
 283                  except:
 284                      # On Windows, even though we close the file, we still can't
 285                      # delete it.  I have no clue why
 286                      pass
 287  
 288      def __enter__(self):
 289          return self
 290  
 291      def __exit__(self, exc_type, exc_value, traceback):
 292          self.tearDown()
 293  
 294  
 295  pair_fail_msg = ("Testing {0}\n\n"
 296                  "In:\n"
 297                  "  {1!r}\n"
 298                  "Expected:\n"
 299                  "  {2!r}\n"
 300                  "Got:\n"
 301                  "  {3!r}\n")
 302  def check_pairs(func, pairs):
 303      """Utility function for the common case of checking a function with a
 304      sequence of input/output pairs.
 305  
 306      Parameters
 307      ----------
 308      func : callable
 309        The function to be tested. Should accept a single argument.
 310      pairs : iterable
 311        A list of (input, expected_output) tuples.
 312  
 313      Returns
 314      -------
 315      None. Raises an AssertionError if any output does not match the expected
 316      value.
 317      """
 318      __tracebackhide__ = True
 319  
 320      name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
 321      for inp, expected in pairs:
 322          out = func(inp)
 323          assert out == expected, pair_fail_msg.format(name, inp, expected, out)
 324  
 325  
 326  MyStringIO = StringIO
 327  
 328  _re_type = type(re.compile(r''))
 329  
 330  notprinted_msg = """Did not find {0!r} in printed output (on {1}):
 331  -------
 332  {2!s}
 333  -------
 334  """
 335  
 336  class AssertPrints(object):
 337      """Context manager for testing that code prints certain text.
 338  
 339      Examples
 340      --------
 341      >>> with AssertPrints("abc", suppress=False):
 342      ...     print("abcd")
 343      ...     print("def")
 344      ...
 345      abcd
 346      def
 347      """
 348      def __init__(self, s, channel='stdout', suppress=True):
 349          self.s = s
 350          if isinstance(self.s, (str, _re_type)):
 351              self.s = [self.s]
 352          self.channel = channel
 353          self.suppress = suppress
 354  
 355      def __enter__(self):
 356          self.orig_stream = getattr(sys, self.channel)
 357          self.buffer = MyStringIO()
 358          self.tee = Tee(self.buffer, channel=self.channel)
 359          setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
 360  
 361      def __exit__(self, etype, value, traceback):
 362          __tracebackhide__ = True
 363  
 364          try:
 365              if value is not None:
 366                  # If an error was raised, don't check anything else
 367                  return False
 368              self.tee.flush()
 369              setattr(sys, self.channel, self.orig_stream)
 370              printed = self.buffer.getvalue()
 371              for s in self.s:
 372                  if isinstance(s, _re_type):
 373                      assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed)
 374                  else:
 375                      assert s in printed, notprinted_msg.format(s, self.channel, printed)
 376              return False
 377          finally:
 378              self.tee.close()
 379  
 380  printed_msg = """Found {0!r} in printed output (on {1}):
 381  -------
 382  {2!s}
 383  -------
 384  """
 385  
 386  class AssertNotPrints(AssertPrints):
 387      """Context manager for checking that certain output *isn't* produced.
 388  
 389      Counterpart of AssertPrints"""
 390      def __exit__(self, etype, value, traceback):
 391          __tracebackhide__ = True
 392  
 393          try:
 394              if value is not None:
 395                  # If an error was raised, don't check anything else
 396                  self.tee.close()
 397                  return False
 398              self.tee.flush()
 399              setattr(sys, self.channel, self.orig_stream)
 400              printed = self.buffer.getvalue()
 401              for s in self.s:
 402                  if isinstance(s, _re_type):
 403                      assert not s.search(printed),printed_msg.format(
 404                          s.pattern, self.channel, printed)
 405                  else:
 406                      assert s not in printed, printed_msg.format(
 407                          s, self.channel, printed)
 408              return False
 409          finally:
 410              self.tee.close()
 411  
 412  @contextmanager
 413  def mute_warn():
 414      from IPython.utils import warn
 415      save_warn = warn.warn
 416      warn.warn = lambda *a, **kw: None
 417      try:
 418          yield
 419      finally:
 420          warn.warn = save_warn
 421  
 422  @contextmanager
 423  def make_tempfile(name):
 424      """Create an empty, named, temporary file for the duration of the context."""
 425      open(name, "w", encoding="utf-8").close()
 426      try:
 427          yield
 428      finally:
 429          os.unlink(name)
 430  
 431  def fake_input(inputs):
 432      """Temporarily replace the input() function to return the given values
 433  
 434      Use as a context manager:
 435  
 436      with fake_input(['result1', 'result2']):
 437          ...
 438  
 439      Values are returned in order. If input() is called again after the last value
 440      was used, EOFError is raised.
 441      """
 442      it = iter(inputs)
 443      def mock_input(prompt=''):
 444          try:
 445              return next(it)
 446          except StopIteration as e:
 447              raise EOFError('No more inputs given') from e
 448  
 449      return patch('builtins.input', mock_input)
 450  
 451  def help_output_test(subcommand=''):
 452      """test that `ipython [subcommand] -h` works"""
 453      cmd = get_ipython_cmd() + [subcommand, '-h']
 454      out, err, rc = get_output_error_code(cmd)
 455      assert rc == 0, err
 456      assert "Traceback" not in err
 457      assert "Options" in out
 458      assert "--help-all" in out
 459      return out, err
 460  
 461  
 462  def help_all_output_test(subcommand=''):
 463      """test that `ipython [subcommand] --help-all` works"""
 464      cmd = get_ipython_cmd() + [subcommand, '--help-all']
 465      out, err, rc = get_output_error_code(cmd)
 466      assert rc == 0, err
 467      assert "Traceback" not in err
 468      assert "Options" in out
 469      assert "Class" in out
 470      return out, err
 471  
 472