1  """ History related magics and functionality """
   2  
   3  # Copyright (c) IPython Development Team.
   4  # Distributed under the terms of the Modified BSD License.
   5  
   6  
   7  import atexit
   8  import datetime
   9  import re
  10  import sqlite3
  11  import threading
  12  from pathlib import Path
  13  
  14  from decorator import decorator
  15  from traitlets import (
  16      Any,
  17      Bool,
  18      Dict,
  19      Instance,
  20      Integer,
  21      List,
  22      TraitError,
  23      Unicode,
  24      Union,
  25      default,
  26      observe,
  27  )
  28  from traitlets.config.configurable import LoggingConfigurable
  29  
  30  from IPython.paths import locate_profile
  31  from IPython.utils.decorators import undoc
  32  
  33  #-----------------------------------------------------------------------------
  34  # Classes and functions
  35  #-----------------------------------------------------------------------------
  36  
  37  @undoc
  38  class DummyDB(object):
  39      """Dummy DB that will act as a black hole for history.
  40  
  41      Only used in the absence of sqlite"""
  42      def execute(*args, **kwargs):
  43          return []
  44  
  45      def commit(self, *args, **kwargs):
  46          pass
  47  
  48      def __enter__(self, *args, **kwargs):
  49          pass
  50  
  51      def __exit__(self, *args, **kwargs):
  52          pass
  53  
  54  
  55  @decorator
  56  def only_when_enabled(f, self, *a, **kw):
  57      """Decorator: return an empty list in the absence of sqlite."""
  58      if not self.enabled:
  59          return []
  60      else:
  61          return f(self, *a, **kw)
  62  
  63  
  64  # use 16kB as threshold for whether a corrupt history db should be saved
  65  # that should be at least 100 entries or so
  66  _SAVE_DB_SIZE = 16384
  67  
  68  @decorator
  69  def catch_corrupt_db(f, self, *a, **kw):
  70      """A decorator which wraps HistoryAccessor method calls to catch errors from
  71      a corrupt SQLite database, move the old database out of the way, and create
  72      a new one.
  73  
  74      We avoid clobbering larger databases because this may be triggered due to filesystem issues,
  75      not just a corrupt file.
  76      """
  77      try:
  78          return f(self, *a, **kw)
  79      except (sqlite3.DatabaseError, sqlite3.OperationalError) as e:
  80          self._corrupt_db_counter += 1
  81          self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
  82          if self.hist_file != ':memory:':
  83              if self._corrupt_db_counter > self._corrupt_db_limit:
  84                  self.hist_file = ':memory:'
  85                  self.log.error("Failed to load history too many times, history will not be saved.")
  86              elif self.hist_file.is_file():
  87                  # move the file out of the way
  88                  base = str(self.hist_file.parent / self.hist_file.stem)
  89                  ext = self.hist_file.suffix
  90                  size = self.hist_file.stat().st_size
  91                  if size >= _SAVE_DB_SIZE:
  92                      # if there's significant content, avoid clobbering
  93                      now = datetime.datetime.now().isoformat().replace(':', '.')
  94                      newpath = base + '-corrupt-' + now + ext
  95                      # don't clobber previous corrupt backups
  96                      for i in range(100):
  97                          if not Path(newpath).exists():
  98                              break
  99                          else:
 100                              newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
 101                  else:
 102                      # not much content, possibly empty; don't worry about clobbering
 103                      # maybe we should just delete it?
 104                      newpath = base + '-corrupt' + ext
 105                  self.hist_file.rename(newpath)
 106                  self.log.error("History file was moved to %s and a new file created.", newpath)
 107              self.init_db()
 108              return []
 109          else:
 110              # Failed with :memory:, something serious is wrong
 111              raise
 112  
 113  
 114  class HistoryAccessorBase(LoggingConfigurable):
 115      """An abstract class for History Accessors """
 116  
 117      def get_tail(self, n=10, raw=True, output=False, include_latest=False):
 118          raise NotImplementedError
 119  
 120      def search(self, pattern="*", raw=True, search_raw=True,
 121                 output=False, n=None, unique=False):
 122          raise NotImplementedError
 123  
 124      def get_range(self, session, start=1, stop=None, raw=True,output=False):
 125          raise NotImplementedError
 126  
 127      def get_range_by_str(self, rangestr, raw=True, output=False):
 128          raise NotImplementedError
 129  
 130  
 131  class HistoryAccessor(HistoryAccessorBase):
 132      """Access the history database without adding to it.
 133  
 134      This is intended for use by standalone history tools. IPython shells use
 135      HistoryManager, below, which is a subclass of this."""
 136  
 137      # counter for init_db retries, so we don't keep trying over and over
 138      _corrupt_db_counter = 0
 139       # after two failures, fallback on :memory:
 140      _corrupt_db_limit = 2
 141  
 142      # String holding the path to the history file
 143      hist_file = Union(
 144          [Instance(Path), Unicode()],
 145          help="""Path to file to use for SQLite history database.
 146  
 147          By default, IPython will put the history database in the IPython
 148          profile directory.  If you would rather share one history among
 149          profiles, you can set this value in each, so that they are consistent.
 150  
 151          Due to an issue with fcntl, SQLite is known to misbehave on some NFS
 152          mounts.  If you see IPython hanging, try setting this to something on a
 153          local disk, e.g::
 154  
 155              ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
 156  
 157          you can also use the specific value `:memory:` (including the colon
 158          at both end but not the back ticks), to avoid creating an history file.
 159  
 160          """,
 161      ).tag(config=True)
 162  
 163      enabled = Bool(True,
 164          help="""enable the SQLite history
 165  
 166          set enabled=False to disable the SQLite history,
 167          in which case there will be no stored history, no SQLite connection,
 168          and no background saving thread.  This may be necessary in some
 169          threaded environments where IPython is embedded.
 170          """,
 171      ).tag(config=True)
 172  
 173      connection_options = Dict(
 174          help="""Options for configuring the SQLite connection
 175  
 176          These options are passed as keyword args to sqlite3.connect
 177          when establishing database connections.
 178          """
 179      ).tag(config=True)
 180  
 181      @default("connection_options")
 182      def _default_connection_options(self):
 183          return dict(check_same_thread=False)
 184  
 185      # The SQLite database
 186      db = Any()
 187      @observe('db')
 188      def _db_changed(self, change):
 189          """validate the db, since it can be an Instance of two different types"""
 190          new = change['new']
 191          connection_types = (DummyDB, sqlite3.Connection)
 192          if not isinstance(new, connection_types):
 193              msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
 194                      (self.__class__.__name__, new)
 195              raise TraitError(msg)
 196  
 197      def __init__(self, profile="default", hist_file="", **traits):
 198          """Create a new history accessor.
 199  
 200          Parameters
 201          ----------
 202          profile : str
 203              The name of the profile from which to open history.
 204          hist_file : str
 205              Path to an SQLite history database stored by IPython. If specified,
 206              hist_file overrides profile.
 207          config : :class:`~traitlets.config.loader.Config`
 208              Config object. hist_file can also be set through this.
 209          """
 210          super(HistoryAccessor, self).__init__(**traits)
 211          # defer setting hist_file from kwarg until after init,
 212          # otherwise the default kwarg value would clobber any value
 213          # set by config
 214          if hist_file:
 215              self.hist_file = hist_file
 216  
 217          try:
 218              self.hist_file
 219          except TraitError:
 220              # No one has set the hist_file, yet.
 221              self.hist_file = self._get_hist_file_name(profile)
 222  
 223          self.init_db()
 224  
 225      def _get_hist_file_name(self, profile='default'):
 226          """Find the history file for the given profile name.
 227  
 228          This is overridden by the HistoryManager subclass, to use the shell's
 229          active profile.
 230  
 231          Parameters
 232          ----------
 233          profile : str
 234              The name of a profile which has a history file.
 235          """
 236          return Path(locate_profile(profile)) / "history.sqlite"
 237  
 238      @catch_corrupt_db
 239      def init_db(self):
 240          """Connect to the database, and create tables if necessary."""
 241          if not self.enabled:
 242              self.db = DummyDB()
 243              return
 244  
 245          # use detect_types so that timestamps return datetime objects
 246          kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
 247          kwargs.update(self.connection_options)
 248          self.db = sqlite3.connect(str(self.hist_file), **kwargs)
 249          with self.db:
 250              self.db.execute(
 251                  """CREATE TABLE IF NOT EXISTS sessions (session integer
 252                              primary key autoincrement, start timestamp,
 253                              end timestamp, num_cmds integer, remark text)"""
 254              )
 255              self.db.execute(
 256                  """CREATE TABLE IF NOT EXISTS history
 257                      (session integer, line integer, source text, source_raw text,
 258                      PRIMARY KEY (session, line))"""
 259              )
 260              # Output history is optional, but ensure the table's there so it can be
 261              # enabled later.
 262              self.db.execute(
 263                  """CREATE TABLE IF NOT EXISTS output_history
 264                              (session integer, line integer, output text,
 265                              PRIMARY KEY (session, line))"""
 266              )
 267          # success! reset corrupt db count
 268          self._corrupt_db_counter = 0
 269  
 270      def writeout_cache(self):
 271          """Overridden by HistoryManager to dump the cache before certain
 272          database lookups."""
 273          pass
 274  
 275      ## -------------------------------
 276      ## Methods for retrieving history:
 277      ## -------------------------------
 278      def _run_sql(self, sql, params, raw=True, output=False, latest=False):
 279          """Prepares and runs an SQL query for the history database.
 280  
 281          Parameters
 282          ----------
 283          sql : str
 284              Any filtering expressions to go after SELECT ... FROM ...
 285          params : tuple
 286              Parameters passed to the SQL query (to replace "?")
 287          raw, output : bool
 288              See :meth:`get_range`
 289          latest : bool
 290              Select rows with max (session, line)
 291  
 292          Returns
 293          -------
 294          Tuples as :meth:`get_range`
 295          """
 296          toget = 'source_raw' if raw else 'source'
 297          sqlfrom = "history"
 298          if output:
 299              sqlfrom = "history LEFT JOIN output_history USING (session, line)"
 300              toget = "history.%s, output_history.output" % toget
 301          if latest:
 302              toget += ", MAX(session * 128 * 1024 + line)"
 303          this_querry = "SELECT session, line, %s FROM %s " % (toget, sqlfrom) + sql
 304          cur = self.db.execute(this_querry, params)
 305          if latest:
 306              cur = (row[:-1] for row in cur)
 307          if output:    # Regroup into 3-tuples, and parse JSON
 308              return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
 309          return cur
 310  
 311      @only_when_enabled
 312      @catch_corrupt_db
 313      def get_session_info(self, session):
 314          """Get info about a session.
 315  
 316          Parameters
 317          ----------
 318          session : int
 319              Session number to retrieve.
 320  
 321          Returns
 322          -------
 323          session_id : int
 324              Session ID number
 325          start : datetime
 326              Timestamp for the start of the session.
 327          end : datetime
 328              Timestamp for the end of the session, or None if IPython crashed.
 329          num_cmds : int
 330              Number of commands run, or None if IPython crashed.
 331          remark : unicode
 332              A manually set description.
 333          """
 334          query = "SELECT * from sessions where session == ?"
 335          return self.db.execute(query, (session,)).fetchone()
 336  
 337      @catch_corrupt_db
 338      def get_last_session_id(self):
 339          """Get the last session ID currently in the database.
 340  
 341          Within IPython, this should be the same as the value stored in
 342          :attr:`HistoryManager.session_number`.
 343          """
 344          for record in self.get_tail(n=1, include_latest=True):
 345              return record[0]
 346  
 347      @catch_corrupt_db
 348      def get_tail(self, n=10, raw=True, output=False, include_latest=False):
 349          """Get the last n lines from the history database.
 350  
 351          Parameters
 352          ----------
 353          n : int
 354              The number of lines to get
 355          raw, output : bool
 356              See :meth:`get_range`
 357          include_latest : bool
 358              If False (default), n+1 lines are fetched, and the latest one
 359              is discarded. This is intended to be used where the function
 360              is called by a user command, which it should not return.
 361  
 362          Returns
 363          -------
 364          Tuples as :meth:`get_range`
 365          """
 366          self.writeout_cache()
 367          if not include_latest:
 368              n += 1
 369          cur = self._run_sql(
 370              "ORDER BY session DESC, line DESC LIMIT ?", (n,), raw=raw, output=output
 371          )
 372          if not include_latest:
 373              return reversed(list(cur)[1:])
 374          return reversed(list(cur))
 375  
 376      @catch_corrupt_db
 377      def search(self, pattern="*", raw=True, search_raw=True,
 378                 output=False, n=None, unique=False):
 379          """Search the database using unix glob-style matching (wildcards
 380          * and ?).
 381  
 382          Parameters
 383          ----------
 384          pattern : str
 385              The wildcarded pattern to match when searching
 386          search_raw : bool
 387              If True, search the raw input, otherwise, the parsed input
 388          raw, output : bool
 389              See :meth:`get_range`
 390          n : None or int
 391              If an integer is given, it defines the limit of
 392              returned entries.
 393          unique : bool
 394              When it is true, return only unique entries.
 395  
 396          Returns
 397          -------
 398          Tuples as :meth:`get_range`
 399          """
 400          tosearch = "source_raw" if search_raw else "source"
 401          if output:
 402              tosearch = "history." + tosearch
 403          self.writeout_cache()
 404          sqlform = "WHERE %s GLOB ?" % tosearch
 405          params = (pattern,)
 406          if unique:
 407              sqlform += ' GROUP BY {0}'.format(tosearch)
 408          if n is not None:
 409              sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
 410              params += (n,)
 411          elif unique:
 412              sqlform += " ORDER BY session, line"
 413          cur = self._run_sql(sqlform, params, raw=raw, output=output, latest=unique)
 414          if n is not None:
 415              return reversed(list(cur))
 416          return cur
 417  
 418      @catch_corrupt_db
 419      def get_range(self, session, start=1, stop=None, raw=True,output=False):
 420          """Retrieve input by session.
 421  
 422          Parameters
 423          ----------
 424          session : int
 425              Session number to retrieve.
 426          start : int
 427              First line to retrieve.
 428          stop : int
 429              End of line range (excluded from output itself). If None, retrieve
 430              to the end of the session.
 431          raw : bool
 432              If True, return untranslated input
 433          output : bool
 434              If True, attempt to include output. This will be 'real' Python
 435              objects for the current session, or text reprs from previous
 436              sessions if db_log_output was enabled at the time. Where no output
 437              is found, None is used.
 438  
 439          Returns
 440          -------
 441          entries
 442              An iterator over the desired lines. Each line is a 3-tuple, either
 443              (session, line, input) if output is False, or
 444              (session, line, (input, output)) if output is True.
 445          """
 446          if stop:
 447              lineclause = "line >= ? AND line < ?"
 448              params = (session, start, stop)
 449          else:
 450              lineclause = "line>=?"
 451              params = (session, start)
 452  
 453          return self._run_sql("WHERE session==? AND %s" % lineclause,
 454                                      params, raw=raw, output=output)
 455  
 456      def get_range_by_str(self, rangestr, raw=True, output=False):
 457          """Get lines of history from a string of ranges, as used by magic
 458          commands %hist, %save, %macro, etc.
 459  
 460          Parameters
 461          ----------
 462          rangestr : str
 463              A string specifying ranges, e.g. "5 ~2/1-4". If empty string is used,
 464              this will return everything from current session's history.
 465  
 466              See the documentation of :func:`%history` for the full details.
 467  
 468          raw, output : bool
 469              As :meth:`get_range`
 470  
 471          Returns
 472          -------
 473          Tuples as :meth:`get_range`
 474          """
 475          for sess, s, e in extract_hist_ranges(rangestr):
 476              for line in self.get_range(sess, s, e, raw=raw, output=output):
 477                  yield line
 478  
 479  
 480  class HistoryManager(HistoryAccessor):
 481      """A class to organize all history-related functionality in one place.
 482      """
 483      # Public interface
 484  
 485      # An instance of the IPython shell we are attached to
 486      shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
 487                       allow_none=True)
 488      # Lists to hold processed and raw history. These start with a blank entry
 489      # so that we can index them starting from 1
 490      input_hist_parsed = List([""])
 491      input_hist_raw = List([""])
 492      # A list of directories visited during session
 493      dir_hist: List = List()
 494  
 495      @default("dir_hist")
 496      def _dir_hist_default(self):
 497          try:
 498              return [Path.cwd()]
 499          except OSError:
 500              return []
 501  
 502      # A dict of output history, keyed with ints from the shell's
 503      # execution count.
 504      output_hist = Dict()
 505      # The text/plain repr of outputs.
 506      output_hist_reprs = Dict()
 507  
 508      # The number of the current session in the history database
 509      session_number = Integer()
 510  
 511      db_log_output = Bool(False,
 512          help="Should the history database include output? (default: no)"
 513      ).tag(config=True)
 514      db_cache_size = Integer(0,
 515          help="Write to database every x commands (higher values save disk access & power).\n"
 516          "Values of 1 or less effectively disable caching."
 517      ).tag(config=True)
 518      # The input and output caches
 519      db_input_cache: List = List()
 520      db_output_cache: List = List()
 521  
 522      # History saving in separate thread
 523      save_thread = Instance('IPython.core.history.HistorySavingThread',
 524                             allow_none=True)
 525      save_flag = Instance(threading.Event, allow_none=True)
 526  
 527      # Private interface
 528      # Variables used to store the three last inputs from the user.  On each new
 529      # history update, we populate the user's namespace with these, shifted as
 530      # necessary.
 531      _i00 = Unicode("")
 532      _i = Unicode("")
 533      _ii = Unicode("")
 534      _iii = Unicode("")
 535  
 536      # A regex matching all forms of the exit command, so that we don't store
 537      # them in the history (it's annoying to rewind the first entry and land on
 538      # an exit call).
 539      _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
 540  
 541      def __init__(self, shell=None, config=None, **traits):
 542          """Create a new history manager associated with a shell instance.
 543          """
 544          super(HistoryManager, self).__init__(shell=shell, config=config,
 545              **traits)
 546          self.save_flag = threading.Event()
 547          self.db_input_cache_lock = threading.Lock()
 548          self.db_output_cache_lock = threading.Lock()
 549  
 550          try:
 551              self.new_session()
 552          except sqlite3.OperationalError:
 553              self.log.error("Failed to create history session in %s. History will not be saved.",
 554                  self.hist_file, exc_info=True)
 555              self.hist_file = ':memory:'
 556  
 557          if self.enabled and self.hist_file != ':memory:':
 558              self.save_thread = HistorySavingThread(self)
 559              try:
 560                  self.save_thread.start()
 561              except RuntimeError:
 562                  self.log.error(
 563                      "Failed to start history saving thread. History will not be saved.",
 564                      exc_info=True,
 565                  )
 566                  self.hist_file = ":memory:"
 567  
 568      def _get_hist_file_name(self, profile=None):
 569          """Get default history file name based on the Shell's profile.
 570  
 571          The profile parameter is ignored, but must exist for compatibility with
 572          the parent class."""
 573          profile_dir = self.shell.profile_dir.location
 574          return Path(profile_dir) / "history.sqlite"
 575  
 576      @only_when_enabled
 577      def new_session(self, conn=None):
 578          """Get a new session number."""
 579          if conn is None:
 580              conn = self.db
 581  
 582          with conn:
 583              cur = conn.execute(
 584                  """INSERT INTO sessions VALUES (NULL, ?, NULL,
 585                              NULL, '') """,
 586                  (datetime.datetime.now().isoformat(" "),),
 587              )
 588              self.session_number = cur.lastrowid
 589  
 590      def end_session(self):
 591          """Close the database session, filling in the end time and line count."""
 592          self.writeout_cache()
 593          with self.db:
 594              self.db.execute(
 595                  """UPDATE sessions SET end=?, num_cmds=? WHERE
 596                              session==?""",
 597                  (
 598                      datetime.datetime.now().isoformat(" "),
 599                      len(self.input_hist_parsed) - 1,
 600                      self.session_number,
 601                  ),
 602              )
 603          self.session_number = 0
 604  
 605      def name_session(self, name):
 606          """Give the current session a name in the history database."""
 607          with self.db:
 608              self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
 609                              (name, self.session_number))
 610  
 611      def reset(self, new_session=True):
 612          """Clear the session history, releasing all object references, and
 613          optionally open a new session."""
 614          self.output_hist.clear()
 615          # The directory history can't be completely empty
 616          self.dir_hist[:] = [Path.cwd()]
 617  
 618          if new_session:
 619              if self.session_number:
 620                  self.end_session()
 621              self.input_hist_parsed[:] = [""]
 622              self.input_hist_raw[:] = [""]
 623              self.new_session()
 624  
 625      # ------------------------------
 626      # Methods for retrieving history
 627      # ------------------------------
 628      def get_session_info(self, session=0):
 629          """Get info about a session.
 630  
 631          Parameters
 632          ----------
 633          session : int
 634              Session number to retrieve. The current session is 0, and negative
 635              numbers count back from current session, so -1 is the previous session.
 636  
 637          Returns
 638          -------
 639          session_id : int
 640              Session ID number
 641          start : datetime
 642              Timestamp for the start of the session.
 643          end : datetime
 644              Timestamp for the end of the session, or None if IPython crashed.
 645          num_cmds : int
 646              Number of commands run, or None if IPython crashed.
 647          remark : unicode
 648              A manually set description.
 649          """
 650          if session <= 0:
 651              session += self.session_number
 652  
 653          return super(HistoryManager, self).get_session_info(session=session)
 654  
 655      @catch_corrupt_db
 656      def get_tail(self, n=10, raw=True, output=False, include_latest=False):
 657          """Get the last n lines from the history database.
 658  
 659          Most recent entry last.
 660  
 661          Completion will be reordered so that that the last ones are when
 662          possible from current session.
 663  
 664          Parameters
 665          ----------
 666          n : int
 667              The number of lines to get
 668          raw, output : bool
 669              See :meth:`get_range`
 670          include_latest : bool
 671              If False (default), n+1 lines are fetched, and the latest one
 672              is discarded. This is intended to be used where the function
 673              is called by a user command, which it should not return.
 674  
 675          Returns
 676          -------
 677          Tuples as :meth:`get_range`
 678          """
 679          self.writeout_cache()
 680          if not include_latest:
 681              n += 1
 682          # cursor/line/entry
 683          this_cur = list(
 684              self._run_sql(
 685                  "WHERE session == ? ORDER BY line DESC LIMIT ?  ",
 686                  (self.session_number, n),
 687                  raw=raw,
 688                  output=output,
 689              )
 690          )
 691          other_cur = list(
 692              self._run_sql(
 693                  "WHERE session != ? ORDER BY session DESC, line DESC LIMIT ?",
 694                  (self.session_number, n),
 695                  raw=raw,
 696                  output=output,
 697              )
 698          )
 699  
 700          everything = this_cur + other_cur
 701  
 702          everything = everything[:n]
 703  
 704          if not include_latest:
 705              return list(everything)[:0:-1]
 706          return list(everything)[::-1]
 707  
 708      def _get_range_session(self, start=1, stop=None, raw=True, output=False):
 709          """Get input and output history from the current session. Called by
 710          get_range, and takes similar parameters."""
 711          input_hist = self.input_hist_raw if raw else self.input_hist_parsed
 712  
 713          n = len(input_hist)
 714          if start < 0:
 715              start += n
 716          if not stop or (stop > n):
 717              stop = n
 718          elif stop < 0:
 719              stop += n
 720  
 721          for i in range(start, stop):
 722              if output:
 723                  line = (input_hist[i], self.output_hist_reprs.get(i))
 724              else:
 725                  line = input_hist[i]
 726              yield (0, i, line)
 727  
 728      def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
 729          """Retrieve input by session.
 730  
 731          Parameters
 732          ----------
 733          session : int
 734              Session number to retrieve. The current session is 0, and negative
 735              numbers count back from current session, so -1 is previous session.
 736          start : int
 737              First line to retrieve.
 738          stop : int
 739              End of line range (excluded from output itself). If None, retrieve
 740              to the end of the session.
 741          raw : bool
 742              If True, return untranslated input
 743          output : bool
 744              If True, attempt to include output. This will be 'real' Python
 745              objects for the current session, or text reprs from previous
 746              sessions if db_log_output was enabled at the time. Where no output
 747              is found, None is used.
 748  
 749          Returns
 750          -------
 751          entries
 752              An iterator over the desired lines. Each line is a 3-tuple, either
 753              (session, line, input) if output is False, or
 754              (session, line, (input, output)) if output is True.
 755          """
 756          if session <= 0:
 757              session += self.session_number
 758          if session==self.session_number:          # Current session
 759              return self._get_range_session(start, stop, raw, output)
 760          return super(HistoryManager, self).get_range(session, start, stop, raw,
 761                                                       output)
 762  
 763      ## ----------------------------
 764      ## Methods for storing history:
 765      ## ----------------------------
 766      def store_inputs(self, line_num, source, source_raw=None):
 767          """Store source and raw input in history and create input cache
 768          variables ``_i*``.
 769  
 770          Parameters
 771          ----------
 772          line_num : int
 773              The prompt number of this input.
 774          source : str
 775              Python input.
 776          source_raw : str, optional
 777              If given, this is the raw input without any IPython transformations
 778              applied to it.  If not given, ``source`` is used.
 779          """
 780          if source_raw is None:
 781              source_raw = source
 782          source = source.rstrip('\n')
 783          source_raw = source_raw.rstrip('\n')
 784  
 785          # do not store exit/quit commands
 786          if self._exit_re.match(source_raw.strip()):
 787              return
 788  
 789          self.input_hist_parsed.append(source)
 790          self.input_hist_raw.append(source_raw)
 791  
 792          with self.db_input_cache_lock:
 793              self.db_input_cache.append((line_num, source, source_raw))
 794              # Trigger to flush cache and write to DB.
 795              if len(self.db_input_cache) >= self.db_cache_size:
 796                  self.save_flag.set()
 797  
 798          # update the auto _i variables
 799          self._iii = self._ii
 800          self._ii = self._i
 801          self._i = self._i00
 802          self._i00 = source_raw
 803  
 804          # hackish access to user namespace to create _i1,_i2... dynamically
 805          new_i = '_i%s' % line_num
 806          to_main = {'_i': self._i,
 807                     '_ii': self._ii,
 808                     '_iii': self._iii,
 809                     new_i : self._i00 }
 810  
 811          if self.shell is not None:
 812              self.shell.push(to_main, interactive=False)
 813  
 814      def store_output(self, line_num):
 815          """If database output logging is enabled, this saves all the
 816          outputs from the indicated prompt number to the database. It's
 817          called by run_cell after code has been executed.
 818  
 819          Parameters
 820          ----------
 821          line_num : int
 822              The line number from which to save outputs
 823          """
 824          if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
 825              return
 826          output = self.output_hist_reprs[line_num]
 827  
 828          with self.db_output_cache_lock:
 829              self.db_output_cache.append((line_num, output))
 830          if self.db_cache_size <= 1:
 831              self.save_flag.set()
 832  
 833      def _writeout_input_cache(self, conn):
 834          with conn:
 835              for line in self.db_input_cache:
 836                  conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
 837                                  (self.session_number,)+line)
 838  
 839      def _writeout_output_cache(self, conn):
 840          with conn:
 841              for line in self.db_output_cache:
 842                  conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
 843                                  (self.session_number,)+line)
 844  
 845      @only_when_enabled
 846      def writeout_cache(self, conn=None):
 847          """Write any entries in the cache to the database."""
 848          if conn is None:
 849              conn = self.db
 850  
 851          with self.db_input_cache_lock:
 852              try:
 853                  self._writeout_input_cache(conn)
 854              except sqlite3.IntegrityError:
 855                  self.new_session(conn)
 856                  print("ERROR! Session/line number was not unique in",
 857                        "database. History logging moved to new session",
 858                                                  self.session_number)
 859                  try:
 860                      # Try writing to the new session. If this fails, don't
 861                      # recurse
 862                      self._writeout_input_cache(conn)
 863                  except sqlite3.IntegrityError:
 864                      pass
 865              finally:
 866                  self.db_input_cache = []
 867  
 868          with self.db_output_cache_lock:
 869              try:
 870                  self._writeout_output_cache(conn)
 871              except sqlite3.IntegrityError:
 872                  print("!! Session/line number for output was not unique",
 873                        "in database. Output will not be stored.")
 874              finally:
 875                  self.db_output_cache = []
 876  
 877  
 878  class HistorySavingThread(threading.Thread):
 879      """This thread takes care of writing history to the database, so that
 880      the UI isn't held up while that happens.
 881  
 882      It waits for the HistoryManager's save_flag to be set, then writes out
 883      the history cache. The main thread is responsible for setting the flag when
 884      the cache size reaches a defined threshold."""
 885      daemon = True
 886      stop_now = False
 887      enabled = True
 888      def __init__(self, history_manager):
 889          super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
 890          self.history_manager = history_manager
 891          self.enabled = history_manager.enabled
 892  
 893      @only_when_enabled
 894      def run(self):
 895          atexit.register(self.stop)
 896          # We need a separate db connection per thread:
 897          try:
 898              self.db = sqlite3.connect(
 899                  str(self.history_manager.hist_file),
 900                  **self.history_manager.connection_options,
 901              )
 902              while True:
 903                  self.history_manager.save_flag.wait()
 904                  if self.stop_now:
 905                      self.db.close()
 906                      return
 907                  self.history_manager.save_flag.clear()
 908                  self.history_manager.writeout_cache(self.db)
 909          except Exception as e:
 910              print(("The history saving thread hit an unexpected error (%s)."
 911                     "History will not be written to the database.") % repr(e))
 912          finally:
 913              atexit.unregister(self.stop)
 914  
 915      def stop(self):
 916          """This can be called from the main thread to safely stop this thread.
 917  
 918          Note that it does not attempt to write out remaining history before
 919          exiting. That should be done by calling the HistoryManager's
 920          end_session method."""
 921          self.stop_now = True
 922          self.history_manager.save_flag.set()
 923          self.join()
 924  
 925  
 926  # To match, e.g. ~5/8-~2/3
 927  range_re = re.compile(r"""
 928  ((?P<startsess>~?\d+)/)?
 929  (?P<start>\d+)?
 930  ((?P<sep>[\-:])
 931   ((?P<endsess>~?\d+)/)?
 932   (?P<end>\d+))?
 933  $""", re.VERBOSE)
 934  
 935  
 936  def extract_hist_ranges(ranges_str):
 937      """Turn a string of history ranges into 3-tuples of (session, start, stop).
 938  
 939      Empty string results in a `[(0, 1, None)]`, i.e. "everything from current
 940      session".
 941  
 942      Examples
 943      --------
 944      >>> list(extract_hist_ranges("~8/5-~7/4 2"))
 945      [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
 946      """
 947      if ranges_str == "":
 948          yield (0, 1, None)  # Everything from current session
 949          return
 950  
 951      for range_str in ranges_str.split():
 952          rmatch = range_re.match(range_str)
 953          if not rmatch:
 954              continue
 955          start = rmatch.group("start")
 956          if start:
 957              start = int(start)
 958              end = rmatch.group("end")
 959              # If no end specified, get (a, a + 1)
 960              end = int(end) if end else start + 1
 961          else:  # start not specified
 962              if not rmatch.group('startsess'):  # no startsess
 963                  continue
 964              start = 1
 965              end = None  # provide the entire session hist
 966  
 967          if rmatch.group("sep") == "-":       # 1-3 == 1:4 --> [1, 2, 3]
 968              end += 1
 969          startsess = rmatch.group("startsess") or "0"
 970          endsess = rmatch.group("endsess") or startsess
 971          startsess = int(startsess.replace("~","-"))
 972          endsess = int(endsess.replace("~","-"))
 973          assert endsess >= startsess, "start session must be earlier than end session"
 974  
 975          if endsess == startsess:
 976              yield (startsess, start, end)
 977              continue
 978          # Multiple sessions in one range:
 979          yield (startsess, start, None)
 980          for sess in range(startsess+1, endsess):
 981              yield (sess, 1, None)
 982          yield (endsess, 1, end)
 983  
 984  
 985  def _format_lineno(session, line):
 986      """Helper function to format line numbers properly."""
 987      if session == 0:
 988          return str(line)
 989      return "%s#%s" % (session, line)
 990