1 """A ZMQ-based subclass of InteractiveShell. 2 3 This code is meant to ease the refactoring of the base InteractiveShell into 4 something with a cleaner architecture for 2-process use, without actually 5 breaking InteractiveShell itself. So we're doing something a bit ugly, where 6 we subclass and override what we want to fix. Once this is working well, we 7 can go back to the base class and refactor the code for a cleaner inheritance 8 implementation that doesn't rely on so much monkeypatching. 9 10 But this lets us maintain a fully working IPython as we develop the new 11 machinery. This should thus be thought of as scaffolding. 12 """ 13 14 # Copyright (c) IPython Development Team. 15 # Distributed under the terms of the Modified BSD License. 16 17 import os 18 import sys 19 import warnings 20 from pathlib import Path 21 from threading import local 22 23 from IPython.core import page, payloadpage 24 from IPython.core.autocall import ZMQExitAutocall 25 from IPython.core.displaypub import DisplayPublisher 26 from IPython.core.error import UsageError 27 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC 28 from IPython.core.magic import Magics, line_magic, magics_class 29 from IPython.core.magics import CodeMagics, MacroToEdit # type:ignore[attr-defined] 30 from IPython.core.usage import default_banner 31 from IPython.display import Javascript, display 32 from IPython.utils import openpy 33 from IPython.utils.process import arg_split, system # type:ignore[attr-defined] 34 from jupyter_client.session import Session, extract_header 35 from jupyter_core.paths import jupyter_runtime_dir 36 from traitlets import Any, CBool, CBytes, Dict, Instance, Type, default, observe 37 38 from ipykernel import connect_qtconsole, get_connection_file, get_connection_info 39 from ipykernel.displayhook import ZMQShellDisplayHook 40 from ipykernel.jsonutil import encode_images, json_clean 41 42 # ----------------------------------------------------------------------------- 43 # Functions and classes 44 # ----------------------------------------------------------------------------- 45 46 47 class ZMQDisplayPublisher(DisplayPublisher): 48 """A display publisher that publishes data using a ZeroMQ PUB socket.""" 49 50 session = Instance(Session, allow_none=True) 51 pub_socket = Any(allow_none=True) 52 parent_header = Dict({}) 53 topic = CBytes(b"display_data") 54 55 # thread_local: 56 # An attribute used to ensure the correct output message 57 # is processed. See ipykernel Issue 113 for a discussion. 58 _thread_local = Any() 59 60 def set_parent(self, parent): 61 """Set the parent for outbound messages.""" 62 self.parent_header = extract_header(parent) 63 64 def _flush_streams(self): 65 """flush IO Streams prior to display""" 66 sys.stdout.flush() 67 sys.stderr.flush() 68 69 @default("_thread_local") 70 def _default_thread_local(self): 71 """Initialize our thread local storage""" 72 return local() 73 74 @property 75 def _hooks(self): 76 if not hasattr(self._thread_local, "hooks"): 77 # create new list for a new thread 78 self._thread_local.hooks = [] 79 return self._thread_local.hooks 80 81 def publish( 82 self, 83 data, 84 metadata=None, 85 transient=None, 86 update=False, 87 ): 88 """Publish a display-data message 89 90 Parameters 91 ---------- 92 data : dict 93 A mime-bundle dict, keyed by mime-type. 94 metadata : dict, optional 95 Metadata associated with the data. 96 transient : dict, optional, keyword-only 97 Transient data that may only be relevant during a live display, 98 such as display_id. 99 Transient data should not be persisted to documents. 100 update : bool, optional, keyword-only 101 If True, send an update_display_data message instead of display_data. 102 """ 103 self._flush_streams() 104 if metadata is None: 105 metadata = {} 106 if transient is None: 107 transient = {} 108 self._validate_data(data, metadata) 109 content = {} 110 content["data"] = encode_images(data) 111 content["metadata"] = metadata 112 content["transient"] = transient 113 114 msg_type = "update_display_data" if update else "display_data" 115 116 # Use 2-stage process to send a message, 117 # in order to put it through the transform 118 # hooks before potentially sending. 119 assert self.session is not None 120 msg = self.session.msg(msg_type, json_clean(content), parent=self.parent_header) 121 122 # Each transform either returns a new 123 # message or None. If None is returned, 124 # the message has been 'used' and we return. 125 for hook in self._hooks: 126 msg = hook(msg) 127 if msg is None: 128 return # type:ignore[unreachable] 129 130 self.session.send( 131 self.pub_socket, 132 msg, 133 ident=self.topic, 134 ) 135 136 def clear_output(self, wait=False): 137 """Clear output associated with the current execution (cell). 138 139 Parameters 140 ---------- 141 wait : bool (default: False) 142 If True, the output will not be cleared immediately, 143 instead waiting for the next display before clearing. 144 This reduces bounce during repeated clear & display loops. 145 146 """ 147 content = dict(wait=wait) 148 self._flush_streams() 149 assert self.session is not None 150 msg = self.session.msg("clear_output", json_clean(content), parent=self.parent_header) 151 152 # see publish() for details on how this works 153 for hook in self._hooks: 154 msg = hook(msg) 155 if msg is None: 156 return # type:ignore[unreachable] 157 158 self.session.send( 159 self.pub_socket, 160 msg, 161 ident=self.topic, 162 ) 163 164 def register_hook(self, hook): 165 """ 166 Registers a hook with the thread-local storage. 167 168 Parameters 169 ---------- 170 hook : Any callable object 171 172 Returns 173 ------- 174 Either a publishable message, or `None`. 175 The DisplayHook objects must return a message from 176 the __call__ method if they still require the 177 `session.send` method to be called after transformation. 178 Returning `None` will halt that execution path, and 179 session.send will not be called. 180 """ 181 self._hooks.append(hook) 182 183 def unregister_hook(self, hook): 184 """ 185 Un-registers a hook with the thread-local storage. 186 187 Parameters 188 ---------- 189 hook : Any callable object which has previously been 190 registered as a hook. 191 192 Returns 193 ------- 194 bool - `True` if the hook was removed, `False` if it wasn't 195 found. 196 """ 197 try: 198 self._hooks.remove(hook) 199 return True 200 except ValueError: 201 return False 202 203 204 @magics_class 205 class KernelMagics(Magics): 206 """Kernel magics.""" 207 208 # ------------------------------------------------------------------------ 209 # Magic overrides 210 # ------------------------------------------------------------------------ 211 # Once the base class stops inheriting from magic, this code needs to be 212 # moved into a separate machinery as well. For now, at least isolate here 213 # the magics which this class needs to implement differently from the base 214 # class, or that are unique to it. 215 216 @line_magic 217 def edit(self, parameter_s="", last_call=None): 218 """Bring up an editor and execute the resulting code. 219 220 Usage: 221 %edit [options] [args] 222 223 %edit runs an external text editor. You will need to set the command for 224 this editor via the ``TerminalInteractiveShell.editor`` option in your 225 configuration file before it will work. 226 227 This command allows you to conveniently edit multi-line code right in 228 your IPython session. 229 230 If called without arguments, %edit opens up an empty editor with a 231 temporary file and will execute the contents of this file when you 232 close it (don't forget to save it!). 233 234 Options: 235 236 -n <number> 237 Open the editor at a specified line number. By default, the IPython 238 editor hook uses the unix syntax 'editor +N filename', but you can 239 configure this by providing your own modified hook if your favorite 240 editor supports line-number specifications with a different syntax. 241 242 -p 243 Call the editor with the same data as the previous time it was used, 244 regardless of how long ago (in your current session) it was. 245 246 -r 247 Use 'raw' input. This option only applies to input taken from the 248 user's history. By default, the 'processed' history is used, so that 249 magics are loaded in their transformed version to valid Python. If 250 this option is given, the raw input as typed as the command line is 251 used instead. When you exit the editor, it will be executed by 252 IPython's own processor. 253 254 Arguments: 255 256 If arguments are given, the following possibilities exist: 257 258 - The arguments are numbers or pairs of colon-separated numbers (like 259 1 4:8 9). These are interpreted as lines of previous input to be 260 loaded into the editor. The syntax is the same of the %macro command. 261 262 - If the argument doesn't start with a number, it is evaluated as a 263 variable and its contents loaded into the editor. You can thus edit 264 any string which contains python code (including the result of 265 previous edits). 266 267 - If the argument is the name of an object (other than a string), 268 IPython will try to locate the file where it was defined and open the 269 editor at the point where it is defined. You can use ``%edit function`` 270 to load an editor exactly at the point where 'function' is defined, 271 edit it and have the file be executed automatically. 272 273 If the object is a macro (see %macro for details), this opens up your 274 specified editor with a temporary file containing the macro's data. 275 Upon exit, the macro is reloaded with the contents of the file. 276 277 Note: opening at an exact line is only supported under Unix, and some 278 editors (like kedit and gedit up to Gnome 2.8) do not understand the 279 '+NUMBER' parameter necessary for this feature. Good editors like 280 (X)Emacs, vi, jed, pico and joe all do. 281 282 - If the argument is not found as a variable, IPython will look for a 283 file with that name (adding .py if necessary) and load it into the 284 editor. It will execute its contents with execfile() when you exit, 285 loading any code in the file into your interactive namespace. 286 287 Unlike in the terminal, this is designed to use a GUI editor, and we do 288 not know when it has closed. So the file you edit will not be 289 automatically executed or printed. 290 291 Note that %edit is also available through the alias %ed. 292 """ 293 last_call = last_call or ["", ""] 294 opts, args = self.parse_options(parameter_s, "prn:") 295 296 try: 297 filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call) 298 except MacroToEdit: 299 # TODO: Implement macro editing over 2 processes. 300 print("Macro editing not yet implemented in 2-process model.") 301 return 302 303 # Make sure we send to the client an absolute path, in case the working 304 # directory of client and kernel don't match 305 filename = str(Path(filename).resolve()) 306 307 payload = {"source": "edit_magic", "filename": filename, "line_number": lineno} 308 assert self.shell is not None 309 self.shell.payload_manager.write_payload(payload) 310 311 # A few magics that are adapted to the specifics of using pexpect and a 312 # remote terminal 313 314 @line_magic 315 def clear(self, arg_s): 316 """Clear the terminal.""" 317 assert self.shell is not None 318 if os.name == "posix": 319 self.shell.system("clear") 320 else: 321 self.shell.system("cls") 322 323 if os.name == "nt": 324 # This is the usual name in windows 325 cls = line_magic("cls")(clear) 326 327 # Terminal pagers won't work over pexpect, but we do have our own pager 328 329 @line_magic 330 def less(self, arg_s): 331 """Show a file through the pager. 332 333 Files ending in .py are syntax-highlighted.""" 334 if not arg_s: 335 msg = "Missing filename." 336 raise UsageError(msg) 337 338 if arg_s.endswith(".py"): 339 assert self.shell is not None 340 cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False)) 341 else: 342 with open(arg_s) as fid: 343 cont = fid.read() 344 page.page(cont) 345 346 more = line_magic("more")(less) 347 348 # Man calls a pager, so we also need to redefine it 349 if os.name == "posix": 350 351 @line_magic 352 def man(self, arg_s): 353 """Find the man page for the given command and display in pager.""" 354 assert self.shell is not None 355 page.page(self.shell.getoutput("man %s | col -b" % arg_s, split=False)) 356 357 @line_magic 358 def connect_info(self, arg_s): 359 """Print information for connecting other clients to this kernel 360 361 It will print the contents of this session's connection file, as well as 362 shortcuts for local clients. 363 364 In the simplest case, when called from the most recently launched kernel, 365 secondary clients can be connected, simply with: 366 367 $> jupyter <app> --existing 368 369 """ 370 371 try: 372 connection_file = get_connection_file() 373 info = get_connection_info(unpack=False) 374 except Exception as e: 375 warnings.warn("Could not get connection info: %r" % e, stacklevel=2) 376 return 377 378 # if it's in the default dir, truncate to basename 379 if jupyter_runtime_dir() == str(Path(connection_file).parent): 380 connection_file = Path(connection_file).name 381 382 assert isinstance(info, str) 383 print(info + "\n") 384 print( 385 f"Paste the above JSON into a file, and connect with:\n" 386 f" $> jupyter <app> --existing <file>\n" 387 f"or, if you are local, you can connect with just:\n" 388 f" $> jupyter <app> --existing {connection_file}\n" 389 f"or even just:\n" 390 f" $> jupyter <app> --existing\n" 391 f"if this is the most recent Jupyter kernel you have started." 392 ) 393 394 @line_magic 395 def qtconsole(self, arg_s): 396 """Open a qtconsole connected to this kernel. 397 398 Useful for connecting a qtconsole to running notebooks, for better 399 debugging. 400 """ 401 402 # %qtconsole should imply bind_kernel for engines: 403 # FIXME: move to ipyparallel Kernel subclass 404 if "ipyparallel" in sys.modules: 405 from ipyparallel import bind_kernel 406 407 bind_kernel() 408 409 try: 410 connect_qtconsole(argv=arg_split(arg_s, os.name == "posix")) 411 except Exception as e: 412 warnings.warn("Could not start qtconsole: %r" % e, stacklevel=2) 413 return 414 415 @line_magic 416 def autosave(self, arg_s): 417 """Set the autosave interval in the notebook (in seconds). 418 419 The default value is 120, or two minutes. 420 ``%autosave 0`` will disable autosave. 421 422 This magic only has an effect when called from the notebook interface. 423 It has no effect when called in a startup file. 424 """ 425 426 try: 427 interval = int(arg_s) 428 except ValueError as e: 429 raise UsageError("%%autosave requires an integer, got %r" % arg_s) from e 430 431 # javascript wants milliseconds 432 milliseconds = 1000 * interval 433 display( 434 Javascript("IPython.notebook.set_autosave_interval(%i)" % milliseconds), 435 include=["application/javascript"], 436 ) 437 if interval: 438 print("Autosaving every %i seconds" % interval) 439 else: 440 print("Autosave disabled") 441 442 443 class ZMQInteractiveShell(InteractiveShell): 444 """A subclass of InteractiveShell for ZMQ.""" 445 446 displayhook_class = Type(ZMQShellDisplayHook) 447 display_pub_class = Type(ZMQDisplayPublisher) 448 data_pub_class = Any() # type:ignore[assignment] 449 kernel = Any() 450 parent_header = Any() 451 452 @default("banner1") 453 def _default_banner1(self): 454 return default_banner 455 456 # Override the traitlet in the parent class, because there's no point using 457 # readline for the kernel. Can be removed when the readline code is moved 458 # to the terminal frontend. 459 readline_use = CBool(False) 460 # autoindent has no meaning in a zmqshell, and attempting to enable it 461 # will print a warning in the absence of readline. 462 autoindent = CBool(False) 463 464 exiter = Instance(ZMQExitAutocall) 465 466 @default("exiter") 467 def _default_exiter(self): 468 return ZMQExitAutocall(self) 469 470 @observe("exit_now") 471 def _update_exit_now(self, change): 472 """stop eventloop when exit_now fires""" 473 if change["new"]: 474 if hasattr(self.kernel, "io_loop"): 475 loop = self.kernel.io_loop 476 loop.call_later(0.1, loop.stop) 477 if self.kernel.eventloop: 478 exit_hook = getattr(self.kernel.eventloop, "exit_hook", None) 479 if exit_hook: 480 exit_hook(self.kernel) 481 482 keepkernel_on_exit = None 483 484 # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no 485 # interactive input being read; we provide event loop support in ipkernel 486 def enable_gui(self, gui): 487 """Enable a given guil.""" 488 from .eventloops import enable_gui as real_enable_gui 489 490 try: 491 real_enable_gui(gui) 492 self.active_eventloop = gui 493 except ValueError as e: 494 raise UsageError("%s" % e) from e 495 496 def init_environment(self): 497 """Configure the user's environment.""" 498 env = os.environ 499 # These two ensure 'ls' produces nice coloring on BSD-derived systems 500 env["TERM"] = "xterm-color" 501 env["CLICOLOR"] = "1" 502 # These two add terminal color in tools that support it. 503 env["FORCE_COLOR"] = "1" 504 env["CLICOLOR_FORCE"] = "1" 505 # Since normal pagers don't work at all (over pexpect we don't have 506 # single-key control of the subprocess), try to disable paging in 507 # subprocesses as much as possible. 508 env["PAGER"] = "cat" 509 env["GIT_PAGER"] = "cat" 510 511 def init_hooks(self): 512 """Initialize hooks.""" 513 super().init_hooks() 514 self.set_hook("show_in_pager", page.as_hook(payloadpage.page), 99) 515 516 def init_data_pub(self): 517 """Delay datapub init until request, for deprecation warnings""" 518 519 @property 520 def data_pub(self): 521 if not hasattr(self, "_data_pub"): 522 warnings.warn( 523 "InteractiveShell.data_pub is deprecated outside IPython parallel.", 524 DeprecationWarning, 525 stacklevel=2, 526 ) 527 528 self._data_pub = self.data_pub_class(parent=self) # type:ignore[has-type] 529 self._data_pub.session = self.display_pub.session # type:ignore[attr-defined] 530 self._data_pub.pub_socket = self.display_pub.pub_socket # type:ignore[attr-defined] 531 return self._data_pub 532 533 @data_pub.setter 534 def data_pub(self, pub): 535 self._data_pub = pub 536 537 def ask_exit(self): 538 """Engage the exit actions.""" 539 self.exit_now = not self.keepkernel_on_exit 540 payload = dict( 541 source="ask_exit", 542 keepkernel=self.keepkernel_on_exit, 543 ) 544 self.payload_manager.write_payload(payload) # type:ignore[union-attr] 545 546 def run_cell(self, *args, **kwargs): 547 """Run a cell.""" 548 self._last_traceback = None 549 return super().run_cell(*args, **kwargs) 550 551 def _showtraceback(self, etype, evalue, stb): 552 # try to preserve ordering of tracebacks and print statements 553 sys.stdout.flush() 554 sys.stderr.flush() 555 556 exc_content = { 557 "traceback": stb, 558 "ename": str(etype.__name__), 559 "evalue": str(evalue), 560 } 561 562 dh = self.displayhook 563 # Send exception info over pub socket for other clients than the caller 564 # to pick up 565 topic = None 566 if dh.topic: # type:ignore[attr-defined] 567 topic = dh.topic.replace(b"execute_result", b"error") # type:ignore[attr-defined] 568 569 dh.session.send( # type:ignore[attr-defined] 570 dh.pub_socket, # type:ignore[attr-defined] 571 "error", 572 json_clean(exc_content), 573 dh.parent_header, # type:ignore[attr-defined] 574 ident=topic, 575 ) 576 577 # FIXME - Once we rely on Python 3, the traceback is stored on the 578 # exception object, so we shouldn't need to store it here. 579 self._last_traceback = stb 580 581 def set_next_input(self, text, replace=False): 582 """Send the specified text to the frontend to be presented at the next 583 input cell.""" 584 payload = dict( 585 source="set_next_input", 586 text=text, 587 replace=replace, 588 ) 589 self.payload_manager.write_payload(payload) # type:ignore[union-attr] 590 591 def set_parent(self, parent): 592 """Set the parent header for associating output with its triggering input""" 593 self.parent_header = parent 594 self.displayhook.set_parent(parent) # type:ignore[attr-defined] 595 self.display_pub.set_parent(parent) # type:ignore[attr-defined] 596 if hasattr(self, "_data_pub"): 597 self.data_pub.set_parent(parent) 598 try: 599 sys.stdout.set_parent(parent) # type:ignore[attr-defined] 600 except AttributeError: 601 pass 602 try: 603 sys.stderr.set_parent(parent) # type:ignore[attr-defined] 604 except AttributeError: 605 pass 606 607 def get_parent(self): 608 """Get the parent header.""" 609 return self.parent_header 610 611 def init_magics(self): 612 """Initialize magics.""" 613 super().init_magics() 614 self.register_magics(KernelMagics) 615 self.magics_manager.register_alias("ed", "edit") 616 617 def init_virtualenv(self): 618 """Initialize virtual environment.""" 619 # Overridden not to do virtualenv detection, because it's probably 620 # not appropriate in a kernel. To use a kernel in a virtualenv, install 621 # it inside the virtualenv. 622 # https://ipython.readthedocs.io/en/latest/install/kernel_install.html 623 624 def system_piped(self, cmd): 625 """Call the given cmd in a subprocess, piping stdout/err 626 627 Parameters 628 ---------- 629 cmd : str 630 Command to execute (can not end in '&', as background processes are 631 not supported. Should not be a command that expects input 632 other than simple text. 633 """ 634 if cmd.rstrip().endswith("&"): 635 # this is *far* from a rigorous test 636 # We do not support backgrounding processes because we either use 637 # pexpect or pipes to read from. Users can always just call 638 # os.system() or use ip.system=ip.system_raw 639 # if they really want a background process. 640 msg = "Background processes not supported." 641 raise OSError(msg) 642 643 # we explicitly do NOT return the subprocess status code, because 644 # a non-None value would trigger :func:`sys.displayhook` calls. 645 # Instead, we store the exit_code in user_ns. 646 # Also, protect system call from UNC paths on Windows here too 647 # as is done in InteractiveShell.system_raw 648 if sys.platform == "win32": 649 cmd = self.var_expand(cmd, depth=1) 650 from IPython.utils._process_win32 import AvoidUNCPath 651 652 with AvoidUNCPath() as path: 653 if path is not None: 654 cmd = f"pushd {path} &&{cmd}" 655 self.user_ns["_exit_code"] = system(cmd) 656 else: 657 self.user_ns["_exit_code"] = system(self.var_expand(cmd, depth=1)) 658 659 # Ensure new system_piped implementation is used 660 system = system_piped 661 662 663 InteractiveShellABC.register(ZMQInteractiveShell) 664