1 """Implementation of code management magic functions. 2 """ 3 #----------------------------------------------------------------------------- 4 # Copyright (c) 2012 The IPython Development Team. 5 # 6 # Distributed under the terms of the Modified BSD License. 7 # 8 # The full license is in the file COPYING.txt, distributed with this software. 9 #----------------------------------------------------------------------------- 10 11 #----------------------------------------------------------------------------- 12 # Imports 13 #----------------------------------------------------------------------------- 14 15 # Stdlib 16 import inspect 17 import io 18 import os 19 import re 20 import sys 21 import ast 22 from itertools import chain 23 from urllib.request import Request, urlopen 24 from urllib.parse import urlencode 25 from pathlib import Path 26 27 # Our own packages 28 from IPython.core.error import TryNext, StdinNotImplementedError, UsageError 29 from IPython.core.macro import Macro 30 from IPython.core.magic import Magics, magics_class, line_magic 31 from IPython.core.oinspect import find_file, find_source_lines 32 from IPython.core.release import version 33 from IPython.testing.skipdoctest import skip_doctest 34 from IPython.utils.contexts import preserve_keys 35 from IPython.utils.path import get_py_filename 36 from warnings import warn 37 from logging import error 38 from IPython.utils.text import get_text_list 39 40 #----------------------------------------------------------------------------- 41 # Magic implementation classes 42 #----------------------------------------------------------------------------- 43 44 # Used for exception handling in magic_edit 45 class MacroToEdit(ValueError): pass 46 47 ipython_input_pat = re.compile(r"<ipython\-input\-(\d+)-[a-z\d]+>$") 48 49 # To match, e.g. 8-10 1:5 :10 3- 50 range_re = re.compile(r""" 51 (?P<start>\d+)? 52 ((?P<sep>[\-:]) 53 (?P<end>\d+)?)? 54 $""", re.VERBOSE) 55 56 57 def extract_code_ranges(ranges_str): 58 """Turn a string of range for %%load into 2-tuples of (start, stop) 59 ready to use as a slice of the content split by lines. 60 61 Examples 62 -------- 63 list(extract_input_ranges("5-10 2")) 64 [(4, 10), (1, 2)] 65 """ 66 for range_str in ranges_str.split(): 67 rmatch = range_re.match(range_str) 68 if not rmatch: 69 continue 70 sep = rmatch.group("sep") 71 start = rmatch.group("start") 72 end = rmatch.group("end") 73 74 if sep == '-': 75 start = int(start) - 1 if start else None 76 end = int(end) if end else None 77 elif sep == ':': 78 start = int(start) - 1 if start else None 79 end = int(end) - 1 if end else None 80 else: 81 end = int(start) 82 start = int(start) - 1 83 yield (start, end) 84 85 86 def extract_symbols(code, symbols): 87 """ 88 Return a tuple (blocks, not_found) 89 where ``blocks`` is a list of code fragments 90 for each symbol parsed from code, and ``not_found`` are 91 symbols not found in the code. 92 93 For example:: 94 95 In [1]: code = '''a = 10 96 ...: def b(): return 42 97 ...: class A: pass''' 98 99 In [2]: extract_symbols(code, 'A,b,z') 100 Out[2]: (['class A: pass\\n', 'def b(): return 42\\n'], ['z']) 101 """ 102 symbols = symbols.split(',') 103 104 # this will raise SyntaxError if code isn't valid Python 105 py_code = ast.parse(code) 106 107 marks = [(getattr(s, 'name', None), s.lineno) for s in py_code.body] 108 code = code.split('\n') 109 110 symbols_lines = {} 111 112 # we already know the start_lineno of each symbol (marks). 113 # To find each end_lineno, we traverse in reverse order until each 114 # non-blank line 115 end = len(code) 116 for name, start in reversed(marks): 117 while not code[end - 1].strip(): 118 end -= 1 119 if name: 120 symbols_lines[name] = (start - 1, end) 121 end = start - 1 122 123 # Now symbols_lines is a map 124 # {'symbol_name': (start_lineno, end_lineno), ...} 125 126 # fill a list with chunks of codes for each requested symbol 127 blocks = [] 128 not_found = [] 129 for symbol in symbols: 130 if symbol in symbols_lines: 131 start, end = symbols_lines[symbol] 132 blocks.append('\n'.join(code[start:end]) + '\n') 133 else: 134 not_found.append(symbol) 135 136 return blocks, not_found 137 138 def strip_initial_indent(lines): 139 """For %load, strip indent from lines until finding an unindented line. 140 141 https://github.com/ipython/ipython/issues/9775 142 """ 143 indent_re = re.compile(r'\s+') 144 145 it = iter(lines) 146 first_line = next(it) 147 indent_match = indent_re.match(first_line) 148 149 if indent_match: 150 # First line was indented 151 indent = indent_match.group() 152 yield first_line[len(indent):] 153 154 for line in it: 155 if line.startswith(indent): 156 yield line[len(indent) :] 157 elif line in ("\n", "\r\n") or len(line) == 0: 158 yield line 159 else: 160 # Less indented than the first line - stop dedenting 161 yield line 162 break 163 else: 164 yield first_line 165 166 # Pass the remaining lines through without dedenting 167 for line in it: 168 yield line 169 170 171 class InteractivelyDefined(Exception): 172 """Exception for interactively defined variable in magic_edit""" 173 def __init__(self, index): 174 self.index = index 175 176 177 @magics_class 178 class CodeMagics(Magics): 179 """Magics related to code management (loading, saving, editing, ...).""" 180 181 def __init__(self, *args, **kwargs): 182 self._knowntemps = set() 183 super(CodeMagics, self).__init__(*args, **kwargs) 184 185 @line_magic 186 def save(self, parameter_s=''): 187 """Save a set of lines or a macro to a given filename. 188 189 Usage:\\ 190 %save [options] filename [history] 191 192 Options: 193 194 -r: use 'raw' input. By default, the 'processed' history is used, 195 so that magics are loaded in their transformed version to valid 196 Python. If this option is given, the raw input as typed as the 197 command line is used instead. 198 199 -f: force overwrite. If file exists, %save will prompt for overwrite 200 unless -f is given. 201 202 -a: append to the file instead of overwriting it. 203 204 The history argument uses the same syntax as %history for input ranges, 205 then saves the lines to the filename you specify. 206 207 If no ranges are specified, saves history of the current session up to 208 this point. 209 210 It adds a '.py' extension to the file if you don't do so yourself, and 211 it asks for confirmation before overwriting existing files. 212 213 If `-r` option is used, the default extension is `.ipy`. 214 """ 215 216 opts,args = self.parse_options(parameter_s,'fra',mode='list') 217 if not args: 218 raise UsageError('Missing filename.') 219 raw = 'r' in opts 220 force = 'f' in opts 221 append = 'a' in opts 222 mode = 'a' if append else 'w' 223 ext = '.ipy' if raw else '.py' 224 fname, codefrom = args[0], " ".join(args[1:]) 225 if not fname.endswith(('.py','.ipy')): 226 fname += ext 227 fname = os.path.expanduser(fname) 228 file_exists = os.path.isfile(fname) 229 if file_exists and not force and not append: 230 try: 231 overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n') 232 except StdinNotImplementedError: 233 print("File `%s` exists. Use `%%save -f %s` to force overwrite" % (fname, parameter_s)) 234 return 235 if not overwrite : 236 print('Operation cancelled.') 237 return 238 try: 239 cmds = self.shell.find_user_code(codefrom,raw) 240 except (TypeError, ValueError) as e: 241 print(e.args[0]) 242 return 243 with io.open(fname, mode, encoding="utf-8") as f: 244 if not file_exists or not append: 245 f.write("# coding: utf-8\n") 246 f.write(cmds) 247 # make sure we end on a newline 248 if not cmds.endswith('\n'): 249 f.write('\n') 250 print('The following commands were written to file `%s`:' % fname) 251 print(cmds) 252 253 @line_magic 254 def pastebin(self, parameter_s=''): 255 """Upload code to dpaste.com, returning the URL. 256 257 Usage:\\ 258 %pastebin [-d "Custom description"][-e 24] 1-7 259 260 The argument can be an input history range, a filename, or the name of a 261 string or macro. 262 263 If no arguments are given, uploads the history of this session up to 264 this point. 265 266 Options: 267 268 -d: Pass a custom description. The default will say 269 "Pasted from IPython". 270 -e: Pass number of days for the link to be expired. 271 The default will be 7 days. 272 """ 273 opts, args = self.parse_options(parameter_s, "d:e:") 274 275 try: 276 code = self.shell.find_user_code(args) 277 except (ValueError, TypeError) as e: 278 print(e.args[0]) 279 return 280 281 expiry_days = 7 282 try: 283 expiry_days = int(opts.get("e", 7)) 284 except ValueError as e: 285 print(e.args[0].capitalize()) 286 return 287 if expiry_days < 1 or expiry_days > 365: 288 print("Expiry days should be in range of 1 to 365") 289 return 290 291 post_data = urlencode( 292 { 293 "title": opts.get("d", "Pasted from IPython"), 294 "syntax": "python", 295 "content": code, 296 "expiry_days": expiry_days, 297 } 298 ).encode("utf-8") 299 300 request = Request( 301 "https://dpaste.com/api/v2/", 302 headers={"User-Agent": "IPython v{}".format(version)}, 303 ) 304 response = urlopen(request, post_data) 305 return response.headers.get('Location') 306 307 @line_magic 308 def loadpy(self, arg_s): 309 """Alias of `%load` 310 311 `%loadpy` has gained some flexibility and dropped the requirement of a `.py` 312 extension. So it has been renamed simply into %load. You can look at 313 `%load`'s docstring for more info. 314 """ 315 self.load(arg_s) 316 317 @line_magic 318 def load(self, arg_s): 319 """Load code into the current frontend. 320 321 Usage:\\ 322 %load [options] source 323 324 where source can be a filename, URL, input history range, macro, or 325 element in the user namespace 326 327 If no arguments are given, loads the history of this session up to this 328 point. 329 330 Options: 331 332 -r <lines>: Specify lines or ranges of lines to load from the source. 333 Ranges could be specified as x-y (x..y) or in python-style x:y 334 (x..(y-1)). Both limits x and y can be left blank (meaning the 335 beginning and end of the file, respectively). 336 337 -s <symbols>: Specify function or classes to load from python source. 338 339 -y : Don't ask confirmation for loading source above 200 000 characters. 340 341 -n : Include the user's namespace when searching for source code. 342 343 This magic command can either take a local filename, a URL, an history 344 range (see %history) or a macro as argument, it will prompt for 345 confirmation before loading source with more than 200 000 characters, unless 346 -y flag is passed or if the frontend does not support raw_input:: 347 348 %load 349 %load myscript.py 350 %load 7-27 351 %load myMacro 352 %load http://www.example.com/myscript.py 353 %load -r 5-10 myscript.py 354 %load -r 10-20,30,40: foo.py 355 %load -s MyClass,wonder_function myscript.py 356 %load -n MyClass 357 %load -n my_module.wonder_function 358 """ 359 opts,args = self.parse_options(arg_s,'yns:r:') 360 search_ns = 'n' in opts 361 contents = self.shell.find_user_code(args, search_ns=search_ns) 362 363 if 's' in opts: 364 try: 365 blocks, not_found = extract_symbols(contents, opts['s']) 366 except SyntaxError: 367 # non python code 368 error("Unable to parse the input as valid Python code") 369 return 370 371 if len(not_found) == 1: 372 warn('The symbol `%s` was not found' % not_found[0]) 373 elif len(not_found) > 1: 374 warn('The symbols %s were not found' % get_text_list(not_found, 375 wrap_item_with='`') 376 ) 377 378 contents = '\n'.join(blocks) 379 380 if 'r' in opts: 381 ranges = opts['r'].replace(',', ' ') 382 lines = contents.split('\n') 383 slices = extract_code_ranges(ranges) 384 contents = [lines[slice(*slc)] for slc in slices] 385 contents = '\n'.join(strip_initial_indent(chain.from_iterable(contents))) 386 387 l = len(contents) 388 389 # 200 000 is ~ 2500 full 80 character lines 390 # so in average, more than 5000 lines 391 if l > 200000 and 'y' not in opts: 392 try: 393 ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\ 394 " (%d characters). Continue (y/[N]) ?" % l), default='n' ) 395 except StdinNotImplementedError: 396 #assume yes if raw input not implemented 397 ans = True 398 399 if ans is False : 400 print('Operation cancelled.') 401 return 402 403 contents = "# %load {}\n".format(arg_s) + contents 404 405 self.shell.set_next_input(contents, replace=True) 406 407 @staticmethod 408 def _find_edit_target(shell, args, opts, last_call): 409 """Utility method used by magic_edit to find what to edit.""" 410 411 def make_filename(arg): 412 "Make a filename from the given args" 413 try: 414 filename = get_py_filename(arg) 415 except IOError: 416 # If it ends with .py but doesn't already exist, assume we want 417 # a new file. 418 if arg.endswith('.py'): 419 filename = arg 420 else: 421 filename = None 422 return filename 423 424 # Set a few locals from the options for convenience: 425 opts_prev = 'p' in opts 426 opts_raw = 'r' in opts 427 428 # custom exceptions 429 class DataIsObject(Exception): pass 430 431 # Default line number value 432 lineno = opts.get('n',None) 433 434 if opts_prev: 435 args = '_%s' % last_call[0] 436 if args not in shell.user_ns: 437 args = last_call[1] 438 439 # by default this is done with temp files, except when the given 440 # arg is a filename 441 use_temp = True 442 443 data = '' 444 445 # First, see if the arguments should be a filename. 446 filename = make_filename(args) 447 if filename: 448 use_temp = False 449 elif args: 450 # Mode where user specifies ranges of lines, like in %macro. 451 data = shell.extract_input_lines(args, opts_raw) 452 if not data: 453 try: 454 # Load the parameter given as a variable. If not a string, 455 # process it as an object instead (below) 456 457 # print('*** args',args,'type',type(args)) # dbg 458 data = eval(args, shell.user_ns) 459 if not isinstance(data, str): 460 raise DataIsObject 461 462 except (NameError,SyntaxError): 463 # given argument is not a variable, try as a filename 464 filename = make_filename(args) 465 if filename is None: 466 warn("Argument given (%s) can't be found as a variable " 467 "or as a filename." % args) 468 return (None, None, None) 469 use_temp = False 470 471 except DataIsObject as e: 472 # macros have a special edit function 473 if isinstance(data, Macro): 474 raise MacroToEdit(data) from e 475 476 # For objects, try to edit the file where they are defined 477 filename = find_file(data) 478 if filename: 479 if 'fakemodule' in filename.lower() and \ 480 inspect.isclass(data): 481 # class created by %edit? Try to find source 482 # by looking for method definitions instead, the 483 # __module__ in those classes is FakeModule. 484 attrs = [getattr(data, aname) for aname in dir(data)] 485 for attr in attrs: 486 if not inspect.ismethod(attr): 487 continue 488 filename = find_file(attr) 489 if filename and \ 490 'fakemodule' not in filename.lower(): 491 # change the attribute to be the edit 492 # target instead 493 data = attr 494 break 495 496 m = ipython_input_pat.match(os.path.basename(filename)) 497 if m: 498 raise InteractivelyDefined(int(m.groups()[0])) from e 499 500 datafile = 1 501 if filename is None: 502 filename = make_filename(args) 503 datafile = 1 504 if filename is not None: 505 # only warn about this if we get a real name 506 warn('Could not find file where `%s` is defined.\n' 507 'Opening a file named `%s`' % (args, filename)) 508 # Now, make sure we can actually read the source (if it was 509 # in a temp file it's gone by now). 510 if datafile: 511 if lineno is None: 512 lineno = find_source_lines(data) 513 if lineno is None: 514 filename = make_filename(args) 515 if filename is None: 516 warn('The file where `%s` was defined ' 517 'cannot be read or found.' % data) 518 return (None, None, None) 519 use_temp = False 520 521 if use_temp: 522 filename = shell.mktempfile(data) 523 print('IPython will make a temporary file named:',filename) 524 525 # use last_call to remember the state of the previous call, but don't 526 # let it be clobbered by successive '-p' calls. 527 try: 528 last_call[0] = shell.displayhook.prompt_count 529 if not opts_prev: 530 last_call[1] = args 531 except: 532 pass 533 534 535 return filename, lineno, use_temp 536 537 def _edit_macro(self,mname,macro): 538 """open an editor with the macro data in a file""" 539 filename = self.shell.mktempfile(macro.value) 540 self.shell.hooks.editor(filename) 541 542 # and make a new macro object, to replace the old one 543 mvalue = Path(filename).read_text(encoding="utf-8") 544 self.shell.user_ns[mname] = Macro(mvalue) 545 546 @skip_doctest 547 @line_magic 548 def edit(self, parameter_s='',last_call=['','']): 549 """Bring up an editor and execute the resulting code. 550 551 Usage: 552 %edit [options] [args] 553 554 %edit runs IPython's editor hook. The default version of this hook is 555 set to call the editor specified by your $EDITOR environment variable. 556 If this isn't found, it will default to vi under Linux/Unix and to 557 notepad under Windows. See the end of this docstring for how to change 558 the editor hook. 559 560 You can also set the value of this editor via the 561 ``TerminalInteractiveShell.editor`` option in your configuration file. 562 This is useful if you wish to use a different editor from your typical 563 default with IPython (and for Windows users who typically don't set 564 environment variables). 565 566 This command allows you to conveniently edit multi-line code right in 567 your IPython session. 568 569 If called without arguments, %edit opens up an empty editor with a 570 temporary file and will execute the contents of this file when you 571 close it (don't forget to save it!). 572 573 574 Options: 575 576 -n <number>: open the editor at a specified line number. By default, 577 the IPython editor hook uses the unix syntax 'editor +N filename', but 578 you can configure this by providing your own modified hook if your 579 favorite editor supports line-number specifications with a different 580 syntax. 581 582 -p: this will call the editor with the same data as the previous time 583 it was used, regardless of how long ago (in your current session) it 584 was. 585 586 -r: use 'raw' input. This option only applies to input taken from the 587 user's history. By default, the 'processed' history is used, so that 588 magics are loaded in their transformed version to valid Python. If 589 this option is given, the raw input as typed as the command line is 590 used instead. When you exit the editor, it will be executed by 591 IPython's own processor. 592 593 -x: do not execute the edited code immediately upon exit. This is 594 mainly useful if you are editing programs which need to be called with 595 command line arguments, which you can then do using %run. 596 597 598 Arguments: 599 600 If arguments are given, the following possibilities exist: 601 602 - If the argument is a filename, IPython will load that into the 603 editor. It will execute its contents with execfile() when you exit, 604 loading any code in the file into your interactive namespace. 605 606 - The arguments are ranges of input history, e.g. "7 ~1/4-6". 607 The syntax is the same as in the %history magic. 608 609 - If the argument is a string variable, its contents are loaded 610 into the editor. You can thus edit any string which contains 611 python code (including the result of previous edits). 612 613 - If the argument is the name of an object (other than a string), 614 IPython will try to locate the file where it was defined and open the 615 editor at the point where it is defined. You can use `%edit function` 616 to load an editor exactly at the point where 'function' is defined, 617 edit it and have the file be executed automatically. 618 619 - If the object is a macro (see %macro for details), this opens up your 620 specified editor with a temporary file containing the macro's data. 621 Upon exit, the macro is reloaded with the contents of the file. 622 623 Note: opening at an exact line is only supported under Unix, and some 624 editors (like kedit and gedit up to Gnome 2.8) do not understand the 625 '+NUMBER' parameter necessary for this feature. Good editors like 626 (X)Emacs, vi, jed, pico and joe all do. 627 628 After executing your code, %edit will return as output the code you 629 typed in the editor (except when it was an existing file). This way 630 you can reload the code in further invocations of %edit as a variable, 631 via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of 632 the output. 633 634 Note that %edit is also available through the alias %ed. 635 636 This is an example of creating a simple function inside the editor and 637 then modifying it. First, start up the editor:: 638 639 In [1]: edit 640 Editing... done. Executing edited code... 641 Out[1]: 'def foo():\\n print("foo() was defined in an editing 642 session")\\n' 643 644 We can then call the function foo():: 645 646 In [2]: foo() 647 foo() was defined in an editing session 648 649 Now we edit foo. IPython automatically loads the editor with the 650 (temporary) file where foo() was previously defined:: 651 652 In [3]: edit foo 653 Editing... done. Executing edited code... 654 655 And if we call foo() again we get the modified version:: 656 657 In [4]: foo() 658 foo() has now been changed! 659 660 Here is an example of how to edit a code snippet successive 661 times. First we call the editor:: 662 663 In [5]: edit 664 Editing... done. Executing edited code... 665 hello 666 Out[5]: "print('hello')\\n" 667 668 Now we call it again with the previous output (stored in _):: 669 670 In [6]: edit _ 671 Editing... done. Executing edited code... 672 hello world 673 Out[6]: "print('hello world')\\n" 674 675 Now we call it with the output #8 (stored in _8, also as Out[8]):: 676 677 In [7]: edit _8 678 Editing... done. Executing edited code... 679 hello again 680 Out[7]: "print('hello again')\\n" 681 682 683 Changing the default editor hook: 684 685 If you wish to write your own editor hook, you can put it in a 686 configuration file which you load at startup time. The default hook 687 is defined in the IPython.core.hooks module, and you can use that as a 688 starting example for further modifications. That file also has 689 general instructions on how to set a new hook for use once you've 690 defined it.""" 691 opts,args = self.parse_options(parameter_s,'prxn:') 692 693 try: 694 filename, lineno, is_temp = self._find_edit_target(self.shell, 695 args, opts, last_call) 696 except MacroToEdit as e: 697 self._edit_macro(args, e.args[0]) 698 return 699 except InteractivelyDefined as e: 700 print("Editing In[%i]" % e.index) 701 args = str(e.index) 702 filename, lineno, is_temp = self._find_edit_target(self.shell, 703 args, opts, last_call) 704 if filename is None: 705 # nothing was found, warnings have already been issued, 706 # just give up. 707 return 708 709 if is_temp: 710 self._knowntemps.add(filename) 711 elif (filename in self._knowntemps): 712 is_temp = True 713 714 715 # do actual editing here 716 print('Editing...', end=' ') 717 sys.stdout.flush() 718 filepath = Path(filename) 719 try: 720 # Quote filenames that may have spaces in them when opening 721 # the editor 722 quoted = filename = str(filepath.absolute()) 723 if " " in quoted: 724 quoted = "'%s'" % quoted 725 self.shell.hooks.editor(quoted, lineno) 726 except TryNext: 727 warn('Could not open editor') 728 return 729 730 # XXX TODO: should this be generalized for all string vars? 731 # For now, this is special-cased to blocks created by cpaste 732 if args.strip() == "pasted_block": 733 self.shell.user_ns["pasted_block"] = filepath.read_text(encoding="utf-8") 734 735 if 'x' in opts: # -x prevents actual execution 736 print() 737 else: 738 print('done. Executing edited code...') 739 with preserve_keys(self.shell.user_ns, '__file__'): 740 if not is_temp: 741 self.shell.user_ns["__file__"] = filename 742 if "r" in opts: # Untranslated IPython code 743 source = filepath.read_text(encoding="utf-8") 744 self.shell.run_cell(source, store_history=False) 745 else: 746 self.shell.safe_execfile(filename, self.shell.user_ns, 747 self.shell.user_ns) 748 749 if is_temp: 750 try: 751 return filepath.read_text(encoding="utf-8") 752 except IOError as msg: 753 if Path(msg.filename) == filepath: 754 warn('File not found. Did you forget to save?') 755 return 756 else: 757 self.shell.showtraceback() 758