1 # -*- coding: utf-8 -*- 2 """ 3 Defines a variety of Pygments lexers for highlighting IPython code. 4 5 This includes: 6 7 IPythonLexer, IPython3Lexer 8 Lexers for pure IPython (python + magic/shell commands) 9 10 IPythonPartialTracebackLexer, IPythonTracebackLexer 11 Supports 2.x and 3.x via keyword `python3`. The partial traceback 12 lexer reads everything but the Python code appearing in a traceback. 13 The full lexer combines the partial lexer with an IPython lexer. 14 15 IPythonConsoleLexer 16 A lexer for IPython console sessions, with support for tracebacks. 17 18 IPyLexer 19 A friendly lexer which examines the first line of text and from it, 20 decides whether to use an IPython lexer or an IPython console lexer. 21 This is probably the only lexer that needs to be explicitly added 22 to Pygments. 23 24 """ 25 #----------------------------------------------------------------------------- 26 # Copyright (c) 2013, the IPython Development Team. 27 # 28 # Distributed under the terms of the Modified BSD License. 29 # 30 # The full license is in the file COPYING.txt, distributed with this software. 31 #----------------------------------------------------------------------------- 32 33 # Standard library 34 import re 35 36 # Third party 37 from pygments.lexers import ( 38 BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer, 39 Python3Lexer, TexLexer) 40 from pygments.lexer import ( 41 Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using, 42 ) 43 from pygments.token import ( 44 Generic, Keyword, Literal, Name, Operator, Other, Text, Error, 45 ) 46 from pygments.util import get_bool_opt 47 48 # Local 49 50 line_re = re.compile('.*?\n') 51 52 __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer', 53 'IPythonPartialTracebackLexer', 'IPythonTracebackLexer', 54 'IPythonConsoleLexer', 'IPyLexer'] 55 56 57 def build_ipy_lexer(python3): 58 """Builds IPython lexers depending on the value of `python3`. 59 60 The lexer inherits from an appropriate Python lexer and then adds 61 information about IPython specific keywords (i.e. magic commands, 62 shell commands, etc.) 63 64 Parameters 65 ---------- 66 python3 : bool 67 If `True`, then build an IPython lexer from a Python 3 lexer. 68 69 """ 70 # It would be nice to have a single IPython lexer class which takes 71 # a boolean `python3`. But since there are two Python lexer classes, 72 # we will also have two IPython lexer classes. 73 if python3: 74 PyLexer = Python3Lexer 75 name = 'IPython3' 76 aliases = ['ipython3'] 77 doc = """IPython3 Lexer""" 78 else: 79 PyLexer = PythonLexer 80 name = 'IPython' 81 aliases = ['ipython2', 'ipython'] 82 doc = """IPython Lexer""" 83 84 ipython_tokens = [ 85 (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 86 (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 87 (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))), 88 (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))), 89 (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))), 90 (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))), 91 (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))), 92 (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 93 (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 94 (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 95 (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))), 96 (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), 97 (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))), 98 (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 99 (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 100 (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 101 (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), 102 (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)), 103 (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))), 104 (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)), 105 (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)), 106 (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword, 107 using(BashLexer), Text)), 108 (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)), 109 (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)), 110 (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)), 111 (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)), 112 (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)), 113 ] 114 115 tokens = PyLexer.tokens.copy() 116 tokens['root'] = ipython_tokens + tokens['root'] 117 118 attrs = {'name': name, 'aliases': aliases, 'filenames': [], 119 '__doc__': doc, 'tokens': tokens} 120 121 return type(name, (PyLexer,), attrs) 122 123 124 IPython3Lexer = build_ipy_lexer(python3=True) 125 IPythonLexer = build_ipy_lexer(python3=False) 126 127 128 class IPythonPartialTracebackLexer(RegexLexer): 129 """ 130 Partial lexer for IPython tracebacks. 131 132 Handles all the non-python output. 133 134 """ 135 name = 'IPython Partial Traceback' 136 137 tokens = { 138 'root': [ 139 # Tracebacks for syntax errors have a different style. 140 # For both types of tracebacks, we mark the first line with 141 # Generic.Traceback. For syntax errors, we mark the filename 142 # as we mark the filenames for non-syntax tracebacks. 143 # 144 # These two regexps define how IPythonConsoleLexer finds a 145 # traceback. 146 # 147 ## Non-syntax traceback 148 (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)), 149 ## Syntax traceback 150 (r'^( File)(.*)(, line )(\d+\n)', 151 bygroups(Generic.Traceback, Name.Namespace, 152 Generic.Traceback, Literal.Number.Integer)), 153 154 # (Exception Identifier)(Whitespace)(Traceback Message) 155 (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)', 156 bygroups(Name.Exception, Generic.Whitespace, Text)), 157 # (Module/Filename)(Text)(Callee)(Function Signature) 158 # Better options for callee and function signature? 159 (r'(.*)( in )(.*)(\(.*\)\n)', 160 bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)), 161 # Regular line: (Whitespace)(Line Number)(Python Code) 162 (r'(\s*?)(\d+)(.*?\n)', 163 bygroups(Generic.Whitespace, Literal.Number.Integer, Other)), 164 # Emphasized line: (Arrow)(Line Number)(Python Code) 165 # Using Exception token so arrow color matches the Exception. 166 (r'(-*>?\s?)(\d+)(.*?\n)', 167 bygroups(Name.Exception, Literal.Number.Integer, Other)), 168 # (Exception Identifier)(Message) 169 (r'(?u)(^[^\d\W]\w*)(:.*?\n)', 170 bygroups(Name.Exception, Text)), 171 # Tag everything else as Other, will be handled later. 172 (r'.*\n', Other), 173 ], 174 } 175 176 177 class IPythonTracebackLexer(DelegatingLexer): 178 """ 179 IPython traceback lexer. 180 181 For doctests, the tracebacks can be snipped as much as desired with the 182 exception to the lines that designate a traceback. For non-syntax error 183 tracebacks, this is the line of hyphens. For syntax error tracebacks, 184 this is the line which lists the File and line number. 185 186 """ 187 # The lexer inherits from DelegatingLexer. The "root" lexer is an 188 # appropriate IPython lexer, which depends on the value of the boolean 189 # `python3`. First, we parse with the partial IPython traceback lexer. 190 # Then, any code marked with the "Other" token is delegated to the root 191 # lexer. 192 # 193 name = 'IPython Traceback' 194 aliases = ['ipythontb'] 195 196 def __init__(self, **options): 197 """ 198 A subclass of `DelegatingLexer` which delegates to the appropriate to either IPyLexer, 199 IPythonPartialTracebackLexer. 200 """ 201 # note we need a __init__ doc, as otherwise it inherits the doc from the super class 202 # which will fail the documentation build as it references section of the pygments docs that 203 # do not exists when building IPython's docs. 204 self.python3 = get_bool_opt(options, 'python3', False) 205 if self.python3: 206 self.aliases = ['ipython3tb'] 207 else: 208 self.aliases = ['ipython2tb', 'ipythontb'] 209 210 if self.python3: 211 IPyLexer = IPython3Lexer 212 else: 213 IPyLexer = IPythonLexer 214 215 DelegatingLexer.__init__(self, IPyLexer, 216 IPythonPartialTracebackLexer, **options) 217 218 class IPythonConsoleLexer(Lexer): 219 """ 220 An IPython console lexer for IPython code-blocks and doctests, such as: 221 222 .. code-block:: rst 223 224 .. code-block:: ipythonconsole 225 226 In [1]: a = 'foo' 227 228 In [2]: a 229 Out[2]: 'foo' 230 231 In [3]: print(a) 232 foo 233 234 235 Support is also provided for IPython exceptions: 236 237 .. code-block:: rst 238 239 .. code-block:: ipythonconsole 240 241 In [1]: raise Exception 242 Traceback (most recent call last): 243 ... 244 Exception 245 246 """ 247 name = 'IPython console session' 248 aliases = ['ipythonconsole'] 249 mimetypes = ['text/x-ipython-console'] 250 251 # The regexps used to determine what is input and what is output. 252 # The default prompts for IPython are: 253 # 254 # in = 'In [#]: ' 255 # continuation = ' .D.: ' 256 # template = 'Out[#]: ' 257 # 258 # Where '#' is the 'prompt number' or 'execution count' and 'D' 259 # D is a number of dots matching the width of the execution count 260 # 261 in1_regex = r'In \[[0-9]+\]: ' 262 in2_regex = r' \.\.+\.: ' 263 out_regex = r'Out\[[0-9]+\]: ' 264 265 #: The regex to determine when a traceback starts. 266 ipytb_start = re.compile(r'^(\^C)?(-+\n)|^( File)(.*)(, line )(\d+\n)') 267 268 def __init__(self, **options): 269 """Initialize the IPython console lexer. 270 271 Parameters 272 ---------- 273 python3 : bool 274 If `True`, then the console inputs are parsed using a Python 3 275 lexer. Otherwise, they are parsed using a Python 2 lexer. 276 in1_regex : RegexObject 277 The compiled regular expression used to detect the start 278 of inputs. Although the IPython configuration setting may have a 279 trailing whitespace, do not include it in the regex. If `None`, 280 then the default input prompt is assumed. 281 in2_regex : RegexObject 282 The compiled regular expression used to detect the continuation 283 of inputs. Although the IPython configuration setting may have a 284 trailing whitespace, do not include it in the regex. If `None`, 285 then the default input prompt is assumed. 286 out_regex : RegexObject 287 The compiled regular expression used to detect outputs. If `None`, 288 then the default output prompt is assumed. 289 290 """ 291 self.python3 = get_bool_opt(options, 'python3', False) 292 if self.python3: 293 self.aliases = ['ipython3console'] 294 else: 295 self.aliases = ['ipython2console', 'ipythonconsole'] 296 297 in1_regex = options.get('in1_regex', self.in1_regex) 298 in2_regex = options.get('in2_regex', self.in2_regex) 299 out_regex = options.get('out_regex', self.out_regex) 300 301 # So that we can work with input and output prompts which have been 302 # rstrip'd (possibly by editors) we also need rstrip'd variants. If 303 # we do not do this, then such prompts will be tagged as 'output'. 304 # The reason can't just use the rstrip'd variants instead is because 305 # we want any whitespace associated with the prompt to be inserted 306 # with the token. This allows formatted code to be modified so as hide 307 # the appearance of prompts, with the whitespace included. One example 308 # use of this is in copybutton.js from the standard lib Python docs. 309 in1_regex_rstrip = in1_regex.rstrip() + '\n' 310 in2_regex_rstrip = in2_regex.rstrip() + '\n' 311 out_regex_rstrip = out_regex.rstrip() + '\n' 312 313 # Compile and save them all. 314 attrs = ['in1_regex', 'in2_regex', 'out_regex', 315 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip'] 316 for attr in attrs: 317 self.__setattr__(attr, re.compile(locals()[attr])) 318 319 Lexer.__init__(self, **options) 320 321 if self.python3: 322 pylexer = IPython3Lexer 323 tblexer = IPythonTracebackLexer 324 else: 325 pylexer = IPythonLexer 326 tblexer = IPythonTracebackLexer 327 328 self.pylexer = pylexer(**options) 329 self.tblexer = tblexer(**options) 330 331 self.reset() 332 333 def reset(self): 334 self.mode = 'output' 335 self.index = 0 336 self.buffer = u'' 337 self.insertions = [] 338 339 def buffered_tokens(self): 340 """ 341 Generator of unprocessed tokens after doing insertions and before 342 changing to a new state. 343 344 """ 345 if self.mode == 'output': 346 tokens = [(0, Generic.Output, self.buffer)] 347 elif self.mode == 'input': 348 tokens = self.pylexer.get_tokens_unprocessed(self.buffer) 349 else: # traceback 350 tokens = self.tblexer.get_tokens_unprocessed(self.buffer) 351 352 for i, t, v in do_insertions(self.insertions, tokens): 353 # All token indexes are relative to the buffer. 354 yield self.index + i, t, v 355 356 # Clear it all 357 self.index += len(self.buffer) 358 self.buffer = u'' 359 self.insertions = [] 360 361 def get_mci(self, line): 362 """ 363 Parses the line and returns a 3-tuple: (mode, code, insertion). 364 365 `mode` is the next mode (or state) of the lexer, and is always equal 366 to 'input', 'output', or 'tb'. 367 368 `code` is a portion of the line that should be added to the buffer 369 corresponding to the next mode and eventually lexed by another lexer. 370 For example, `code` could be Python code if `mode` were 'input'. 371 372 `insertion` is a 3-tuple (index, token, text) representing an 373 unprocessed "token" that will be inserted into the stream of tokens 374 that are created from the buffer once we change modes. This is usually 375 the input or output prompt. 376 377 In general, the next mode depends on current mode and on the contents 378 of `line`. 379 380 """ 381 # To reduce the number of regex match checks, we have multiple 382 # 'if' blocks instead of 'if-elif' blocks. 383 384 # Check for possible end of input 385 in2_match = self.in2_regex.match(line) 386 in2_match_rstrip = self.in2_regex_rstrip.match(line) 387 if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \ 388 in2_match_rstrip: 389 end_input = True 390 else: 391 end_input = False 392 if end_input and self.mode != 'tb': 393 # Only look for an end of input when not in tb mode. 394 # An ellipsis could appear within the traceback. 395 mode = 'output' 396 code = u'' 397 insertion = (0, Generic.Prompt, line) 398 return mode, code, insertion 399 400 # Check for output prompt 401 out_match = self.out_regex.match(line) 402 out_match_rstrip = self.out_regex_rstrip.match(line) 403 if out_match or out_match_rstrip: 404 mode = 'output' 405 if out_match: 406 idx = out_match.end() 407 else: 408 idx = out_match_rstrip.end() 409 code = line[idx:] 410 # Use the 'heading' token for output. We cannot use Generic.Error 411 # since it would conflict with exceptions. 412 insertion = (0, Generic.Heading, line[:idx]) 413 return mode, code, insertion 414 415 416 # Check for input or continuation prompt (non stripped version) 417 in1_match = self.in1_regex.match(line) 418 if in1_match or (in2_match and self.mode != 'tb'): 419 # New input or when not in tb, continued input. 420 # We do not check for continued input when in tb since it is 421 # allowable to replace a long stack with an ellipsis. 422 mode = 'input' 423 if in1_match: 424 idx = in1_match.end() 425 else: # in2_match 426 idx = in2_match.end() 427 code = line[idx:] 428 insertion = (0, Generic.Prompt, line[:idx]) 429 return mode, code, insertion 430 431 # Check for input or continuation prompt (stripped version) 432 in1_match_rstrip = self.in1_regex_rstrip.match(line) 433 if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'): 434 # New input or when not in tb, continued input. 435 # We do not check for continued input when in tb since it is 436 # allowable to replace a long stack with an ellipsis. 437 mode = 'input' 438 if in1_match_rstrip: 439 idx = in1_match_rstrip.end() 440 else: # in2_match 441 idx = in2_match_rstrip.end() 442 code = line[idx:] 443 insertion = (0, Generic.Prompt, line[:idx]) 444 return mode, code, insertion 445 446 # Check for traceback 447 if self.ipytb_start.match(line): 448 mode = 'tb' 449 code = line 450 insertion = None 451 return mode, code, insertion 452 453 # All other stuff... 454 if self.mode in ('input', 'output'): 455 # We assume all other text is output. Multiline input that 456 # does not use the continuation marker cannot be detected. 457 # For example, the 3 in the following is clearly output: 458 # 459 # In [1]: print(3) 460 # 3 461 # 462 # But the following second line is part of the input: 463 # 464 # In [2]: while True: 465 # print(True) 466 # 467 # In both cases, the 2nd line will be 'output'. 468 # 469 mode = 'output' 470 else: 471 mode = 'tb' 472 473 code = line 474 insertion = None 475 476 return mode, code, insertion 477 478 def get_tokens_unprocessed(self, text): 479 self.reset() 480 for match in line_re.finditer(text): 481 line = match.group() 482 mode, code, insertion = self.get_mci(line) 483 484 if mode != self.mode: 485 # Yield buffered tokens before transitioning to new mode. 486 for token in self.buffered_tokens(): 487 yield token 488 self.mode = mode 489 490 if insertion: 491 self.insertions.append((len(self.buffer), [insertion])) 492 self.buffer += code 493 494 for token in self.buffered_tokens(): 495 yield token 496 497 class IPyLexer(Lexer): 498 r""" 499 Primary lexer for all IPython-like code. 500 501 This is a simple helper lexer. If the first line of the text begins with 502 "In \[[0-9]+\]:", then the entire text is parsed with an IPython console 503 lexer. If not, then the entire text is parsed with an IPython lexer. 504 505 The goal is to reduce the number of lexers that are registered 506 with Pygments. 507 508 """ 509 name = 'IPy session' 510 aliases = ['ipy'] 511 512 def __init__(self, **options): 513 """ 514 Create a new IPyLexer instance which dispatch to either an 515 IPythonCOnsoleLexer (if In prompts are present) or and IPythonLexer (if 516 In prompts are not present). 517 """ 518 # init docstring is necessary for docs not to fail to build do to parent 519 # docs referenceing a section in pygments docs. 520 self.python3 = get_bool_opt(options, 'python3', False) 521 if self.python3: 522 self.aliases = ['ipy3'] 523 else: 524 self.aliases = ['ipy2', 'ipy'] 525 526 Lexer.__init__(self, **options) 527 528 self.IPythonLexer = IPythonLexer(**options) 529 self.IPythonConsoleLexer = IPythonConsoleLexer(**options) 530 531 def get_tokens_unprocessed(self, text): 532 # Search for the input prompt anywhere...this allows code blocks to 533 # begin with comments as well. 534 if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL): 535 lex = self.IPythonConsoleLexer 536 else: 537 lex = self.IPythonLexer 538 for token in lex.get_tokens_unprocessed(text): 539 yield token 540 541