1 # -*- coding: utf-8 -*- 2 """ 3 Class and program to colorize python source code for ANSI terminals. 4 5 Based on an HTML code highlighter by Jurgen Hermann found at: 6 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298 7 8 Modifications by Fernando Perez (fperez@colorado.edu). 9 10 Information on the original HTML highlighter follows: 11 12 MoinMoin - Python Source Parser 13 14 Title: Colorize Python source using the built-in tokenizer 15 16 Submitter: Jurgen Hermann 17 Last Updated:2001/04/06 18 19 Version no:1.2 20 21 Description: 22 23 This code is part of MoinMoin (http://moin.sourceforge.net/) and converts 24 Python source code to HTML markup, rendering comments, keywords, 25 operators, numeric and string literals in different colors. 26 27 It shows how to use the built-in keyword, token and tokenize modules to 28 scan Python source code and re-emit it with no changes to its original 29 formatting (which is the hard part). 30 """ 31 32 __all__ = ['ANSICodeColors', 'Parser'] 33 34 _scheme_default = 'Linux' 35 36 37 # Imports 38 import keyword 39 import os 40 import sys 41 import token 42 import tokenize 43 44 generate_tokens = tokenize.generate_tokens 45 46 from IPython.utils.coloransi import TermColors, InputTermColors,ColorScheme, ColorSchemeTable 47 from .colorable import Colorable 48 from io import StringIO 49 50 ############################################################################# 51 ### Python Source Parser (does Highlighting) 52 ############################################################################# 53 54 _KEYWORD = token.NT_OFFSET + 1 55 _TEXT = token.NT_OFFSET + 2 56 57 #**************************************************************************** 58 # Builtin color schemes 59 60 Colors = TermColors # just a shorthand 61 62 # Build a few color schemes 63 NoColor = ColorScheme( 64 'NoColor',{ 65 'header' : Colors.NoColor, 66 token.NUMBER : Colors.NoColor, 67 token.OP : Colors.NoColor, 68 token.STRING : Colors.NoColor, 69 tokenize.COMMENT : Colors.NoColor, 70 token.NAME : Colors.NoColor, 71 token.ERRORTOKEN : Colors.NoColor, 72 73 _KEYWORD : Colors.NoColor, 74 _TEXT : Colors.NoColor, 75 76 'in_prompt' : InputTermColors.NoColor, # Input prompt 77 'in_number' : InputTermColors.NoColor, # Input prompt number 78 'in_prompt2' : InputTermColors.NoColor, # Continuation prompt 79 'in_normal' : InputTermColors.NoColor, # color off (usu. Colors.Normal) 80 81 'out_prompt' : Colors.NoColor, # Output prompt 82 'out_number' : Colors.NoColor, # Output prompt number 83 84 'normal' : Colors.NoColor # color off (usu. Colors.Normal) 85 } ) 86 87 LinuxColors = ColorScheme( 88 'Linux',{ 89 'header' : Colors.LightRed, 90 token.NUMBER : Colors.LightCyan, 91 token.OP : Colors.Yellow, 92 token.STRING : Colors.LightBlue, 93 tokenize.COMMENT : Colors.LightRed, 94 token.NAME : Colors.Normal, 95 token.ERRORTOKEN : Colors.Red, 96 97 _KEYWORD : Colors.LightGreen, 98 _TEXT : Colors.Yellow, 99 100 'in_prompt' : InputTermColors.Green, 101 'in_number' : InputTermColors.LightGreen, 102 'in_prompt2' : InputTermColors.Green, 103 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) 104 105 'out_prompt' : Colors.Red, 106 'out_number' : Colors.LightRed, 107 108 'normal' : Colors.Normal # color off (usu. Colors.Normal) 109 } ) 110 111 NeutralColors = ColorScheme( 112 'Neutral',{ 113 'header' : Colors.Red, 114 token.NUMBER : Colors.Cyan, 115 token.OP : Colors.Blue, 116 token.STRING : Colors.Blue, 117 tokenize.COMMENT : Colors.Red, 118 token.NAME : Colors.Normal, 119 token.ERRORTOKEN : Colors.Red, 120 121 _KEYWORD : Colors.Green, 122 _TEXT : Colors.Blue, 123 124 'in_prompt' : InputTermColors.Blue, 125 'in_number' : InputTermColors.LightBlue, 126 'in_prompt2' : InputTermColors.Blue, 127 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) 128 129 'out_prompt' : Colors.Red, 130 'out_number' : Colors.LightRed, 131 132 'normal' : Colors.Normal # color off (usu. Colors.Normal) 133 } ) 134 135 # Hack: the 'neutral' colours are not very visible on a dark background on 136 # Windows. Since Windows command prompts have a dark background by default, and 137 # relatively few users are likely to alter that, we will use the 'Linux' colours, 138 # designed for a dark background, as the default on Windows. Changing it here 139 # avoids affecting the prompt colours rendered by prompt_toolkit, where the 140 # neutral defaults do work OK. 141 142 if os.name == 'nt': 143 NeutralColors = LinuxColors.copy(name='Neutral') 144 145 LightBGColors = ColorScheme( 146 'LightBG',{ 147 'header' : Colors.Red, 148 token.NUMBER : Colors.Cyan, 149 token.OP : Colors.Blue, 150 token.STRING : Colors.Blue, 151 tokenize.COMMENT : Colors.Red, 152 token.NAME : Colors.Normal, 153 token.ERRORTOKEN : Colors.Red, 154 155 156 _KEYWORD : Colors.Green, 157 _TEXT : Colors.Blue, 158 159 'in_prompt' : InputTermColors.Blue, 160 'in_number' : InputTermColors.LightBlue, 161 'in_prompt2' : InputTermColors.Blue, 162 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) 163 164 'out_prompt' : Colors.Red, 165 'out_number' : Colors.LightRed, 166 167 'normal' : Colors.Normal # color off (usu. Colors.Normal) 168 } ) 169 170 # Build table of color schemes (needed by the parser) 171 ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors, NeutralColors], 172 _scheme_default) 173 174 Undefined = object() 175 176 class Parser(Colorable): 177 """ Format colored Python source. 178 """ 179 180 def __init__(self, color_table=None, out = sys.stdout, parent=None, style=None): 181 """ Create a parser with a specified color table and output channel. 182 183 Call format() to process code. 184 """ 185 186 super(Parser, self).__init__(parent=parent) 187 188 self.color_table = color_table if color_table else ANSICodeColors 189 self.out = out 190 self.pos = None 191 self.lines = None 192 self.raw = None 193 if not style: 194 self.style = self.default_style 195 else: 196 self.style = style 197 198 199 def format(self, raw, out=None, scheme=Undefined): 200 import warnings 201 if scheme is not Undefined: 202 warnings.warn('The `scheme` argument of IPython.utils.PyColorize:Parser.format is deprecated since IPython 6.0.' 203 'It will have no effect. Set the parser `style` directly.', 204 stacklevel=2) 205 return self.format2(raw, out)[0] 206 207 def format2(self, raw, out = None): 208 """ Parse and send the colored source. 209 210 If out and scheme are not specified, the defaults (given to 211 constructor) are used. 212 213 out should be a file-type object. Optionally, out can be given as the 214 string 'str' and the parser will automatically return the output in a 215 string.""" 216 217 string_output = 0 218 if out == 'str' or self.out == 'str' or \ 219 isinstance(self.out, StringIO): 220 # XXX - I don't really like this state handling logic, but at this 221 # point I don't want to make major changes, so adding the 222 # isinstance() check is the simplest I can do to ensure correct 223 # behavior. 224 out_old = self.out 225 self.out = StringIO() 226 string_output = 1 227 elif out is not None: 228 self.out = out 229 else: 230 raise ValueError('`out` or `self.out` should be file-like or the value `"str"`') 231 232 # Fast return of the unmodified input for NoColor scheme 233 if self.style == 'NoColor': 234 error = False 235 self.out.write(raw) 236 if string_output: 237 return raw, error 238 return None, error 239 240 # local shorthands 241 colors = self.color_table[self.style].colors 242 self.colors = colors # put in object so __call__ sees it 243 244 # Remove trailing whitespace and normalize tabs 245 self.raw = raw.expandtabs().rstrip() 246 247 # store line offsets in self.lines 248 self.lines = [0, 0] 249 pos = 0 250 raw_find = self.raw.find 251 lines_append = self.lines.append 252 while True: 253 pos = raw_find('\n', pos) + 1 254 if not pos: 255 break 256 lines_append(pos) 257 lines_append(len(self.raw)) 258 259 # parse the source and write it 260 self.pos = 0 261 text = StringIO(self.raw) 262 263 error = False 264 try: 265 for atoken in generate_tokens(text.readline): 266 self(*atoken) 267 except tokenize.TokenError as ex: 268 msg = ex.args[0] 269 line = ex.args[1][0] 270 self.out.write("%s\n\n*** ERROR: %s%s%s\n" % 271 (colors[token.ERRORTOKEN], 272 msg, self.raw[self.lines[line]:], 273 colors.normal) 274 ) 275 error = True 276 self.out.write(colors.normal+'\n') 277 if string_output: 278 output = self.out.getvalue() 279 self.out = out_old 280 return (output, error) 281 return (None, error) 282 283 284 def _inner_call_(self, toktype, toktext, start_pos): 285 """like call but write to a temporary buffer""" 286 buff = StringIO() 287 srow, scol = start_pos 288 colors = self.colors 289 owrite = buff.write 290 291 # line separator, so this works across platforms 292 linesep = os.linesep 293 294 # calculate new positions 295 oldpos = self.pos 296 newpos = self.lines[srow] + scol 297 self.pos = newpos + len(toktext) 298 299 # send the original whitespace, if needed 300 if newpos > oldpos: 301 owrite(self.raw[oldpos:newpos]) 302 303 # skip indenting tokens 304 if toktype in [token.INDENT, token.DEDENT]: 305 self.pos = newpos 306 buff.seek(0) 307 return buff.read() 308 309 # map token type to a color group 310 if token.LPAR <= toktype <= token.OP: 311 toktype = token.OP 312 elif toktype == token.NAME and keyword.iskeyword(toktext): 313 toktype = _KEYWORD 314 color = colors.get(toktype, colors[_TEXT]) 315 316 # Triple quoted strings must be handled carefully so that backtracking 317 # in pagers works correctly. We need color terminators on _each_ line. 318 if linesep in toktext: 319 toktext = toktext.replace(linesep, '%s%s%s' % 320 (colors.normal,linesep,color)) 321 322 # send text 323 owrite('%s%s%s' % (color,toktext,colors.normal)) 324 buff.seek(0) 325 return buff.read() 326 327 328 def __call__(self, toktype, toktext, start_pos, end_pos, line): 329 """ Token handler, with syntax highlighting.""" 330 self.out.write( 331 self._inner_call_(toktype, toktext, start_pos)) 332