1 """Test lexers module""" 2 3 # Copyright (c) IPython Development Team. 4 # Distributed under the terms of the Modified BSD License. 5 6 from unittest import TestCase 7 from pygments import __version__ as pygments_version 8 from pygments.token import Token 9 from pygments.lexers import BashLexer 10 11 from .. import lexers 12 13 pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14) 14 15 16 class TestLexers(TestCase): 17 """Collection of lexers tests""" 18 def setUp(self): 19 self.lexer = lexers.IPythonLexer() 20 self.bash_lexer = BashLexer() 21 22 def testIPythonLexer(self): 23 fragment = '!echo $HOME\n' 24 bash_tokens = [ 25 (Token.Operator, '!'), 26 ] 27 bash_tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) 28 ipylex_token = list(self.lexer.get_tokens(fragment)) 29 assert bash_tokens[:-1] == ipylex_token[:-1] 30 31 fragment_2 = "!" + fragment 32 tokens_2 = [ 33 (Token.Operator, '!!'), 34 ] + bash_tokens[1:] 35 assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] 36 37 fragment_2 = '\t %%!\n' + fragment[1:] 38 tokens_2 = [ 39 (Token.Text, '\t '), 40 (Token.Operator, '%%!'), 41 (Token.Text, '\n'), 42 ] + bash_tokens[1:] 43 assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) 44 45 fragment_2 = 'x = ' + fragment 46 tokens_2 = [ 47 (Token.Name, 'x'), 48 (Token.Text, ' '), 49 (Token.Operator, '='), 50 (Token.Text, ' '), 51 ] + bash_tokens 52 assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] 53 54 fragment_2 = 'x, = ' + fragment 55 tokens_2 = [ 56 (Token.Name, 'x'), 57 (Token.Punctuation, ','), 58 (Token.Text, ' '), 59 (Token.Operator, '='), 60 (Token.Text, ' '), 61 ] + bash_tokens 62 assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] 63 64 fragment_2 = 'x, = %sx ' + fragment[1:] 65 tokens_2 = [ 66 (Token.Name, 'x'), 67 (Token.Punctuation, ','), 68 (Token.Text, ' '), 69 (Token.Operator, '='), 70 (Token.Text, ' '), 71 (Token.Operator, '%'), 72 (Token.Keyword, 'sx'), 73 (Token.Text, ' '), 74 ] + bash_tokens[1:] 75 if tokens_2[7] == (Token.Text, " ") and pyg214: # pygments 2.14+ 76 tokens_2[7] = (Token.Text.Whitespace, " ") 77 assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] 78 79 fragment_2 = 'f = %R function () {}\n' 80 tokens_2 = [ 81 (Token.Name, 'f'), 82 (Token.Text, ' '), 83 (Token.Operator, '='), 84 (Token.Text, ' '), 85 (Token.Operator, '%'), 86 (Token.Keyword, 'R'), 87 (Token.Text, ' function () {}\n'), 88 ] 89 assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) 90 91 fragment_2 = '\t%%xyz\n$foo\n' 92 tokens_2 = [ 93 (Token.Text, '\t'), 94 (Token.Operator, '%%'), 95 (Token.Keyword, 'xyz'), 96 (Token.Text, '\n$foo\n'), 97 ] 98 assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) 99 100 fragment_2 = '%system?\n' 101 tokens_2 = [ 102 (Token.Operator, '%'), 103 (Token.Keyword, 'system'), 104 (Token.Operator, '?'), 105 (Token.Text, '\n'), 106 ] 107 assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] 108 109 fragment_2 = 'x != y\n' 110 tokens_2 = [ 111 (Token.Name, 'x'), 112 (Token.Text, ' '), 113 (Token.Operator, '!='), 114 (Token.Text, ' '), 115 (Token.Name, 'y'), 116 (Token.Text, '\n'), 117 ] 118 assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] 119 120 fragment_2 = ' ?math.sin\n' 121 tokens_2 = [ 122 (Token.Text, ' '), 123 (Token.Operator, '?'), 124 (Token.Text, 'math.sin'), 125 (Token.Text, '\n'), 126 ] 127 assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] 128 129 fragment = ' *int*?\n' 130 tokens = [ 131 (Token.Text, ' *int*'), 132 (Token.Operator, '?'), 133 (Token.Text, '\n'), 134 ] 135 assert tokens == list(self.lexer.get_tokens(fragment)) 136 137 fragment = '%%writefile -a foo.py\nif a == b:\n pass' 138 tokens = [ 139 (Token.Operator, '%%writefile'), 140 (Token.Text, ' -a foo.py\n'), 141 (Token.Keyword, 'if'), 142 (Token.Text, ' '), 143 (Token.Name, 'a'), 144 (Token.Text, ' '), 145 (Token.Operator, '=='), 146 (Token.Text, ' '), 147 (Token.Name, 'b'), 148 (Token.Punctuation, ':'), 149 (Token.Text, '\n'), 150 (Token.Text, ' '), 151 (Token.Keyword, 'pass'), 152 (Token.Text, '\n'), 153 ] 154 if tokens[10] == (Token.Text, "\n") and pyg214: # pygments 2.14+ 155 tokens[10] = (Token.Text.Whitespace, "\n") 156 assert tokens[:-1] == list(self.lexer.get_tokens(fragment))[:-1] 157 158 fragment = '%%timeit\nmath.sin(0)' 159 tokens = [ 160 (Token.Operator, '%%timeit\n'), 161 (Token.Name, 'math'), 162 (Token.Operator, '.'), 163 (Token.Name, 'sin'), 164 (Token.Punctuation, '('), 165 (Token.Literal.Number.Integer, '0'), 166 (Token.Punctuation, ')'), 167 (Token.Text, '\n'), 168 ] 169 170 fragment = '%%HTML\n<div>foo</div>' 171 tokens = [ 172 (Token.Operator, '%%HTML'), 173 (Token.Text, '\n'), 174 (Token.Punctuation, '<'), 175 (Token.Name.Tag, 'div'), 176 (Token.Punctuation, '>'), 177 (Token.Text, 'foo'), 178 (Token.Punctuation, '<'), 179 (Token.Punctuation, '/'), 180 (Token.Name.Tag, 'div'), 181 (Token.Punctuation, '>'), 182 (Token.Text, '\n'), 183 ] 184 assert tokens == list(self.lexer.get_tokens(fragment)) 185