-
Notifications
You must be signed in to change notification settings - Fork 1
/
tests.py
executable file
·93 lines (72 loc) · 3.88 KB
/
tests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
#!/usr/bin/python3
import unittest
import tokenizers
from functools import partial
from tokenizers.PyTokenizer import PyTokenizer
from tokenizers.CTokenizer import CTokenizer
from tokenizers.BashTokenizer import BashTokenizer
from utils import tokenize_source_code, Token
class TestTokenizer(unittest.TestCase):
TOKENIZE = None
def _test_generator(self, text, result):
self.assertEqual(list(self.TOKENIZE(text)), result)
class TestPyTokenizer(TestTokenizer):
TOKENIZE = partial(tokenize_source_code, tokenizer=PyTokenizer)
def test_comments(self):
self._test_generator('# no', [Token('# no', 'comment')])
self._test_generator('#', [Token('#', 'comment')])
def test_numbers(self):
self._test_generator('42', [Token('42', 'number')])
self._test_generator('0x15', [Token('0x15', 'number')])
self._test_generator('0b001', [Token('0b001', 'number')])
self._test_generator('0o175', [Token('0o175', 'number')])
def test_keywords(self):
self._test_generator('break', [Token('break', 'keyword')])
self._test_generator('while True',
[Token(content='while', type='keyword'),
Token(content=' ', type=''),
Token(content='True', type='keyword')])
def test_builtins(self):
self._test_generator('help()', [Token(content='help', type='builtin'),
Token(content='(', type='operator'),
Token(content=')', type='operator')])
def test_variables(self):
self._test_generator('x = 5', [Token(content='x = ', type=''),
Token(content='5', type='number')])
def test_indent(self):
data = 'try:\n pass\nexcept:\n pass'
self._test_generator(data, [Token(content='try', type='keyword'),
Token(content=':', type='operator'),
Token(content='\n', type='linebreak'),
Token(content=' ', type='new-indent'),
Token(content='pass', type='keyword'),
Token(content='\n', type='linebreak'),
Token(content='except', type='keyword'),
Token(content=':', type='operator'),
Token(content='\n', type='linebreak'),
Token(content=' ', type='new-indent'),
Token(content='pass', type='keyword')])
class TestCTokenizer(TestTokenizer):
TOKENIZE = partial(tokenize_source_code, tokenizer=CTokenizer)
def test_comments(self):
self._test_generator('// no', [Token('// no', 'comment')])
self._test_generator('//', [Token('//', 'comment')])
def test_numbers(self):
self._test_generator('42', [Token('42', 'number')])
self._test_generator('0x15', [Token('0x15', 'number')])
self._test_generator('0b001', [Token('0b001', 'number')])
self._test_generator('break', [Token('break', 'builtin')])
def test_keywords(self):
self._test_generator('while (true)',
[Token(content='while', type='builtin'),
Token(content=' ', type=''),
Token(content='(', type='operator'),
Token(content='true', type='keyword'),
Token(content=')', type='operator')])
def test_builtins(self):
self._test_generator('NULL', [Token(content='NULL', type='keyword')])
def test_variables(self):
self._test_generator('x = 5', [Token(content='x = ', type=''),
Token(content='5', type='number')])
if __name__ == '__main__':
unittest.main()