35 "symbol": (0, "symbol", None, None, None), |
35 "symbol": (0, "symbol", None, None, None), |
36 "string": (0, "string", None, None, None), |
36 "string": (0, "string", None, None, None), |
37 "end": (0, None, None, None, None), |
37 "end": (0, None, None, None, None), |
38 } |
38 } |
39 |
39 |
40 keywords = set(['and', 'or', 'not']) |
40 keywords = {'and', 'or', 'not'} |
41 |
41 |
42 globchars = ".*{}[]?/\\_" |
42 globchars = ".*{}[]?/\\_" |
43 |
43 |
44 def tokenize(program): |
44 def tokenize(program): |
45 pos, l = 0, len(program) |
45 pos, l = 0, len(program) |