equal
deleted
inserted
replaced
61 |
61 |
62 # default set of valid characters for the initial letter of symbols |
62 # default set of valid characters for the initial letter of symbols |
63 _syminitletters = set(pycompat.iterbytestr( |
63 _syminitletters = set(pycompat.iterbytestr( |
64 string.ascii_letters.encode('ascii') + |
64 string.ascii_letters.encode('ascii') + |
65 string.digits.encode('ascii') + |
65 string.digits.encode('ascii') + |
66 '._@')) | set(map(pycompat.bytechr, xrange(128, 256))) |
66 '._@')) | set(map(pycompat.bytechr, pycompat.xrange(128, 256))) |
67 |
67 |
68 # default set of valid characters for non-initial letters of symbols |
68 # default set of valid characters for non-initial letters of symbols |
69 _symletters = _syminitletters | set(pycompat.iterbytestr('-/')) |
69 _symletters = _syminitletters | set(pycompat.iterbytestr('-/')) |
70 |
70 |
71 def tokenize(program, lookup=None, syminitletters=None, symletters=None): |
71 def tokenize(program, lookup=None, syminitletters=None, symletters=None): |