equal
deleted
inserted
replaced
103 } |
103 } |
104 |
104 |
105 keywords = set(['and', 'or', 'not']) |
105 keywords = set(['and', 'or', 'not']) |
106 |
106 |
107 def tokenize(program): |
107 def tokenize(program): |
|
108 ''' |
|
109 Parse a revset statement into a stream of tokens |
|
110 |
|
111 Check that @ is a valid unquoted token character (issue3686): |
|
112 >>> list(tokenize("@::")) |
|
113 [('symbol', '@', 0), ('::', None, 1), ('end', None, 3)] |
|
114 |
|
115 ''' |
|
116 |
108 pos, l = 0, len(program) |
117 pos, l = 0, len(program) |
109 while pos < l: |
118 while pos < l: |
110 c = program[pos] |
119 c = program[pos] |
111 if c.isspace(): # skip inter-token whitespace |
120 if c.isspace(): # skip inter-token whitespace |
112 pass |
121 pass |
138 break |
147 break |
139 pos += 1 |
148 pos += 1 |
140 else: |
149 else: |
141 raise error.ParseError(_("unterminated string"), s) |
150 raise error.ParseError(_("unterminated string"), s) |
142 # gather up a symbol/keyword |
151 # gather up a symbol/keyword |
143 elif c.isalnum() or c in '._' or ord(c) > 127: |
152 elif c.isalnum() or c in '._@' or ord(c) > 127: |
144 s = pos |
153 s = pos |
145 pos += 1 |
154 pos += 1 |
146 while pos < l: # find end of symbol |
155 while pos < l: # find end of symbol |
147 d = program[pos] |
156 d = program[pos] |
148 if not (d.isalnum() or d in "._/" or ord(d) > 127): |
157 if not (d.isalnum() or d in "._/@" or ord(d) > 127): |
149 break |
158 break |
150 if d == '.' and program[pos - 1] == '.': # special case for .. |
159 if d == '.' and program[pos - 1] == '.': # special case for .. |
151 pos -= 1 |
160 pos -= 1 |
152 break |
161 break |
153 pos += 1 |
162 pos += 1 |