byteify-strings: add support for ignore comments
Our simple token analysis is sometimes not clever enough, we need to be able
to turn off our script for parts of the code.
This change introduces three special comments:
- `#no-py3-transform` to tell `byteify-strings` ignore the next line
- `#py3-transform: off` to ignore everything until the end of the file
- `#py3-transform: on` to stop ignoring
The last two can be particularly useful within Python 2/3 compatibility files.
--- a/contrib/byteify-strings.py Fri Aug 02 09:48:13 2019 +0200
+++ b/contrib/byteify-strings.py Fri Aug 02 09:55:32 2019 +0200
@@ -95,6 +95,8 @@
coldelta = 0 # column increment for new opening parens
coloffset = -1 # column offset for the current line (-1: TBD)
parens = [(0, 0, 0)] # stack of (line, end-column, column-offset)
+ ignorenextline = False # don't transform the next line
+ insideignoreblock = False # don't transform until turned off
for i, t in enumerate(tokens):
# Compute the column offset for the current line, such that
# the current line will be aligned to the last opening paren
@@ -113,6 +115,21 @@
yield adjusttokenpos(t, coloffset)
coldelta = 0
coloffset = -1
+ if not insideignoreblock:
+ ignorenextline = (
+ tokens[i - 1].type == token.COMMENT
+ and tokens[i - 1].string == "#no-py3-transform"
+ )
+ continue
+
+ if t.type == token.COMMENT:
+ if t.string == "#py3-transform: off":
+ insideignoreblock = True
+ if t.string == "#py3-transform: on":
+ insideignoreblock = False
+
+ if ignorenextline or insideignoreblock:
+ yield adjusttokenpos(t, coloffset)
continue
# Remember the last paren position.