contrib/byteify-strings.py
author Raphaël Gomès <rgomes@octobus.net>
Thu, 28 Mar 2024 14:47:20 +0000
branchstable
changeset 51540 803e61387e86
parent 50773 8250ecb53f30
permissions -rwxr-xr-x
relnotes: add 6.7.2
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
     1
#!/usr/bin/env python3
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
     2
#
38384
1d9c97db465f byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents: 36617
diff changeset
     3
# byteify-strings.py - transform string literals to be Python 3 safe
27220
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     4
#
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     5
# Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com>
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     6
#
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     7
# This software may be used and distributed according to the terms of the
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     8
# GNU General Public License version 2 or any later version.
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
     9
4374d819ccd5 mercurial: implement import hook for handling C/Python modules
Gregory Szorc <gregory.szorc@gmail.com>
parents: 0
diff changeset
    10
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
    11
import argparse
38386
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
    12
import contextlib
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
    13
import errno
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
    14
import os
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
    15
import sys
38386
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
    16
import tempfile
38384
1d9c97db465f byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents: 36617
diff changeset
    17
import token
1d9c97db465f byteify-strings: fork py3 code transformer to make it a standalone command
Yuya Nishihara <yuya@tcha.org>
parents: 36617
diff changeset
    18
import tokenize
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
    19
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
    20
38390
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38389
diff changeset
    21
def adjusttokenpos(t, ofs):
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38389
diff changeset
    22
    """Adjust start/end column of the given token"""
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
    23
    return t._replace(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
    24
        start=(t.start[0], t.start[1] + ofs), end=(t.end[0], t.end[1] + ofs)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
    25
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
    26
38390
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38389
diff changeset
    27
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    28
def replacetokens(tokens, opts):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    29
    """Transform a stream of tokens from raw to Python 3.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    30
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    31
    Returns a generator of possibly rewritten tokens.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    32
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    33
    The input token list may be mutated as part of processing. However,
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    34
    its changes do not necessarily match the output token stream.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    35
    """
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    36
    sysstrtokens = set()
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
    37
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    38
    # The following utility functions access the tokens list and i index of
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    39
    # the for i, t enumerate(tokens) loop below
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    40
    def _isop(j, *o):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    41
        """Assert that tokens[j] is an OP with one of the given values"""
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    42
        try:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    43
            return tokens[j].type == token.OP and tokens[j].string in o
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    44
        except IndexError:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    45
            return False
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
    46
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    47
    def _findargnofcall(n):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    48
        """Find arg n of a call expression (start at 0)
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    49
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    50
        Returns index of the first token of that argument, or None if
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    51
        there is not that many arguments.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    52
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    53
        Assumes that token[i + 1] is '('.
38389
1d68fd5f614a byteify-strings: do not rewrite system string literals to u''
Yuya Nishihara <yuya@tcha.org>
parents: 38388
diff changeset
    54
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    55
        """
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    56
        nested = 0
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    57
        for j in range(i + 2, len(tokens)):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    58
            if _isop(j, ')', ']', '}'):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    59
                # end of call, tuple, subscription or dict / set
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    60
                nested -= 1
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    61
                if nested < 0:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    62
                    return None
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    63
            elif n == 0:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    64
                # this is the starting position of arg
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    65
                return j
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    66
            elif _isop(j, '(', '[', '{'):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    67
                nested += 1
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    68
            elif _isop(j, ',') and nested == 0:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    69
                n -= 1
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
    70
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    71
        return None
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    72
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    73
    def _ensuresysstr(j):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    74
        """Make sure the token at j is a system string
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
    75
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    76
        Remember the given token so the string transformer won't add
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    77
        the byte prefix.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
    78
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    79
        Ignores tokens that are not strings. Assumes bounds checking has
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    80
        already been done.
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
    81
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
    82
        """
42674
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    83
        k = j
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    84
        currtoken = tokens[k]
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    85
        while currtoken.type in (token.STRING, token.NEWLINE, tokenize.NL):
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    86
            k += 1
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
    87
            if currtoken.type == token.STRING and currtoken.string.startswith(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
    88
                ("'", '"')
42674
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    89
            ):
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    90
                sysstrtokens.add(currtoken)
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    91
            try:
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    92
                currtoken = tokens[k]
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    93
            except IndexError:
70bd1965bd07 byteify-strings: handle multi-line strings in _ensuresysstr
Raphaël Gomès <rgomes@octobus.net>
parents: 42247
diff changeset
    94
                break
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
    95
42677
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
    96
    def _isitemaccess(j):
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
    97
        """Assert the next tokens form an item access on `tokens[j]` and that
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
    98
        `tokens[j]` is a name.
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
    99
        """
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   100
        try:
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   101
            return (
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   102
                tokens[j].type == token.NAME
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   103
                and _isop(j + 1, '[')
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   104
                and tokens[j + 2].type == token.STRING
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   105
                and _isop(j + 3, ']')
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   106
            )
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   107
        except IndexError:
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   108
            return False
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   109
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   110
    def _ismethodcall(j, *methodnames):
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   111
        """Assert the next tokens form a call to `methodname` with a string
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   112
        as first argument on `tokens[j]` and that `tokens[j]` is a name.
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   113
        """
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   114
        try:
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   115
            return (
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   116
                tokens[j].type == token.NAME
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   117
                and _isop(j + 1, '.')
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   118
                and tokens[j + 2].type == token.NAME
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   119
                and tokens[j + 2].string in methodnames
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   120
                and _isop(j + 3, '(')
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   121
                and tokens[j + 4].type == token.STRING
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   122
            )
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   123
        except IndexError:
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   124
            return False
c9fd8163131f byteify-strings: add helpers to check for item access or method call
Raphaël Gomès <rgomes@octobus.net>
parents: 42676
diff changeset
   125
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   126
    coldelta = 0  # column increment for new opening parens
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   127
    coloffset = -1  # column offset for the current line (-1: TBD)
42687
26a31c88e1a5 byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents: 42683
diff changeset
   128
    parens = [(0, 0, 0, -1)]  # stack of (line, end-column, column-offset, type)
42676
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   129
    ignorenextline = False  # don't transform the next line
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   130
    insideignoreblock = False  # don't transform until turned off
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   131
    for i, t in enumerate(tokens):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   132
        # Compute the column offset for the current line, such that
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   133
        # the current line will be aligned to the last opening paren
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   134
        # as before.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   135
        if coloffset < 0:
42687
26a31c88e1a5 byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents: 42683
diff changeset
   136
            lastparen = parens[-1]
26a31c88e1a5 byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents: 42683
diff changeset
   137
            if t.start[1] == lastparen[1]:
26a31c88e1a5 byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents: 42683
diff changeset
   138
                coloffset = lastparen[2]
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   139
            elif t.start[1] + 1 == lastparen[1] and lastparen[3] not in (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   140
                token.NEWLINE,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   141
                tokenize.NL,
42687
26a31c88e1a5 byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents: 42683
diff changeset
   142
            ):
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   143
                # fix misaligned indent of s/util.Abort/error.Abort/
42687
26a31c88e1a5 byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents: 42683
diff changeset
   144
                coloffset = lastparen[2] + (lastparen[1] - t.start[1])
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   145
            else:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   146
                coloffset = 0
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   147
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   148
        # Reset per-line attributes at EOL.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   149
        if t.type in (token.NEWLINE, tokenize.NL):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   150
            yield adjusttokenpos(t, coloffset)
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   151
            coldelta = 0
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   152
            coloffset = -1
42676
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   153
            if not insideignoreblock:
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   154
                ignorenextline = (
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   155
                    tokens[i - 1].type == token.COMMENT
42700
f9b64ff9d26b byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents: 42687
diff changeset
   156
                    and tokens[i - 1].string == "# no-py3-transform"
42676
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   157
                )
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   158
            continue
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   159
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   160
        if t.type == token.COMMENT:
42700
f9b64ff9d26b byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents: 42687
diff changeset
   161
            if t.string == "# py3-transform: off":
42676
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   162
                insideignoreblock = True
42700
f9b64ff9d26b byteify-strings: add space in special comments to silence flake8 error
Raphaël Gomès <rgomes@octobus.net>
parents: 42687
diff changeset
   163
            if t.string == "# py3-transform: on":
42676
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   164
                insideignoreblock = False
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   165
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   166
        if ignorenextline or insideignoreblock:
b9a200477edf byteify-strings: add support for ignore comments
Raphaël Gomès <rgomes@octobus.net>
parents: 42675
diff changeset
   167
            yield adjusttokenpos(t, coloffset)
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   168
            continue
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   169
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   170
        # Remember the last paren position.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   171
        if _isop(i, '(', '[', '{'):
42687
26a31c88e1a5 byteify-strings: fix misalignment with multi-line parenthesis
Raphaël Gomès <rgomes@octobus.net>
parents: 42683
diff changeset
   172
            parens.append(t.end + (coloffset + coldelta, tokens[i + 1].type))
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   173
        elif _isop(i, ')', ']', '}'):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   174
            parens.pop()
30165
423377290a3a py3: refactor token parsing to handle call args properly
Martijn Pieters <mjpieters@fb.com>
parents: 30118
diff changeset
   175
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   176
        # Convert most string literals to byte literals. String literals
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   177
        # in Python 2 are bytes. String literals in Python 3 are unicode.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   178
        # Most strings in Mercurial are bytes and unicode strings are rare.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   179
        # Rather than rewrite all string literals to use ``b''`` to indicate
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   180
        # byte strings, we apply this token transformer to insert the ``b``
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   181
        # prefix nearly everywhere.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   182
        if t.type == token.STRING and t not in sysstrtokens:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   183
            s = t.string
38390
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38389
diff changeset
   184
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   185
            # Preserve docstrings as string literals. This is inconsistent
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   186
            # with regular unprefixed strings. However, the
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   187
            # "from __future__" parsing (which allows a module docstring to
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   188
            # exist before it) doesn't properly handle the docstring if it
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   189
            # is b''' prefixed, leading to a SyntaxError. We leave all
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   190
            # docstrings as unprefixed to avoid this. This means Mercurial
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   191
            # components touching docstrings need to handle unicode,
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   192
            # unfortunately.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   193
            if s[0:3] in ("'''", '"""'):
42675
e9592e113c31 byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents: 42674
diff changeset
   194
                # If it's assigned to something, it's not a docstring
e9592e113c31 byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents: 42674
diff changeset
   195
                if not _isop(i - 1, '='):
e9592e113c31 byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents: 42674
diff changeset
   196
                    yield adjusttokenpos(t, coloffset)
e9592e113c31 byteify-strings: handle triple quoted strings if they are not docstrings
Raphaël Gomès <rgomes@octobus.net>
parents: 42674
diff changeset
   197
                    continue
38390
47dd23e6b116 byteify-strings: try to preserve column alignment
Yuya Nishihara <yuya@tcha.org>
parents: 38389
diff changeset
   198
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   199
            # If the first character isn't a quote, it is likely a string
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   200
            # prefixing character (such as 'b', 'u', or 'r'. Ignore.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   201
            if s[0] not in ("'", '"'):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   202
                yield adjusttokenpos(t, coloffset)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   203
                continue
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   204
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   205
            # String literal. Prefix to make a b'' string.
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   206
            yield adjusttokenpos(t._replace(string='b%s' % t.string), coloffset)
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   207
            coldelta += 1
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   208
            continue
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   209
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   210
        # This looks like a function call.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   211
        if t.type == token.NAME and _isop(i + 1, '('):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   212
            fn = t.string
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   213
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   214
            # *attr() builtins don't accept byte strings to 2nd argument.
50773
8250ecb53f30 byteify-strings: passe sysstr to attr function and wrapper
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48875
diff changeset
   215
            if fn in (
8250ecb53f30 byteify-strings: passe sysstr to attr function and wrapper
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48875
diff changeset
   216
                'getattr',
8250ecb53f30 byteify-strings: passe sysstr to attr function and wrapper
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48875
diff changeset
   217
                'setattr',
8250ecb53f30 byteify-strings: passe sysstr to attr function and wrapper
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48875
diff changeset
   218
                'hasattr',
8250ecb53f30 byteify-strings: passe sysstr to attr function and wrapper
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48875
diff changeset
   219
                'safehasattr',
8250ecb53f30 byteify-strings: passe sysstr to attr function and wrapper
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48875
diff changeset
   220
                'wrapfunction',
8250ecb53f30 byteify-strings: passe sysstr to attr function and wrapper
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48875
diff changeset
   221
                'wrapclass',
8250ecb53f30 byteify-strings: passe sysstr to attr function and wrapper
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48875
diff changeset
   222
                'addattr',
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 44559
diff changeset
   223
            ):
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   224
                arg1idx = _findargnofcall(1)
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   225
                if arg1idx is not None:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   226
                    _ensuresysstr(arg1idx)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   227
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   228
            # .encode() and .decode() on str/bytes/unicode don't accept
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   229
            # byte strings on Python 3.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   230
            elif fn in ('encode', 'decode') and _isop(i - 1, '.'):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   231
                for argn in range(2):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   232
                    argidx = _findargnofcall(argn)
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   233
                    if argidx is not None:
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   234
                        _ensuresysstr(argidx)
29550
1c22400db72d mercurial: implement a source transforming module loader on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29490
diff changeset
   235
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   236
            # It changes iteritems/values to items/values as they are not
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   237
            # present in Python 3 world.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   238
            elif opts['dictiter'] and fn in ('iteritems', 'itervalues'):
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   239
                yield adjusttokenpos(t._replace(string=fn[4:]), coloffset)
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   240
                continue
30052
eaaedad68011 py3: switch to .items() using transformer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30051
diff changeset
   241
42678
f95b59ffc307 byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents: 42677
diff changeset
   242
        if t.type == token.NAME and t.string in opts['treat-as-kwargs']:
f95b59ffc307 byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents: 42677
diff changeset
   243
            if _isitemaccess(i):
f95b59ffc307 byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents: 42677
diff changeset
   244
                _ensuresysstr(i + 2)
f95b59ffc307 byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents: 42677
diff changeset
   245
            if _ismethodcall(i, 'get', 'pop', 'setdefault', 'popitem'):
f95b59ffc307 byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents: 42677
diff changeset
   246
                _ensuresysstr(i + 4)
f95b59ffc307 byteify-strings: add --treat-as-kwargs argument to handle kwargs-like objects
Raphaël Gomès <rgomes@octobus.net>
parents: 42677
diff changeset
   247
39103
da130c5cef90 byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
   248
        # Looks like "if __name__ == '__main__'".
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   249
        if (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   250
            t.type == token.NAME
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   251
            and t.string == '__name__'
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   252
            and _isop(i + 1, '==')
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   253
        ):
39103
da130c5cef90 byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
   254
            _ensuresysstr(i + 2)
da130c5cef90 byteify-strings: prevent "__name__ == '__main__'" from being transformed
Yuya Nishihara <yuya@tcha.org>
parents: 38391
diff changeset
   255
38391
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   256
        # Emit unmodified token.
f77bbd34a1df byteify-strings: remove superfluous "if True" block
Yuya Nishihara <yuya@tcha.org>
parents: 38390
diff changeset
   257
        yield adjusttokenpos(t, coloffset)
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   258
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   259
38388
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38387
diff changeset
   260
def process(fin, fout, opts):
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   261
    tokens = tokenize.tokenize(fin.readline)
38388
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38387
diff changeset
   262
    tokens = replacetokens(list(tokens), opts)
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   263
    fout.write(tokenize.untokenize(tokens))
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   264
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   265
38386
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   266
def tryunlink(fname):
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   267
    try:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   268
        os.unlink(fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   269
    except OSError as err:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   270
        if err.errno != errno.ENOENT:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   271
            raise
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   272
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   273
38386
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   274
@contextlib.contextmanager
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   275
def editinplace(fname):
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   276
    n = os.path.basename(fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   277
    d = os.path.dirname(fname)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   278
    fp = tempfile.NamedTemporaryFile(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   279
        prefix='.%s-' % n, suffix='~', dir=d, delete=False
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   280
    )
38386
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   281
    try:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   282
        yield fp
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   283
        fp.close()
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   284
        if os.name == 'nt':
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   285
            tryunlink(fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   286
        os.rename(fp.name, fname)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   287
    finally:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   288
        fp.close()
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   289
        tryunlink(fp.name)
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   290
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   291
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   292
def main():
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   293
    ap = argparse.ArgumentParser()
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   294
    ap.add_argument(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   295
        '--version', action='version', version='Byteify strings 1.0'
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   296
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   297
    ap.add_argument(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   298
        '-i',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   299
        '--inplace',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   300
        action='store_true',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   301
        default=False,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   302
        help='edit files in place',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   303
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   304
    ap.add_argument(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   305
        '--dictiter',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   306
        action='store_true',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   307
        default=False,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   308
        help='rewrite iteritems() and itervalues()',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   309
    ),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   310
    ap.add_argument(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   311
        '--treat-as-kwargs',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   312
        nargs="+",
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   313
        default=[],
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   314
        help="ignore kwargs-like objects",
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   315
    ),
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   316
    ap.add_argument('files', metavar='FILE', nargs='+', help='source file')
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   317
    args = ap.parse_args()
38388
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38387
diff changeset
   318
    opts = {
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38387
diff changeset
   319
        'dictiter': args.dictiter,
42679
5e296f618920 byteify-strings: simplify default value for `--treat-as-kwargs`
Raphaël Gomès <rgomes@octobus.net>
parents: 42678
diff changeset
   320
        'treat-as-kwargs': set(args.treat_as_kwargs),
38388
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38387
diff changeset
   321
    }
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   322
    for fname in args.files:
44559
bc9a9016467d byteify-string: resolve symlink before byteifying
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43407
diff changeset
   323
        fname = os.path.realpath(fname)
38386
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   324
        if args.inplace:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   325
            with editinplace(fname) as fout:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   326
                with open(fname, 'rb') as fin:
38388
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38387
diff changeset
   327
                    process(fin, fout, opts)
38386
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   328
        else:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   329
            with open(fname, 'rb') as fin:
9f42e4a83676 byteify-strings: add --inplace option to write back result
Yuya Nishihara <yuya@tcha.org>
parents: 38385
diff changeset
   330
                fout = sys.stdout.buffer
38388
f701bc936e7f byteify-strings: do not rewrite iteritems() and itervalues() by default
Yuya Nishihara <yuya@tcha.org>
parents: 38387
diff changeset
   331
                process(fin, fout, opts)
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   332
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42701
diff changeset
   333
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   334
if __name__ == '__main__':
43407
09234eeae206 contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents: 43379
diff changeset
   335
    if sys.version_info[0:2] < (3, 7):
09234eeae206 contrib: require Python 3.7 for byteify-strings.py
Ian Moody <moz-ian@perix.co.uk>
parents: 43379
diff changeset
   336
        print('This script must be run under Python 3.7+')
42247
970aaf38c3fc contrib: have byteify-strings explode if run in Python 2
Augie Fackler <augie@google.com>
parents: 39103
diff changeset
   337
        sys.exit(3)
38385
a2976c27dac4 byteify-strings: add basic command interface
Yuya Nishihara <yuya@tcha.org>
parents: 38384
diff changeset
   338
    main()