mercurial/pycompat.py
author Gregory Szorc <gregory.szorc@gmail.com>
Wed, 14 Mar 2018 11:52:35 -0700
changeset 36958 644a02f6b34f
parent 36934 dbae581010ea
child 37064 434e520adb8c
permissions -rw-r--r--
util: prefer "bytesio" to "stringio" The io.BytesIO and io.StringIO types enforce the type of data being operated on. On Python 2, we use cStringIO.StringIO(), which is lax about mixing types. On Python 3, we actually use io.BytesIO. Ideally, we'd use io.BytesIO on Python 2. But I believe its performance is poor compared to cString.StringIO(). Anyway, we canonically define our pycompat type as "stringio." That name is misleading, especially on Python 3. This commit renames the canonical symbols to "bytesio." "stringio" is preserved as an alias for API compatibility. There are a lot of callers in the repo and I hesitate to take away the old name. I also don't feel like changing everything at this time. But at least new callers can use a "proper" name. Differential Revision: https://phab.mercurial-scm.org/D2868
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
28818
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     1
# pycompat.py - portability shim for python 3
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     2
#
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     3
# This software may be used and distributed according to the terms of the
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     4
# GNU General Public License version 2 or any later version.
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     5
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     6
"""Mercurial portability shim for python 3.
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     7
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     8
This contains aliases to hide python version-specific details from the core.
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
     9
"""
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
    10
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
    11
from __future__ import absolute_import
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
diff changeset
    12
30578
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
    13
import getopt
36178
646002338365 py3: introduce and use pycompat.getargspec
Augie Fackler <augie@google.com>
parents: 36045
diff changeset
    14
import inspect
30302
3874ddba1ab4 py3: add a bytes version of os.name
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30300
diff changeset
    15
import os
30678
caf7e1c5efe4 py3: have a bytes version of shlex.split()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30668
diff changeset
    16
import shlex
29584
06587edd1233 pycompat: make pycompat demandimport friendly
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29566
diff changeset
    17
import sys
06587edd1233 pycompat: make pycompat demandimport friendly
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29566
diff changeset
    18
30031
0f6d6fdd3c2a pycompat: provide 'ispy3' constant
Yuya Nishihara <yuya@tcha.org>
parents: 29801
diff changeset
    19
ispy3 = (sys.version_info[0] >= 3)
33626
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 32865
diff changeset
    20
ispypy = (r'__pypy__' in sys.builtin_module_names)
30031
0f6d6fdd3c2a pycompat: provide 'ispy3' constant
Yuya Nishihara <yuya@tcha.org>
parents: 29801
diff changeset
    21
0f6d6fdd3c2a pycompat: provide 'ispy3' constant
Yuya Nishihara <yuya@tcha.org>
parents: 29801
diff changeset
    22
if not ispy3:
31934
12aca6770046 util: make cookielib module available
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31843
diff changeset
    23
    import cookielib
29324
b501579147f1 py3: conditionalize cPickle import by adding in util
Pulkit Goyal <7895pulkit@gmail.com>
parents: 28882
diff changeset
    24
    import cPickle as pickle
29455
0c741fd6158a py3: conditionalize httplib import
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29433
diff changeset
    25
    import httplib
29584
06587edd1233 pycompat: make pycompat demandimport friendly
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29566
diff changeset
    26
    import Queue as _queue
29433
33770d2b6cf9 py3: conditionalize SocketServer import
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29432
diff changeset
    27
    import SocketServer as socketserver
29432
34b914ac573e py3: conditionalize xmlrpclib import
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29431
diff changeset
    28
    import xmlrpclib
29584
06587edd1233 pycompat: make pycompat demandimport friendly
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29566
diff changeset
    29
else:
31942
bc0579a25f82 pycompat: import correct cookie module on Python 3
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31934
diff changeset
    30
    import http.cookiejar as cookielib
29584
06587edd1233 pycompat: make pycompat demandimport friendly
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29566
diff changeset
    31
    import http.client as httplib
06587edd1233 pycompat: make pycompat demandimport friendly
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29566
diff changeset
    32
    import pickle
06587edd1233 pycompat: make pycompat demandimport friendly
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29566
diff changeset
    33
    import queue as _queue
06587edd1233 pycompat: make pycompat demandimport friendly
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29566
diff changeset
    34
    import socketserver
29432
34b914ac573e py3: conditionalize xmlrpclib import
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29431
diff changeset
    35
    import xmlrpc.client as xmlrpclib
29431
80880ad3fccd py3: conditionalize the urlparse import
Pulkit Goyal <7895pulkit@gmail.com>
parents: 29414
diff changeset
    36
32865
6e38b4212661 pycompat: move the queue related definitions below queue import
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32864
diff changeset
    37
empty = _queue.Empty
6e38b4212661 pycompat: move the queue related definitions below queue import
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32864
diff changeset
    38
queue = _queue.Queue
6e38b4212661 pycompat: move the queue related definitions below queue import
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32864
diff changeset
    39
31774
7d2cbe11ae48 pycompat: introduce identity function as a compat stub
Yuya Nishihara <yuya@tcha.org>
parents: 31573
diff changeset
    40
def identity(a):
7d2cbe11ae48 pycompat: introduce identity function as a compat stub
Yuya Nishihara <yuya@tcha.org>
parents: 31573
diff changeset
    41
    return a
7d2cbe11ae48 pycompat: introduce identity function as a compat stub
Yuya Nishihara <yuya@tcha.org>
parents: 31573
diff changeset
    42
30031
0f6d6fdd3c2a pycompat: provide 'ispy3' constant
Yuya Nishihara <yuya@tcha.org>
parents: 29801
diff changeset
    43
if ispy3:
29797
965c91bad9e3 py3: move xrange alias next to import lines
Yuya Nishihara <yuya@tcha.org>
parents: 29779
diff changeset
    44
    import builtins
29799
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
    45
    import functools
31372
06440ba06bc0 pycompat: move imports of cStringIO/io to where they are used
Yuya Nishihara <yuya@tcha.org>
parents: 31359
diff changeset
    46
    import io
31424
4acc49335a6e py3: optimize py3 compat.bytechr using Struct.pack
Martin von Zweigbergk <martinvonz@google.com>
parents: 31400
diff changeset
    47
    import struct
31372
06440ba06bc0 pycompat: move imports of cStringIO/io to where they are used
Yuya Nishihara <yuya@tcha.org>
parents: 31359
diff changeset
    48
30119
f4a5e0e86a7e py3: add an os.fsencode backport to ease path handling
Martijn Pieters <mjpieters@fb.com>
parents: 30086
diff changeset
    49
    fsencode = os.fsencode
30300
42af0590f4b9 py3: add os.fsdecode() as pycompat.fsdecode()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30133
diff changeset
    50
    fsdecode = os.fsdecode
36648
6585ac350fd9 py3: make os.curdir a bytes
Yuya Nishihara <yuya@tcha.org>
parents: 36647
diff changeset
    51
    oscurdir = os.curdir.encode('ascii')
31775
8181f378b073 pycompat: provide bytes os.linesep
Yuya Nishihara <yuya@tcha.org>
parents: 31774
diff changeset
    52
    oslinesep = os.linesep.encode('ascii')
30302
3874ddba1ab4 py3: add a bytes version of os.name
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30300
diff changeset
    53
    osname = os.name.encode('ascii')
30303
ad40d307a9f0 py3: have pycompat.ospathsep and pycompat.ossep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30302
diff changeset
    54
    ospathsep = os.pathsep.encode('ascii')
36647
052351e3e1cd py3: make os.pardir a bytes
Yuya Nishihara <yuya@tcha.org>
parents: 36644
diff changeset
    55
    ospardir = os.pardir.encode('ascii')
30303
ad40d307a9f0 py3: have pycompat.ospathsep and pycompat.ossep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30302
diff changeset
    56
    ossep = os.sep.encode('ascii')
30623
c6026c20a3ce py3: have a bytes version of os.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30579
diff changeset
    57
    osaltsep = os.altsep
c6026c20a3ce py3: have a bytes version of os.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30579
diff changeset
    58
    if osaltsep:
c6026c20a3ce py3: have a bytes version of os.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30579
diff changeset
    59
        osaltsep = osaltsep.encode('ascii')
30500
fc0cfe6c87d7 py3: add os.getcwdb() to have bytes path
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30472
diff changeset
    60
    # os.getcwd() on Python 3 returns string, but it has os.getcwdb() which
fc0cfe6c87d7 py3: add os.getcwdb() to have bytes path
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30472
diff changeset
    61
    # returns bytes.
fc0cfe6c87d7 py3: add os.getcwdb() to have bytes path
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30472
diff changeset
    62
    getcwd = os.getcwdb
30624
a82a6eee2613 py3: have a bytes version of sys.platform
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30623
diff changeset
    63
    sysplatform = sys.platform.encode('ascii')
30668
3fcaf0f660ce py3: have bytes version of sys.executable
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30663
diff changeset
    64
    sysexecutable = sys.executable
3fcaf0f660ce py3: have bytes version of sys.executable
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30663
diff changeset
    65
    if sysexecutable:
3fcaf0f660ce py3: have bytes version of sys.executable
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30663
diff changeset
    66
        sysexecutable = os.fsencode(sysexecutable)
36958
644a02f6b34f util: prefer "bytesio" to "stringio"
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36934
diff changeset
    67
    bytesio = io.BytesIO
644a02f6b34f util: prefer "bytesio" to "stringio"
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36934
diff changeset
    68
    # TODO deprecate stringio name, as it is a lie on Python 3.
644a02f6b34f util: prefer "bytesio" to "stringio"
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36934
diff changeset
    69
    stringio = bytesio
36934
dbae581010ea pycompat: name maplist() and ziplist() for better traceback message
Yuya Nishihara <yuya@tcha.org>
parents: 36648
diff changeset
    70
dbae581010ea pycompat: name maplist() and ziplist() for better traceback message
Yuya Nishihara <yuya@tcha.org>
parents: 36648
diff changeset
    71
    def maplist(*args):
dbae581010ea pycompat: name maplist() and ziplist() for better traceback message
Yuya Nishihara <yuya@tcha.org>
parents: 36648
diff changeset
    72
        return list(map(*args))
dbae581010ea pycompat: name maplist() and ziplist() for better traceback message
Yuya Nishihara <yuya@tcha.org>
parents: 36648
diff changeset
    73
dbae581010ea pycompat: name maplist() and ziplist() for better traceback message
Yuya Nishihara <yuya@tcha.org>
parents: 36648
diff changeset
    74
    def ziplist(*args):
dbae581010ea pycompat: name maplist() and ziplist() for better traceback message
Yuya Nishihara <yuya@tcha.org>
parents: 36648
diff changeset
    75
        return list(zip(*args))
dbae581010ea pycompat: name maplist() and ziplist() for better traceback message
Yuya Nishihara <yuya@tcha.org>
parents: 36648
diff changeset
    76
33853
cfcfbe6c96f8 py3: select input or raw_input by pycompat
Yuya Nishihara <yuya@tcha.org>
parents: 33626
diff changeset
    77
    rawinput = input
36178
646002338365 py3: introduce and use pycompat.getargspec
Augie Fackler <augie@google.com>
parents: 36045
diff changeset
    78
    getargspec = inspect.getfullargspec
30334
19d8e19fde5b py3: document why os.fsencode() can be used to get back bytes argv
Yuya Nishihara <yuya@tcha.org>
parents: 30330
diff changeset
    79
30472
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
    80
    # TODO: .buffer might not exist if std streams were replaced; we'll need
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
    81
    # a silly wrapper to make a bytes stream backed by a unicode one.
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
    82
    stdin = sys.stdin.buffer
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
    83
    stdout = sys.stdout.buffer
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
    84
    stderr = sys.stderr.buffer
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
    85
30334
19d8e19fde5b py3: document why os.fsencode() can be used to get back bytes argv
Yuya Nishihara <yuya@tcha.org>
parents: 30330
diff changeset
    86
    # Since Python 3 converts argv to wchar_t type by Py_DecodeLocale() on Unix,
19d8e19fde5b py3: document why os.fsencode() can be used to get back bytes argv
Yuya Nishihara <yuya@tcha.org>
parents: 30330
diff changeset
    87
    # we can use os.fsencode() to get back bytes argv.
19d8e19fde5b py3: document why os.fsencode() can be used to get back bytes argv
Yuya Nishihara <yuya@tcha.org>
parents: 30330
diff changeset
    88
    #
19d8e19fde5b py3: document why os.fsencode() can be used to get back bytes argv
Yuya Nishihara <yuya@tcha.org>
parents: 30330
diff changeset
    89
    # https://hg.python.org/cpython/file/v3.5.1/Programs/python.c#l55
19d8e19fde5b py3: document why os.fsencode() can be used to get back bytes argv
Yuya Nishihara <yuya@tcha.org>
parents: 30330
diff changeset
    90
    #
19d8e19fde5b py3: document why os.fsencode() can be used to get back bytes argv
Yuya Nishihara <yuya@tcha.org>
parents: 30330
diff changeset
    91
    # TODO: On Windows, the native argv is wchar_t, so we'll need a different
19d8e19fde5b py3: document why os.fsencode() can be used to get back bytes argv
Yuya Nishihara <yuya@tcha.org>
parents: 30330
diff changeset
    92
    # workaround to simulate the Python 2 (i.e. ANSI Win32 API) behavior.
31277
86cd1f2cfff5 pycompat: verify sys.argv exists before forwarding it (issue5493)
Augie Fackler <augie@google.com>
parents: 30820
diff changeset
    93
    if getattr(sys, 'argv', None) is not None:
86cd1f2cfff5 pycompat: verify sys.argv exists before forwarding it (issue5493)
Augie Fackler <augie@google.com>
parents: 30820
diff changeset
    94
        sysargv = list(map(os.fsencode, sys.argv))
29797
965c91bad9e3 py3: move xrange alias next to import lines
Yuya Nishihara <yuya@tcha.org>
parents: 29779
diff changeset
    95
31424
4acc49335a6e py3: optimize py3 compat.bytechr using Struct.pack
Martin von Zweigbergk <martinvonz@google.com>
parents: 31400
diff changeset
    96
    bytechr = struct.Struct('>B').pack
36265
b44fac3a49fb py3: factor out byterepr() which returns an asciified value on py3
Yuya Nishihara <yuya@tcha.org>
parents: 36178
diff changeset
    97
    byterepr = b'%r'.__mod__
31253
64596338ba10 py3: factor out bytechr() function
Yuya Nishihara <yuya@tcha.org>
parents: 31149
diff changeset
    98
31439
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
    99
    class bytestr(bytes):
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   100
        """A bytes which mostly acts as a Python 2 str
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   101
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   102
        >>> bytestr(), bytestr(bytearray(b'foo')), bytestr(u'ascii'), bytestr(1)
35903
1a31111e6239 py3: always drop b'' prefix from repr() of bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 35405
diff changeset
   103
        ('', 'foo', 'ascii', '1')
31439
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   104
        >>> s = bytestr(b'foo')
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   105
        >>> assert s is bytestr(s)
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   106
32450
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   107
        __bytes__() should be called if provided:
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   108
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   109
        >>> class bytesable(object):
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   110
        ...     def __bytes__(self):
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   111
        ...         return b'bytes'
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   112
        >>> bytestr(bytesable())
35903
1a31111e6239 py3: always drop b'' prefix from repr() of bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 35405
diff changeset
   113
        'bytes'
32450
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   114
31439
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   115
        There's no implicit conversion from non-ascii str as its encoding is
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   116
        unknown:
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   117
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   118
        >>> bytestr(chr(0x80)) # doctest: +ELLIPSIS
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   119
        Traceback (most recent call last):
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   120
          ...
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   121
        UnicodeEncodeError: ...
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   122
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   123
        Comparison between bytestr and bytes should work:
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   124
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   125
        >>> assert bytestr(b'foo') == b'foo'
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   126
        >>> assert b'foo' == bytestr(b'foo')
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   127
        >>> assert b'f' in bytestr(b'foo')
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   128
        >>> assert bytestr(b'f') in b'foo'
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   129
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   130
        Sliced elements should be bytes, not integer:
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   131
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   132
        >>> s[1], s[:2]
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   133
        (b'o', b'fo')
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   134
        >>> list(s), list(reversed(s))
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   135
        ([b'f', b'o', b'o'], [b'o', b'o', b'f'])
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   136
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   137
        As bytestr type isn't propagated across operations, you need to cast
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   138
        bytes to bytestr explicitly:
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   139
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   140
        >>> s = bytestr(b'foo').upper()
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   141
        >>> t = bytestr(s)
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   142
        >>> s[0], t[0]
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   143
        (70, b'F')
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   144
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   145
        Be careful to not pass a bytestr object to a function which expects
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   146
        bytearray-like behavior.
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   147
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   148
        >>> t = bytes(t)  # cast to bytes
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   149
        >>> assert type(t) is bytes
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   150
        """
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   151
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   152
        def __new__(cls, s=b''):
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   153
            if isinstance(s, bytestr):
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   154
                return s
32450
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   155
            if (not isinstance(s, (bytes, bytearray))
548478efc46c pycompat: try __bytes__() to convert object to bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 32186
diff changeset
   156
                and not hasattr(s, u'__bytes__')):  # hasattr-py3-only
31439
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   157
                s = str(s).encode(u'ascii')
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   158
            return bytes.__new__(cls, s)
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   159
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   160
        def __getitem__(self, key):
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   161
            s = bytes.__getitem__(self, key)
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   162
            if not isinstance(s, bytes):
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   163
                s = bytechr(s)
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   164
            return s
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   165
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   166
        def __iter__(self):
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   167
            return iterbytestr(bytes.__iter__(self))
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   168
35903
1a31111e6239 py3: always drop b'' prefix from repr() of bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 35405
diff changeset
   169
        def __repr__(self):
1a31111e6239 py3: always drop b'' prefix from repr() of bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 35405
diff changeset
   170
            return bytes.__repr__(self)[1:]  # drop b''
1a31111e6239 py3: always drop b'' prefix from repr() of bytestr
Yuya Nishihara <yuya@tcha.org>
parents: 35405
diff changeset
   171
31382
c9fd842dc886 pycompat: add helper to iterate each char in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31372
diff changeset
   172
    def iterbytestr(s):
c9fd842dc886 pycompat: add helper to iterate each char in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31372
diff changeset
   173
        """Iterate bytes as if it were a str object of Python 2"""
31425
63a39d647888 py3: make py3 compat.iterbytestr simpler and faster
Martin von Zweigbergk <martinvonz@google.com>
parents: 31424
diff changeset
   174
        return map(bytechr, s)
31382
c9fd842dc886 pycompat: add helper to iterate each char in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31372
diff changeset
   175
35904
fc44c2657dc5 py3: drop b'' from repr() of smartset
Yuya Nishihara <yuya@tcha.org>
parents: 35903
diff changeset
   176
    def maybebytestr(s):
fc44c2657dc5 py3: drop b'' from repr() of smartset
Yuya Nishihara <yuya@tcha.org>
parents: 35903
diff changeset
   177
        """Promote bytes to bytestr"""
fc44c2657dc5 py3: drop b'' from repr() of smartset
Yuya Nishihara <yuya@tcha.org>
parents: 35903
diff changeset
   178
        if isinstance(s, bytes):
fc44c2657dc5 py3: drop b'' from repr() of smartset
Yuya Nishihara <yuya@tcha.org>
parents: 35903
diff changeset
   179
            return bytestr(s)
fc44c2657dc5 py3: drop b'' from repr() of smartset
Yuya Nishihara <yuya@tcha.org>
parents: 35903
diff changeset
   180
        return s
fc44c2657dc5 py3: drop b'' from repr() of smartset
Yuya Nishihara <yuya@tcha.org>
parents: 35903
diff changeset
   181
31820
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   182
    def sysbytes(s):
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   183
        """Convert an internal str (e.g. keyword, __doc__) back to bytes
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   184
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   185
        This never raises UnicodeEncodeError, but only ASCII characters
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   186
        can be round-trip by sysstr(sysbytes(s)).
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   187
        """
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   188
        return s.encode(u'utf-8')
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   189
30032
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   190
    def sysstr(s):
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   191
        """Return a keyword str to be passed to Python functions such as
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   192
        getattr() and str.encode()
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   193
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   194
        This never raises UnicodeDecodeError. Non-ascii characters are
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   195
        considered invalid and mapped to arbitrary but unique code points
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   196
        such that 'sysstr(a) != sysstr(b)' for all 'a != b'.
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   197
        """
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   198
        if isinstance(s, builtins.str):
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   199
            return s
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   200
        return s.decode(u'latin-1')
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   201
32859
a05f3675c46a py3: add a new strurl() which will convert a bytes url to str
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32615
diff changeset
   202
    def strurl(url):
a05f3675c46a py3: add a new strurl() which will convert a bytes url to str
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32615
diff changeset
   203
        """Converts a bytes url back to str"""
36644
e2b87e19c6ef pycompat: prevent encoding or decoding values if not required
Pulkit Goyal <7895pulkit@gmail.com>
parents: 36556
diff changeset
   204
        if isinstance(url, bytes):
e2b87e19c6ef pycompat: prevent encoding or decoding values if not required
Pulkit Goyal <7895pulkit@gmail.com>
parents: 36556
diff changeset
   205
            return url.decode(u'ascii')
e2b87e19c6ef pycompat: prevent encoding or decoding values if not required
Pulkit Goyal <7895pulkit@gmail.com>
parents: 36556
diff changeset
   206
        return url
32859
a05f3675c46a py3: add a new strurl() which will convert a bytes url to str
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32615
diff changeset
   207
32860
f22f39d56bb5 py3: add a new bytesurl() to convert a str url into bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32859
diff changeset
   208
    def bytesurl(url):
f22f39d56bb5 py3: add a new bytesurl() to convert a str url into bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32859
diff changeset
   209
        """Converts a str url to bytes by encoding in ascii"""
36644
e2b87e19c6ef pycompat: prevent encoding or decoding values if not required
Pulkit Goyal <7895pulkit@gmail.com>
parents: 36556
diff changeset
   210
        if isinstance(url, str):
e2b87e19c6ef pycompat: prevent encoding or decoding values if not required
Pulkit Goyal <7895pulkit@gmail.com>
parents: 36556
diff changeset
   211
            return url.encode(u'ascii')
e2b87e19c6ef pycompat: prevent encoding or decoding values if not required
Pulkit Goyal <7895pulkit@gmail.com>
parents: 36556
diff changeset
   212
        return url
32860
f22f39d56bb5 py3: add a new bytesurl() to convert a str url into bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32859
diff changeset
   213
32186
76f9a0009b4b pycompat: extract helper to raise exception with traceback
Yuya Nishihara <yuya@tcha.org>
parents: 31942
diff changeset
   214
    def raisewithtb(exc, tb):
76f9a0009b4b pycompat: extract helper to raise exception with traceback
Yuya Nishihara <yuya@tcha.org>
parents: 31942
diff changeset
   215
        """Raise exception with the given traceback"""
76f9a0009b4b pycompat: extract helper to raise exception with traceback
Yuya Nishihara <yuya@tcha.org>
parents: 31942
diff changeset
   216
        raise exc.with_traceback(tb)
76f9a0009b4b pycompat: extract helper to raise exception with traceback
Yuya Nishihara <yuya@tcha.org>
parents: 31942
diff changeset
   217
32615
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   218
    def getdoc(obj):
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   219
        """Get docstring as bytes; may be None so gettext() won't confuse it
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   220
        with _('')"""
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   221
        doc = getattr(obj, u'__doc__', None)
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   222
        if doc is None:
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   223
            return doc
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   224
        return sysbytes(doc)
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   225
29799
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   226
    def _wrapattrfunc(f):
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   227
        @functools.wraps(f)
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   228
        def w(object, name, *args):
30032
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   229
            return f(object, sysstr(name), *args)
29799
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   230
        return w
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   231
29800
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29799
diff changeset
   232
    # these wrappers are automagically imported by hgloader
29799
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   233
    delattr = _wrapattrfunc(builtins.delattr)
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   234
    getattr = _wrapattrfunc(builtins.getattr)
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   235
    hasattr = _wrapattrfunc(builtins.hasattr)
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   236
    setattr = _wrapattrfunc(builtins.setattr)
29800
178c89e8519a py3: import builtin wrappers automagically by code transformer
Yuya Nishihara <yuya@tcha.org>
parents: 29799
diff changeset
   237
    xrange = builtins.range
31843
526e4597cca5 py3: add pycompat.unicode and add it to importer
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31842
diff changeset
   238
    unicode = str
29799
45fa8de47a0f py3: provide (del|get|has|set)attr wrappers that accepts bytes
Yuya Nishihara <yuya@tcha.org>
parents: 29798
diff changeset
   239
36556
63fe5ca93b13 pycompat: add support for encoding argument to our wrapper
Augie Fackler <augie@google.com>
parents: 36334
diff changeset
   240
    def open(name, mode='r', buffering=-1, encoding=None):
63fe5ca93b13 pycompat: add support for encoding argument to our wrapper
Augie Fackler <augie@google.com>
parents: 36334
diff changeset
   241
        return builtins.open(name, sysstr(mode), buffering, encoding)
31149
76a64c1e5439 py3: add pycompat.open and replace open() calls
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30820
diff changeset
   242
35226
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   243
    def _getoptbwrapper(orig, args, shortlist, namelist):
32864
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   244
        """
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   245
        Takes bytes arguments, converts them to unicode, pass them to
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   246
        getopt.getopt(), convert the returned values back to bytes and then
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   247
        return them for Python 3 compatibility as getopt.getopt() don't accepts
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   248
        bytes on Python 3.
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   249
        """
30578
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   250
        args = [a.decode('latin-1') for a in args]
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   251
        shortlist = shortlist.decode('latin-1')
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   252
        namelist = [a.decode('latin-1') for a in namelist]
35226
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   253
        opts, args = orig(args, shortlist, namelist)
30578
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   254
        opts = [(a[0].encode('latin-1'), a[1].encode('latin-1'))
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   255
                for a in opts]
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   256
        args = [a.encode('latin-1') for a in args]
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   257
        return opts, args
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   258
30579
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   259
    def strkwargs(dic):
32864
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   260
        """
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   261
        Converts the keys of a python dictonary to str i.e. unicodes so that
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   262
        they can be passed as keyword arguments as dictonaries with bytes keys
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   263
        can't be passed as keyword arguments to functions on Python 3.
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   264
        """
30579
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   265
        dic = dict((k.decode('latin-1'), v) for k, v in dic.iteritems())
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   266
        return dic
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   267
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   268
    def byteskwargs(dic):
32864
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   269
        """
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   270
        Converts keys of python dictonaries to bytes as they were converted to
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   271
        str to pass that dictonary as a keyword argument on Python 3.
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   272
        """
30579
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   273
        dic = dict((k.encode('latin-1'), v) for k, v in dic.iteritems())
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   274
        return dic
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   275
30678
caf7e1c5efe4 py3: have a bytes version of shlex.split()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30668
diff changeset
   276
    # TODO: handle shlex.shlex().
36334
4cd2d1cc2a31 pycompat: correct the shlex.split() proxy method signature in py3
Matt Harbison <matt_harbison@yahoo.com>
parents: 36265
diff changeset
   277
    def shlexsplit(s, comments=False, posix=True):
32864
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   278
        """
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   279
        Takes bytes argument, convert it to str i.e. unicodes, pass that into
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   280
        shlex.split(), convert the returned value to bytes and return that for
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   281
        Python 3 compatibility as shelx.split() don't accept bytes on Python 3.
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   282
        """
36334
4cd2d1cc2a31 pycompat: correct the shlex.split() proxy method signature in py3
Matt Harbison <matt_harbison@yahoo.com>
parents: 36265
diff changeset
   283
        ret = shlex.split(s.decode('latin-1'), comments, posix)
30678
caf7e1c5efe4 py3: have a bytes version of shlex.split()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30668
diff changeset
   284
        return [a.encode('latin-1') for a in ret]
caf7e1c5efe4 py3: have a bytes version of shlex.split()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30668
diff changeset
   285
36045
04984f2e50ae py3: use email parser that operates on bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35904
diff changeset
   286
    def emailparser(*args, **kwargs):
04984f2e50ae py3: use email parser that operates on bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35904
diff changeset
   287
        import email.parser
04984f2e50ae py3: use email parser that operates on bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35904
diff changeset
   288
        return email.parser.BytesParser(*args, **kwargs)
04984f2e50ae py3: use email parser that operates on bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35904
diff changeset
   289
30032
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   290
else:
31372
06440ba06bc0 pycompat: move imports of cStringIO/io to where they are used
Yuya Nishihara <yuya@tcha.org>
parents: 31359
diff changeset
   291
    import cStringIO
06440ba06bc0 pycompat: move imports of cStringIO/io to where they are used
Yuya Nishihara <yuya@tcha.org>
parents: 31359
diff changeset
   292
31253
64596338ba10 py3: factor out bytechr() function
Yuya Nishihara <yuya@tcha.org>
parents: 31149
diff changeset
   293
    bytechr = chr
36265
b44fac3a49fb py3: factor out byterepr() which returns an asciified value on py3
Yuya Nishihara <yuya@tcha.org>
parents: 36178
diff changeset
   294
    byterepr = repr
31439
b70407bd84d5 pycompat: add bytestr wrapper which mostly acts as a Python 2 str
Yuya Nishihara <yuya@tcha.org>
parents: 31425
diff changeset
   295
    bytestr = str
31382
c9fd842dc886 pycompat: add helper to iterate each char in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31372
diff changeset
   296
    iterbytestr = iter
35904
fc44c2657dc5 py3: drop b'' from repr() of smartset
Yuya Nishihara <yuya@tcha.org>
parents: 35903
diff changeset
   297
    maybebytestr = identity
31820
45761ef1bc93 py3: have registrar process docstrings in bytes
Yuya Nishihara <yuya@tcha.org>
parents: 31775
diff changeset
   298
    sysbytes = identity
31774
7d2cbe11ae48 pycompat: introduce identity function as a compat stub
Yuya Nishihara <yuya@tcha.org>
parents: 31573
diff changeset
   299
    sysstr = identity
32859
a05f3675c46a py3: add a new strurl() which will convert a bytes url to str
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32615
diff changeset
   300
    strurl = identity
32860
f22f39d56bb5 py3: add a new bytesurl() to convert a str url into bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32859
diff changeset
   301
    bytesurl = identity
30032
2219f4f82ede pycompat: extract function that converts attribute or encoding name to str
Yuya Nishihara <yuya@tcha.org>
parents: 30031
diff changeset
   302
32186
76f9a0009b4b pycompat: extract helper to raise exception with traceback
Yuya Nishihara <yuya@tcha.org>
parents: 31942
diff changeset
   303
    # this can't be parsed on Python 3
76f9a0009b4b pycompat: extract helper to raise exception with traceback
Yuya Nishihara <yuya@tcha.org>
parents: 31942
diff changeset
   304
    exec('def raisewithtb(exc, tb):\n'
76f9a0009b4b pycompat: extract helper to raise exception with traceback
Yuya Nishihara <yuya@tcha.org>
parents: 31942
diff changeset
   305
         '    raise exc, None, tb\n')
76f9a0009b4b pycompat: extract helper to raise exception with traceback
Yuya Nishihara <yuya@tcha.org>
parents: 31942
diff changeset
   306
30133
f6dcda7505f9 pycompat: only accept a bytestring filepath in Python 2
Martijn Pieters <mjpieters@fb.com>
parents: 30119
diff changeset
   307
    def fsencode(filename):
32864
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   308
        """
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   309
        Partial backport from os.py in Python 3, which only accepts bytes.
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   310
        In Python 2, our paths should only ever be bytes, a unicode path
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   311
        indicates a bug.
f57f1f37290d pycompat: move multiline comments above a function to function doc
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32860
diff changeset
   312
        """
30133
f6dcda7505f9 pycompat: only accept a bytestring filepath in Python 2
Martijn Pieters <mjpieters@fb.com>
parents: 30119
diff changeset
   313
        if isinstance(filename, str):
f6dcda7505f9 pycompat: only accept a bytestring filepath in Python 2
Martijn Pieters <mjpieters@fb.com>
parents: 30119
diff changeset
   314
            return filename
30119
f4a5e0e86a7e py3: add an os.fsencode backport to ease path handling
Martijn Pieters <mjpieters@fb.com>
parents: 30086
diff changeset
   315
        else:
30133
f6dcda7505f9 pycompat: only accept a bytestring filepath in Python 2
Martijn Pieters <mjpieters@fb.com>
parents: 30119
diff changeset
   316
            raise TypeError(
f6dcda7505f9 pycompat: only accept a bytestring filepath in Python 2
Martijn Pieters <mjpieters@fb.com>
parents: 30119
diff changeset
   317
                "expect str, not %s" % type(filename).__name__)
30119
f4a5e0e86a7e py3: add an os.fsencode backport to ease path handling
Martijn Pieters <mjpieters@fb.com>
parents: 30086
diff changeset
   318
30300
42af0590f4b9 py3: add os.fsdecode() as pycompat.fsdecode()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30133
diff changeset
   319
    # In Python 2, fsdecode() has a very chance to receive bytes. So it's
42af0590f4b9 py3: add os.fsdecode() as pycompat.fsdecode()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30133
diff changeset
   320
    # better not to touch Python 2 part as it's already working fine.
31774
7d2cbe11ae48 pycompat: introduce identity function as a compat stub
Yuya Nishihara <yuya@tcha.org>
parents: 31573
diff changeset
   321
    fsdecode = identity
30300
42af0590f4b9 py3: add os.fsdecode() as pycompat.fsdecode()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30133
diff changeset
   322
32615
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   323
    def getdoc(obj):
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   324
        return getattr(obj, '__doc__', None)
c9318beb7c1a py3: convert __doc__ back to bytes in help.py
Yuya Nishihara <yuya@tcha.org>
parents: 32450
diff changeset
   325
35226
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   326
    def _getoptbwrapper(orig, args, shortlist, namelist):
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   327
        return orig(args, shortlist, namelist)
30578
c6ce11f2ee50 py3: make a bytes version of getopt.getopt()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30500
diff changeset
   328
31774
7d2cbe11ae48 pycompat: introduce identity function as a compat stub
Yuya Nishihara <yuya@tcha.org>
parents: 31573
diff changeset
   329
    strkwargs = identity
7d2cbe11ae48 pycompat: introduce identity function as a compat stub
Yuya Nishihara <yuya@tcha.org>
parents: 31573
diff changeset
   330
    byteskwargs = identity
30579
fbc3f73dc802 py3: utility functions to convert keys of kwargs to bytes/unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30578
diff changeset
   331
36648
6585ac350fd9 py3: make os.curdir a bytes
Yuya Nishihara <yuya@tcha.org>
parents: 36647
diff changeset
   332
    oscurdir = os.curdir
31775
8181f378b073 pycompat: provide bytes os.linesep
Yuya Nishihara <yuya@tcha.org>
parents: 31774
diff changeset
   333
    oslinesep = os.linesep
30302
3874ddba1ab4 py3: add a bytes version of os.name
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30300
diff changeset
   334
    osname = os.name
30303
ad40d307a9f0 py3: have pycompat.ospathsep and pycompat.ossep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30302
diff changeset
   335
    ospathsep = os.pathsep
36647
052351e3e1cd py3: make os.pardir a bytes
Yuya Nishihara <yuya@tcha.org>
parents: 36644
diff changeset
   336
    ospardir = os.pardir
30303
ad40d307a9f0 py3: have pycompat.ospathsep and pycompat.ossep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30302
diff changeset
   337
    ossep = os.sep
30623
c6026c20a3ce py3: have a bytes version of os.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30579
diff changeset
   338
    osaltsep = os.altsep
30472
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
   339
    stdin = sys.stdin
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
   340
    stdout = sys.stdout
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30334
diff changeset
   341
    stderr = sys.stderr
31277
86cd1f2cfff5 pycompat: verify sys.argv exists before forwarding it (issue5493)
Augie Fackler <augie@google.com>
parents: 30820
diff changeset
   342
    if getattr(sys, 'argv', None) is not None:
86cd1f2cfff5 pycompat: verify sys.argv exists before forwarding it (issue5493)
Augie Fackler <augie@google.com>
parents: 30820
diff changeset
   343
        sysargv = sys.argv
30624
a82a6eee2613 py3: have a bytes version of sys.platform
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30623
diff changeset
   344
    sysplatform = sys.platform
30500
fc0cfe6c87d7 py3: add os.getcwdb() to have bytes path
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30472
diff changeset
   345
    getcwd = os.getcwd
30668
3fcaf0f660ce py3: have bytes version of sys.executable
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30663
diff changeset
   346
    sysexecutable = sys.executable
30678
caf7e1c5efe4 py3: have a bytes version of shlex.split()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30668
diff changeset
   347
    shlexsplit = shlex.split
36958
644a02f6b34f util: prefer "bytesio" to "stringio"
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36934
diff changeset
   348
    bytesio = cStringIO.StringIO
644a02f6b34f util: prefer "bytesio" to "stringio"
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36934
diff changeset
   349
    stringio = bytesio
31501
a1e40ceee640 pycompat: add maplist alias for old map behavior
Augie Fackler <augie@google.com>
parents: 31439
diff changeset
   350
    maplist = map
35405
e66d6e938d2d py3: introduce pycompat.ziplist as zip is a generator on Python 3
Pulkit Goyal <7895pulkit@gmail.com>
parents: 35226
diff changeset
   351
    ziplist = zip
33853
cfcfbe6c96f8 py3: select input or raw_input by pycompat
Yuya Nishihara <yuya@tcha.org>
parents: 33626
diff changeset
   352
    rawinput = raw_input
36178
646002338365 py3: introduce and use pycompat.getargspec
Augie Fackler <augie@google.com>
parents: 36045
diff changeset
   353
    getargspec = inspect.getargspec
34639
a568a46751b6 selectors2: do not use platform.system()
Jun Wu <quark@fb.com>
parents: 34467
diff changeset
   354
36045
04984f2e50ae py3: use email parser that operates on bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35904
diff changeset
   355
    def emailparser(*args, **kwargs):
04984f2e50ae py3: use email parser that operates on bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35904
diff changeset
   356
        import email.parser
04984f2e50ae py3: use email parser that operates on bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35904
diff changeset
   357
        return email.parser.Parser(*args, **kwargs)
04984f2e50ae py3: use email parser that operates on bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35904
diff changeset
   358
34639
a568a46751b6 selectors2: do not use platform.system()
Jun Wu <quark@fb.com>
parents: 34467
diff changeset
   359
isjython = sysplatform.startswith('java')
34644
c0a6c19690ff pycompat: define operating system constants
Jun Wu <quark@fb.com>
parents: 34639
diff changeset
   360
c0a6c19690ff pycompat: define operating system constants
Jun Wu <quark@fb.com>
parents: 34639
diff changeset
   361
isdarwin = sysplatform == 'darwin'
c0a6c19690ff pycompat: define operating system constants
Jun Wu <quark@fb.com>
parents: 34639
diff changeset
   362
isposix = osname == 'posix'
c0a6c19690ff pycompat: define operating system constants
Jun Wu <quark@fb.com>
parents: 34639
diff changeset
   363
iswindows = osname == 'nt'
35226
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   364
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   365
def getoptb(args, shortlist, namelist):
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   366
    return _getoptbwrapper(getopt.getopt, args, shortlist, namelist)
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   367
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   368
def gnugetoptb(args, shortlist, namelist):
5b569d512fbd fancyopts: use getopt.gnu_getopt()
Yuya Nishihara <yuya@tcha.org>
parents: 34644
diff changeset
   369
    return _getoptbwrapper(getopt.gnu_getopt, args, shortlist, namelist)