mercurial/util.py
branchstable
changeset 49366 288de6f5d724
parent 49308 d2adebe35635
child 49572 c4f07a011714
equal deleted inserted replaced
49364:e8ea403b1c46 49366:288de6f5d724
    11 
    11 
    12 This contains helper routines that are independent of the SCM core and
    12 This contains helper routines that are independent of the SCM core and
    13 hide platform-specific details from the core.
    13 hide platform-specific details from the core.
    14 """
    14 """
    15 
    15 
    16 from __future__ import absolute_import, print_function
       
    17 
    16 
    18 import abc
    17 import abc
    19 import collections
    18 import collections
    20 import contextlib
    19 import contextlib
    21 import errno
    20 import errno
    22 import gc
    21 import gc
    23 import hashlib
    22 import hashlib
       
    23 import io
    24 import itertools
    24 import itertools
    25 import locale
    25 import locale
    26 import mmap
    26 import mmap
    27 import os
    27 import os
    28 import platform as pyplatform
    28 import pickle  # provides util.pickle symbol
    29 import re as remod
    29 import re as remod
    30 import shutil
    30 import shutil
    31 import stat
    31 import stat
    32 import sys
    32 import sys
    33 import time
    33 import time
    40     delattr,
    40     delattr,
    41     getattr,
    41     getattr,
    42     open,
    42     open,
    43     setattr,
    43     setattr,
    44 )
    44 )
    45 from .node import hex
       
    46 from hgdemandimport import tracing
    45 from hgdemandimport import tracing
    47 from . import (
    46 from . import (
    48     encoding,
    47     encoding,
    49     error,
    48     error,
    50     i18n,
    49     i18n,
    74 b85decode = base85.b85decode
    73 b85decode = base85.b85decode
    75 b85encode = base85.b85encode
    74 b85encode = base85.b85encode
    76 
    75 
    77 cookielib = pycompat.cookielib
    76 cookielib = pycompat.cookielib
    78 httplib = pycompat.httplib
    77 httplib = pycompat.httplib
    79 pickle = pycompat.pickle
       
    80 safehasattr = pycompat.safehasattr
    78 safehasattr = pycompat.safehasattr
    81 socketserver = pycompat.socketserver
    79 socketserver = pycompat.socketserver
    82 bytesio = pycompat.bytesio
    80 bytesio = io.BytesIO
    83 # TODO deprecate stringio name, as it is a lie on Python 3.
    81 # TODO deprecate stringio name, as it is a lie on Python 3.
    84 stringio = bytesio
    82 stringio = bytesio
    85 xmlrpclib = pycompat.xmlrpclib
    83 xmlrpclib = pycompat.xmlrpclib
    86 
    84 
    87 httpserver = urllibcompat.httpserver
    85 httpserver = urllibcompat.httpserver
   156 # small compat layer
   154 # small compat layer
   157 compengines = compression.compengines
   155 compengines = compression.compengines
   158 SERVERROLE = compression.SERVERROLE
   156 SERVERROLE = compression.SERVERROLE
   159 CLIENTROLE = compression.CLIENTROLE
   157 CLIENTROLE = compression.CLIENTROLE
   160 
   158 
   161 try:
       
   162     recvfds = osutil.recvfds
       
   163 except AttributeError:
       
   164     pass
       
   165 
       
   166 # Python compatibility
   159 # Python compatibility
   167 
   160 
   168 _notset = object()
   161 _notset = object()
   169 
   162 
   170 
   163 
   187     # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
   180     # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
   188     # makes the whole PYTHONWARNINGS thing useless for our usecase.
   181     # makes the whole PYTHONWARNINGS thing useless for our usecase.
   189     warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
   182     warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial')
   190     warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
   183     warnings.filterwarnings('default', '', DeprecationWarning, 'hgext')
   191     warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
   184     warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd')
   192 if _dowarn and pycompat.ispy3:
   185 if _dowarn:
   193     # silence warning emitted by passing user string to re.sub()
   186     # silence warning emitted by passing user string to re.sub()
   194     warnings.filterwarnings(
   187     warnings.filterwarnings(
   195         'ignore', 'bad escape', DeprecationWarning, 'mercurial'
   188         'ignore', 'bad escape', DeprecationWarning, 'mercurial'
   196     )
   189     )
   197     warnings.filterwarnings(
   190     warnings.filterwarnings(
   231 
   224 
   232 for k in DIGESTS_BY_STRENGTH:
   225 for k in DIGESTS_BY_STRENGTH:
   233     assert k in DIGESTS
   226     assert k in DIGESTS
   234 
   227 
   235 
   228 
   236 class digester(object):
   229 class digester:
   237     """helper to compute digests.
   230     """helper to compute digests.
   238 
   231 
   239     This helper can be used to compute one or more digests given their name.
   232     This helper can be used to compute one or more digests given their name.
   240 
   233 
   241     >>> d = digester([b'md5', b'sha1'])
   234     >>> d = digester([b'md5', b'sha1'])
   279             if k in supported:
   272             if k in supported:
   280                 return k
   273                 return k
   281         return None
   274         return None
   282 
   275 
   283 
   276 
   284 class digestchecker(object):
   277 class digestchecker:
   285     """file handle wrapper that additionally checks content against a given
   278     """file handle wrapper that additionally checks content against a given
   286     size and digests.
   279     size and digests.
   287 
   280 
   288         d = digestchecker(fh, size, {'md5': '...'})
   281         d = digestchecker(fh, size, {'md5': '...'})
   289 
   282 
   329 
   322 
   330 
   323 
   331 _chunksize = 4096
   324 _chunksize = 4096
   332 
   325 
   333 
   326 
   334 class bufferedinputpipe(object):
   327 class bufferedinputpipe:
   335     """a manually buffered input pipe
   328     """a manually buffered input pipe
   336 
   329 
   337     Python will not let us use buffered IO and lazy reading with 'polling' at
   330     Python will not let us use buffered IO and lazy reading with 'polling' at
   338     the same time. We cannot probe the buffer state and select will not detect
   331     the same time. We cannot probe the buffer state and select will not detect
   339     that data are ready to read if they are already buffered.
   332     that data are ready to read if they are already buffered.
   457         if os.fstat(fd).st_size == 0:
   450         if os.fstat(fd).st_size == 0:
   458             return b''
   451             return b''
   459         raise
   452         raise
   460 
   453 
   461 
   454 
   462 class fileobjectproxy(object):
   455 class fileobjectproxy:
   463     """A proxy around file objects that tells a watcher when events occur.
   456     """A proxy around file objects that tells a watcher when events occur.
   464 
   457 
   465     This type is intended to only be used for testing purposes. Think hard
   458     This type is intended to only be used for testing purposes. Think hard
   466     before using it in important code.
   459     before using it in important code.
   467     """
   460     """
   693     'gettimeout',
   686     'gettimeout',
   694     'setsockopt',
   687     'setsockopt',
   695 }
   688 }
   696 
   689 
   697 
   690 
   698 class socketproxy(object):
   691 class socketproxy:
   699     """A proxy around a socket that tells a watcher when events occur.
   692     """A proxy around a socket that tells a watcher when events occur.
   700 
   693 
   701     This is like ``fileobjectproxy`` except for sockets.
   694     This is like ``fileobjectproxy`` except for sockets.
   702 
   695 
   703     This type is intended to only be used for testing purposes. Think hard
   696     This type is intended to only be used for testing purposes. Think hard
   816         return object.__getattribute__(self, '_observedcall')(
   809         return object.__getattribute__(self, '_observedcall')(
   817             'setsockopt', *args, **kwargs
   810             'setsockopt', *args, **kwargs
   818         )
   811         )
   819 
   812 
   820 
   813 
   821 class baseproxyobserver(object):
   814 class baseproxyobserver:
   822     def __init__(self, fh, name, logdata, logdataapis):
   815     def __init__(self, fh, name, logdata, logdataapis):
   823         self.fh = fh
   816         self.fh = fh
   824         self.name = name
   817         self.name = name
   825         self.logdata = logdata
   818         self.logdata = logdata
   826         self.logdataapis = logdataapis
   819         self.logdataapis = logdataapis
  1256             return cache[args]
  1249             return cache[args]
  1257 
  1250 
  1258     return f
  1251     return f
  1259 
  1252 
  1260 
  1253 
  1261 class cow(object):
  1254 class cow:
  1262     """helper class to make copy-on-write easier
  1255     """helper class to make copy-on-write easier
  1263 
  1256 
  1264     Call preparewrite before doing any writes.
  1257     Call preparewrite before doing any writes.
  1265     """
  1258     """
  1266 
  1259 
  1300 
  1293 
  1301     if pycompat.ispypy:
  1294     if pycompat.ispypy:
  1302         # __setitem__() isn't called as of PyPy 5.8.0
  1295         # __setitem__() isn't called as of PyPy 5.8.0
  1303         def update(self, src, **f):
  1296         def update(self, src, **f):
  1304             if isinstance(src, dict):
  1297             if isinstance(src, dict):
  1305                 src = pycompat.iteritems(src)
  1298                 src = src.items()
  1306             for k, v in src:
  1299             for k, v in src:
  1307                 self[k] = v
  1300                 self[k] = v
  1308             for k in f:
  1301             for k in f:
  1309                 self[k] = f[k]
  1302                 self[k] = f[k]
  1310 
  1303 
  1349 
  1342 
  1350     Be sure to call d = d.preparewrite() before writing to d.
  1343     Be sure to call d = d.preparewrite() before writing to d.
  1351     """
  1344     """
  1352 
  1345 
  1353 
  1346 
  1354 class transactional(object):  # pytype: disable=ignored-metaclass
  1347 class transactional:  # pytype: disable=ignored-metaclass
  1355     """Base class for making a transactional type into a context manager."""
  1348     """Base class for making a transactional type into a context manager."""
  1356 
  1349 
  1357     __metaclass__ = abc.ABCMeta
  1350     __metaclass__ = abc.ABCMeta
  1358 
  1351 
  1359     @abc.abstractmethod
  1352     @abc.abstractmethod
  1400 @contextlib.contextmanager
  1393 @contextlib.contextmanager
  1401 def nullcontextmanager(enter_result=None):
  1394 def nullcontextmanager(enter_result=None):
  1402     yield enter_result
  1395     yield enter_result
  1403 
  1396 
  1404 
  1397 
  1405 class _lrucachenode(object):
  1398 class _lrucachenode:
  1406     """A node in a doubly linked list.
  1399     """A node in a doubly linked list.
  1407 
  1400 
  1408     Holds a reference to nodes on either side as well as a key-value
  1401     Holds a reference to nodes on either side as well as a key-value
  1409     pair for the dictionary entry.
  1402     pair for the dictionary entry.
  1410     """
  1403     """
  1424         self.key = _notset
  1417         self.key = _notset
  1425         self.value = None
  1418         self.value = None
  1426         self.cost = 0
  1419         self.cost = 0
  1427 
  1420 
  1428 
  1421 
  1429 class lrucachedict(object):
  1422 class lrucachedict:
  1430     """Dict that caches most recent accesses and sets.
  1423     """Dict that caches most recent accesses and sets.
  1431 
  1424 
  1432     The dict consists of an actual backing dict - indexed by original
  1425     The dict consists of an actual backing dict - indexed by original
  1433     key - and a doubly linked circular list defining the order of entries in
  1426     key - and a doubly linked circular list defining the order of entries in
  1434     the cache.
  1427     the cache.
  1755             return cache[args]
  1748             return cache[args]
  1756 
  1749 
  1757     return f
  1750     return f
  1758 
  1751 
  1759 
  1752 
  1760 class propertycache(object):
  1753 class propertycache:
  1761     def __init__(self, func):
  1754     def __init__(self, func):
  1762         self.func = func
  1755         self.func = func
  1763         self.name = func.__name__
  1756         self.name = func.__name__
  1764 
  1757 
  1765     def __get__(self, obj, type=None):
  1758     def __get__(self, obj, type=None):
  2214     _re2 = None
  2207     _re2 = None
  2215 except ImportError:
  2208 except ImportError:
  2216     _re2 = False
  2209     _re2 = False
  2217 
  2210 
  2218 
  2211 
  2219 class _re(object):
  2212 class _re:
  2220     def _checkre2(self):
  2213     def _checkre2(self):
  2221         global _re2
  2214         global _re2
  2222         global _re2_input
  2215         global _re2_input
  2223 
  2216 
  2224         check_pattern = br'\[([^\[]+)\]'
  2217         check_pattern = br'\[([^\[]+)\]'
  2416             pass
  2409             pass
  2417         raise
  2410         raise
  2418     return temp
  2411     return temp
  2419 
  2412 
  2420 
  2413 
  2421 class filestat(object):
  2414 class filestat:
  2422     """help to exactly detect change of a file
  2415     """help to exactly detect change of a file
  2423 
  2416 
  2424     'stat' attribute is result of 'os.stat()' if specified 'path'
  2417     'stat' attribute is result of 'os.stat()' if specified 'path'
  2425     exists. Otherwise, it is None. This can avoid preparative
  2418     exists. Otherwise, it is None. This can avoid preparative
  2426     'exists()' examination on client side of this class.
  2419     'exists()' examination on client side of this class.
  2431 
  2424 
  2432     @classmethod
  2425     @classmethod
  2433     def frompath(cls, path):
  2426     def frompath(cls, path):
  2434         try:
  2427         try:
  2435             stat = os.stat(path)
  2428             stat = os.stat(path)
  2436         except OSError as err:
  2429         except FileNotFoundError:
  2437             if err.errno != errno.ENOENT:
       
  2438                 raise
       
  2439             stat = None
  2430             stat = None
  2440         return cls(stat)
  2431         return cls(stat)
  2441 
  2432 
  2442     @classmethod
  2433     @classmethod
  2443     def fromfp(cls, fp):
  2434     def fromfp(cls, fp):
  2510         Otherwise, this returns True, as "ambiguity is avoided".
  2501         Otherwise, this returns True, as "ambiguity is avoided".
  2511         """
  2502         """
  2512         advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
  2503         advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF
  2513         try:
  2504         try:
  2514             os.utime(path, (advanced, advanced))
  2505             os.utime(path, (advanced, advanced))
  2515         except OSError as inst:
  2506         except PermissionError:
  2516             if inst.errno == errno.EPERM:
  2507             # utime() on the file created by another user causes EPERM,
  2517                 # utime() on the file created by another user causes EPERM,
  2508             # if a process doesn't have appropriate privileges
  2518                 # if a process doesn't have appropriate privileges
  2509             return False
  2519                 return False
       
  2520             raise
       
  2521         return True
  2510         return True
  2522 
  2511 
  2523     def __ne__(self, other):
  2512     def __ne__(self, other):
  2524         return not self == other
  2513         return not self == other
  2525 
  2514 
  2526 
  2515 
  2527 class atomictempfile(object):
  2516 class atomictempfile:
  2528     """writable file object that atomically updates a file
  2517     """writable file object that atomically updates a file
  2529 
  2518 
  2530     All writes will go to a temporary copy of the original file. Call
  2519     All writes will go to a temporary copy of the original file. Call
  2531     close() when you are done writing, and atomictempfile will rename
  2520     close() when you are done writing, and atomictempfile will rename
  2532     the temporary copy to the original name, making the changes
  2521     the temporary copy to the original name, making the changes
  2592             self.discard()
  2581             self.discard()
  2593         else:
  2582         else:
  2594             self.close()
  2583             self.close()
  2595 
  2584 
  2596 
  2585 
       
  2586 def tryrmdir(f):
       
  2587     try:
       
  2588         removedirs(f)
       
  2589     except OSError as e:
       
  2590         if e.errno != errno.ENOENT and e.errno != errno.ENOTEMPTY:
       
  2591             raise
       
  2592 
       
  2593 
  2597 def unlinkpath(f, ignoremissing=False, rmdir=True):
  2594 def unlinkpath(f, ignoremissing=False, rmdir=True):
  2598     # type: (bytes, bool, bool) -> None
  2595     # type: (bytes, bool, bool) -> None
  2599     """unlink and remove the directory if it is empty"""
  2596     """unlink and remove the directory if it is empty"""
  2600     if ignoremissing:
  2597     if ignoremissing:
  2601         tryunlink(f)
  2598         tryunlink(f)
  2609             pass
  2606             pass
  2610 
  2607 
  2611 
  2608 
  2612 def tryunlink(f):
  2609 def tryunlink(f):
  2613     # type: (bytes) -> None
  2610     # type: (bytes) -> None
  2614     """Attempt to remove a file, ignoring ENOENT errors."""
  2611     """Attempt to remove a file, ignoring FileNotFoundError."""
  2615     try:
  2612     try:
  2616         unlink(f)
  2613         unlink(f)
  2617     except OSError as e:
  2614     except FileNotFoundError:
  2618         if e.errno != errno.ENOENT:
  2615         pass
  2619             raise
       
  2620 
  2616 
  2621 
  2617 
  2622 def makedirs(name, mode=None, notindexed=False):
  2618 def makedirs(name, mode=None, notindexed=False):
  2623     # type: (bytes, Optional[int], bool) -> None
  2619     # type: (bytes, Optional[int], bool) -> None
  2624     """recursive directory creation with parent mode inheritance
  2620     """recursive directory creation with parent mode inheritance
  2665     # type: (bytes, bytes) -> None
  2661     # type: (bytes, bytes) -> None
  2666     with open(path, b'ab') as fp:
  2662     with open(path, b'ab') as fp:
  2667         fp.write(text)
  2663         fp.write(text)
  2668 
  2664 
  2669 
  2665 
  2670 class chunkbuffer(object):
  2666 class chunkbuffer:
  2671     """Allow arbitrary sized chunks of data to be efficiently read from an
  2667     """Allow arbitrary sized chunks of data to be efficiently read from an
  2672     iterator over chunks of arbitrary size."""
  2668     iterator over chunks of arbitrary size."""
  2673 
  2669 
  2674     def __init__(self, in_iter):
  2670     def __init__(self, in_iter):
  2675         """in_iter is the iterator that's iterating over the input chunks."""
  2671         """in_iter is the iterator that's iterating over the input chunks."""
  2770         if limit:
  2766         if limit:
  2771             limit -= len(s)
  2767             limit -= len(s)
  2772         yield s
  2768         yield s
  2773 
  2769 
  2774 
  2770 
  2775 class cappedreader(object):
  2771 class cappedreader:
  2776     """A file object proxy that allows reading up to N bytes.
  2772     """A file object proxy that allows reading up to N bytes.
  2777 
  2773 
  2778     Given a source file object, instances of this type allow reading up to
  2774     Given a source file object, instances of this type allow reading up to
  2779     N bytes from that source file object. Attempts to read past the allowed
  2775     N bytes from that source file object. Attempts to read past the allowed
  2780     limit are treated as EOF.
  2776     limit are treated as EOF.
  2858     (1, 1 << 10, _(b'%.2f KB')),
  2854     (1, 1 << 10, _(b'%.2f KB')),
  2859     (1, 1, _(b'%.0f bytes')),
  2855     (1, 1, _(b'%.0f bytes')),
  2860 )
  2856 )
  2861 
  2857 
  2862 
  2858 
  2863 class transformingwriter(object):
  2859 class transformingwriter:
  2864     """Writable file wrapper to transform data by function"""
  2860     """Writable file wrapper to transform data by function"""
  2865 
  2861 
  2866     def __init__(self, fp, encode):
  2862     def __init__(self, fp, encode):
  2867         self._fp = fp
  2863         self._fp = fp
  2868         self._encode = encode
  2864         self._encode = encode
  2904 else:
  2900 else:
  2905     tonativeeol = pycompat.identity
  2901     tonativeeol = pycompat.identity
  2906     fromnativeeol = pycompat.identity
  2902     fromnativeeol = pycompat.identity
  2907     nativeeolwriter = pycompat.identity
  2903     nativeeolwriter = pycompat.identity
  2908 
  2904 
  2909 if pyplatform.python_implementation() == b'CPython' and sys.version_info < (
  2905 
  2910     3,
  2906 # TODO delete since workaround variant for Python 2 no longer needed.
  2911     0,
  2907 def iterfile(fp):
  2912 ):
  2908     return fp
  2913     # There is an issue in CPython that some IO methods do not handle EINTR
       
  2914     # correctly. The following table shows what CPython version (and functions)
       
  2915     # are affected (buggy: has the EINTR bug, okay: otherwise):
       
  2916     #
       
  2917     #                | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
       
  2918     #   --------------------------------------------------
       
  2919     #    fp.__iter__ | buggy   | buggy           | okay
       
  2920     #    fp.read*    | buggy   | okay [1]        | okay
       
  2921     #
       
  2922     # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
       
  2923     #
       
  2924     # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
       
  2925     # like "read*" work fine, as we do not support Python < 2.7.4.
       
  2926     #
       
  2927     # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
       
  2928     # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
       
  2929     # CPython 2, because CPython 2 maintains an internal readahead buffer for
       
  2930     # fp.__iter__ but not other fp.read* methods.
       
  2931     #
       
  2932     # On modern systems like Linux, the "read" syscall cannot be interrupted
       
  2933     # when reading "fast" files like on-disk files. So the EINTR issue only
       
  2934     # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
       
  2935     # files approximately as "fast" files and use the fast (unsafe) code path,
       
  2936     # to minimize the performance impact.
       
  2937 
       
  2938     def iterfile(fp):
       
  2939         fastpath = True
       
  2940         if type(fp) is file:
       
  2941             fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
       
  2942         if fastpath:
       
  2943             return fp
       
  2944         else:
       
  2945             # fp.readline deals with EINTR correctly, use it as a workaround.
       
  2946             return iter(fp.readline, b'')
       
  2947 
       
  2948 
       
  2949 else:
       
  2950     # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
       
  2951     def iterfile(fp):
       
  2952         return fp
       
  2953 
  2909 
  2954 
  2910 
  2955 def iterlines(iterator):
  2911 def iterlines(iterator):
  2956     # type: (Iterator[bytes]) -> Iterator[bytes]
  2912     # type: (Iterator[bytes]) -> Iterator[bytes]
  2957     for chunk in iterator:
  2913     for chunk in iterator:
  3006     (1, 0.000000001, _(b'%.3f ns')),
  2962     (1, 0.000000001, _(b'%.3f ns')),
  3007 )
  2963 )
  3008 
  2964 
  3009 
  2965 
  3010 @attr.s
  2966 @attr.s
  3011 class timedcmstats(object):
  2967 class timedcmstats:
  3012     """Stats information produced by the timedcm context manager on entering."""
  2968     """Stats information produced by the timedcm context manager on entering."""
  3013 
  2969 
  3014     # the starting value of the timer as a float (meaning and resulution is
  2970     # the starting value of the timer as a float (meaning and resulution is
  3015     # platform dependent, see util.timer)
  2971     # platform dependent, see util.timer)
  3016     start = attr.ib(default=attr.Factory(lambda: timer()))
  2972     start = attr.ib(default=attr.Factory(lambda: timer()))
  3107         return int(t)
  3063         return int(t)
  3108     except ValueError:
  3064     except ValueError:
  3109         raise error.ParseError(_(b"couldn't parse size: %s") % s)
  3065         raise error.ParseError(_(b"couldn't parse size: %s") % s)
  3110 
  3066 
  3111 
  3067 
  3112 class hooks(object):
  3068 class hooks:
  3113     """A collection of hook functions that can be used to extend a
  3069     """A collection of hook functions that can be used to extend a
  3114     function's behavior. Hooks are called in lexicographic order,
  3070     function's behavior. Hooks are called in lexicographic order,
  3115     based on the names of their sources."""
  3071     based on the names of their sources."""
  3116 
  3072 
  3117     def __init__(self):
  3073     def __init__(self):