hgext/remotefilelog/basepack.py
changeset 43077 687b865b95ad
parent 43076 2372284d9457
child 43085 eef9a2d67051
equal deleted inserted replaced
43076:2372284d9457 43077:687b865b95ad
    54 
    54 
    55 if pycompat.isposix and not pycompat.ispy3:
    55 if pycompat.isposix and not pycompat.ispy3:
    56     # With glibc 2.7+ the 'e' flag uses O_CLOEXEC when opening.
    56     # With glibc 2.7+ the 'e' flag uses O_CLOEXEC when opening.
    57     # The 'e' flag will be ignored on older versions of glibc.
    57     # The 'e' flag will be ignored on older versions of glibc.
    58     # Python 3 can't handle the 'e' flag.
    58     # Python 3 can't handle the 'e' flag.
    59     PACKOPENMODE = 'rbe'
    59     PACKOPENMODE = b'rbe'
    60 else:
    60 else:
    61     PACKOPENMODE = 'rb'
    61     PACKOPENMODE = b'rb'
    62 
    62 
    63 
    63 
    64 class _cachebackedpacks(object):
    64 class _cachebackedpacks(object):
    65     def __init__(self, packs, cachesize):
    65     def __init__(self, packs, cachesize):
    66         self._packs = set(packs)
    66         self._packs = set(packs)
   130                 #
   130                 #
   131                 # If this is an ENOENT error then don't even bother logging.
   131                 # If this is an ENOENT error then don't even bother logging.
   132                 # Someone could have removed the file since we retrieved the
   132                 # Someone could have removed the file since we retrieved the
   133                 # list of paths.
   133                 # list of paths.
   134                 if getattr(ex, 'errno', None) != errno.ENOENT:
   134                 if getattr(ex, 'errno', None) != errno.ENOENT:
   135                     ui.warn(_('unable to load pack %s: %s\n') % (filepath, ex))
   135                     ui.warn(_(b'unable to load pack %s: %s\n') % (filepath, ex))
   136                 continue
   136                 continue
   137             packs.append(pack)
   137             packs.append(pack)
   138 
   138 
   139         self.packs = _cachebackedpacks(packs, self.DEFAULTCACHESIZE)
   139         self.packs = _cachebackedpacks(packs, self.DEFAULTCACHESIZE)
   140 
   140 
   208 
   208 
   209     def getmetrics(self):
   209     def getmetrics(self):
   210         """Returns metrics on the state of this store."""
   210         """Returns metrics on the state of this store."""
   211         size, count = self.gettotalsizeandcount()
   211         size, count = self.gettotalsizeandcount()
   212         return {
   212         return {
   213             'numpacks': count,
   213             b'numpacks': count,
   214             'totalpacksize': size,
   214             b'totalpacksize': size,
   215         }
   215         }
   216 
   216 
   217     def getpack(self, path):
   217     def getpack(self, path):
   218         raise NotImplementedError()
   218         raise NotImplementedError()
   219 
   219 
   274         if version in self.SUPPORTED_VERSIONS:
   274         if version in self.SUPPORTED_VERSIONS:
   275             if self.VERSION is None:
   275             if self.VERSION is None:
   276                 # only affect this instance
   276                 # only affect this instance
   277                 self.VERSION = version
   277                 self.VERSION = version
   278             elif self.VERSION != version:
   278             elif self.VERSION != version:
   279                 raise RuntimeError('inconsistent version: %d' % version)
   279                 raise RuntimeError(b'inconsistent version: %d' % version)
   280         else:
   280         else:
   281             raise RuntimeError('unsupported version: %d' % version)
   281             raise RuntimeError(b'unsupported version: %d' % version)
   282 
   282 
   283 
   283 
   284 class basepack(versionmixin):
   284 class basepack(versionmixin):
   285     # The maximum amount we should read via mmap before remmaping so the old
   285     # The maximum amount we should read via mmap before remmaping so the old
   286     # pages can be released (100MB)
   286     # pages can be released (100MB)
   298 
   298 
   299         self._index = None
   299         self._index = None
   300         self._data = None
   300         self._data = None
   301         self.freememory()  # initialize the mmap
   301         self.freememory()  # initialize the mmap
   302 
   302 
   303         version = struct.unpack('!B', self._data[:PACKVERSIONSIZE])[0]
   303         version = struct.unpack(b'!B', self._data[:PACKVERSIONSIZE])[0]
   304         self._checkversion(version)
   304         self._checkversion(version)
   305 
   305 
   306         version, config = struct.unpack('!BB', self._index[:INDEXVERSIONSIZE])
   306         version, config = struct.unpack(b'!BB', self._index[:INDEXVERSIONSIZE])
   307         self._checkversion(version)
   307         self._checkversion(version)
   308 
   308 
   309         if 0b10000000 & config:
   309         if 0b10000000 & config:
   310             self.params = indexparams(LARGEFANOUTPREFIX, version)
   310             self.params = indexparams(LARGEFANOUTPREFIX, version)
   311         else:
   311         else:
   316         params = self.params
   316         params = self.params
   317         rawfanout = self._index[FANOUTSTART : FANOUTSTART + params.fanoutsize]
   317         rawfanout = self._index[FANOUTSTART : FANOUTSTART + params.fanoutsize]
   318         fanouttable = []
   318         fanouttable = []
   319         for i in pycompat.xrange(0, params.fanoutcount):
   319         for i in pycompat.xrange(0, params.fanoutcount):
   320             loc = i * 4
   320             loc = i * 4
   321             fanoutentry = struct.unpack('!I', rawfanout[loc : loc + 4])[0]
   321             fanoutentry = struct.unpack(b'!I', rawfanout[loc : loc + 4])[0]
   322             fanouttable.append(fanoutentry)
   322             fanouttable.append(fanoutentry)
   323         return fanouttable
   323         return fanouttable
   324 
   324 
   325     @util.propertycache
   325     @util.propertycache
   326     def _indexend(self):
   326     def _indexend(self):
   327         nodecount = struct.unpack_from(
   327         nodecount = struct.unpack_from(
   328             '!Q', self._index, self.params.indexstart - 8
   328             b'!Q', self._index, self.params.indexstart - 8
   329         )[0]
   329         )[0]
   330         return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH
   330         return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH
   331 
   331 
   332     def freememory(self):
   332     def freememory(self):
   333         """Unmap and remap the memory to free it up after known expensive
   333         """Unmap and remap the memory to free it up after known expensive
   370 
   370 
   371 class mutablebasepack(versionmixin):
   371 class mutablebasepack(versionmixin):
   372     def __init__(self, ui, packdir, version=2):
   372     def __init__(self, ui, packdir, version=2):
   373         self._checkversion(version)
   373         self._checkversion(version)
   374         # TODO(augie): make this configurable
   374         # TODO(augie): make this configurable
   375         self._compressor = 'GZ'
   375         self._compressor = b'GZ'
   376         opener = vfsmod.vfs(packdir)
   376         opener = vfsmod.vfs(packdir)
   377         opener.createmode = 0o444
   377         opener.createmode = 0o444
   378         self.opener = opener
   378         self.opener = opener
   379 
   379 
   380         self.entries = {}
   380         self.entries = {}
   381 
   381 
   382         shallowutil.mkstickygroupdir(ui, packdir)
   382         shallowutil.mkstickygroupdir(ui, packdir)
   383         self.packfp, self.packpath = opener.mkstemp(
   383         self.packfp, self.packpath = opener.mkstemp(
   384             suffix=self.PACKSUFFIX + '-tmp'
   384             suffix=self.PACKSUFFIX + b'-tmp'
   385         )
   385         )
   386         self.idxfp, self.idxpath = opener.mkstemp(
   386         self.idxfp, self.idxpath = opener.mkstemp(
   387             suffix=self.INDEXSUFFIX + '-tmp'
   387             suffix=self.INDEXSUFFIX + b'-tmp'
   388         )
   388         )
   389         self.packfp = os.fdopen(self.packfp, r'wb+')
   389         self.packfp = os.fdopen(self.packfp, r'wb+')
   390         self.idxfp = os.fdopen(self.idxfp, r'wb+')
   390         self.idxfp = os.fdopen(self.idxfp, r'wb+')
   391         self.sha = hashlib.sha1()
   391         self.sha = hashlib.sha1()
   392         self._closed = False
   392         self._closed = False
   398         opener._fixfilemode(opener.join(self.idxpath))
   398         opener._fixfilemode(opener.join(self.idxpath))
   399 
   399 
   400         # Write header
   400         # Write header
   401         # TODO: make it extensible (ex: allow specifying compression algorithm,
   401         # TODO: make it extensible (ex: allow specifying compression algorithm,
   402         # a flexible key/value header, delta algorithm, fanout size, etc)
   402         # a flexible key/value header, delta algorithm, fanout size, etc)
   403         versionbuf = struct.pack('!B', self.VERSION)  # unsigned 1 byte int
   403         versionbuf = struct.pack(b'!B', self.VERSION)  # unsigned 1 byte int
   404         self.writeraw(versionbuf)
   404         self.writeraw(versionbuf)
   405 
   405 
   406     def __enter__(self):
   406     def __enter__(self):
   407         return self
   407         return self
   408 
   408 
   489                 params.fanoutstruct, node[: params.fanoutprefix]
   489                 params.fanoutstruct, node[: params.fanoutprefix]
   490             )[0]
   490             )[0]
   491             if fanouttable[fanoutkey] == EMPTYFANOUT:
   491             if fanouttable[fanoutkey] == EMPTYFANOUT:
   492                 fanouttable[fanoutkey] = location
   492                 fanouttable[fanoutkey] = location
   493 
   493 
   494         rawfanouttable = ''
   494         rawfanouttable = b''
   495         last = 0
   495         last = 0
   496         for offset in fanouttable:
   496         for offset in fanouttable:
   497             offset = offset if offset != EMPTYFANOUT else last
   497             offset = offset if offset != EMPTYFANOUT else last
   498             last = offset
   498             last = offset
   499             rawfanouttable += struct.pack('!I', offset)
   499             rawfanouttable += struct.pack(b'!I', offset)
   500 
   500 
   501         rawentrieslength = struct.pack('!Q', len(self.entries))
   501         rawentrieslength = struct.pack(b'!Q', len(self.entries))
   502 
   502 
   503         # The index offset is the it's location in the file. So after the 2 byte
   503         # The index offset is the it's location in the file. So after the 2 byte
   504         # header and the fanouttable.
   504         # header and the fanouttable.
   505         rawindex = self.createindex(locations, 2 + len(rawfanouttable))
   505         rawindex = self.createindex(locations, 2 + len(rawfanouttable))
   506 
   506 
   519         #    <large fanout: 1 bit> # 1 means 2^16, 0 means 2^8
   519         #    <large fanout: 1 bit> # 1 means 2^16, 0 means 2^8
   520         #    <unused: 7 bit> # future use (compression, delta format, etc)
   520         #    <unused: 7 bit> # future use (compression, delta format, etc)
   521         config = 0
   521         config = 0
   522         if indexparams.fanoutprefix == LARGEFANOUTPREFIX:
   522         if indexparams.fanoutprefix == LARGEFANOUTPREFIX:
   523             config = 0b10000000
   523             config = 0b10000000
   524         self.idxfp.write(struct.pack('!BB', self.VERSION, config))
   524         self.idxfp.write(struct.pack(b'!BB', self.VERSION, config))
   525 
   525 
   526 
   526 
   527 class indexparams(object):
   527 class indexparams(object):
   528     __slots__ = (
   528     __slots__ = (
   529         r'fanoutprefix',
   529         r'fanoutprefix',
   538 
   538 
   539         # The struct pack format for fanout table location (i.e. the format that
   539         # The struct pack format for fanout table location (i.e. the format that
   540         # converts the node prefix into an integer location in the fanout
   540         # converts the node prefix into an integer location in the fanout
   541         # table).
   541         # table).
   542         if prefixsize == SMALLFANOUTPREFIX:
   542         if prefixsize == SMALLFANOUTPREFIX:
   543             self.fanoutstruct = '!B'
   543             self.fanoutstruct = b'!B'
   544         elif prefixsize == LARGEFANOUTPREFIX:
   544         elif prefixsize == LARGEFANOUTPREFIX:
   545             self.fanoutstruct = '!H'
   545             self.fanoutstruct = b'!H'
   546         else:
   546         else:
   547             raise ValueError("invalid fanout prefix size: %s" % prefixsize)
   547             raise ValueError(b"invalid fanout prefix size: %s" % prefixsize)
   548 
   548 
   549         # The number of fanout table entries
   549         # The number of fanout table entries
   550         self.fanoutcount = 2 ** (prefixsize * 8)
   550         self.fanoutcount = 2 ** (prefixsize * 8)
   551 
   551 
   552         # The total bytes used by the fanout table
   552         # The total bytes used by the fanout table