mercurial: comparison hgext/remotefilelog/repack.py

equal deleted inserted replaced

-:2372284d9457
+:687b865b95ad
 def backgroundrepack(
 repo, incremental=True, packsonly=False, ensurestart=False
 ):
-cmd = [procutil.hgexecutable(), '-R', repo.origroot, 'repack']
+cmd = [procutil.hgexecutable(), b'-R', repo.origroot, b'repack']
-msg = _("(running background repack)\n")
+msg = _(b"(running background repack)\n")
 if incremental:
-cmd.append('--incremental')
+cmd.append(b'--incremental')
-msg = _("(running background incremental repack)\n")
+msg = _(b"(running background incremental repack)\n")
 if packsonly:
-cmd.append('--packsonly')
+cmd.append(b'--packsonly')
 repo.ui.warn(msg)
 # We know this command will find a binary, so don't block on it starting.
 procutil.runbgcommand(cmd, encoding.environ, ensurestart=ensurestart)
 def fullrepack(repo, options=None):
 """If ``packsonly`` is True, stores creating only loose objects are skipped.
 """
-if util.safehasattr(repo, 'shareddatastores'):
+if util.safehasattr(repo, b'shareddatastores'):
 datasource = contentstore.unioncontentstore(*repo.shareddatastores)
 historysource = metadatastore.unionmetadatastore(
 *repo.sharedhistorystores, allowincomplete=True
 )
 packpath,
 constants.FILEPACK_CATEGORY,
 options=options,
 )
-if util.safehasattr(repo.manifestlog, 'datastore'):
+if util.safehasattr(repo.manifestlog, b'datastore'):
 localdata, shareddata = _getmanifeststores(repo)
 lpackpath, ldstores, lhstores = localdata
 spackpath, sdstores, shstores = shareddata
 # Repack the shared manifest store
 def incrementalrepack(repo, options=None):
 """This repacks the repo by looking at the distribution of pack files in the
 repo and performing the most minimal repack to keep the repo in good shape.
 """
-if util.safehasattr(repo, 'shareddatastores'):
+if util.safehasattr(repo, b'shareddatastores'):
 packpath = shallowutil.getcachepackpath(
 repo, constants.FILEPACK_CATEGORY
 )
 _incrementalrepack(
 repo,
 packpath,
 constants.FILEPACK_CATEGORY,
 options=options,
 )
-if util.safehasattr(repo.manifestlog, 'datastore'):
+if util.safehasattr(repo.manifestlog, b'datastore'):
 localdata, shareddata = _getmanifeststores(repo)
 lpackpath, ldstores, lhstores = localdata
 spackpath, sdstores, shstores = shareddata
 # Repack the shared manifest store
 def _deletebigpacks(repo, folder, files):
 """Deletes packfiles that are bigger than ``packs.maxpacksize``.
 Returns ``files` with the removed files omitted."""
-maxsize = repo.ui.configbytes("packs", "maxpacksize")
+maxsize = repo.ui.configbytes(b"packs", b"maxpacksize")
 if maxsize <= 0:
 return files
 # This only considers datapacks today, but we could broaden it to include
 # historypacks.
-VALIDEXTS = [".datapack", ".dataidx"]
+VALIDEXTS = [b".datapack", b".dataidx"]
 # Either an oversize index or datapack will trigger cleanup of the whole
 # pack:
 oversized = {
 os.path.splitext(path)[0]
 for rootfname in oversized:
 rootpath = os.path.join(folder, rootfname)
 for ext in VALIDEXTS:
 path = rootpath + ext
 repo.ui.debug(
-'removing oversize packfile %s (%s)\n'
+b'removing oversize packfile %s (%s)\n'
 % (path, util.bytecount(os.stat(path).st_size))
 )
 os.unlink(path)
 return [row for row in files if os.path.basename(row[0]) not in oversized]
 )
 def _computeincrementaldatapack(ui, files):
 opts = {
-'gencountlimit': ui.configint('remotefilelog', 'data.gencountlimit'),
+b'gencountlimit': ui.configint(b'remotefilelog', b'data.gencountlimit'),
-'generations': ui.configlist('remotefilelog', 'data.generations'),
+b'generations': ui.configlist(b'remotefilelog', b'data.generations'),
-'maxrepackpacks': ui.configint('remotefilelog', 'data.maxrepackpacks'),
+b'maxrepackpacks': ui.configint(
-'repackmaxpacksize': ui.configbytes(
+b'remotefilelog', b'data.maxrepackpacks'
-'remotefilelog', 'data.repackmaxpacksize'
 ),
-'repacksizelimit': ui.configbytes(
+b'repackmaxpacksize': ui.configbytes(
-'remotefilelog', 'data.repacksizelimit'
+b'remotefilelog', b'data.repackmaxpacksize'
+),
+b'repacksizelimit': ui.configbytes(
+b'remotefilelog', b'data.repacksizelimit'
 ),
 }
 packfiles = _allpackfileswithsuffix(
 files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX
 return _computeincrementalpack(packfiles, opts)
 def _computeincrementalhistorypack(ui, files):
 opts = {
-'gencountlimit': ui.configint('remotefilelog', 'history.gencountlimit'),
+b'gencountlimit': ui.configint(
-'generations': ui.configlist(
+b'remotefilelog', b'history.gencountlimit'
-'remotefilelog', 'history.generations', ['100MB']
 ),
-'maxrepackpacks': ui.configint(
+b'generations': ui.configlist(
-'remotefilelog', 'history.maxrepackpacks'
+b'remotefilelog', b'history.generations', [b'100MB']
 ),
-'repackmaxpacksize': ui.configbytes(
+b'maxrepackpacks': ui.configint(
-'remotefilelog', 'history.repackmaxpacksize', '400MB'
+b'remotefilelog', b'history.maxrepackpacks'
 ),
-'repacksizelimit': ui.configbytes(
+b'repackmaxpacksize': ui.configbytes(
-'remotefilelog', 'history.repacksizelimit'
+b'remotefilelog', b'history.repackmaxpacksize', b'400MB'
+),
+b'repacksizelimit': ui.configbytes(
+b'remotefilelog', b'history.repacksizelimit'
 ),
 }
 packfiles = _allpackfileswithsuffix(
 files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX
 packing small things when possible, and rolling the packs up to the big ones
 over time).
 """
 limits = list(
-sorted((util.sizetoint(s) for s in opts['generations']), reverse=True)
+sorted((util.sizetoint(s) for s in opts[b'generations']), reverse=True)
 )
 limits.append(0)
 # Group the packs by generation (i.e. by size)
 generations = []
 generations.append([])
 sizes = {}
 for prefix, mode, stat in files:
 size = stat.st_size
-if size > opts['repackmaxpacksize']:
+if size > opts[b'repackmaxpacksize']:
 continue
 sizes[prefix] = size
 for i, limit in enumerate(limits):
 if size > limit:
 # 3. While total-size-of-packs < repacksizelimit: add another pack
 # Find the largest generation with more than gencountlimit packs
 genpacks = []
 for i, limit in enumerate(limits):
-if len(generations[i]) > opts['gencountlimit']:
+if len(generations[i]) > opts[b'gencountlimit']:
 # Sort to be smallest last, for easy popping later
 genpacks.extend(
 sorted(generations[i], reverse=True, key=lambda x: sizes[x])
 )
 break
 # Take as many packs from the generation as we can
 chosenpacks = genpacks[-3:]
 genpacks = genpacks[:-3]
 repacksize = sum(sizes[n] for n in chosenpacks)
 while (
-repacksize < opts['repacksizelimit']
+repacksize < opts[b'repacksizelimit']
 and genpacks
-and len(chosenpacks) < opts['maxrepackpacks']
+and len(chosenpacks) < opts[b'maxrepackpacks']
 ):
 chosenpacks.append(genpacks.pop())
 repacksize += sizes[chosenpacks[-1]]
 return chosenpacks
 Unless a limit is specified in the config the default limit is taken.
 """
 filectx = repo.filectx(filename, fileid=node)
 filetime = repo[filectx.linkrev()].date()
-ttl = repo.ui.configint('remotefilelog', 'nodettl')
+ttl = repo.ui.configint(b'remotefilelog', b'nodettl')
 limit = time.time() - ttl
 return filetime[0] < limit
-garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack')
+garbagecollect = repo.ui.configbool(b'remotefilelog', b'gcrepack')
 if not fullhistory:
 fullhistory = history
 packer = repacker(
 repo,
 data,
 with historypack.mutablehistorypack(repo.ui, packpath) as hpack:
 try:
 packer.run(dpack, hpack)
 except error.LockHeld:
 raise RepackAlreadyRunning(
-_("skipping repack - another repack " "is already running")
+_(
+b"skipping repack - another repack "
+b"is already running"
+)
 )
 def keepset(repo, keyfn, lastkeepkeys=None):
 """Computes a keepset which is not garbage collected.
 # We want to keep:
 # 1. Working copy parent
 # 2. Draft commits
 # 3. Parents of draft commits
 # 4. Pullprefetch and bgprefetchrevs revsets if specified
-revs = ['.', 'draft()', 'parents(draft())']
+revs = [b'.', b'draft()', b'parents(draft())']
-prefetchrevs = repo.ui.config('remotefilelog', 'pullprefetch', None)
+prefetchrevs = repo.ui.config(b'remotefilelog', b'pullprefetch', None)
 if prefetchrevs:
-revs.append('(%s)' % prefetchrevs)
+revs.append(b'(%s)' % prefetchrevs)
-prefetchrevs = repo.ui.config('remotefilelog', 'bgprefetchrevs', None)
+prefetchrevs = repo.ui.config(b'remotefilelog', b'bgprefetchrevs', None)
 if prefetchrevs:
-revs.append('(%s)' % prefetchrevs)
+revs.append(b'(%s)' % prefetchrevs)
-revs = '+'.join(revs)
+revs = b'+'.join(revs)
-revs = ['sort((%s), "topo")' % revs]
+revs = [b'sort((%s), "topo")' % revs]
 keep = scmutil.revrange(repo, revs)
 processed = set()
 lastmanifest = None
 self.unit = constants.getunits(category)
 self.garbagecollect = gc
 self.options = options
 if self.garbagecollect:
 if not isold:
-raise ValueError("Function 'isold' is not properly specified")
+raise ValueError(b"Function 'isold' is not properly specified")
 # use (filename, node) tuple as a keepset key
 self.keepkeys = keepset(repo, lambda f, n: (f, n))
 self.isold = isold
 def run(self, targetdata, targethistory):
 ledger = repackledger()
 with lockmod.lock(
-repacklockvfs(self.repo), "repacklock", desc=None, timeout=0
+repacklockvfs(self.repo), b"repacklock", desc=None, timeout=0
 ):
-self.repo.hook('prerepack')
+self.repo.hook(b'prerepack')
 # Populate ledger from source
 self.data.markledger(ledger, options=self.options)
 self.history.markledger(ledger, options=self.options)
 # (Sort by node first to ensure the sort is stable.)
 orphans = sorted(orphans)
 orphans = list(sorted(orphans, key=getsize, reverse=True))
 if ui.debugflag:
 ui.debug(
-"%s: orphan chain: %s\n"
+b"%s: orphan chain: %s\n"
-% (filename, ", ".join([short(s) for s in orphans]))
+% (filename, b", ".join([short(s) for s in orphans]))
 )
 # Create one contiguous chain and reassign deltabases.
 for i, node in enumerate(orphans):
 if i == 0:
 nodes += orphans
 return nodes
 def repackdata(self, ledger, target):
 ui = self.repo.ui
-maxchainlen = ui.configint('packs', 'maxchainlen', 1000)
+maxchainlen = ui.configint(b'packs', b'maxchainlen', 1000)
 byfile = {}
 for entry in ledger.entries.itervalues():
 if entry.datasource:
 byfile.setdefault(entry.filename, {})[entry.node] = entry
 count = 0
 repackprogress = ui.makeprogress(
-_("repacking data"), unit=self.unit, total=len(byfile)
+_(b"repacking data"), unit=self.unit, total=len(byfile)
 )
 for filename, entries in sorted(byfile.iteritems()):
 repackprogress.update(count)
 ancestors = {}
 nodes = list(node for node in entries)
 nohistory = []
 buildprogress = ui.makeprogress(
-_("building history"), unit='nodes', total=len(nodes)
+_(b"building history"), unit=b'nodes', total=len(nodes)
 )
 for i, node in enumerate(nodes):
 if node in ancestors:
 continue
 buildprogress.update(i)
 # Order the nodes children first, so we can produce reverse deltas
 orderednodes = list(reversed(self._toposort(ancestors)))
 if len(nohistory) > 0:
 ui.debug(
-'repackdata: %d nodes without history\n' % len(nohistory)
+b'repackdata: %d nodes without history\n' % len(nohistory)
 )
 orderednodes.extend(sorted(nohistory))
 # Filter orderednodes to just the nodes we want to serialize (it
 # currently also has the edge nodes' ancestors).
 deltabases = {}
 nobase = set()
 referenced = set()
 nodes = set(nodes)
 processprogress = ui.makeprogress(
-_("processing nodes"), unit='nodes', total=len(orderednodes)
+_(b"processing nodes"), unit=b'nodes', total=len(orderednodes)
 )
 for i, node in enumerate(orderednodes):
 processprogress.update(i)
 # Find delta base
 # TODO: allow delta'ing against most recent descendant instead
 deltabases[p1] = (node, chainlen + 1)
 if p2 != nullid:
 deltabases[p2] = (node, chainlen + 1)
 # experimental config: repack.chainorphansbysize
-if ui.configbool('repack', 'chainorphansbysize'):
+if ui.configbool(b'repack', b'chainorphansbysize'):
 orphans = nobase - referenced
 orderednodes = self._chainorphans(
 ui, filename, orderednodes, orphans, deltabases
 )
 for entry in ledger.entries.itervalues():
 if entry.historysource:
 byfile.setdefault(entry.filename, {})[entry.node] = entry
 progress = ui.makeprogress(
-_("repacking history"), unit=self.unit, total=len(byfile)
+_(b"repacking history"), unit=self.unit, total=len(byfile)
 )
 for filename, entries in sorted(byfile.iteritems()):
 ancestors = {}
 nodes = list(node for node in entries)
 # If garbage collected
 self.gced = False
 def repacklockvfs(repo):
-if util.safehasattr(repo, 'name'):
+if util.safehasattr(repo, b'name'):
 # Lock in the shared cache so repacks across multiple copies of the same
 # repo are coordinated.
 sharedcachepath = shallowutil.getcachepackpath(
 repo, constants.FILEPACK_CATEGORY
 )

changeset 43077	687b865b95ad
parent 43076	2372284d9457
child 43104	74802979dd9d