mercurial: comparison hgext/remotefilelog/repack.py

equal deleted inserted replaced

-:57875cf423c9
+:2372284d9457
 shallowutil,
 )
 osutil = policy.importmod(r'osutil')
 class RepackAlreadyRunning(error.Abort):
 pass
-def backgroundrepack(repo, incremental=True, packsonly=False,
-ensurestart=False):
+def backgroundrepack(
+repo, incremental=True, packsonly=False, ensurestart=False
+):
 cmd = [procutil.hgexecutable(), '-R', repo.origroot, 'repack']
 msg = _("(running background repack)\n")
 if incremental:
 cmd.append('--incremental')
 msg = _("(running background incremental repack)\n")
 cmd.append('--packsonly')
 repo.ui.warn(msg)
 # We know this command will find a binary, so don't block on it starting.
 procutil.runbgcommand(cmd, encoding.environ, ensurestart=ensurestart)
 def fullrepack(repo, options=None):
 """If ``packsonly`` is True, stores creating only loose objects are skipped.
 """
 if util.safehasattr(repo, 'shareddatastores'):
-datasource = contentstore.unioncontentstore(
+datasource = contentstore.unioncontentstore(*repo.shareddatastores)
-*repo.shareddatastores)
 historysource = metadatastore.unionmetadatastore(
-*repo.sharedhistorystores,
+*repo.sharedhistorystores, allowincomplete=True
-allowincomplete=True)
+)
 packpath = shallowutil.getcachepackpath(
+repo, constants.FILEPACK_CATEGORY
+)
+_runrepack(
 repo,
-constants.FILEPACK_CATEGORY)
+datasource,
-_runrepack(repo, datasource, historysource, packpath,
+historysource,
-constants.FILEPACK_CATEGORY, options=options)
+packpath,
+constants.FILEPACK_CATEGORY,
+options=options,
+)
 if util.safehasattr(repo.manifestlog, 'datastore'):
 localdata, shareddata = _getmanifeststores(repo)
 lpackpath, ldstores, lhstores = localdata
 spackpath, sdstores, shstores = shareddata
 # Repack the shared manifest store
 datasource = contentstore.unioncontentstore(*sdstores)
 historysource = metadatastore.unionmetadatastore(
-*shstores,
+*shstores, allowincomplete=True
-allowincomplete=True)
+)
-_runrepack(repo, datasource, historysource, spackpath,
+_runrepack(
-constants.TREEPACK_CATEGORY, options=options)
+repo,
+datasource,
+historysource,
+spackpath,
+constants.TREEPACK_CATEGORY,
+options=options,
+)
 # Repack the local manifest store
 datasource = contentstore.unioncontentstore(
-*ldstores,
+*ldstores, allowincomplete=True
-allowincomplete=True)
+)
 historysource = metadatastore.unionmetadatastore(
-*lhstores,
+*lhstores, allowincomplete=True
-allowincomplete=True)
+)
-_runrepack(repo, datasource, historysource, lpackpath,
+_runrepack(
-constants.TREEPACK_CATEGORY, options=options)
+repo,
+datasource,
+historysource,
+lpackpath,
+constants.TREEPACK_CATEGORY,
+options=options,
+)
 def incrementalrepack(repo, options=None):
 """This repacks the repo by looking at the distribution of pack files in the
 repo and performing the most minimal repack to keep the repo in good shape.
 """
 if util.safehasattr(repo, 'shareddatastores'):
 packpath = shallowutil.getcachepackpath(
+repo, constants.FILEPACK_CATEGORY
+)
+_incrementalrepack(
 repo,
-constants.FILEPACK_CATEGORY)
+repo.shareddatastores,
-_incrementalrepack(repo,
+repo.sharedhistorystores,
-repo.shareddatastores,
+packpath,
-repo.sharedhistorystores,
+constants.FILEPACK_CATEGORY,
-packpath,
+options=options,
-constants.FILEPACK_CATEGORY,
+)
-options=options)
 if util.safehasattr(repo.manifestlog, 'datastore'):
 localdata, shareddata = _getmanifeststores(repo)
 lpackpath, ldstores, lhstores = localdata
 spackpath, sdstores, shstores = shareddata
 # Repack the shared manifest store
-_incrementalrepack(repo,
+_incrementalrepack(
-sdstores,
+repo,
-shstores,
+sdstores,
-spackpath,
+shstores,
-constants.TREEPACK_CATEGORY,
+spackpath,
-options=options)
+constants.TREEPACK_CATEGORY,
+options=options,
+)
 # Repack the local manifest store
-_incrementalrepack(repo,
+_incrementalrepack(
-ldstores,
+repo,
-lhstores,
+ldstores,
-lpackpath,
+lhstores,
-constants.TREEPACK_CATEGORY,
+lpackpath,
-allowincompletedata=True,
+constants.TREEPACK_CATEGORY,
-options=options)
+allowincompletedata=True,
+options=options,
+)
 def _getmanifeststores(repo):
 shareddatastores = repo.manifestlog.shareddatastores
 localdatastores = repo.manifestlog.localdatastores
 sharedhistorystores = repo.manifestlog.sharedhistorystores
 localhistorystores = repo.manifestlog.localhistorystores
-sharedpackpath = shallowutil.getcachepackpath(repo,
+sharedpackpath = shallowutil.getcachepackpath(
-constants.TREEPACK_CATEGORY)
+repo, constants.TREEPACK_CATEGORY
-localpackpath = shallowutil.getlocalpackpath(repo.svfs.vfs.base,
+)
-constants.TREEPACK_CATEGORY)
+localpackpath = shallowutil.getlocalpackpath(
+repo.svfs.vfs.base, constants.TREEPACK_CATEGORY
-return ((localpackpath, localdatastores, localhistorystores),
+)
-(sharedpackpath, shareddatastores, sharedhistorystores))
+return (
+(localpackpath, localdatastores, localhistorystores),
+(sharedpackpath, shareddatastores, sharedhistorystores),
+)
 def _topacks(packpath, files, constructor):
 paths = list(os.path.join(packpath, p) for p in files)
 packs = list(constructor(p) for p in paths)
 return packs
 def _deletebigpacks(repo, folder, files):
 """Deletes packfiles that are bigger than ``packs.maxpacksize``.
 Returns ``files` with the removed files omitted."""
 maxsize = repo.ui.configbytes("packs", "maxpacksize")
 # historypacks.
 VALIDEXTS = [".datapack", ".dataidx"]
 # Either an oversize index or datapack will trigger cleanup of the whole
 # pack:
-oversized = {os.path.splitext(path)[0] for path, ftype, stat in files
+oversized = {
-if (stat.st_size > maxsize and (os.path.splitext(path)[1]
+os.path.splitext(path)[0]
-in VALIDEXTS))}
+for path, ftype, stat in files
+if (stat.st_size > maxsize and (os.path.splitext(path)[1] in VALIDEXTS))
+}
 for rootfname in oversized:
 rootpath = os.path.join(folder, rootfname)
 for ext in VALIDEXTS:
 path = rootpath + ext
-repo.ui.debug('removing oversize packfile %s (%s)\n' %
+repo.ui.debug(
-(path, util.bytecount(os.stat(path).st_size)))
+'removing oversize packfile %s (%s)\n'
+% (path, util.bytecount(os.stat(path).st_size))
+)
 os.unlink(path)
 return [row for row in files if os.path.basename(row[0]) not in oversized]
-def _incrementalrepack(repo, datastore, historystore, packpath, category,
-allowincompletedata=False, options=None):
+def _incrementalrepack(
+repo,
+datastore,
+historystore,
+packpath,
+category,
+allowincompletedata=False,
+options=None,
+):
 shallowutil.mkstickygroupdir(repo.ui, packpath)
 files = osutil.listdir(packpath, stat=True)
 files = _deletebigpacks(repo, packpath, files)
-datapacks = _topacks(packpath,
+datapacks = _topacks(
-_computeincrementaldatapack(repo.ui, files),
+packpath, _computeincrementaldatapack(repo.ui, files), datapack.datapack
-datapack.datapack)
+)
-datapacks.extend(s for s in datastore
+datapacks.extend(
-if not isinstance(s, datapack.datapackstore))
+s for s in datastore if not isinstance(s, datapack.datapackstore)
+)
-historypacks = _topacks(packpath,
+historypacks = _topacks(
+packpath,
 _computeincrementalhistorypack(repo.ui, files),
-historypack.historypack)
+historypack.historypack,
-historypacks.extend(s for s in historystore
+)
-if not isinstance(s, historypack.historypackstore))
+historypacks.extend(
+s
+for s in historystore
+if not isinstance(s, historypack.historypackstore)
+)
 # ``allhistory{files,packs}`` contains all known history packs, even ones we
 # don't plan to repack. They are used during the datapack repack to ensure
 # good ordering of nodes.
-allhistoryfiles = _allpackfileswithsuffix(files, historypack.PACKSUFFIX,
+allhistoryfiles = _allpackfileswithsuffix(
-historypack.INDEXSUFFIX)
+files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX
-allhistorypacks = _topacks(packpath,
+)
+allhistorypacks = _topacks(
+packpath,
 (f for f, mode, stat in allhistoryfiles),
-historypack.historypack)
+historypack.historypack,
-allhistorypacks.extend(s for s in historystore
+)
-if not isinstance(s, historypack.historypackstore))
+allhistorypacks.extend(
-_runrepack(repo,
+s
-contentstore.unioncontentstore(
+for s in historystore
-*datapacks,
+if not isinstance(s, historypack.historypackstore)
-allowincomplete=allowincompletedata),
+)
-metadatastore.unionmetadatastore(
+_runrepack(
-*historypacks,
+repo,
-allowincomplete=True),
+contentstore.unioncontentstore(
-packpath, category,
+*datapacks, allowincomplete=allowincompletedata
-fullhistory=metadatastore.unionmetadatastore(
+),
-*allhistorypacks,
+metadatastore.unionmetadatastore(*historypacks, allowincomplete=True),
-allowincomplete=True),
+packpath,
-options=options)
+category,
+fullhistory=metadatastore.unionmetadatastore(
+*allhistorypacks, allowincomplete=True
+),
+options=options,
+)
 def _computeincrementaldatapack(ui, files):
 opts = {
-'gencountlimit' : ui.configint(
+'gencountlimit': ui.configint('remotefilelog', 'data.gencountlimit'),
-'remotefilelog', 'data.gencountlimit'),
+'generations': ui.configlist('remotefilelog', 'data.generations'),
-'generations' : ui.configlist(
+'maxrepackpacks': ui.configint('remotefilelog', 'data.maxrepackpacks'),
-'remotefilelog', 'data.generations'),
+'repackmaxpacksize': ui.configbytes(
-'maxrepackpacks' : ui.configint(
+'remotefilelog', 'data.repackmaxpacksize'
-'remotefilelog', 'data.maxrepackpacks'),
+),
-'repackmaxpacksize' : ui.configbytes(
+'repacksizelimit': ui.configbytes(
-'remotefilelog', 'data.repackmaxpacksize'),
+'remotefilelog', 'data.repacksizelimit'
-'repacksizelimit' : ui.configbytes(
+),
-'remotefilelog', 'data.repacksizelimit'),
 }
 packfiles = _allpackfileswithsuffix(
-files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX)
+files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX
+)
 return _computeincrementalpack(packfiles, opts)
 def _computeincrementalhistorypack(ui, files):
 opts = {
-'gencountlimit' : ui.configint(
+'gencountlimit': ui.configint('remotefilelog', 'history.gencountlimit'),
-'remotefilelog', 'history.gencountlimit'),
+'generations': ui.configlist(
-'generations' : ui.configlist(
+'remotefilelog', 'history.generations', ['100MB']
-'remotefilelog', 'history.generations', ['100MB']),
+),
-'maxrepackpacks' : ui.configint(
+'maxrepackpacks': ui.configint(
-'remotefilelog', 'history.maxrepackpacks'),
+'remotefilelog', 'history.maxrepackpacks'
-'repackmaxpacksize' : ui.configbytes(
+),
-'remotefilelog', 'history.repackmaxpacksize', '400MB'),
+'repackmaxpacksize': ui.configbytes(
-'repacksizelimit' : ui.configbytes(
+'remotefilelog', 'history.repackmaxpacksize', '400MB'
-'remotefilelog', 'history.repacksizelimit'),
+),
+'repacksizelimit': ui.configbytes(
+'remotefilelog', 'history.repacksizelimit'
+),
 }
 packfiles = _allpackfileswithsuffix(
-files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX)
+files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX
+)
 return _computeincrementalpack(packfiles, opts)
 def _allpackfileswithsuffix(files, packsuffix, indexsuffix):
 result = []
 fileset = set(fn for fn, mode, stat in files)
 for filename, mode, stat in files:
 if not filename.endswith(packsuffix):
 continue
-prefix = filename[:-len(packsuffix)]
+prefix = filename[: -len(packsuffix)]
 # Don't process a pack if it doesn't have an index.
 if (prefix + indexsuffix) not in fileset:
 continue
 result.append((prefix, mode, stat))
 return result
 def _computeincrementalpack(files, opts):
 """Given a set of pack files along with the configuration options, this
 function computes the list of files that should be packed as part of an
 incremental repack.
 It tries to strike a balance between keeping incremental repacks cheap (i.e.
 packing small things when possible, and rolling the packs up to the big ones
 over time).
 """
-limits = list(sorted((util.sizetoint(s) for s in opts['generations']),
+limits = list(
-reverse=True))
+sorted((util.sizetoint(s) for s in opts['generations']), reverse=True)
+)
 limits.append(0)
 # Group the packs by generation (i.e. by size)
 generations = []
 for i in pycompat.xrange(len(limits)):
 # Find the largest generation with more than gencountlimit packs
 genpacks = []
 for i, limit in enumerate(limits):
 if len(generations[i]) > opts['gencountlimit']:
 # Sort to be smallest last, for easy popping later
-genpacks.extend(sorted(generations[i], reverse=True,
+genpacks.extend(
-key=lambda x: sizes[x]))
+sorted(generations[i], reverse=True, key=lambda x: sizes[x])
+)
 break
 # Take as many packs from the generation as we can
 chosenpacks = genpacks[-3:]
 genpacks = genpacks[:-3]
 repacksize = sum(sizes[n] for n in chosenpacks)
-while (repacksize < opts['repacksizelimit'] and genpacks and
+while (
-len(chosenpacks) < opts['maxrepackpacks']):
+repacksize < opts['repacksizelimit']
+and genpacks
+and len(chosenpacks) < opts['maxrepackpacks']
+):
 chosenpacks.append(genpacks.pop())
 repacksize += sizes[chosenpacks[-1]]
 return chosenpacks
-def _runrepack(repo, data, history, packpath, category, fullhistory=None,
-options=None):
+def _runrepack(
+repo, data, history, packpath, category, fullhistory=None, options=None
+):
 shallowutil.mkstickygroupdir(repo.ui, packpath)
 def isold(repo, filename, node):
 """Check if the file node is older than a limit.
 Unless a limit is specified in the config the default limit is taken.
 return filetime[0] < limit
 garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack')
 if not fullhistory:
 fullhistory = history
-packer = repacker(repo, data, history, fullhistory, category,
+packer = repacker(
-gc=garbagecollect, isold=isold, options=options)
+repo,
+data,
+history,
+fullhistory,
+category,
+gc=garbagecollect,
+isold=isold,
+options=options,
+)
 with datapack.mutabledatapack(repo.ui, packpath) as dpack:
 with historypack.mutablehistorypack(repo.ui, packpath) as hpack:
 try:
 packer.run(dpack, hpack)
 except error.LockHeld:
-raise RepackAlreadyRunning(_("skipping repack - another repack "
+raise RepackAlreadyRunning(
-"is already running"))
+_("skipping repack - another repack " "is already running")
+)
 def keepset(repo, keyfn, lastkeepkeys=None):
 """Computes a keepset which is not garbage collected.
 'keyfn' is a function that maps filename, node to a unique key.
 'lastkeepkeys' is an optional argument and if provided the keepset
 for filename, filenode in m.iteritems():
 keepkeys.add(keyfn(filename, filenode))
 return keepkeys
 class repacker(object):
 """Class for orchestrating the repack of data and history information into a
 new format.
 """
-def __init__(self, repo, data, history, fullhistory, category, gc=False,
-isold=None, options=None):
+def __init__(
+self,
+repo,
+data,
+history,
+fullhistory,
+category,
+gc=False,
+isold=None,
+options=None,
+):
 self.repo = repo
 self.data = data
 self.history = history
 self.fullhistory = fullhistory
 self.unit = constants.getunits(category)
 self.options = options
 if self.garbagecollect:
 if not isold:
 raise ValueError("Function 'isold' is not properly specified")
 # use (filename, node) tuple as a keepset key
-self.keepkeys = keepset(repo, lambda f, n : (f, n))
+self.keepkeys = keepset(repo, lambda f, n: (f, n))
 self.isold = isold
 def run(self, targetdata, targethistory):
 ledger = repackledger()
-with lockmod.lock(repacklockvfs(self.repo), "repacklock", desc=None,
+with lockmod.lock(
-timeout=0):
+repacklockvfs(self.repo), "repacklock", desc=None, timeout=0
+):
 self.repo.hook('prerepack')
 # Populate ledger from source
 self.data.markledger(ledger, options=self.options)
 self.history.markledger(ledger, options=self.options)
 # likely to be the newest version assuming files grow over time.
 # (Sort by node first to ensure the sort is stable.)
 orphans = sorted(orphans)
 orphans = list(sorted(orphans, key=getsize, reverse=True))
 if ui.debugflag:
-ui.debug("%s: orphan chain: %s\n" % (filename,
+ui.debug(
-", ".join([short(s) for s in orphans])))
+"%s: orphan chain: %s\n"
+% (filename, ", ".join([short(s) for s in orphans]))
+)
 # Create one contiguous chain and reassign deltabases.
 for i, node in enumerate(orphans):
 if i == 0:
 deltabases[node] = (nullid, 0)
 for entry in ledger.entries.itervalues():
 if entry.datasource:
 byfile.setdefault(entry.filename, {})[entry.node] = entry
 count = 0
-repackprogress = ui.makeprogress(_("repacking data"), unit=self.unit,
+repackprogress = ui.makeprogress(
-total=len(byfile))
+_("repacking data"), unit=self.unit, total=len(byfile)
+)
 for filename, entries in sorted(byfile.iteritems()):
 repackprogress.update(count)
 ancestors = {}
 nodes = list(node for node in entries)
 nohistory = []
-buildprogress = ui.makeprogress(_("building history"), unit='nodes',
+buildprogress = ui.makeprogress(
-total=len(nodes))
+_("building history"), unit='nodes', total=len(nodes)
+)
 for i, node in enumerate(nodes):
 if node in ancestors:
 continue
 buildprogress.update(i)
 try:
-ancestors.update(self.fullhistory.getancestors(filename,
+ancestors.update(
-node, known=ancestors))
+self.fullhistory.getancestors(
+filename, node, known=ancestors
+)
+)
 except KeyError:
 # Since we're packing data entries, we may not have the
 # corresponding history entries for them. It's not a big
 # deal, but the entries won't be delta'd perfectly.
 nohistory.append(node)
 buildprogress.complete()
 # Order the nodes children first, so we can produce reverse deltas
 orderednodes = list(reversed(self._toposort(ancestors)))
 if len(nohistory) > 0:
-ui.debug('repackdata: %d nodes without history\n' %
+ui.debug(
-len(nohistory))
+'repackdata: %d nodes without history\n' % len(nohistory)
+)
 orderednodes.extend(sorted(nohistory))
 # Filter orderednodes to just the nodes we want to serialize (it
 # currently also has the edge nodes' ancestors).
-orderednodes = list(filter(lambda node: node in nodes,
+orderednodes = list(
-orderednodes))
+filter(lambda node: node in nodes, orderednodes)
+)
 # Garbage collect old nodes:
 if self.garbagecollect:
 neworderednodes = []
 for node in orderednodes:
 # If the node is old and is not in the keepset, we skip it,
 # and mark as garbage collected
-if ((filename, node) not in self.keepkeys and
+if (filename, node) not in self.keepkeys and self.isold(
-self.isold(self.repo, filename, node)):
+self.repo, filename, node
+):
 entries[node].gced = True
 continue
 neworderednodes.append(node)
 orderednodes = neworderednodes
 # Compute delta bases for nodes:
 deltabases = {}
 nobase = set()
 referenced = set()
 nodes = set(nodes)
-processprogress = ui.makeprogress(_("processing nodes"),
+processprogress = ui.makeprogress(
-unit='nodes',
+_("processing nodes"), unit='nodes', total=len(orderednodes)
-total=len(orderednodes))
+)
 for i, node in enumerate(orderednodes):
 processprogress.update(i)
 # Find delta base
 # TODO: allow delta'ing against most recent descendant instead
 # of immediate child
 deltabases[p2] = (node, chainlen + 1)
 # experimental config: repack.chainorphansbysize
 if ui.configbool('repack', 'chainorphansbysize'):
 orphans = nobase - referenced
-orderednodes = self._chainorphans(ui, filename, orderednodes,
+orderednodes = self._chainorphans(
-orphans, deltabases)
+ui, filename, orderednodes, orphans, deltabases
+)
 # Compute deltas and write to the pack
 for i, node in enumerate(orderednodes):
 deltabase, chainlen = deltabases[node]
 # Compute delta
 # be fetching the same deltachain over and over again.
 if deltabase != nullid:
 deltaentry = self.data.getdelta(filename, node)
 delta, deltabasename, origdeltabase, meta = deltaentry
 size = meta.get(constants.METAKEYSIZE)
-if (deltabasename != filename or origdeltabase != deltabase
+if (
-or size is None):
+deltabasename != filename
+or origdeltabase != deltabase
+or size is None
+):
 deltabasetext = self.data.get(filename, deltabase)
 original = self.data.get(filename, node)
 size = len(original)
 delta = mdiff.textdiff(deltabasetext, original)
 else:
 byfile = {}
 for entry in ledger.entries.itervalues():
 if entry.historysource:
 byfile.setdefault(entry.filename, {})[entry.node] = entry
-progress = ui.makeprogress(_("repacking history"), unit=self.unit,
+progress = ui.makeprogress(
-total=len(byfile))
+_("repacking history"), unit=self.unit, total=len(byfile)
+)
 for filename, entries in sorted(byfile.iteritems()):
 ancestors = {}
 nodes = list(node for node in entries)
 for node in nodes:
 if node in ancestors:
 continue
-ancestors.update(self.history.getancestors(filename, node,
+ancestors.update(
-known=ancestors))
+self.history.getancestors(filename, node, known=ancestors)
+)
 # Order the nodes children first
 orderednodes = reversed(self._toposort(ancestors))
 # Write to the pack
 return parents
 sortednodes = shallowutil.sortnodes(ancestors.keys(), parentfunc)
 return sortednodes
 class repackledger(object):
 """Storage for all the bookkeeping that happens during a repack. It contains
 the list of revisions being repacked, what happened to each revision, and
 which source store contained which revision originally (for later cleanup).
 """
 def __init__(self):
 self.entries = {}
 self.sources = {}
 self.created = set()
 return value
 def addcreated(self, value):
 self.created.add(value)
 class repackentry(object):
 """Simple class representing a single revision entry in the repackledger.
 """
-__slots__ = (r'filename', r'node', r'datasource', r'historysource',
-r'datarepacked', r'historyrepacked', r'gced')
+__slots__ = (
+r'filename',
+r'node',
+r'datasource',
+r'historysource',
+r'datarepacked',
+r'historyrepacked',
+r'gced',
+)
 def __init__(self, filename, node):
 self.filename = filename
 self.node = node
 # If the revision has a data entry in the source
 self.datasource = False
 # If the revision's history entry was repacked into the repack target
 self.historyrepacked = False
 # If garbage collected
 self.gced = False
 def repacklockvfs(repo):
 if util.safehasattr(repo, 'name'):
 # Lock in the shared cache so repacks across multiple copies of the same
 # repo are coordinated.
 sharedcachepath = shallowutil.getcachepackpath(
-repo,
+repo, constants.FILEPACK_CATEGORY
-constants.FILEPACK_CATEGORY)
+)
 return vfs.vfs(sharedcachepath)
 else:
 return repo.svfs

changeset 43076	2372284d9457
parent 42943	5fadf6103790
child 43077	687b865b95ad