mercurial: comparison hgext/largefiles/lfutil.py

equal deleted inserted replaced

-:57875cf423c9
+:2372284d9457
 shortnameslash = shortname + '/'
 longname = 'largefiles'
 # -- Private worker functions ------------------------------------------
 def getminsize(ui, assumelfiles, opt, default=10):
 lfsize = opt
 if not lfsize and assumelfiles:
 lfsize = ui.config(longname, 'minsize', default=default)
 if lfsize:
 try:
 lfsize = float(lfsize)
 except ValueError:
-raise error.Abort(_('largefiles: size must be number (not %s)\n')
+raise error.Abort(
-% lfsize)
+_('largefiles: size must be number (not %s)\n') % lfsize
+)
 if lfsize is None:
 raise error.Abort(_('minimum size for largefiles must be specified'))
 return lfsize
 def link(src, dest):
 """Try to create hardlink - if that fails, efficiently make a copy."""
 util.makedirs(os.path.dirname(dest))
 try:
 with open(src, 'rb') as srcf, util.atomictempfile(dest) as dstf:
 for chunk in util.filechunkiter(srcf):
 dstf.write(chunk)
 os.chmod(dest, os.stat(src).st_mode)
 def usercachepath(ui, hash):
 '''Return the correct location in the "global" largefiles cache for a file
 with the given hash.
 This cache is used for sharing of largefiles across repositories - both
 to preserve download bandwidth and storage space.'''
 return os.path.join(_usercachedir(ui), hash)
 def _usercachedir(ui, name=longname):
 '''Return the location of the "global" largefiles cache.'''
 path = ui.configpath(name, 'usercache')
 if path:
 return path
 if pycompat.iswindows:
-appdata = encoding.environ.get('LOCALAPPDATA',
+appdata = encoding.environ.get(
-encoding.environ.get('APPDATA'))
+'LOCALAPPDATA', encoding.environ.get('APPDATA')
+)
 if appdata:
 return os.path.join(appdata, name)
 elif pycompat.isdarwin:
 home = encoding.environ.get('HOME')
 if home:
 return os.path.join(path, name)
 home = encoding.environ.get('HOME')
 if home:
 return os.path.join(home, '.cache', name)
 else:
-raise error.Abort(_('unknown operating system: %s\n')
+raise error.Abort(_('unknown operating system: %s\n') % pycompat.osname)
-% pycompat.osname)
 raise error.Abort(_('unknown %s usercache location') % name)
 def inusercache(ui, hash):
 path = usercachepath(ui, hash)
 return os.path.exists(path)
 def findfile(repo, hash):
 '''Return store path of the largefile with the specified hash.
 As a side effect, the file might be linked from user cache.
 Return None if the file can't be found locally.'''
 path = storepath(repo, hash)
 link(usercachepath(repo.ui, hash), path)
 return path
 return None
 class largefilesdirstate(dirstate.dirstate):
 def __getitem__(self, key):
 return super(largefilesdirstate, self).__getitem__(unixpath(key))
 def normal(self, f):
 return super(largefilesdirstate, self).normal(unixpath(f))
 def remove(self, f):
 return super(largefilesdirstate, self).remove(unixpath(f))
 def add(self, f):
 return super(largefilesdirstate, self).add(unixpath(f))
 def drop(self, f):
 return super(largefilesdirstate, self).drop(unixpath(f))
 def forget(self, f):
 return super(largefilesdirstate, self).forget(unixpath(f))
 def normallookup(self, f):
 return super(largefilesdirstate, self).normallookup(unixpath(f))
 def _ignore(self, f):
 return False
 def write(self, tr=False):
 # (1) disable PENDING mode always
 #     (lfdirstate isn't yet managed as a part of the transaction)
 # (2) avoid develwarn 'use dirstate.write with ....'
 super(largefilesdirstate, self).write(None)
 def openlfdirstate(ui, repo, create=True):
 '''
 Return a dirstate object that tracks largefiles: i.e. its root is
 the repo root, but it is saved in .hg/largefiles/dirstate.
 '''
 vfs = repo.vfs
 lfstoredir = longname
 opener = vfsmod.vfs(vfs.join(lfstoredir))
-lfdirstate = largefilesdirstate(opener, ui, repo.root,
+lfdirstate = largefilesdirstate(
-repo.dirstate._validate,
+opener,
-lambda: sparse.matcher(repo))
+ui,
+repo.root,
+repo.dirstate._validate,
+lambda: sparse.matcher(repo),
+)
 # If the largefiles dirstate does not exist, populate and create
 # it. This ensures that we create it on the first meaningful
 # largefiles operation in a new clone.
 if create and not vfs.exists(vfs.join(lfstoredir, 'dirstate')):
 matcher = getstandinmatcher(repo)
-standins = repo.dirstate.walk(matcher, subrepos=[], unknown=False,
+standins = repo.dirstate.walk(
-ignored=False)
+matcher, subrepos=[], unknown=False, ignored=False
+)
 if len(standins) > 0:
 vfs.makedirs(lfstoredir)
 for standin in standins:
 lfile = splitstandin(standin)
 lfdirstate.normallookup(lfile)
 return lfdirstate
 def lfdirstatestatus(lfdirstate, repo):
 pctx = repo['.']
 match = matchmod.always()
-unsure, s = lfdirstate.status(match, subrepos=[], ignored=False,
+unsure, s = lfdirstate.status(
-clean=False, unknown=False)
+match, subrepos=[], ignored=False, clean=False, unknown=False
+)
 modified, clean = s.modified, s.clean
 for lfile in unsure:
 try:
 fctx = pctx[standin(lfile)]
 except LookupError:
 else:
 clean.append(lfile)
 lfdirstate.normal(lfile)
 return s
 def listlfiles(repo, rev=None, matcher=None):
 '''return a list of largefiles in the working copy or the
 specified changeset'''
 if matcher is None:
 matcher = getstandinmatcher(repo)
 # ignore unknown files in working directory
-return [splitstandin(f)
+return [
-for f in repo[rev].walk(matcher)
+splitstandin(f)
-if rev is not None or repo.dirstate[f] != '?']
+for f in repo[rev].walk(matcher)
+if rev is not None or repo.dirstate[f] != '?'
+]
 def instore(repo, hash, forcelocal=False):
 '''Return true if a largefile with the given hash exists in the store'''
 return os.path.exists(storepath(repo, hash, forcelocal))
 def storepath(repo, hash, forcelocal=False):
 '''Return the correct location in the repository largefiles store for a
 file with the given hash.'''
 if not forcelocal and repo.shared():
 return repo.vfs.reljoin(repo.sharedpath, longname, hash)
 return repo.vfs.join(longname, hash)
 def findstorepath(repo, hash):
 '''Search through the local store path(s) to find the file for the given
 hash.  If the file is not found, its path in the primary store is returned.
 The return value is a tuple of (path, exists(path)).
 return (path, True)
 elif repo.shared() and instore(repo, hash, True):
 return storepath(repo, hash, True), True
 return (path, False)
 def copyfromcache(repo, hash, filename):
 '''Copy the specified largefile from the repo or system cache to
 filename in the repository. Return true on success or false if the
 file was not found in either cache (which should not happened:
 return False
 wvfs.makedirs(wvfs.dirname(wvfs.join(filename)))
 # The write may fail before the file is fully written, but we
 # don't use atomic writes in the working copy.
 with open(path, 'rb') as srcfd, wvfs(filename, 'wb') as destfd:
-gothash = copyandhash(
+gothash = copyandhash(util.filechunkiter(srcfd), destfd)
-util.filechunkiter(srcfd), destfd)
 if gothash != hash:
-repo.ui.warn(_('%s: data corruption in %s with hash %s\n')
+repo.ui.warn(
-% (filename, path, gothash))
+_('%s: data corruption in %s with hash %s\n')
+% (filename, path, gothash)
+)
 wvfs.unlink(filename)
 return False
 return True
 def copytostore(repo, ctx, file, fstandin):
 wvfs = repo.wvfs
 hash = readasstandin(ctx[fstandin])
 if instore(repo, hash):
 return
 if wvfs.exists(file):
 copytostoreabsolute(repo, wvfs.join(file), hash)
 else:
-repo.ui.warn(_("%s: largefile %s not available from local store\n") %
+repo.ui.warn(
-(file, hash))
+_("%s: largefile %s not available from local store\n")
+% (file, hash)
+)
 def copyalltostore(repo, node):
 '''Copy all largefiles in a given revision to the store'''
 ctx = repo[node]
 for filename in ctx.files():
 realfile = splitstandin(filename)
 if realfile is not None and filename in ctx.manifest():
 copytostore(repo, ctx, realfile, filename)
 def copytostoreabsolute(repo, file, hash):
 if inusercache(repo.ui, hash):
 link(usercachepath(repo.ui, hash), storepath(repo, hash))
 else:
 util.makedirs(os.path.dirname(storepath(repo, hash)))
 with open(file, 'rb') as srcf:
-with util.atomictempfile(storepath(repo, hash),
+with util.atomictempfile(
-createmode=repo.store.createmode) as dstf:
+storepath(repo, hash), createmode=repo.store.createmode
+) as dstf:
 for chunk in util.filechunkiter(srcf):
 dstf.write(chunk)
 linktousercache(repo, hash)
 def linktousercache(repo, hash):
 '''Link / copy the largefile with the specified hash from the store
 to the cache.'''
 path = usercachepath(repo.ui, hash)
 link(storepath(repo, hash), path)
 def getstandinmatcher(repo, rmatcher=None):
 '''Return a match object that applies rmatcher to the standin directory'''
 wvfs = repo.wvfs
 standindir = shortname
 else:
 # no patterns: relative to repo root
 match = scmutil.match(repo[None], [wvfs.join(standindir)], badfn=badfn)
 return match
 def composestandinmatcher(repo, rmatcher):
 '''Return a matcher that accepts standins corresponding to the
 files accepted by rmatcher. Pass the list of files in the matcher
 as the paths specified by the user.'''
 smatcher = getstandinmatcher(repo, rmatcher)
 isstandin = smatcher.matchfn
 def composedmatchfn(f):
 return isstandin(f) and rmatcher.matchfn(splitstandin(f))
 smatcher.matchfn = composedmatchfn
 return smatcher
 def standin(filename):
 '''Return the repo-relative path to the standin for the specified big
 file.'''
 # Notes:
 # 2) Join with '/' because that's what dirstate always uses, even on
 #    Windows. Change existing separator to '/' first in case we are
 #    passed filenames from an external source (like the command line).
 return shortnameslash + util.pconvert(filename)
 def isstandin(filename):
 '''Return true if filename is a big file standin. filename must be
 in Mercurial's internal form (slash-separated).'''
 return filename.startswith(shortnameslash)
 def splitstandin(filename):
 # Split on / because that's what dirstate always uses, even on Windows.
 # Change local separator to / first just in case we are passed filenames
 # from an external source (like the command line).
 if len(bits) == 2 and bits[0] == shortname:
 return bits[1]
 else:
 return None
 def updatestandin(repo, lfile, standin):
 """Re-calculate hash value of lfile and write it into standin
 This assumes that "lfutil.standin(lfile) == standin", for efficiency.
 """
 executable = getexecutable(file)
 writestandin(repo, standin, hash, executable)
 else:
 raise error.Abort(_('%s: file not found!') % lfile)
 def readasstandin(fctx):
 '''read hex hash from given filectx of standin file
 This encapsulates how "standin" data is stored into storage layer.'''
 return fctx.data().strip()
 def writestandin(repo, standin, hash, executable):
 '''write hash to <repo.root>/<standin>'''
 repo.wwrite(standin, hash + '\n', executable and 'x' or '')
 def copyandhash(instream, outfile):
 '''Read bytes from instream (iterable) and write them to outfile,
 computing the SHA-1 hash of the data along the way. Return the hash.'''
 hasher = hashlib.sha1('')
 for data in instream:
 hasher.update(data)
 outfile.write(data)
 return hex(hasher.digest())
 def hashfile(file):
 if not os.path.exists(file):
 return ''
 with open(file, 'rb') as fd:
 return hexsha1(fd)
 def getexecutable(filename):
 mode = os.stat(filename).st_mode
-return ((mode & stat.S_IXUSR) and
+return (
-(mode & stat.S_IXGRP) and
+(mode & stat.S_IXUSR)
-(mode & stat.S_IXOTH))
+and (mode & stat.S_IXGRP)
+and (mode & stat.S_IXOTH)
+)
 def urljoin(first, second, *arg):
 def join(left, right):
 if not left.endswith('/'):
 left += '/'
 url = join(first, second)
 for a in arg:
 url = join(url, a)
 return url
 def hexsha1(fileobj):
 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
 object data"""
 h = hashlib.sha1()
 for chunk in util.filechunkiter(fileobj):
 h.update(chunk)
 return hex(h.digest())
 def httpsendfile(ui, filename):
 return httpconnection.httpsendfile(ui, filename, 'rb')
 def unixpath(path):
 '''Return a version of path normalized for use with the lfdirstate.'''
 return util.pconvert(os.path.normpath(path))
 def islfilesrepo(repo):
 '''Return true if the repo is a largefile repo.'''
-if ('largefiles' in repo.requirements and
+if 'largefiles' in repo.requirements and any(
-any(shortnameslash in f[0] for f in repo.store.datafiles())):
+shortnameslash in f[0] for f in repo.store.datafiles()
+):
 return True
 return any(openlfdirstate(repo.ui, repo, False))
 class storeprotonotcapable(Exception):
 def __init__(self, storetypes):
 self.storetypes = storetypes
 def getstandinsstate(repo):
 standins = []
 matcher = getstandinmatcher(repo)
 wctx = repo[None]
-for standin in repo.dirstate.walk(matcher, subrepos=[], unknown=False,
+for standin in repo.dirstate.walk(
-ignored=False):
+matcher, subrepos=[], unknown=False, ignored=False
+):
 lfile = splitstandin(standin)
 try:
 hash = readasstandin(wctx[standin])
 except IOError:
 hash = None
 standins.append((lfile, hash))
 return standins
 def synclfdirstate(repo, lfdirstate, lfile, normallookup):
 lfstandin = standin(lfile)
 if lfstandin in repo.dirstate:
 stat = repo.dirstate._map[lfstandin]
 state, mtime = stat[0], stat[3]
 else:
 state, mtime = '?', -1
 if state == 'n':
-if (normallookup or mtime < 0 or
+if normallookup or mtime < 0 or not repo.wvfs.exists(lfile):
-not repo.wvfs.exists(lfile)):
 # state 'n' doesn't ensure 'clean' in this case
 lfdirstate.normallookup(lfile)
 else:
 lfdirstate.normal(lfile)
 elif state == 'm':
 elif state == 'a':
 lfdirstate.add(lfile)
 elif state == '?':
 lfdirstate.drop(lfile)
 def markcommitted(orig, ctx, node):
 repo = ctx.repo()
 orig(node)
 # lookup while copyalltostore(), but can omit redundant check for
 # files comming from the 2nd parent, which should exist in store
 # at merging.
 copyalltostore(repo, node)
 def getlfilestoupdate(oldstandins, newstandins):
 changedstandins = set(oldstandins).symmetric_difference(set(newstandins))
 filelist = []
 for f in changedstandins:
 if f[0] not in filelist:
 filelist.append(f[0])
 return filelist
 def getlfilestoupload(repo, missing, addfunc):
 makeprogress = repo.ui.makeprogress
-with makeprogress(_('finding outgoing largefiles'),
+with makeprogress(
-unit=_('revisions'), total=len(missing)) as progress:
+_('finding outgoing largefiles'),
+unit=_('revisions'),
+total=len(missing),
+) as progress:
 for i, n in enumerate(missing):
 progress.update(i)
 parents = [p for p in repo[n].parents() if p != node.nullid]
 oldlfstatus = repo.lfstatus
 files.add(f)
 for fn in files:
 if isstandin(fn) and fn in ctx:
 addfunc(fn, readasstandin(ctx[fn]))
 def updatestandinsbymatch(repo, match):
 '''Update standins in the working directory according to specified match
 This returns (possibly modified) ``match`` object to be used for
 subsequent commit process.
 # It can cost a lot of time (several seconds)
 # otherwise to update all standins if the largefiles are
 # large.
 lfdirstate = openlfdirstate(ui, repo)
 dirtymatch = matchmod.always()
-unsure, s = lfdirstate.status(dirtymatch, subrepos=[], ignored=False,
+unsure, s = lfdirstate.status(
-clean=False, unknown=False)
+dirtymatch, subrepos=[], ignored=False, clean=False, unknown=False
+)
 modifiedfiles = unsure + s.modified + s.added + s.removed
 lfiles = listlfiles(repo)
 # this only loops through largefiles that exist (not
 # removed/renamed)
 for lfile in lfiles:
 match._files = repo._subdirlfs(match.files(), lfiles)
 # Case 2: user calls commit with specified patterns: refresh
 # any matching big files.
 smatcher = composestandinmatcher(repo, match)
-standins = repo.dirstate.walk(smatcher, subrepos=[], unknown=False,
+standins = repo.dirstate.walk(
-ignored=False)
+smatcher, subrepos=[], unknown=False, ignored=False
+)
 # No matching big files: get out of the way and pass control to
 # the usual commit() method.
 if not standins:
 return match
 match.matchfn = matchfn
 return match
 class automatedcommithook(object):
 '''Stateful hook to update standins at the 1st commit of resuming
 For efficiency, updating standins in the working directory should
 be avoided while automated committing (like rebase, transplant and
 But the 1st commit of resuming automated committing (e.g. ``rebase
 --continue``) should update them, because largefiles may be
 modified manually.
 '''
 def __init__(self, resuming):
 self.resuming = resuming
 def __call__(self, repo, match):
 if self.resuming:
-self.resuming = False # avoids updating at subsequent commits
+self.resuming = False  # avoids updating at subsequent commits
 return updatestandinsbymatch(repo, match)
 else:
 return match
 def getstatuswriter(ui, repo, forcibly=None):
 '''Return the function to write largefiles specific status out
 If ``forcibly`` is ``None``, this returns the last element of
 '''
 if forcibly is None and util.safehasattr(repo, '_largefilesenabled'):
 return repo._lfstatuswriters[-1]
 else:
 if forcibly:
-return ui.status # forcibly WRITE OUT
+return ui.status  # forcibly WRITE OUT
 else:
-return lambda *msg, **opts: None # forcibly IGNORE
+return lambda *msg, **opts: None  # forcibly IGNORE

changeset 43076	2372284d9457
parent 41759	aaad36b88298
child 43077	687b865b95ad