mercurial: comparison hgext/remotefilelog/fileserverclient.py

equal deleted inserted replaced

-:2372284d9457
+:687b865b95ad
 def peersetup(ui, peer):
 class remotefilepeer(peer.__class__):
 @wireprotov1peer.batchable
 def x_rfl_getfile(self, file, node):
-if not self.capable('x_rfl_getfile'):
+if not self.capable(b'x_rfl_getfile'):
 raise error.Abort(
-'configured remotefile server does not support getfile'
+b'configured remotefile server does not support getfile'
 )
 f = wireprotov1peer.future()
-yield {'file': file, 'node': node}, f
+yield {b'file': file, b'node': node}, f
-code, data = f.value.split('\0', 1)
+code, data = f.value.split(b'\0', 1)
 if int(code):
 raise error.LookupError(file, node, data)
 yield data
 @wireprotov1peer.batchable
 def x_rfl_getflogheads(self, path):
-if not self.capable('x_rfl_getflogheads'):
+if not self.capable(b'x_rfl_getflogheads'):
 raise error.Abort(
-'configured remotefile server does not '
+b'configured remotefile server does not '
-'support getflogheads'
+b'support getflogheads'
 )
 f = wireprotov1peer.future()
-yield {'path': path}, f
+yield {b'path': path}, f
-heads = f.value.split('\n') if f.value else []
+heads = f.value.split(b'\n') if f.value else []
 yield heads
 def _updatecallstreamopts(self, command, opts):
-if command != 'getbundle':
+if command != b'getbundle':
 return
 if (
 constants.NETWORK_CAP_LEGACY_SSH_GETFILES
 not in self.capabilities()
 ):
 return
-if not util.safehasattr(self, '_localrepo'):
+if not util.safehasattr(self, b'_localrepo'):
 return
 if (
 constants.SHALLOWREPO_REQUIREMENT
 not in self._localrepo.requirements
 ):
 return
-bundlecaps = opts.get('bundlecaps')
+bundlecaps = opts.get(b'bundlecaps')
 if bundlecaps:
 bundlecaps = [bundlecaps]
 else:
 bundlecaps = []
 # getbundle args before it goes across the wire. Once we get rid
 # of bundle1, we can use bundle2's _pullbundle2extraprepare to
 # do this more cleanly.
 bundlecaps.append(constants.BUNDLE2_CAPABLITY)
 if self._localrepo.includepattern:
-patterns = '\0'.join(self._localrepo.includepattern)
+patterns = b'\0'.join(self._localrepo.includepattern)
-includecap = "includepattern=" + patterns
+includecap = b"includepattern=" + patterns
 bundlecaps.append(includecap)
 if self._localrepo.excludepattern:
-patterns = '\0'.join(self._localrepo.excludepattern)
+patterns = b'\0'.join(self._localrepo.excludepattern)
-excludecap = "excludepattern=" + patterns
+excludecap = b"excludepattern=" + patterns
 bundlecaps.append(excludecap)
-opts['bundlecaps'] = ','.join(bundlecaps)
+opts[b'bundlecaps'] = b','.join(bundlecaps)
 def _sendrequest(self, command, args, **opts):
 self._updatecallstreamopts(command, args)
 return super(remotefilepeer, self)._sendrequest(
 command, args, **opts
 )
 def _callstream(self, command, **opts):
 supertype = super(remotefilepeer, self)
-if not util.safehasattr(supertype, '_sendrequest'):
+if not util.safehasattr(supertype, b'_sendrequest'):
 self._updatecallstreamopts(command, pycompat.byteskwargs(opts))
 return super(remotefilepeer, self)._callstream(command, **opts)
 peer.__class__ = remotefilepeer
 self.subprocess = None
 self.connected = False
 def connect(self, cachecommand):
 if self.pipeo:
-raise error.Abort(_("cache connection already open"))
+raise error.Abort(_(b"cache connection already open"))
 self.pipei, self.pipeo, self.pipee, self.subprocess = procutil.popen4(
 cachecommand
 )
 self.connected = True
 except Exception:
 pass
 if self.connected:
 try:
-self.pipei.write("exit\n")
+self.pipei.write(b"exit\n")
 except Exception:
 pass
 tryclose(self.pipei)
 self.pipei = None
 tryclose(self.pipeo)
 with remote.commandexecutor() as e:
 futures = []
 for m in missed:
 futures.append(
 e.callcommand(
-'x_rfl_getfile', {'file': idmap[m], 'node': m[-40:]}
+b'x_rfl_getfile', {b'file': idmap[m], b'node': m[-40:]}
 )
 )
 for i, m in enumerate(missed):
 r = futures[i].result()
 futures[i] = None  # release memory
 file_ = idmap[m]
 node = m[-40:]
-receivemissing(io.BytesIO('%d\n%s' % (len(r), r)), file_, node)
+receivemissing(io.BytesIO(b'%d\n%s' % (len(r), r)), file_, node)
 progresstick()
 def _getfiles_optimistic(
 remote, receivemissing, progresstick, missed, idmap, step
 ):
-remote._callstream("x_rfl_getfiles")
+remote._callstream(b"x_rfl_getfiles")
 i = 0
 pipeo = remote._pipeo
 pipei = remote._pipei
 while i < len(missed):
 # issue a batch of requests
 i = end
 for missingid in missed[start:end]:
 # issue new request
 versionid = missingid[-40:]
 file = idmap[missingid]
-sshrequest = "%s%s\n" % (versionid, file)
+sshrequest = b"%s%s\n" % (versionid, file)
 pipeo.write(sshrequest)
 pipeo.flush()
 # receive batch results
 for missingid in missed[start:end]:
 file = idmap[missingid]
 receivemissing(pipei, file, versionid)
 progresstick()
 # End the command
-pipeo.write('\n')
+pipeo.write(b'\n')
 pipeo.flush()
 def _getfiles_threaded(
 remote, receivemissing, progresstick, missed, idmap, step
 ):
-remote._callstream("getfiles")
+remote._callstream(b"getfiles")
 pipeo = remote._pipeo
 pipei = remote._pipei
 def writer():
 for missingid in missed:
 versionid = missingid[-40:]
 file = idmap[missingid]
-sshrequest = "%s%s\n" % (versionid, file)
+sshrequest = b"%s%s\n" % (versionid, file)
 pipeo.write(sshrequest)
 pipeo.flush()
 writerthread = threading.Thread(target=writer)
 writerthread.daemon = True
 receivemissing(pipei, file, versionid)
 progresstick()
 writerthread.join()
 # End the command
-pipeo.write('\n')
+pipeo.write(b'\n')
 pipeo.flush()
 class fileserverclient(object):
 """A client for requesting files from the remote file server.
 def __init__(self, repo):
 ui = repo.ui
 self.repo = repo
 self.ui = ui
-self.cacheprocess = ui.config("remotefilelog", "cacheprocess")
+self.cacheprocess = ui.config(b"remotefilelog", b"cacheprocess")
 if self.cacheprocess:
 self.cacheprocess = util.expandpath(self.cacheprocess)
 # This option causes remotefilelog to pass the full file path to the
 # cacheprocess instead of a hashed key.
 self.cacheprocesspasspath = ui.configbool(
-"remotefilelog", "cacheprocess.includepath"
+b"remotefilelog", b"cacheprocess.includepath"
 )
-self.debugoutput = ui.configbool("remotefilelog", "debug")
+self.debugoutput = ui.configbool(b"remotefilelog", b"debug")
 self.remotecache = cacheconnection()
 def setstore(self, datastore, historystore, writedata, writehistory):
 self.datastore = datastore
 cache = self.remotecache
 writedata = self.writedata
 repo = self.repo
 total = len(fileids)
-request = "get\n%d\n" % total
+request = b"get\n%d\n" % total
 idmap = {}
 reponame = repo.name
 for file, id in fileids:
 fullid = getcachekey(reponame, file, id)
 if self.cacheprocesspasspath:
-request += file + '\0'
+request += file + b'\0'
-request += fullid + "\n"
+request += fullid + b"\n"
 idmap[fullid] = file
 cache.request(request)
-progress = self.ui.makeprogress(_('downloading'), total=total)
+progress = self.ui.makeprogress(_(b'downloading'), total=total)
 progress.update(0)
 missed = []
 while True:
 missingid = cache.receiveline()
 for missingid in idmap:
 if not missingid in missedset:
 missed.append(missingid)
 self.ui.warn(
 _(
-"warning: cache connection closed early - "
+b"warning: cache connection closed early - "
-+ "falling back to server\n"
++ b"falling back to server\n"
 )
 )
 break
-if missingid == "0":
+if missingid == b"0":
 break
-if missingid.startswith("_hits_"):
+if missingid.startswith(b"_hits_"):
 # receive progress reports
-parts = missingid.split("_")
+parts = missingid.split(b"_")
 progress.increment(int(parts[2]))
 continue
 missed.append(missingid)
 fetchmisses += len(missed)
 fromcache = total - len(missed)
 progress.update(fromcache, total=total)
 self.ui.log(
-"remotefilelog",
+b"remotefilelog",
-"remote cache hit rate is %r of %r\n",
+b"remote cache hit rate is %r of %r\n",
 fromcache,
 total,
 hit=fromcache,
 total=total,
 )
 if remote.capable(
 constants.NETWORK_CAP_LEGACY_SSH_GETFILES
 ):
 if not isinstance(remote, _sshv1peer):
 raise error.Abort(
-'remotefilelog requires ssh ' 'servers'
+b'remotefilelog requires ssh ' b'servers'
 )
 step = self.ui.configint(
-'remotefilelog', 'getfilesstep'
+b'remotefilelog', b'getfilesstep'
 )
 getfilestype = self.ui.config(
-'remotefilelog', 'getfilestype'
+b'remotefilelog', b'getfilestype'
 )
-if getfilestype == 'threaded':
+if getfilestype == b'threaded':
 _getfiles = _getfiles_threaded
 else:
 _getfiles = _getfiles_optimistic
 _getfiles(
 remote,
 progress.increment,
 missed,
 idmap,
 step,
 )
-elif remote.capable("x_rfl_getfile"):
+elif remote.capable(b"x_rfl_getfile"):
-if remote.capable('batch'):
+if remote.capable(b'batch'):
 batchdefault = 100
 else:
 batchdefault = 10
 batchsize = self.ui.configint(
-'remotefilelog', 'batchsize', batchdefault
+b'remotefilelog', b'batchsize', batchdefault
 )
 self.ui.debug(
 b'requesting %d files from '
 b'remotefilelog server...\n' % len(missed)
 )
 idmap,
 batchsize,
 )
 else:
 raise error.Abort(
-"configured remotefilelog server"
+b"configured remotefilelog server"
-" does not support remotefilelog"
+b" does not support remotefilelog"
 )
 self.ui.log(
-"remotefilefetchlog",
+b"remotefilefetchlog",
-"Success\n",
+b"Success\n",
 fetched_files=progress.pos - fromcache,
 total_to_fetch=total - fromcache,
 )
 except Exception:
 self.ui.log(
-"remotefilefetchlog",
+b"remotefilefetchlog",
-"Fail\n",
+b"Fail\n",
 fetched_files=progress.pos - fromcache,
 total_to_fetch=total - fromcache,
 )
 raise
 finally:
 self.ui.verbose = verbose
 # send to memcache
-request = "set\n%d\n%s\n" % (len(missed), "\n".join(missed))
+request = b"set\n%d\n%s\n" % (len(missed), b"\n".join(missed))
 cache.request(request)
 progress.complete()
 # mark ourselves as a user of this cache
 def receivemissing(self, pipe, filename, node):
 line = pipe.readline()[:-1]
 if not line:
 raise error.ResponseError(
-_("error downloading file contents:"),
+_(b"error downloading file contents:"),
-_("connection closed early"),
+_(b"connection closed early"),
 )
 size = int(line)
 data = pipe.read(size)
 if len(data) != size:
 raise error.ResponseError(
-_("error downloading file contents:"),
+_(b"error downloading file contents:"),
-_("only received %s of %s bytes") % (len(data), size),
+_(b"only received %s of %s bytes") % (len(data), size),
 )
 self.writedata.addremotefilelognode(
 filename, bin(node), zlib.decompress(data)
 )
 def connect(self):
 if self.cacheprocess:
-cmd = "%s %s" % (self.cacheprocess, self.writedata._path)
+cmd = b"%s %s" % (self.cacheprocess, self.writedata._path)
 self.remotecache.connect(cmd)
 else:
 # If no cache process is specified, we fake one that always
 # returns cache misses.  This enables tests to run easily
 # and may eventually allow us to be a drop in replacement
 def close(self):
 pass
 def request(self, value, flush=True):
-lines = value.split("\n")
+lines = value.split(b"\n")
-if lines[0] != "get":
+if lines[0] != b"get":
 return
 self.missingids = lines[2:-1]
-self.missingids.append('0')
+self.missingids.append(b'0')
 def receiveline(self):
 if len(self.missingids) > 0:
 return self.missingids.pop(0)
 return None
 self.remotecache = simplecache()
 def close(self):
 if fetches:
 msg = (
-"%d files fetched over %d fetches - "
+b"%d files fetched over %d fetches - "
-+ "(%d misses, %0.2f%% hit ratio) over %0.2fs\n"
++ b"(%d misses, %0.2f%% hit ratio) over %0.2fs\n"
 ) % (
 fetched,
 fetches,
 fetchmisses,
 float(fetched - fetchmisses) / float(fetched) * 100.0,
 fetchcost,
 )
 if self.debugoutput:
 self.ui.warn(msg)
 self.ui.log(
-"remotefilelog.prefetch",
+b"remotefilelog.prefetch",
-msg.replace("%", "%%"),
+msg.replace(b"%", b"%%"),
 remotefilelogfetched=fetched,
 remotefilelogfetches=fetches,
 remotefilelogfetchmisses=fetchmisses,
 remotefilelogfetchtime=fetchcost * 1000,
 )
 # hack
 # - we don't use .hgtags
 # - workingctx produces ids with length 42,
 #   which we skip since they aren't in any cache
 if (
-file == '.hgtags'
+file == b'.hgtags'
 or len(id) == 42
 or not repo.shallowmatch(file)
 ):
 continue
 nullids = len([None for unused, id in missingids if id == nullid])
 if nullids:
 missingids = [(f, id) for f, id in missingids if id != nullid]
 repo.ui.develwarn(
 (
-'remotefilelog not fetching %d null revs'
+b'remotefilelog not fetching %d null revs'
-' - this is likely hiding bugs' % nullids
+b' - this is likely hiding bugs' % nullids
 ),
-config='remotefilelog-ext',
+config=b'remotefilelog-ext',
 )
 if missingids:
 global fetches, fetched, fetchcost
 fetches += 1
 # We want to be able to detect excess individual file downloads, so
 # let's log that information for debugging.
 if fetches >= 15 and fetches < 18:
 if fetches == 15:
 fetchwarning = self.ui.config(
-'remotefilelog', 'fetchwarning'
+b'remotefilelog', b'fetchwarning'
 )
 if fetchwarning:
-self.ui.warn(fetchwarning + '\n')
+self.ui.warn(fetchwarning + b'\n')
 self.logstacktrace()
 missingids = [(file, hex(id)) for file, id in sorted(missingids)]
 fetched += len(missingids)
 start = time.time()
 missingids = self.request(missingids)
 if missingids:
 raise error.Abort(
-_("unable to download %d files") % len(missingids)
+_(b"unable to download %d files") % len(missingids)
 )
 fetchcost += time.time() - start
 self._lfsprefetch(fileids)
 def _lfsprefetch(self, fileids):
 if not _lfsmod or not util.safehasattr(
-self.repo.svfs, 'lfslocalblobstore'
+self.repo.svfs, b'lfslocalblobstore'
 ):
 return
 if not _lfsmod.wrapper.candownload(self.repo):
 return
 pointers = []
 def logstacktrace(self):
 import traceback
 self.ui.log(
-'remotefilelog',
+b'remotefilelog',
-'excess remotefilelog fetching:\n%s\n',
+b'excess remotefilelog fetching:\n%s\n',
-''.join(traceback.format_stack()),
+b''.join(traceback.format_stack()),
 )

changeset 43077	687b865b95ad
parent 43076	2372284d9457
child 43115	4aa72cdf616f