stream-clone: introduce the notion of an experimental "v3" version
We introduce a new experimental "v3" stream protocol, disabled by default. In
practice the "v3-exp" protocol introduced in this changeset is identical to v2,
but this changeset, lay the groundwork for having a new protocol:
configuration, capability exchange, test coverage, etc.
The actual protocol work will starts in the coming changesets.
--- a/hgext/remotefilelog/__init__.py Sat May 20 01:39:13 2023 +0200
+++ b/hgext/remotefilelog/__init__.py Fri May 19 14:49:50 2023 +0200
@@ -408,9 +408,7 @@
# bundle2 flavor of streamclones, so force us to use
# v1 instead.
if b'v2' in pullop.remotebundle2caps.get(b'stream', []):
- pullop.remotebundle2caps[b'stream'] = [
- c for c in pullop.remotebundle2caps[b'stream'] if c != b'v2'
- ]
+ pullop.remotebundle2caps[b'stream'] = []
if bundle2:
return False, None
supported, requirements = orig(pullop, bundle2=bundle2)
--- a/mercurial/bundle2.py Sat May 20 01:39:13 2023 +0200
+++ b/mercurial/bundle2.py Fri May 19 14:49:50 2023 +0200
@@ -1671,6 +1671,10 @@
# Else always advertise support on client, because payload support
# should always be advertised.
+ if repo.ui.configbool(b'experimental', b'stream-v3'):
+ if b'stream' in caps:
+ caps[b'stream'] += (b'v3-exp',)
+
# b'rev-branch-cache is no longer advertised, but still supported
# for legacy clients.
@@ -1892,8 +1896,16 @@
)
hint = _(b'the client seems buggy')
raise error.Abort(msg, hint=hint)
- if not (b'v2' in bundler.capabilities[b'stream']):
- raise error.Abort(_(b'the client does not support streamclone v2'))
+ client_supported = set(bundler.capabilities[b'stream'])
+ server_supported = set(getrepocaps(repo, role=b'client').get(b'stream', []))
+ common_supported = client_supported & server_supported
+ if not common_supported:
+ msg = _(b'no common supported version with the client: %s; %s')
+ str_server = b','.join(sorted(server_supported))
+ str_client = b','.join(sorted(client_supported))
+ msg %= (str_server, str_client)
+ raise error.Abort(msg)
+ version = max(common_supported)
# Stream clones don't compress well. And compression undermines a
# goal of stream clones, which is to be fast. Communicate the desire
@@ -1924,15 +1936,26 @@
elif repo.obsstore._version in remoteversions:
includeobsmarkers = True
- filecount, bytecount, it = streamclone.generatev2(
- repo, includepats, excludepats, includeobsmarkers
- )
- requirements = streamclone.streamed_requirements(repo)
- requirements = _formatrequirementsspec(requirements)
- part = bundler.newpart(b'stream2', data=it)
- part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True)
- part.addparam(b'filecount', b'%d' % filecount, mandatory=True)
- part.addparam(b'requirements', requirements, mandatory=True)
+ if version == b"v2":
+ filecount, bytecount, it = streamclone.generatev2(
+ repo, includepats, excludepats, includeobsmarkers
+ )
+ requirements = streamclone.streamed_requirements(repo)
+ requirements = _formatrequirementsspec(requirements)
+ part = bundler.newpart(b'stream2', data=it)
+ part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True)
+ part.addparam(b'filecount', b'%d' % filecount, mandatory=True)
+ part.addparam(b'requirements', requirements, mandatory=True)
+ elif version == b"v3-exp":
+ filecount, bytecount, it = streamclone.generatev2(
+ repo, includepats, excludepats, includeobsmarkers
+ )
+ requirements = streamclone.streamed_requirements(repo)
+ requirements = _formatrequirementsspec(requirements)
+ part = bundler.newpart(b'stream3', data=it)
+ part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True)
+ part.addparam(b'filecount', b'%d' % filecount, mandatory=True)
+ part.addparam(b'requirements', requirements, mandatory=True)
def buildobsmarkerspart(bundler, markers, mandatory=True):
@@ -2588,6 +2611,11 @@
streamclone.applybundlev2(repo, part, filecount, bytecount, requirements)
+@parthandler(b'stream3', (b'requirements', b'filecount', b'bytecount'))
+def handlestreamv3bundle(op, part):
+ return handlestreamv2bundle(op, part)
+
+
def widen_bundle(
bundler, repo, oldmatcher, newmatcher, common, known, cgversion, ellipses
):
--- a/mercurial/configitems.py Sat May 20 01:39:13 2023 +0200
+++ b/mercurial/configitems.py Fri May 19 14:49:50 2023 +0200
@@ -1298,6 +1298,11 @@
)
coreconfigitem(
b'experimental',
+ b'stream-v3',
+ default=False,
+)
+coreconfigitem(
+ b'experimental',
b'treemanifest',
default=False,
)
--- a/mercurial/exchange.py Sat May 20 01:39:13 2023 +0200
+++ b/mercurial/exchange.py Fri May 19 14:49:50 2023 +0200
@@ -2421,7 +2421,7 @@
return info, bundler.getchunks()
-@getbundle2partsgenerator(b'stream2')
+@getbundle2partsgenerator(b'stream')
def _getbundlestream2(bundler, repo, *args, **kwargs):
return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
--- a/mercurial/streamclone.py Sat May 20 01:39:13 2023 +0200
+++ b/mercurial/streamclone.py Fri May 19 14:49:50 2023 +0200
@@ -758,6 +758,10 @@
return len(entries), totalfilesize, chunks
+def generatev3(repo, includes, excludes, includeobsmarkers):
+ return generatev2(repo, includes, excludes, includeobsmarkers)
+
+
@contextlib.contextmanager
def nested(*ctxs):
this = ctxs[0]
--- a/tests/test-clone-stream.t Sat May 20 01:39:13 2023 +0200
+++ b/tests/test-clone-stream.t Fri May 19 14:49:50 2023 +0200
@@ -1,6 +1,6 @@
#require serve no-reposimplestore no-chg
-#testcases stream-legacy stream-bundle2
+#testcases stream-legacy stream-bundle2-v2 stream-bundle2-v3
#if stream-legacy
$ cat << EOF >> $HGRCPATH
@@ -8,6 +8,12 @@
> bundle2.stream = no
> EOF
#endif
+#if stream-bundle2-v3
+ $ cat << EOF >> $HGRCPATH
+ > [experimental]
+ > stream-v3 = yes
+ > EOF
+#endif
Initialize repository
@@ -174,7 +180,75 @@
0060: 69 73 20 66 |is f|
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
+ $ hg debugcapabilities http://localhost:$HGPORT
+ Main capabilities:
+ batch
+ branchmap
+ $USUAL_BUNDLE2_CAPS_SERVER$
+ changegroupsubset
+ compression=$BUNDLE2_COMPRESSIONS$
+ getbundle
+ httpheader=1024
+ httpmediatype=0.1rx,0.1tx,0.2tx
+ known
+ lookup
+ pushkey
+ unbundle=HG10GZ,HG10BZ,HG10UN
+ unbundlehash
+ Bundle2 capabilities:
+ HG20
+ bookmarks
+ changegroup
+ 01
+ 02
+ 03
+ checkheads
+ related
+ digests
+ md5
+ sha1
+ sha512
+ error
+ abort
+ unsupportedcontent
+ pushraced
+ pushkey
+ hgtagsfnodes
+ listkeys
+ phases
+ heads
+ pushkey
+ remote-changegroup
+ http
+ https
+
+ $ hg clone --stream -U http://localhost:$HGPORT server-disabled
+ warning: stream clone requested but server has them disabled
+ requesting all changes
+ adding changesets
+ adding manifests
+ adding file changes
+ added 3 changesets with 1088 changes to 1088 files
+ new changesets 96ee1d7354c4:5223b5e3265f
+
+ $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=getbundle' content-type --bodyfile body --hgproto 0.2 --requestheader "x-hgarg-1=bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=0&common=0000000000000000000000000000000000000000&heads=c17445101a72edac06facd130d14808dfbd5c7c2&stream=1"
+ 200 Script output follows
+ content-type: application/mercurial-0.2
+
+
+ $ f --size body --hexdump --bytes 100
+ body: size=140
+ 0000: 04 6e 6f 6e 65 48 47 32 30 00 00 00 00 00 00 00 |.noneHG20.......|
+ 0010: 73 0b 45 52 52 4f 52 3a 41 42 4f 52 54 00 00 00 |s.ERROR:ABORT...|
+ 0020: 00 01 01 07 3c 04 16 6d 65 73 73 61 67 65 73 74 |....<..messagest|
+ 0030: 72 65 61 6d 20 64 61 74 61 20 72 65 71 75 65 73 |ream data reques|
+ 0040: 74 65 64 20 62 75 74 20 73 65 72 76 65 72 20 64 |ted but server d|
+ 0050: 6f 65 73 20 6e 6f 74 20 61 6c 6c 6f 77 20 74 68 |oes not allow th|
+ 0060: 69 73 20 66 |is f|
+
+#endif
+#if stream-bundle2-v3
$ hg debugcapabilities http://localhost:$HGPORT
Main capabilities:
batch
@@ -262,7 +336,28 @@
no changes found
$ cat server/errors.txt
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
+ $ hg clone --stream -U http://localhost:$HGPORT clone1
+ streaming all changes
+ 1093 files to transfer, 102 KB of data (no-zstd !)
+ transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+ 1093 files to transfer, 98.9 KB of data (zstd !)
+ transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !)
+
+ $ ls -1 clone1/.hg/cache
+ branch2-base
+ branch2-immutable
+ branch2-served
+ branch2-served.hidden
+ branch2-visible
+ branch2-visible-hidden
+ rbc-names-v1
+ rbc-revs-v1
+ tags2
+ tags2-served
+ $ cat server/errors.txt
+#endif
+#if stream-bundle2-v3
$ hg clone --stream -U http://localhost:$HGPORT clone1
streaming all changes
1093 files to transfer, 102 KB of data (no-zstd !)
@@ -386,7 +481,15 @@
searching for changes
no changes found
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
+ $ hg clone --uncompressed -U http://localhost:$HGPORT clone1-uncompressed
+ streaming all changes
+ 1093 files to transfer, 102 KB of data (no-zstd !)
+ transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+ 1093 files to transfer, 98.9 KB of data (zstd !)
+ transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !)
+#endif
+#if stream-bundle2-v3
$ hg clone --uncompressed -U http://localhost:$HGPORT clone1-uncompressed
streaming all changes
1093 files to transfer, 102 KB of data (no-zstd !)
@@ -425,7 +528,7 @@
updating the branch cache
(sent 5 HTTP requests and * bytes; received * bytes in responses) (glob)
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
$ hg --debug --config worker.backgroundclose=true --config worker.backgroundcloseminfilecount=1 clone --stream -U http://localhost:$HGPORT clone-background | grep -v adding
using http://localhost:$HGPORT/
sending capabilities command
@@ -452,6 +555,33 @@
updating the branch cache
(sent 3 HTTP requests and * bytes; received * bytes in responses) (glob)
#endif
+#if stream-bundle2-v3
+ $ hg --debug --config worker.backgroundclose=true --config worker.backgroundcloseminfilecount=1 clone --stream -U http://localhost:$HGPORT clone-background | grep -v adding
+ using http://localhost:$HGPORT/
+ sending capabilities command
+ query 1; heads
+ sending batch command
+ streaming all changes
+ sending getbundle command
+ bundle2-input-bundle: with-transaction
+ bundle2-input-part: "stream3" (params: 3 mandatory) supported
+ applying stream bundle
+ 1093 files to transfer, 102 KB of data (no-zstd !)
+ 1093 files to transfer, 98.9 KB of data (zstd !)
+ starting 4 threads for background file closing
+ starting 4 threads for background file closing
+ updating the branch cache
+ transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+ bundle2-input-part: total payload size 118984 (no-zstd !)
+ transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !)
+ bundle2-input-part: total payload size 116145 (zstd no-bigendian !)
+ bundle2-input-part: total payload size 116140 (zstd bigendian !)
+ bundle2-input-part: "listkeys" (params: 1 mandatory) supported
+ bundle2-input-bundle: 2 parts total
+ checking for updated bookmarks
+ updating the branch cache
+ (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob)
+#endif
Cannot stream clone when there are secret changesets
@@ -484,7 +614,15 @@
searching for changes
no changes found
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
+ $ hg clone --stream -U http://localhost:$HGPORT secret-allowed
+ streaming all changes
+ 1093 files to transfer, 102 KB of data (no-zstd !)
+ transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+ 1093 files to transfer, 98.9 KB of data (zstd !)
+ transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !)
+#endif
+#if stream-bundle2-v3
$ hg clone --stream -U http://localhost:$HGPORT secret-allowed
streaming all changes
1093 files to transfer, 102 KB of data (no-zstd !)
@@ -638,7 +776,17 @@
updating to branch default
1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
+ $ hg clone --stream http://localhost:$HGPORT with-bookmarks
+ streaming all changes
+ 1096 files to transfer, 102 KB of data (no-zstd !)
+ transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+ 1096 files to transfer, 99.1 KB of data (zstd !)
+ transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !)
+ updating to branch default
+ 1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
+#endif
+#if stream-bundle2-v3
$ hg clone --stream http://localhost:$HGPORT with-bookmarks
streaming all changes
1096 files to transfer, 102 KB of data (no-zstd !)
@@ -674,7 +822,17 @@
updating to branch default
1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
+ $ hg clone --stream http://localhost:$HGPORT phase-publish
+ streaming all changes
+ 1096 files to transfer, 102 KB of data (no-zstd !)
+ transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+ 1096 files to transfer, 99.1 KB of data (zstd !)
+ transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !)
+ updating to branch default
+ 1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
+#endif
+#if stream-bundle2-v3
$ hg clone --stream http://localhost:$HGPORT phase-publish
streaming all changes
1096 files to transfer, 102 KB of data (no-zstd !)
@@ -720,7 +878,21 @@
1: public
2: public
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
+ $ hg clone --stream http://localhost:$HGPORT phase-no-publish
+ streaming all changes
+ 1097 files to transfer, 102 KB of data (no-zstd !)
+ transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+ 1097 files to transfer, 99.1 KB of data (zstd !)
+ transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !)
+ updating to branch default
+ 1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ $ hg -R phase-no-publish phase -r 'all()'
+ 0: draft
+ 1: draft
+ 2: draft
+#endif
+#if stream-bundle2-v3
$ hg clone --stream http://localhost:$HGPORT phase-no-publish
streaming all changes
1097 files to transfer, 102 KB of data (no-zstd !)
@@ -744,7 +916,57 @@
no obsolescence markers exchange in stream v1.
#endif
-#if stream-bundle2
+#if stream-bundle2-v2
+
+Stream repository with obsolescence
+-----------------------------------
+
+Clone non-publishing with obsolescence
+
+ $ cat >> $HGRCPATH << EOF
+ > [experimental]
+ > evolution=all
+ > EOF
+
+ $ cd server
+ $ echo foo > foo
+ $ hg -q commit -m 'about to be pruned'
+ $ hg debugobsolete `hg log -r . -T '{node}'` -d '0 0' -u test --record-parents
+ 1 new obsolescence markers
+ obsoleted 1 changesets
+ $ hg up null -q
+ $ hg log -T '{rev}: {phase}\n'
+ 2: draft
+ 1: draft
+ 0: draft
+ $ hg serve -p $HGPORT -d --pid-file=hg.pid
+ $ cat hg.pid > $DAEMON_PIDS
+ $ cd ..
+
+ $ hg clone -U --stream http://localhost:$HGPORT with-obsolescence
+ streaming all changes
+ 1098 files to transfer, 102 KB of data (no-zstd !)
+ transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+ 1098 files to transfer, 99.5 KB of data (zstd !)
+ transferred 99.5 KB in * seconds (* */sec) (glob) (zstd !)
+ $ hg -R with-obsolescence log -T '{rev}: {phase}\n'
+ 2: draft
+ 1: draft
+ 0: draft
+ $ hg debugobsolete -R with-obsolescence
+ 8c206a663911c1f97f2f9d7382e417ae55872cfa 0 {5223b5e3265f0df40bb743da62249413d74ac70f} (Thu Jan 01 00:00:00 1970 +0000) {'user': 'test'}
+ $ hg verify -R with-obsolescence -q
+
+ $ hg clone -U --stream --config experimental.evolution=0 http://localhost:$HGPORT with-obsolescence-no-evolution
+ streaming all changes
+ remote: abort: server has obsolescence markers, but client cannot receive them via stream clone
+ abort: pull failed on remote
+ [100]
+
+ $ killdaemons.py
+
+#endif
+#if stream-bundle2-v3
Stream repository with obsolescence
-----------------------------------