changegroup: add flags field to cg3 delta header
authorMike Edgar <adgar@google.com>
Mon, 14 Dec 2015 15:55:12 -0500
changeset 27433 12f727a5b434
parent 27432 77d25b913f80
child 27434 11150176a000
changegroup: add flags field to cg3 delta header This lets revlog flags be transmitted over the wire. Right now this is useful for censored nodes and for narrowhg's ellipsis nodes.
mercurial/changegroup.py
mercurial/revlog.py
mercurial/unionrepo.py
tests/test-acl.t
tests/test-largefiles.t
tests/test-obsolete-changeset-exchange.t
tests/test-phases-exchange.t
tests/test-push-warn.t
--- a/mercurial/changegroup.py	Fri Dec 11 11:23:49 2015 -0500
+++ b/mercurial/changegroup.py	Mon Dec 14 15:55:12 2015 -0500
@@ -32,6 +32,7 @@
 
 _CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s"
 _CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s"
+_CHANGEGROUPV3_DELTA_HEADER = ">20s20s20s20s20sH"
 
 def readexactly(stream, n):
     '''read n bytes from stream.read and abort if less was available'''
@@ -246,7 +247,8 @@
             deltabase = p1
         else:
             deltabase = prevnode
-        return node, p1, p2, deltabase, cs
+        flags = 0
+        return node, p1, p2, deltabase, cs, flags
 
     def deltachunk(self, prevnode):
         l = self._chunklength()
@@ -255,9 +257,9 @@
         headerdata = readexactly(self._stream, self.deltaheadersize)
         header = struct.unpack(self.deltaheader, headerdata)
         delta = readexactly(self._stream, l - self.deltaheadersize)
-        node, p1, p2, deltabase, cs = self._deltaheader(header, prevnode)
+        node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
         return {'node': node, 'p1': p1, 'p2': p2, 'cs': cs,
-                'deltabase': deltabase, 'delta': delta}
+                'deltabase': deltabase, 'delta': delta, 'flags': flags}
 
     def getchunks(self):
         """returns all the chunks contains in the bundle
@@ -296,6 +298,7 @@
         # no new manifest will be created and the manifest group will
         # be empty during the pull
         self.manifestheader()
+        repo.manifest.narrowdebug = repo.ui.warn
         repo.manifest.addgroup(self, revmap, trp)
         repo.ui.progress(_('manifests'), None)
 
@@ -495,16 +498,24 @@
 
     def _deltaheader(self, headertuple, prevnode):
         node, p1, p2, deltabase, cs = headertuple
-        return node, p1, p2, deltabase, cs
+        flags = 0
+        return node, p1, p2, deltabase, cs, flags
 
 class cg3unpacker(cg2unpacker):
     """Unpacker for cg3 streams.
 
-    cg3 streams add support for exchanging treemanifests, so the only
-    thing that changes is the version number.
+    cg3 streams add support for exchanging treemanifests and revlog
+    flags, so the only changes from cg2 are the delta header and
+    version number.
     """
+    deltaheader = _CHANGEGROUPV3_DELTA_HEADER
+    deltaheadersize = struct.calcsize(deltaheader)
     version = '03'
 
+    def _deltaheader(self, headertuple, prevnode):
+        node, p1, p2, deltabase, cs, flags = headertuple
+        return node, p1, p2, deltabase, cs, flags
+
 class headerlessfixup(object):
     def __init__(self, fh, h):
         self._h = h
@@ -841,14 +852,16 @@
             delta = revlog.revdiff(base, rev)
         p1n, p2n = revlog.parents(node)
         basenode = revlog.node(base)
-        meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode)
+        flags = revlog.flags(rev)
+        meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags)
         meta += prefix
         l = len(meta) + len(delta)
         yield chunkheader(l)
         yield meta
         yield delta
-    def builddeltaheader(self, node, p1n, p2n, basenode, linknode):
+    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
         # do nothing with basenode, it is implicitly the previous one in HG10
+        # do nothing with flags, it is implicitly 0 for cg1 and cg2
         return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
 
 class cg2packer(cg1packer):
@@ -871,11 +884,13 @@
             return prev
         return dp
 
-    def builddeltaheader(self, node, p1n, p2n, basenode, linknode):
+    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
+        # Do nothing with flags, it is implicitly 0 in cg1 and cg2
         return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode)
 
 class cg3packer(cg2packer):
     version = '03'
+    deltaheader = _CHANGEGROUPV3_DELTA_HEADER
 
     def _packmanifests(self, mfnodes, tmfnodes, lookuplinknode):
         # Note that debug prints are super confusing in this code, as
@@ -894,6 +909,9 @@
             for chunk in self.group(nodes, dirlog(name), nodes.get):
                 yield chunk
 
+    def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
+        return struct.pack(
+            self.deltaheader, node, p1n, p2n, basenode, linknode, flags)
 
 packermap = {'01': (cg1packer, cg1unpacker),
              # cg2 adds support for exchanging generaldelta
--- a/mercurial/revlog.py	Fri Dec 11 11:23:49 2015 -0500
+++ b/mercurial/revlog.py	Mon Dec 14 15:55:12 2015 -0500
@@ -1572,6 +1572,7 @@
                 cs = chunkdata['cs']
                 deltabase = chunkdata['deltabase']
                 delta = chunkdata['delta']
+                flags = chunkdata['flags'] or REVIDX_DEFAULT_FLAGS
 
                 content.append(node)
 
@@ -1602,8 +1603,7 @@
                         raise error.CensoredBaseError(self.indexfile,
                                                       self.node(baserev))
 
-                flags = REVIDX_DEFAULT_FLAGS
-                if self._peek_iscensored(baserev, delta, flush):
+                if not flags and self._peek_iscensored(baserev, delta, flush):
                     flags |= REVIDX_ISCENSORED
 
                 # We assume consumers of addrevisioncb will want to retrieve
--- a/mercurial/unionrepo.py	Fri Dec 11 11:23:49 2015 -0500
+++ b/mercurial/unionrepo.py	Mon Dec 14 15:55:12 2015 -0500
@@ -51,6 +51,7 @@
             rev = self.revlog2.index[rev2]
             # rev numbers - in revlog2, very different from self.rev
             _start, _csize, _rsize, base, linkrev, p1rev, p2rev, node = rev
+            flags = _start & 0xFFFF
 
             if linkmapper is None: # link is to same revlog
                 assert linkrev == rev2 # we never link back
@@ -69,7 +70,7 @@
             p1node = self.revlog2.node(p1rev)
             p2node = self.revlog2.node(p2rev)
 
-            e = (None, None, None, base,
+            e = (flags, None, None, base,
                  link, self.rev(p1node), self.rev(p2node), node)
             self.index.insert(-1, e)
             self.nodemap[node] = n
--- a/tests/test-acl.t	Fri Dec 11 11:23:49 2015 -0500
+++ b/tests/test-acl.t	Mon Dec 14 15:55:12 2015 -0500
@@ -119,7 +119,7 @@
   adding foo/file.txt revisions
   adding quux/file.py revisions
   added 3 changesets with 3 changes to 3 files
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -184,7 +184,7 @@
   added 3 changesets with 3 changes to 3 files
   calling hook pretxnchangegroup.acl: hgext.acl.hook
   acl: changes have source "push" - skipping
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -260,7 +260,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -332,7 +332,7 @@
   acl: acl.deny not enabled
   acl: branch access granted: "ef1ea85a6374" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" not allowed on "foo/file.txt" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -401,7 +401,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" not allowed on "quux/file.py" (changeset "911600dab2ae")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -467,7 +467,7 @@
   acl: acl.deny enabled, 0 entries for user barney
   acl: branch access granted: "ef1ea85a6374" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "barney" not allowed on "foo/file.txt" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -538,7 +538,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" not allowed on "quux/file.py" (changeset "911600dab2ae")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -608,7 +608,7 @@
   acl: path access granted: "ef1ea85a6374"
   acl: branch access granted: "f9cafe1212c8" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" denied on "foo/Bar/file.txt" (changeset "f9cafe1212c8")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -675,7 +675,7 @@
   acl: acl.deny enabled, 0 entries for user barney
   acl: branch access granted: "ef1ea85a6374" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "barney" not allowed on "foo/file.txt" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -750,7 +750,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -833,7 +833,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "wilma" not allowed on "quux/file.py" (changeset "911600dab2ae")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -902,7 +902,7 @@
   calling hook pretxnchangegroup.acl: hgext.acl.hook
   acl: checking access for user "barney"
   error: pretxnchangegroup.acl hook raised an exception: [Errno 2] No such file or directory: '../acl.config'
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -984,7 +984,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "betty" not allowed on "quux/file.py" (changeset "911600dab2ae")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -1068,7 +1068,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -1154,7 +1154,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -1234,7 +1234,7 @@
   acl: path access granted: "ef1ea85a6374"
   acl: branch access granted: "f9cafe1212c8" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" denied on "foo/Bar/file.txt" (changeset "f9cafe1212c8")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -1313,7 +1313,7 @@
   acl: path access granted: "f9cafe1212c8"
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-bundle: 3 parts total
@@ -1395,7 +1395,7 @@
   acl: path access granted: "ef1ea85a6374"
   acl: branch access granted: "f9cafe1212c8" on branch "default"
   error: pretxnchangegroup.acl hook failed: acl: user "fred" denied on "foo/Bar/file.txt" (changeset "f9cafe1212c8")
-  bundle2-input-part: total payload size 1606
+  bundle2-input-part: total payload size 1624
   bundle2-input-bundle: 3 parts total
   transaction abort!
   rollback completed
@@ -1515,7 +1515,7 @@
   acl: path access granted: "911600dab2ae"
   acl: branch access granted: "e8fc755d4d82" on branch "foobar"
   acl: path access granted: "e8fc755d4d82"
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
@@ -1602,7 +1602,7 @@
   acl: branch access granted: "911600dab2ae" on branch "default"
   acl: path access granted: "911600dab2ae"
   error: pretxnchangegroup.acl hook failed: acl: user "astro" denied on branch "foobar" (changeset "e8fc755d4d82")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
@@ -1670,7 +1670,7 @@
   acl: acl.allow not enabled
   acl: acl.deny not enabled
   error: pretxnchangegroup.acl hook failed: acl: user "astro" not allowed on branch "default" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
@@ -1740,7 +1740,7 @@
   acl: acl.allow not enabled
   acl: acl.deny not enabled
   error: pretxnchangegroup.acl hook failed: acl: user "astro" not allowed on branch "default" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
@@ -1811,7 +1811,7 @@
   acl: path access granted: "911600dab2ae"
   acl: branch access granted: "e8fc755d4d82" on branch "foobar"
   acl: path access granted: "e8fc755d4d82"
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
@@ -1904,7 +1904,7 @@
   acl: path access granted: "911600dab2ae"
   acl: branch access granted: "e8fc755d4d82" on branch "foobar"
   acl: path access granted: "e8fc755d4d82"
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
@@ -1989,7 +1989,7 @@
   acl: acl.allow not enabled
   acl: acl.deny not enabled
   error: pretxnchangegroup.acl hook failed: acl: user "george" denied on branch "default" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
@@ -2065,7 +2065,7 @@
   acl: path access granted: "911600dab2ae"
   acl: branch access granted: "e8fc755d4d82" on branch "foobar"
   acl: path access granted: "e8fc755d4d82"
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
   pushing key for "phases:911600dab2ae7a9baff75958b84fe606851ce955"
   bundle2-input-part: "pushkey" (params: 4 mandatory) supported
@@ -2144,7 +2144,7 @@
   acl: acl.allow not enabled
   acl: acl.deny not enabled
   error: pretxnchangegroup.acl hook failed: acl: user "george" denied on branch "default" (changeset "ef1ea85a6374")
-  bundle2-input-part: total payload size 2101
+  bundle2-input-part: total payload size 2125
   bundle2-input-bundle: 4 parts total
   transaction abort!
   rollback completed
--- a/tests/test-largefiles.t	Fri Dec 11 11:23:49 2015 -0500
+++ b/tests/test-largefiles.t	Mon Dec 14 15:55:12 2015 -0500
@@ -1103,16 +1103,16 @@
   all local heads known remotely
   6 changesets found
   uncompressed size of bundle content:
-      1333 (changelog)
-      1599 (manifests)
-       254  .hglf/large1
-       564  .hglf/large3
-       572  .hglf/sub/large4
-       182  .hglf/sub2/large6
-       182  .hglf/sub2/large7
-       212  normal1
-       457  normal3
-       465  sub/normal4
+      1345 (changelog)
+      1611 (manifests)
+       256  .hglf/large1
+       570  .hglf/large3
+       578  .hglf/sub/large4
+       184  .hglf/sub2/large6
+       184  .hglf/sub2/large7
+       214  normal1
+       463  normal3
+       471  sub/normal4
   adding changesets
   adding manifests
   adding file changes
--- a/tests/test-obsolete-changeset-exchange.t	Fri Dec 11 11:23:49 2015 -0500
+++ b/tests/test-obsolete-changeset-exchange.t	Mon Dec 14 15:55:12 2015 -0500
@@ -144,7 +144,7 @@
   adding file changes
   adding foo revisions
   added 1 changesets with 1 changes to 1 files (+1 heads)
-  bundle2-input-part: total payload size 474
+  bundle2-input-part: total payload size 480
   bundle2-input-part: "listkeys" (params: 1 mandatory) supported
   bundle2-input-part: "listkeys" (params: 1 mandatory) supported
   bundle2-input-bundle: 2 parts total
--- a/tests/test-phases-exchange.t	Fri Dec 11 11:23:49 2015 -0500
+++ b/tests/test-phases-exchange.t	Mon Dec 14 15:55:12 2015 -0500
@@ -772,9 +772,9 @@
   searching for changes
   1 changesets found
   uncompressed size of bundle content:
-       192 (changelog)
-       165 (manifests)
-       131  a-H
+       194 (changelog)
+       167 (manifests)
+       133  a-H
   adding changesets
   adding manifests
   adding file changes
--- a/tests/test-push-warn.t	Fri Dec 11 11:23:49 2015 -0500
+++ b/tests/test-push-warn.t	Mon Dec 14 15:55:12 2015 -0500
@@ -156,9 +156,9 @@
   searching for changes
   2 changesets found
   uncompressed size of bundle content:
-       348 (changelog)
-       326 (manifests)
-       253  foo
+       352 (changelog)
+       330 (manifests)
+       257  foo
   adding changesets
   adding manifests
   adding file changes