bundle2: support for bundling and unbundling payload
authorPierre-Yves David <pierre-yves.david@fb.com>
Wed, 19 Mar 2014 23:36:15 -0700
changeset 20876 ddd56f3eb786
parent 20875 cc62c9d6887a
child 20877 9e9e3a4e9261
bundle2: support for bundling and unbundling payload We add the ability to bundle and unbundle a payload in parts. The payload is the actual binary data of the part. It is used to convey all the applicative data. For now we stick to very simple implementation with all the data fit in single chunk. This open the door to some bundle2 testing usage. This will be improved before bundle2 get used for real. We need to be able to stream the payload in multiple part to exchange any changegroup efficiently. This simple version will do for now. Bundling and unbundling are done in the same changeset because the test for parts is less modular. However, the result is not too complex.
mercurial/bundle2.py
tests/test-bundle2.t
--- a/mercurial/bundle2.py	Tue Apr 01 17:59:06 2014 -0500
+++ b/mercurial/bundle2.py	Wed Mar 19 23:36:15 2014 -0700
@@ -89,8 +89,13 @@
 
 :payload:
 
-    The current payload is a 32bit integer with a value of 0. This is
-    considered an "empty" payload.
+    payload is a series of `<chunksize><chunkdata>`.
+
+    `chunksize` is a 32 bits integer, `chunkdata` are plain bytes (as much as
+    `chunksize` says)` The payload part is concluded by a zero size chunk.
+
+    The current implementation always produces either zero or one chunk.
+    This is an implementation limitation that will ultimatly be lifted.
 """
 
 import util
@@ -109,6 +114,7 @@
 _fstreamparamsize = '>H'
 _fpartheadersize = '>H'
 _fparttypesize = '>B'
+_fpayloadsize = '>I'
 
 class bundle20(object):
     """represent an outgoing bundle2 container
@@ -257,12 +263,18 @@
         typesize = _unpack(_fparttypesize, fromheader(1))[0]
         parttype = fromheader(typesize)
         self.ui.debug('part type: "%s"\n' % parttype)
-        current = part(parttype)
         assert fromheader(2) == '\0\0' # no option for now
         del self._offset # clean up layer, nobody saw anything.
         self.ui.debug('part parameters: 0\n')
-        assert self._readexact(4) == '\0\0\0\0' #empty payload
-        self.ui.debug('payload chunk size: 0\n')
+        payload = []
+        payloadsize = self._unpack(_fpayloadsize)[0]
+        self.ui.debug('payload chunk size: %i\n' % payloadsize)
+        while payloadsize:
+            payload.append(self._readexact(payloadsize))
+            payloadsize = self._unpack(_fpayloadsize)[0]
+            self.ui.debug('payload chunk size: %i\n' % payloadsize)
+        payload = ''.join(payload)
+        current = part(parttype, data=payload)
         return current
 
 
@@ -286,7 +298,12 @@
         headerchunk = ''.join(header)
         yield _pack(_fpartheadersize, len(headerchunk))
         yield headerchunk
-        # force empty part for now
-        yield '\0\0\0\0'
+        # we only support fixed size data now.
+        # This will be improved in the future.
+        if len(self.data):
+            yield _pack(_fpayloadsize, len(self.data))
+            yield self.data
+        # end of payload
+        yield _pack(_fpayloadsize, 0)
 
 
--- a/tests/test-bundle2.t	Tue Apr 01 17:59:06 2014 -0500
+++ b/tests/test-bundle2.t	Wed Mar 19 23:36:15 2014 -0700
@@ -15,6 +15,11 @@
   > cmdtable = {}
   > command = cmdutil.command(cmdtable)
   > 
+  > ELEPHANTSSONG = """Patali Dirapata, Cromda Cromda Ripalo, Pata Pata, Ko Ko Ko
+  > Bokoro Dipoulito, Rondi Rondi Pepino, Pata Pata, Ko Ko Ko
+  > Emana Karassoli, Loucra Loucra Ponponto, Pata Pata, Ko Ko Ko."""
+  > assert len(ELEPHANTSSONG) == 178 # future test say 178 bytes, trust it.
+  > 
   > @command('bundle2',
   >          [('', 'param', [], 'stream level parameter'),
   >           ('', 'parts', False, 'include some arbitrary parts to the bundle'),],
@@ -35,6 +40,8 @@
   >        # add a second one to make sure we handle multiple parts
   >        part = bundle2.part('test:empty')
   >        bundler.addpart(part)
+  >        part = bundle2.part('test:song', data=ELEPHANTSSONG)
+  >        bundler.addpart(part)
   > 
   >     if path is None:
   >        file = sys.stdout
@@ -62,6 +69,7 @@
   >     ui.write('parts count:   %i\n' % len(parts))
   >     for p in parts:
   >         ui.write('  :%s:\n' % p.type)
+  >         ui.write('    payload: %i bytes\n' % len(p.data))
   > EOF
   $ cat >> $HGRCPATH << EOF
   > [extensions]
@@ -238,19 +246,26 @@
   start of parts
   bundle part: "test:empty"
   bundle part: "test:empty"
+  bundle part: "test:song"
   end of bundle
 
   $ cat ../parts.hg2
   HG20\x00\x00\x00\r (esc)
   test:empty\x00\x00\x00\x00\x00\x00\x00\r (esc)
-  test:empty\x00\x00\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
+  test:empty\x00\x00\x00\x00\x00\x00\x00\x0c	test:song\x00\x00\x00\x00\x00\xb2Patali Dirapata, Cromda Cromda Ripalo, Pata Pata, Ko Ko Ko (esc)
+  Bokoro Dipoulito, Rondi Rondi Pepino, Pata Pata, Ko Ko Ko
+  Emana Karassoli, Loucra Loucra Ponponto, Pata Pata, Ko Ko Ko.\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
 
 
   $ hg unbundle2 < ../parts.hg2
   options count: 0
-  parts count:   2
+  parts count:   3
+    :test:empty:
+      payload: 0 bytes
     :test:empty:
-    :test:empty:
+      payload: 0 bytes
+    :test:song:
+      payload: 178 bytes
 
   $ hg unbundle2 --debug < ../parts.hg2
   start processing of HG20 stream
@@ -265,8 +280,17 @@
   part type: "test:empty"
   part parameters: 0
   payload chunk size: 0
+  part header size: 12
+  part type: "test:song"
+  part parameters: 0
+  payload chunk size: 178
+  payload chunk size: 0
   part header size: 0
   end of bundle2 stream
-  parts count:   2
+  parts count:   3
+    :test:empty:
+      payload: 0 bytes
     :test:empty:
-    :test:empty:
+      payload: 0 bytes
+    :test:song:
+      payload: 178 bytes