This test is dedicated to test the bundle2 container format

It test multiple existing parts to test different feature of the container. You

probably do not need to touch this test unless you change the binary encoding

of the bundle2 format itself.

Create an extension to test bundle2 API

  $ cat > bundle2.py << EOF

  > """A small extension to test bundle2 implementation

  > 

  > This extension allows detailed testing of the various bundle2 API and

  > behaviors.

  > """

  > import gc

  > import os

  > import sys

  > from mercurial import util

  > from mercurial import bundle2

  > from mercurial import scmutil

  > from mercurial import discovery

  > from mercurial import changegroup

  > from mercurial import error

  > from mercurial import obsolete

  > from mercurial import pycompat

  > from mercurial import registrar

  > from mercurial.utils import procutil

  > 

  > 

  > try:

  >     import msvcrt

  >     msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)

  >     msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)

  >     msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)

  > except ImportError:

  >     pass

  > 

  > cmdtable = {}

  > command = registrar.command(cmdtable)

  > 

  > ELEPHANTSSONG = b"""Patali Dirapata, Cromda Cromda Ripalo, Pata Pata, Ko Ko Ko

  > Bokoro Dipoulito, Rondi Rondi Pepino, Pata Pata, Ko Ko Ko

  > Emana Karassoli, Loucra Loucra Ponponto, Pata Pata, Ko Ko Ko."""

  > assert len(ELEPHANTSSONG) == 178 # future test say 178 bytes, trust it.

  > 

  > @bundle2.parthandler(b'test:song')

  > def songhandler(op, part):

  >     """handle a "test:song" bundle2 part, printing the lyrics on stdin"""

  >     op.ui.write(b'The choir starts singing:\n')

  >     verses = 0

  >     for line in part.read().split(b'\n'):

  >         op.ui.write(b'    %s\n' % line)

  >         verses += 1

  >     op.records.add(b'song', {b'verses': verses})

  > 

  > @bundle2.parthandler(b'test:ping')

  > def pinghandler(op, part):

  >     op.ui.write(b'received ping request (id %i)\n' % part.id)

  >     if op.reply is not None and b'ping-pong' in op.reply.capabilities:

  >         op.ui.write_err(b'replying to ping request (id %i)\n' % part.id)

  >         op.reply.newpart(b'test:pong', [(b'in-reply-to', b'%d' % part.id)],

  >                          mandatory=False)

  > 

  > @bundle2.parthandler(b'test:debugreply')

  > def debugreply(op, part):

  >     """print data about the capacity of the bundle reply"""

  >     if op.reply is None:

  >         op.ui.write(b'debugreply: no reply\n')

  >     else:

  >         op.ui.write(b'debugreply: capabilities:\n')

  >         for cap in sorted(op.reply.capabilities):

  >             op.ui.write(b"debugreply:     '%s'\n" % cap)

  >             for val in op.reply.capabilities[cap]:

  >                 op.ui.write(b"debugreply:         '%s'\n" % val)

  > 

  > @command(b'bundle2',

  >          [(b'', b'param', [], b'stream level parameter'),

  >           (b'', b'unknown', False, b'include an unknown mandatory part in the bundle'),

  >           (b'', b'unknownparams', False, b'include an unknown part parameters in the bundle'),

  >           (b'', b'parts', False, b'include some arbitrary parts to the bundle'),

  >           (b'', b'reply', False, b'produce a reply bundle'),

  >           (b'', b'pushrace', False, b'includes a check:head part with unknown nodes'),

  >           (b'', b'genraise', False, b'includes a part that raise an exception during generation'),

  >           (b'', b'timeout', False, b'emulate a timeout during bundle generation'),

  >           (b'r', b'rev', [], b'includes those changeset in the bundle'),

  >           (b'', b'compress', b'', b'compress the stream'),

  >          ],

  >          b'[OUTPUTFILE]')

  > def cmdbundle2(ui, repo, path=None, **opts):

  >     """write a bundle2 container on standard output"""

  >     bundler = bundle2.bundle20(ui)

  >     for p in opts['param']:

  >         p = p.split(b'=', 1)

  >         try:

  >             bundler.addparam(*p)

  >         except error.ProgrammingError as exc:

  >             raise error.Abort(b'%s' % exc)

  > 

  >     if opts['compress']:

  >         bundler.setcompression(opts['compress'])

  > 

  >     if opts['reply']:

  >         capsstring = b'ping-pong\nelephants=babar,celeste\ncity%3D%21=celeste%2Cville'

  >         bundler.newpart(b'replycaps', data=capsstring)

  > 

  >     if opts['pushrace']:

  >         # also serve to test the assignement of data outside of init

  >         part = bundler.newpart(b'check:heads')

  >         part.data = b'01234567890123456789'

  > 

  >     revs = opts['rev']

  >     if 'rev' in opts:

  >         revs = scmutil.revrange(repo, opts['rev'])

  >         if revs:

  >             # very crude version of a changegroup part creation

  >             bundled = repo.revs('%ld::%ld', revs, revs)

  >             headmissing = [c.node() for c in repo.set('heads(%ld)', revs)]

  >             headcommon  = [c.node() for c in repo.set('parents(%ld) - %ld', revs, revs)]

  >             outgoing = discovery.outgoing(repo, headcommon, headmissing)

  >             cg = changegroup.makechangegroup(repo, outgoing, b'01',

  >                                              b'test:bundle2')

  >             bundler.newpart(b'changegroup', data=cg.getchunks(),

  >                             mandatory=False)

  > 

  >     if opts['parts']:

  >        bundler.newpart(b'test:empty', mandatory=False)

  >        # add a second one to make sure we handle multiple parts

  >        bundler.newpart(b'test:empty', mandatory=False)

  >        bundler.newpart(b'test:song', data=ELEPHANTSSONG, mandatory=False)

  >        bundler.newpart(b'test:debugreply', mandatory=False)

  >        mathpart = bundler.newpart(b'test:math')

  >        mathpart.addparam(b'pi', b'3.14')

  >        mathpart.addparam(b'e', b'2.72')

  >        mathpart.addparam(b'cooking', b'raw', mandatory=False)

  >        mathpart.data = b'42'

  >        mathpart.mandatory = False

  >        # advisory known part with unknown mandatory param

  >        bundler.newpart(b'test:song', [(b'randomparam', b'')], mandatory=False)

  >     if opts['unknown']:

  >        bundler.newpart(b'test:unknown', data=b'some random content')

  >     if opts['unknownparams']:

  >        bundler.newpart(b'test:song', [(b'randomparams', b'')])

  >     if opts['parts']:

  >        bundler.newpart(b'test:ping', mandatory=False)

  >     if opts['genraise']:

  >        def genraise():

  >            yield b'first line\n'

  >            raise RuntimeError('Someone set up us the bomb!')

  >        bundler.newpart(b'output', data=genraise(), mandatory=False)

  > 

  >     if path is None:

  >         file = procutil.stdout

  >     else:

  >         file = open(path, 'wb')

  > 

  >     if opts['timeout']:

  >         bundler.newpart(b'test:song', data=ELEPHANTSSONG, mandatory=False)

  >         for idx, junk in enumerate(bundler.getchunks()):

  >             ui.write(b'%d chunk\n' % idx)

  >             if idx > 4:

  >                 # This throws a GeneratorExit inside the generator, which

  >                 # can cause problems if the exception-recovery code is

  >                 # too zealous. It's important for this test that the break

  >                 # occur while we're in the middle of a part.

  >                 break

  >         gc.collect()

  >         ui.write(b'fake timeout complete.\n')

  >         return

  >     try:

  >         for chunk in bundler.getchunks():

  >             file.write(chunk)

  >     except RuntimeError as exc:

  >         raise error.Abort(pycompat.bytestr(exc))

  >     finally:

  >         file.flush()

  > 

  > @command(b'unbundle2', [], b'')

  > def cmdunbundle2(ui, repo, replypath=None):

  >     """process a bundle2 stream from stdin on the current repo"""

  >     try:

  >         tr = None

  >         lock = repo.lock()

  >         tr = repo.transaction(b'processbundle')

  >         try:

  >             unbundler = bundle2.getunbundler(ui, procutil.stdin)

  >             op = bundle2.processbundle(repo, unbundler, lambda: tr)

  >             tr.close()

  >         except error.BundleValueError as exc:

  >             raise error.Abort(b'missing support for %s' % exc)

  >         except error.PushRaced as exc:

  >             raise error.Abort(b'push race: %s' % exc)

  >     finally:

  >         if tr is not None:

  >             tr.release()

  >         lock.release()

  >         remains = procutil.stdin.read()

  >         ui.write(b'%i unread bytes\n' % len(remains))

  >     if op.records[b'song']:

  >         totalverses = sum(r[b'verses'] for r in op.records[b'song'])

  >         ui.write(b'%i total verses sung\n' % totalverses)

  >     for rec in op.records[b'changegroup']:

  >         ui.write(b'addchangegroup return: %i\n' % rec[b'return'])

  >     if op.reply is not None and replypath is not None:

  >         with open(replypath, 'wb') as file:

  >             for chunk in op.reply.getchunks():

  >                 file.write(chunk)

  > 

  > @command(b'statbundle2', [], b'')

  > def cmdstatbundle2(ui, repo):

  >     """print statistic on the bundle2 container read from stdin"""

  >     unbundler = bundle2.getunbundler(ui, procutil.stdin)

  >     try:

  >         params = unbundler.params

  >     except error.BundleValueError as exc:

  >        raise error.Abort(b'unknown parameters: %s' % exc)

  >     ui.write(b'options count: %i\n' % len(params))

  >     for key in sorted(params):

  >         ui.write(b'- %s\n' % key)

  >         value = params[key]

  >         if value is not None:

  >             ui.write(b'    %s\n' % value)

  >     count = 0

  >     for p in unbundler.iterparts():

  >         count += 1

  >         ui.write(b'  :%s:\n' % p.type)

  >         ui.write(b'    mandatory: %i\n' % len(p.mandatoryparams))

  >         ui.write(b'    advisory: %i\n' % len(p.advisoryparams))

  >         ui.write(b'    payload: %i bytes\n' % len(p.read()))

  >     ui.write(b'parts count:   %i\n' % count)

  > EOF

  $ cat >> $HGRCPATH << EOF

  > [extensions]

  > bundle2=$TESTTMP/bundle2.py

  > [experimental]

  > evolution.createmarkers=True

  > [ui]

  > ssh="$PYTHON" "$TESTDIR/dummyssh"

  > [command-templates]

  > log={rev}:{node|short} {phase} {author} {bookmarks} {desc|firstline}

  > [web]

  > push_ssl = false

  > allow_push = *

  > [phases]

  > publish=False

  > EOF

The extension requires a repo (currently unused)

  $ hg init main

  $ cd main

  $ touch a

  $ hg add a

  $ hg commit -m 'a'

Empty bundle

=================

- no option

- no parts

Test bundling

  $ hg bundle2 | f --hexdump

  0000: 48 47 32 30 00 00 00 00 00 00 00 00             |HG20........|

Test timeouts during bundling

  $ hg bundle2 --timeout --debug --config devel.bundle2.debug=yes

  bundle2-output-bundle: "HG20", 1 parts total

  bundle2-output: start emission of HG20 stream

  0 chunk

  bundle2-output: bundle parameter: 

  1 chunk

  bundle2-output: start of parts

  bundle2-output: bundle part: "test:song"

  bundle2-output-part: "test:song" (advisory) 178 bytes payload

  bundle2-output: part 0: "test:song"

  bundle2-output: header chunk size: 16

  2 chunk

  3 chunk

  bundle2-output: payload chunk size: 178

  4 chunk

  5 chunk

  bundle2-generatorexit

  fake timeout complete.

Test unbundling

  $ hg bundle2 | hg statbundle2

  options count: 0

  parts count:   0

Test old style bundle are detected and refused

  $ hg bundle --all --type v1 ../bundle.hg

  1 changesets found

  $ hg statbundle2 < ../bundle.hg

  abort: unknown bundle version 10

  [255]

Test parameters

=================

- some options

- no parts

advisory parameters, no value

-------------------------------

Simplest possible parameters form

Test generation simple option

  $ hg bundle2 --param 'caution' | f --hexdump

  0000: 48 47 32 30 00 00 00 07 63 61 75 74 69 6f 6e 00 |HG20....caution.|

  0010: 00 00 00                                        |...|

Test unbundling

  $ hg bundle2 --param 'caution' | hg statbundle2

  options count: 1

  - caution

  parts count:   0

Test generation multiple option

  $ hg bundle2 --param 'caution' --param 'meal' | f --hexdump

  0000: 48 47 32 30 00 00 00 0c 63 61 75 74 69 6f 6e 20 |HG20....caution |

  0010: 6d 65 61 6c 00 00 00 00                         |meal....|

Test unbundling

  $ hg bundle2 --param 'caution' --param 'meal' | hg statbundle2