wireproto: advertise supported media types and compression formats
authorGregory Szorc <gregory.szorc@gmail.com>
Sat, 24 Dec 2016 15:21:46 -0700
changeset 30762 35b516f800e0
parent 30761 7283719e2bfd
child 30763 a520aefb96f1
wireproto: advertise supported media types and compression formats This commit introduces support for advertising a server's support for media types and compression formats in accordance with the spec defined in internals.wireproto. The bulk of the new code is a helper function in wireproto.py to obtain a prioritized list of compression engines available to the wire protocol. While not utilized yet, we implement support for obtaining the list of compression engines advertised by the client. The upcoming HTTP protocol enhancements are a bit lower-level than existing tests (most existing tests are command centric). So, this commit establishes a new test file that will be appropriate for holding tests around the functionality of the HTTP protocol itself. Rounding out this change, `hg debuginstall` now prints compression engines available to the server.
mercurial/commands.py
mercurial/help/config.txt
mercurial/wireproto.py
tests/test-hgweb-commands.t
tests/test-http-protocol.t
tests/test-install.t
--- a/mercurial/commands.py	Sat Dec 24 13:51:12 2016 -0700
+++ b/mercurial/commands.py	Sat Dec 24 15:21:46 2016 -0700
@@ -1959,6 +1959,12 @@
              fm.formatlist(sorted(e.name() for e in compengines
                                   if e.available()),
                            name='compengine', fmt='%s', sep=', '))
+    wirecompengines = util.compengines.supportedwireengines(util.SERVERROLE)
+    fm.write('compenginesserver', _('checking available compression engines '
+                                    'for wire protocol (%s)\n'),
+             fm.formatlist([e.name() for e in wirecompengines
+                            if e.wireprotosupport()],
+                           name='compengine', fmt='%s', sep=', '))
 
     # templates
     p = templater.templatepaths()
--- a/mercurial/help/config.txt	Sat Dec 24 13:51:12 2016 -0700
+++ b/mercurial/help/config.txt	Sat Dec 24 15:21:46 2016 -0700
@@ -1545,6 +1545,21 @@
 
 Controls generic server settings.
 
+``compressionengines``
+    List of compression engines and their relative priority to advertise
+    to clients.
+
+    The order of compression engines determines their priority, the first
+    having the highest priority. If a compression engine is not listed
+    here, it won't be advertised to clients.
+
+    If not set (the default), built-in defaults are used. Run
+    :hg:`debuginstall` to list available compression engines and their
+    default wire protocol priority.
+
+    Older Mercurial clients only support zlib compression and this setting
+    has no effect for legacy clients.
+
 ``uncompressed``
     Whether to allow clients to clone a repository using the
     uncompressed streaming protocol. This transfers about 40% more
--- a/mercurial/wireproto.py	Sat Dec 24 13:51:12 2016 -0700
+++ b/mercurial/wireproto.py	Sat Dec 24 15:21:46 2016 -0700
@@ -607,6 +607,55 @@
 
     return ui.configbool('server', 'bundle1', True)
 
+def supportedcompengines(ui, proto, role):
+    """Obtain the list of supported compression engines for a request."""
+    assert role in (util.CLIENTROLE, util.SERVERROLE)
+
+    compengines = util.compengines.supportedwireengines(role)
+
+    # Allow config to override default list and ordering.
+    if role == util.SERVERROLE:
+        configengines = ui.configlist('server', 'compressionengines')
+        config = 'server.compressionengines'
+    else:
+        # This is currently implemented mainly to facilitate testing. In most
+        # cases, the server should be in charge of choosing a compression engine
+        # because a server has the most to lose from a sub-optimal choice. (e.g.
+        # CPU DoS due to an expensive engine or a network DoS due to poor
+        # compression ratio).
+        configengines = ui.configlist('experimental',
+                                      'clientcompressionengines')
+        config = 'experimental.clientcompressionengines'
+
+    # No explicit config. Filter out the ones that aren't supposed to be
+    # advertised and return default ordering.
+    if not configengines:
+        attr = 'serverpriority' if role == util.SERVERROLE else 'clientpriority'
+        return [e for e in compengines
+                if getattr(e.wireprotosupport(), attr) > 0]
+
+    # If compression engines are listed in the config, assume there is a good
+    # reason for it (like server operators wanting to achieve specific
+    # performance characteristics). So fail fast if the config references
+    # unusable compression engines.
+    validnames = set(e.name() for e in compengines)
+    invalidnames = set(e for e in configengines if e not in validnames)
+    if invalidnames:
+        raise error.Abort(_('invalid compression engine defined in %s: %s') %
+                          (config, ', '.join(sorted(invalidnames))))
+
+    compengines = [e for e in compengines if e.name() in configengines]
+    compengines = sorted(compengines,
+                         key=lambda e: configengines.index(e.name()))
+
+    if not compengines:
+        raise error.Abort(_('%s config option does not specify any known '
+                            'compression engines') % config,
+                          hint=_('usable compression engines: %s') %
+                          ', '.sorted(validnames))
+
+    return compengines
+
 # list of commands
 commands = {}
 
@@ -723,6 +772,16 @@
         if repo.ui.configbool('experimental', 'httppostargs', False):
             caps.append('httppostargs')
 
+        # FUTURE advertise 0.2rx once support is implemented
+        # FUTURE advertise minrx and mintx after consulting config option
+        caps.append('httpmediatype=0.1rx,0.1tx,0.2tx')
+
+        compengines = supportedcompengines(repo.ui, proto, util.SERVERROLE)
+        if compengines:
+            comptypes = ','.join(urlreq.quote(e.wireprotosupport().name)
+                                 for e in compengines)
+            caps.append('compression=%s' % comptypes)
+
     return caps
 
 # If you are writing an extension and consider wrapping this function. Wrap
--- a/tests/test-hgweb-commands.t	Sat Dec 24 13:51:12 2016 -0700
+++ b/tests/test-hgweb-commands.t	Sat Dec 24 15:21:46 2016 -0700
@@ -1903,7 +1903,7 @@
   $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities'; echo
   200 Script output follows
   
-  lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024
+  lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024 httpmediatype=0.1rx,0.1tx,0.2tx compression=*zlib (glob)
 
 heads
 
@@ -2154,6 +2154,8 @@
   bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps
   unbundle=HG10GZ,HG10BZ,HG10UN
   httpheader=1024
+  httpmediatype=0.1rx,0.1tx,0.2tx
+  compression=*zlib (glob)
 
 heads
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-http-protocol.t	Sat Dec 24 15:21:46 2016 -0700
@@ -0,0 +1,44 @@
+  $ cat >> $HGRCPATH << EOF
+  > [web]
+  > push_ssl = false
+  > allow_push = *
+  > EOF
+
+  $ hg init server
+  $ cd server
+  $ touch a
+  $ hg -q commit -A -m initial
+  $ cd ..
+
+  $ hg -R server serve -p $HGPORT -d --pid-file hg.pid
+  $ cat hg.pid >> $DAEMON_PIDS
+
+compression formats are advertised in compression capability
+
+#if zstd
+  $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep compression
+  compression=zstd,zlib
+#else
+  $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep compression
+  compression=ZL
+#endif
+
+  $ killdaemons.py
+
+server.compressionengines can replace engines list wholesale
+
+  $ hg --config server.compressionengines=none -R server serve -p $HGPORT -d --pid-file hg.pid
+  $ cat hg.pid > $DAEMON_PIDS
+  $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep compression
+  compression=none
+
+  $ killdaemons.py
+
+Order of engines can also change
+
+  $ hg --config server.compressionengines=none,zlib -R server serve -p $HGPORT -d --pid-file hg.pid
+  $ cat hg.pid > $DAEMON_PIDS
+  $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep compression
+  compression=none,zlib
+
+  $ killdaemons.py
--- a/tests/test-install.t	Sat Dec 24 13:51:12 2016 -0700
+++ b/tests/test-install.t	Sat Dec 24 15:21:46 2016 -0700
@@ -13,6 +13,7 @@
   checking installed modules (*mercurial)... (glob)
   checking registered compression engines (*zlib*) (glob)
   checking available compression engines (*zlib*) (glob)
+  checking available compression engines for wire protocol (*zlib*) (glob)
   checking templates (*mercurial?templates)... (glob)
   checking default template (*mercurial?templates?map-cmdline.default) (glob)
   checking commit editor... (* -c "import sys; sys.exit(0)") (glob)
@@ -25,6 +26,7 @@
    {
     "compengines": ["bz2", "bz2truncated", "none", "zlib"*], (glob)
     "compenginesavail": ["bz2", "bz2truncated", "none", "zlib"*], (glob)
+    "compenginesserver": [*"zlib"*], (glob)
     "defaulttemplate": "*mercurial?templates?map-cmdline.default", (glob)
     "defaulttemplateerror": null,
     "defaulttemplatenotfound": "default",
@@ -64,6 +66,7 @@
   checking installed modules (*mercurial)... (glob)
   checking registered compression engines (*zlib*) (glob)
   checking available compression engines (*zlib*) (glob)
+  checking available compression engines for wire protocol (*zlib*) (glob)
   checking templates (*mercurial?templates)... (glob)
   checking default template (*mercurial?templates?map-cmdline.default) (glob)
   checking commit editor... (* -c "import sys; sys.exit(0)") (glob)
@@ -93,6 +96,7 @@
   checking installed modules (*mercurial)... (glob)
   checking registered compression engines (*zlib*) (glob)
   checking available compression engines (*zlib*) (glob)
+  checking available compression engines for wire protocol (*zlib*) (glob)
   checking templates (*mercurial?templates)... (glob)
   checking default template (*mercurial?templates?map-cmdline.default) (glob)
   checking commit editor... (* -c "import sys; sys.exit(0)") (glob)