sidedata-exchange: add `wanted_sidedata` and `sidedata_computers` to repos
authorRaphaël Gomès <rgomes@octobus.net>
Fri, 19 Feb 2021 10:53:27 +0100
changeset 46713 bc2519513ae0
parent 46712 e8c11a2c96c0
child 46714 f63299ee7e4d
sidedata-exchange: add `wanted_sidedata` and `sidedata_computers` to repos Each repo will advertise the sidedata categories it requires (categories being unique and canonical), and have a set of "computers", functions to generate sidedata from `(repo, revlog, rev, previous_sidedata)`, for a given category. The set of computers can be a superset of the set of the wanted categories, but not smaller: repos are expected to be coherent in their handling of sidedata. Differential Revision: https://phab.mercurial-scm.org/D10028
mercurial/bundle2.py
mercurial/changegroup.py
mercurial/exchange.py
mercurial/interfaces/repository.py
mercurial/localrepo.py
mercurial/metadata.py
mercurial/statichttprepo.py
tests/test-check-interfaces.py
--- a/mercurial/bundle2.py	Thu Feb 18 18:18:35 2021 +0100
+++ b/mercurial/bundle2.py	Fri Feb 19 10:53:27 2021 +0100
@@ -1812,6 +1812,28 @@
     return params
 
 
+def format_remote_wanted_sidedata(repo):
+    """Formats a repo's wanted sidedata categories into a bytestring for
+    capabilities exchange."""
+    wanted = b""
+    if repo._wanted_sidedata:
+        wanted = b','.join(
+            pycompat.bytestr(c) for c in sorted(repo._wanted_sidedata)
+        )
+    return wanted
+
+
+def read_remote_wanted_sidedata(remote):
+    sidedata_categories = remote.capable(b'exp-wanted-sidedata')
+    return read_wanted_sidedata(sidedata_categories)
+
+
+def read_wanted_sidedata(formatted):
+    if formatted:
+        return set(formatted.split(b','))
+    return set()
+
+
 def addpartbundlestream2(bundler, repo, **kwargs):
     if not kwargs.get('stream', False):
         return
@@ -1957,6 +1979,7 @@
         b'version',
         b'nbchanges',
         b'exp-sidedata',
+        b'exp-wanted-sidedata',
         b'treemanifest',
         b'targetphase',
     ),
@@ -1999,6 +2022,10 @@
     targetphase = inpart.params.get(b'targetphase')
     if targetphase is not None:
         extrakwargs['targetphase'] = int(targetphase)
+
+    remote_sidedata = inpart.params.get(b'exp-wanted-sidedata')
+    extrakwargs['sidedata_categories'] = read_wanted_sidedata(remote_sidedata)
+
     ret = _processchangegroup(
         op,
         cg,
@@ -2559,5 +2586,7 @@
             part.addparam(b'treemanifest', b'1')
         if b'exp-sidedata-flag' in repo.requirements:
             part.addparam(b'exp-sidedata', b'1')
+            wanted = format_remote_wanted_sidedata(repo)
+            part.addparam(b'exp-wanted-sidedata', wanted)
 
     return bundler
--- a/mercurial/changegroup.py	Thu Feb 18 18:18:35 2021 +0100
+++ b/mercurial/changegroup.py	Fri Feb 19 10:53:27 2021 +0100
@@ -945,6 +945,9 @@
         if bundlecaps is None:
             bundlecaps = set()
         self._bundlecaps = bundlecaps
+        if remote_sidedata is None:
+            remote_sidedata = set()
+        self._remote_sidedata = remote_sidedata
         self._isshallow = shallow
         self._fullclnodes = fullnodes
 
--- a/mercurial/exchange.py	Thu Feb 18 18:18:35 2021 +0100
+++ b/mercurial/exchange.py	Fri Feb 19 10:53:27 2021 +0100
@@ -420,7 +420,20 @@
                 b'unbundle wire protocol command'
             )
         )
-
+    for category in sorted(bundle2.read_remote_wanted_sidedata(pushop.remote)):
+        # Check that a computer is registered for that category for at least
+        # one revlog kind.
+        for kind, computers in repo._sidedata_computers.items():
+            if computers.get(category):
+                break
+        else:
+            raise error.Abort(
+                _(
+                    b'cannot push: required sidedata category not supported'
+                    b" by this client: '%s'"
+                )
+                % pycompat.bytestr(category)
+            )
     # get lock as we might write phase data
     wlock = lock = None
     try:
@@ -865,8 +878,15 @@
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
+
+    remote_sidedata = bundle2.read_remote_wanted_sidedata(pushop.remote)
     cgstream = changegroup.makestream(
-        pushop.repo, pushop.outgoing, version, b'push'
+        pushop.repo,
+        pushop.outgoing,
+        version,
+        b'push',
+        bundlecaps=b2caps,
+        remote_sidedata=remote_sidedata,
     )
     cgpart = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
@@ -1607,6 +1627,23 @@
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
+    for category in repo._wanted_sidedata:
+        # Check that a computer is registered for that category for at least
+        # one revlog kind.
+        for kind, computers in repo._sidedata_computers.items():
+            if computers.get(category):
+                break
+        else:
+            # This should never happen since repos are supposed to be able to
+            # generate the sidedata they require.
+            raise error.ProgrammingError(
+                _(
+                    b'sidedata category requested by local side without local'
+                    b"support: '%s'"
+                )
+                % pycompat.bytestr(category)
+            )
+
     pullop.trmanager = transactionmanager(repo, b'pull', remote.url())
     wlock = util.nullcontextmanager()
     if not bookmod.bookmarksinstore(repo):
@@ -1820,6 +1857,10 @@
             pullop.stepsdone.add(b'obsmarkers')
     _pullbundle2extraprepare(pullop, kwargs)
 
+    remote_sidedata = bundle2.read_remote_wanted_sidedata(pullop.remote)
+    if remote_sidedata:
+        kwargs[b'remote_sidedata'] = remote_sidedata
+
     with pullop.remote.commandexecutor() as e:
         args = dict(kwargs)
         args[b'source'] = b'pull'
@@ -2388,6 +2429,8 @@
 
     if b'exp-sidedata-flag' in repo.requirements:
         part.addparam(b'exp-sidedata', b'1')
+        sidedata = bundle2.format_remote_wanted_sidedata(repo)
+        part.addparam(b'exp-wanted-sidedata', sidedata)
 
     if (
         kwargs.get('narrow', False)
--- a/mercurial/interfaces/repository.py	Thu Feb 18 18:18:35 2021 +0100
+++ b/mercurial/interfaces/repository.py	Fri Feb 19 10:53:27 2021 +0100
@@ -1832,6 +1832,12 @@
     def savecommitmessage(text):
         pass
 
+    def register_sidedata_computer(kind, category, keys, computer):
+        pass
+
+    def register_wanted_sidedata(category):
+        pass
+
 
 class completelocalrepository(
     ilocalrepositorymain, ilocalrepositoryfilestorage
--- a/mercurial/localrepo.py	Thu Feb 18 18:18:35 2021 +0100
+++ b/mercurial/localrepo.py	Fri Feb 19 10:53:27 2021 +0100
@@ -49,6 +49,7 @@
     match as matchmod,
     mergestate as mergestatemod,
     mergeutil,
+    metadata as metadatamod,
     namespaces,
     narrowspec,
     obsolete,
@@ -273,6 +274,11 @@
             caps = moderncaps.copy()
         self._repo = repo.filtered(b'served')
         self.ui = repo.ui
+
+        if repo._wanted_sidedata:
+            formatted = bundle2.format_remote_wanted_sidedata(repo)
+            caps.add(b'exp-wanted-sidedata=' + formatted)
+
         self._caps = repo._restrictcapabilities(caps)
 
     # Begin of _basepeer interface.
@@ -1395,6 +1401,10 @@
         if requirementsmod.COPIESSDC_REQUIREMENT in self.requirements:
             self.filecopiesmode = b'changeset-sidedata'
 
+        self._wanted_sidedata = set()
+        self._sidedata_computers = {}
+        metadatamod.set_sidedata_spec_for_repo(self)
+
     def _getvfsward(self, origfunc):
         """build a ward for self.vfs"""
         rref = weakref.ref(self)
@@ -3332,6 +3342,22 @@
             fp.close()
         return self.pathto(fp.name[len(self.root) + 1 :])
 
+    def register_wanted_sidedata(self, category):
+        self._wanted_sidedata.add(pycompat.bytestr(category))
+
+    def register_sidedata_computer(self, kind, category, keys, computer):
+        if kind not in (b"changelog", b"manifest", b"filelog"):
+            msg = _(b"unexpected revlog kind '%s'.")
+            raise error.ProgrammingError(msg % kind)
+        category = pycompat.bytestr(category)
+        if category in self._sidedata_computers.get(kind, []):
+            msg = _(
+                b"cannot register a sidedata computer twice for category '%s'."
+            )
+            raise error.ProgrammingError(msg % category)
+        self._sidedata_computers.setdefault(kind, {})
+        self._sidedata_computers[kind][category] = (keys, computer)
+
 
 # used to avoid circular references so destructors work
 def aftertrans(files):
--- a/mercurial/metadata.py	Thu Feb 18 18:18:35 2021 +0100
+++ b/mercurial/metadata.py	Fri Feb 19 10:53:27 2021 +0100
@@ -18,6 +18,7 @@
 from . import (
     error,
     pycompat,
+    requirements as requirementsmod,
     util,
 )
 
@@ -804,6 +805,21 @@
     return encode_files_sidedata(files), files.has_copies_info
 
 
+def copies_sidedata_computer(repo, revlog, rev, existing_sidedata):
+    return _getsidedata(repo, rev)[0]
+
+
+def set_sidedata_spec_for_repo(repo):
+    if requirementsmod.COPIESSDC_REQUIREMENT in repo.requirements:
+        repo.register_wanted_sidedata(sidedatamod.SD_FILES)
+        repo.register_sidedata_computer(
+            b"changelog",
+            sidedatamod.SD_FILES,
+            (sidedatamod.SD_FILES,),
+            copies_sidedata_computer,
+        )
+
+
 def getsidedataadder(srcrepo, destrepo):
     use_w = srcrepo.ui.configbool(b'experimental', b'worker.repository-upgrade')
     if pycompat.iswindows or not use_w:
--- a/mercurial/statichttprepo.py	Thu Feb 18 18:18:35 2021 +0100
+++ b/mercurial/statichttprepo.py	Fri Feb 19 10:53:27 2021 +0100
@@ -172,6 +172,7 @@
         self.names = namespaces.namespaces()
         self.filtername = None
         self._extrafilterid = None
+        self._wanted_sidedata = set()
 
         try:
             requirements = set(self.vfs.read(b'requires').splitlines())
--- a/tests/test-check-interfaces.py	Thu Feb 18 18:18:35 2021 +0100
+++ b/tests/test-check-interfaces.py	Fri Feb 19 10:53:27 2021 +0100
@@ -85,6 +85,7 @@
 class dummyrepo(object):
     def __init__(self):
         self.ui = uimod.ui()
+        self._wanted_sidedata = set()
 
     def filtered(self, name):
         pass