mercurial/exchangev2.py
changeset 43077 687b865b95ad
parent 43076 2372284d9457
child 43104 74802979dd9d
equal deleted inserted replaced
43076:2372284d9457 43077:687b865b95ad
    37     # If this is a clone and it was requested to perform a "stream clone",
    37     # If this is a clone and it was requested to perform a "stream clone",
    38     # we obtain the raw files data from the remote then fall back to an
    38     # we obtain the raw files data from the remote then fall back to an
    39     # incremental pull. This is somewhat hacky and is not nearly robust enough
    39     # incremental pull. This is somewhat hacky and is not nearly robust enough
    40     # for long-term usage.
    40     # for long-term usage.
    41     if usingrawchangelogandmanifest:
    41     if usingrawchangelogandmanifest:
    42         with repo.transaction('clone'):
    42         with repo.transaction(b'clone'):
    43             _fetchrawstorefiles(repo, remote)
    43             _fetchrawstorefiles(repo, remote)
    44             repo.invalidate(clearfilecache=True)
    44             repo.invalidate(clearfilecache=True)
    45 
    45 
    46     tr = pullop.trmanager.transaction()
    46     tr = pullop.trmanager.transaction()
    47 
    47 
    49     # to exchange.pull() could be different.
    49     # to exchange.pull() could be different.
    50     narrowmatcher = narrowspec.match(
    50     narrowmatcher = narrowspec.match(
    51         repo.root,
    51         repo.root,
    52         # Empty maps to nevermatcher. So always
    52         # Empty maps to nevermatcher. So always
    53         # set includes if missing.
    53         # set includes if missing.
    54         pullop.includepats or {'path:.'},
    54         pullop.includepats or {b'path:.'},
    55         pullop.excludepats,
    55         pullop.excludepats,
    56     )
    56     )
    57 
    57 
    58     if pullop.includepats or pullop.excludepats:
    58     if pullop.includepats or pullop.excludepats:
    59         pathfilter = {}
    59         pathfilter = {}
    76     # New revisions are written to the changelog. But all other updates
    76     # New revisions are written to the changelog. But all other updates
    77     # are deferred. Do those now.
    77     # are deferred. Do those now.
    78 
    78 
    79     # Ensure all new changesets are draft by default. If the repo is
    79     # Ensure all new changesets are draft by default. If the repo is
    80     # publishing, the phase will be adjusted by the loop below.
    80     # publishing, the phase will be adjusted by the loop below.
    81     if csetres['added']:
    81     if csetres[b'added']:
    82         phases.registernew(repo, tr, phases.draft, csetres['added'])
    82         phases.registernew(repo, tr, phases.draft, csetres[b'added'])
    83 
    83 
    84     # And adjust the phase of all changesets accordingly.
    84     # And adjust the phase of all changesets accordingly.
    85     for phase in phases.phasenames:
    85     for phase in phases.phasenames:
    86         if phase == b'secret' or not csetres['nodesbyphase'][phase]:
    86         if phase == b'secret' or not csetres[b'nodesbyphase'][phase]:
    87             continue
    87             continue
    88 
    88 
    89         phases.advanceboundary(
    89         phases.advanceboundary(
    90             repo,
    90             repo,
    91             tr,
    91             tr,
    92             phases.phasenames.index(phase),
    92             phases.phasenames.index(phase),
    93             csetres['nodesbyphase'][phase],
    93             csetres[b'nodesbyphase'][phase],
    94         )
    94         )
    95 
    95 
    96     # Write bookmark updates.
    96     # Write bookmark updates.
    97     bookmarks.updatefromremote(
    97     bookmarks.updatefromremote(
    98         repo.ui,
    98         repo.ui,
    99         repo,
    99         repo,
   100         csetres['bookmarks'],
   100         csetres[b'bookmarks'],
   101         remote.url(),
   101         remote.url(),
   102         pullop.gettransaction,
   102         pullop.gettransaction,
   103         explicit=pullop.explicitbookmarks,
   103         explicit=pullop.explicitbookmarks,
   104     )
   104     )
   105 
   105 
   106     manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes'])
   106     manres = _fetchmanifests(repo, tr, remote, csetres[b'manifestnodes'])
   107 
   107 
   108     # We don't properly support shallow changeset and manifest yet. So we apply
   108     # We don't properly support shallow changeset and manifest yet. So we apply
   109     # depth limiting locally.
   109     # depth limiting locally.
   110     if pullop.depth:
   110     if pullop.depth:
   111         relevantcsetnodes = set()
   111         relevantcsetnodes = set()
   140             csetsforfiles.append(node)
   140             csetsforfiles.append(node)
   141             mnodesforfiles.append(mnode)
   141             mnodesforfiles.append(mnode)
   142             manifestlinkrevs[mnode] = rev
   142             manifestlinkrevs[mnode] = rev
   143 
   143 
   144     else:
   144     else:
   145         csetsforfiles = [n for n in csetres['added'] if csetrelevantfilter(n)]
   145         csetsforfiles = [n for n in csetres[b'added'] if csetrelevantfilter(n)]
   146         mnodesforfiles = manres['added']
   146         mnodesforfiles = manres[b'added']
   147         manifestlinkrevs = manres['linkrevs']
   147         manifestlinkrevs = manres[b'linkrevs']
   148 
   148 
   149     # Find all file nodes referenced by added manifests and fetch those
   149     # Find all file nodes referenced by added manifests and fetch those
   150     # revisions.
   150     # revisions.
   151     fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
   151     fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
   152     _fetchfilesfromcsets(
   152     _fetchfilesfromcsets(
   195 
   195 
   196         # First object is a summary of files data that follows.
   196         # First object is a summary of files data that follows.
   197         overall = next(objs)
   197         overall = next(objs)
   198 
   198 
   199         progress = repo.ui.makeprogress(
   199         progress = repo.ui.makeprogress(
   200             _('clone'), total=overall[b'totalsize'], unit=_('bytes')
   200             _(b'clone'), total=overall[b'totalsize'], unit=_(b'bytes')
   201         )
   201         )
   202         with progress:
   202         with progress:
   203             progress.update(0)
   203             progress.update(0)
   204 
   204 
   205             # Next are pairs of file metadata, data.
   205             # Next are pairs of file metadata, data.
   328         # we need to remain in the context manager in order to stream data.
   328         # we need to remain in the context manager in order to stream data.
   329         return _processchangesetdata(repo, tr, objs)
   329         return _processchangesetdata(repo, tr, objs)
   330 
   330 
   331 
   331 
   332 def _processchangesetdata(repo, tr, objs):
   332 def _processchangesetdata(repo, tr, objs):
   333     repo.hook('prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs))
   333     repo.hook(b'prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs))
   334 
   334 
   335     urepo = repo.unfiltered()
   335     urepo = repo.unfiltered()
   336     cl = urepo.changelog
   336     cl = urepo.changelog
   337 
   337 
   338     cl.delayupdate(tr)
   338     cl.delayupdate(tr)
   340     # The first emitted object is a header describing the data that
   340     # The first emitted object is a header describing the data that
   341     # follows.
   341     # follows.
   342     meta = next(objs)
   342     meta = next(objs)
   343 
   343 
   344     progress = repo.ui.makeprogress(
   344     progress = repo.ui.makeprogress(
   345         _('changesets'), unit=_('chunks'), total=meta.get(b'totalitems')
   345         _(b'changesets'), unit=_(b'chunks'), total=meta.get(b'totalitems')
   346     )
   346     )
   347 
   347 
   348     manifestnodes = {}
   348     manifestnodes = {}
   349 
   349 
   350     def linkrev(node):
   350     def linkrev(node):
   351         repo.ui.debug('add changeset %s\n' % short(node))
   351         repo.ui.debug(b'add changeset %s\n' % short(node))
   352         # Linkrev for changelog is always self.
   352         # Linkrev for changelog is always self.
   353         return len(cl)
   353         return len(cl)
   354 
   354 
   355     def onchangeset(cl, node):
   355     def onchangeset(cl, node):
   356         progress.increment()
   356         progress.increment()
   411     )
   411     )
   412 
   412 
   413     progress.complete()
   413     progress.complete()
   414 
   414 
   415     return {
   415     return {
   416         'added': added,
   416         b'added': added,
   417         'nodesbyphase': nodesbyphase,
   417         b'nodesbyphase': nodesbyphase,
   418         'bookmarks': remotebookmarks,
   418         b'bookmarks': remotebookmarks,
   419         'manifestnodes': manifestnodes,
   419         b'manifestnodes': manifestnodes,
   420     }
   420     }
   421 
   421 
   422 
   422 
   423 def _fetchmanifests(repo, tr, remote, manifestnodes):
   423 def _fetchmanifests(repo, tr, remote, manifestnodes):
   424     rootmanifest = repo.manifestlog.getstorage(b'')
   424     rootmanifest = repo.manifestlog.getstorage(b'')
   481             )
   481             )
   482 
   482 
   483             progress.increment()
   483             progress.increment()
   484 
   484 
   485     progress = repo.ui.makeprogress(
   485     progress = repo.ui.makeprogress(
   486         _('manifests'), unit=_('chunks'), total=len(fetchnodes)
   486         _(b'manifests'), unit=_(b'chunks'), total=len(fetchnodes)
   487     )
   487     )
   488 
   488 
   489     commandmeta = remote.apidescriptor[b'commands'][b'manifestdata']
   489     commandmeta = remote.apidescriptor[b'commands'][b'manifestdata']
   490     batchsize = commandmeta.get(b'recommendedbatchsize', 10000)
   490     batchsize = commandmeta.get(b'recommendedbatchsize', 10000)
   491     # TODO make size configurable on client?
   491     # TODO make size configurable on client?
   528             )
   528             )
   529 
   529 
   530     progress.complete()
   530     progress.complete()
   531 
   531 
   532     return {
   532     return {
   533         'added': added,
   533         b'added': added,
   534         'linkrevs': linkrevs,
   534         b'linkrevs': linkrevs,
   535     }
   535     }
   536 
   536 
   537 
   537 
   538 def _derivefilesfrommanifests(repo, matcher, manifestnodes):
   538 def _derivefilesfrommanifests(repo, matcher, manifestnodes):
   539     """Determine what file nodes are relevant given a set of manifest nodes.
   539     """Determine what file nodes are relevant given a set of manifest nodes.
   543     """
   543     """
   544     ml = repo.manifestlog
   544     ml = repo.manifestlog
   545     fnodes = collections.defaultdict(dict)
   545     fnodes = collections.defaultdict(dict)
   546 
   546 
   547     progress = repo.ui.makeprogress(
   547     progress = repo.ui.makeprogress(
   548         _('scanning manifests'), total=len(manifestnodes)
   548         _(b'scanning manifests'), total=len(manifestnodes)
   549     )
   549     )
   550 
   550 
   551     with progress:
   551     with progress:
   552         for manifestnode in manifestnodes:
   552         for manifestnode in manifestnodes:
   553             m = ml.get(b'', manifestnode)
   553             m = ml.get(b'', manifestnode)
   603             )
   603             )
   604 
   604 
   605             progress.increment()
   605             progress.increment()
   606 
   606 
   607     progress = repo.ui.makeprogress(
   607     progress = repo.ui.makeprogress(
   608         _('files'),
   608         _(b'files'),
   609         unit=_('chunks'),
   609         unit=_(b'chunks'),
   610         total=sum(len(v) for v in fnodes.itervalues()),
   610         total=sum(len(v) for v in fnodes.itervalues()),
   611     )
   611     )
   612 
   612 
   613     # TODO make batch size configurable
   613     # TODO make batch size configurable
   614     batchsize = 10000
   614     batchsize = 10000
   702 
   702 
   703             progress.increment()
   703             progress.increment()
   704             remaining -= 1
   704             remaining -= 1
   705 
   705 
   706     progress = repo.ui.makeprogress(
   706     progress = repo.ui.makeprogress(
   707         _('files'),
   707         _(b'files'),
   708         unit=_('chunks'),
   708         unit=_(b'chunks'),
   709         total=sum(len(v) for v in fnodes.itervalues()),
   709         total=sum(len(v) for v in fnodes.itervalues()),
   710     )
   710     )
   711 
   711 
   712     commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
   712     commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
   713     batchsize = commandmeta.get(b'recommendedbatchsize', 50000)
   713     batchsize = commandmeta.get(b'recommendedbatchsize', 50000)