mercurial/exchangev2.py
author Pulkit Goyal <pulkit@yandex-team.ru>
Sun, 18 Aug 2019 00:45:33 +0300
changeset 42813 268662aac075
parent 41217 afa884015e66
child 43076 2372284d9457
permissions -rw-r--r--
interfaces: create a new folder for interfaces and move repository.py in it I was trying to understand current interfaces and write new ones and I realized we need to improve how current interfaces are organised. This creates a dedicated folder for defining interfaces and move `repository.py` which defines all the current interfaces inside it. Differential Revision: https://phab.mercurial-scm.org/D6741
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     1
# exchangev2.py - repository exchange for wire protocol version 2
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     2
#
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     3
# Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     4
#
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     5
# This software may be used and distributed according to the terms of the
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     6
# GNU General Public License version 2 or any later version.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     7
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     8
from __future__ import absolute_import
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     9
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
    10
import collections
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    11
import weakref
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    12
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    13
from .i18n import _
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    14
from .node import (
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    15
    nullid,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    16
    short,
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    17
)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    18
from . import (
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
    19
    bookmarks,
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
    20
    error,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    21
    mdiff,
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    22
    narrowspec,
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    23
    phases,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    24
    pycompat,
42813
268662aac075 interfaces: create a new folder for interfaces and move repository.py in it
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41217
diff changeset
    25
    setdiscovery,
268662aac075 interfaces: create a new folder for interfaces and move repository.py in it
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41217
diff changeset
    26
)
268662aac075 interfaces: create a new folder for interfaces and move repository.py in it
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41217
diff changeset
    27
from .interfaces import (
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
    28
    repository,
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    29
)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    30
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    31
def pull(pullop):
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    32
    """Pull using wire protocol version 2."""
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    33
    repo = pullop.repo
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    34
    remote = pullop.remote
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    35
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    36
    usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    37
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    38
    # If this is a clone and it was requested to perform a "stream clone",
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    39
    # we obtain the raw files data from the remote then fall back to an
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    40
    # incremental pull. This is somewhat hacky and is not nearly robust enough
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    41
    # for long-term usage.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    42
    if usingrawchangelogandmanifest:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    43
        with repo.transaction('clone'):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    44
            _fetchrawstorefiles(repo, remote)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    45
            repo.invalidate(clearfilecache=True)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    46
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    47
    tr = pullop.trmanager.transaction()
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    48
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    49
    # We don't use the repo's narrow matcher here because the patterns passed
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    50
    # to exchange.pull() could be different.
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    51
    narrowmatcher = narrowspec.match(repo.root,
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    52
                                     # Empty maps to nevermatcher. So always
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    53
                                     # set includes if missing.
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    54
                                     pullop.includepats or {'path:.'},
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    55
                                     pullop.excludepats)
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    56
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    57
    if pullop.includepats or pullop.excludepats:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    58
        pathfilter = {}
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    59
        if pullop.includepats:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    60
            pathfilter[b'include'] = sorted(pullop.includepats)
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    61
        if pullop.excludepats:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    62
            pathfilter[b'exclude'] = sorted(pullop.excludepats)
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    63
    else:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    64
        pathfilter = None
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    65
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    66
    # Figure out what needs to be fetched.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    67
    common, fetch, remoteheads = _pullchangesetdiscovery(
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    68
        repo, remote, pullop.heads, abortwhenunrelated=pullop.force)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    69
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    70
    # And fetch the data.
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    71
    pullheads = pullop.heads or remoteheads
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    72
    csetres = _fetchchangesets(repo, tr, remote, common, fetch, pullheads)
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    73
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    74
    # New revisions are written to the changelog. But all other updates
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    75
    # are deferred. Do those now.
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    76
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    77
    # Ensure all new changesets are draft by default. If the repo is
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    78
    # publishing, the phase will be adjusted by the loop below.
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    79
    if csetres['added']:
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    80
        phases.registernew(repo, tr, phases.draft, csetres['added'])
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    81
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    82
    # And adjust the phase of all changesets accordingly.
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    83
    for phase in phases.phasenames:
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    84
        if phase == b'secret' or not csetres['nodesbyphase'][phase]:
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    85
            continue
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    86
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    87
        phases.advanceboundary(repo, tr, phases.phasenames.index(phase),
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    88
                               csetres['nodesbyphase'][phase])
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    89
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
    90
    # Write bookmark updates.
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
    91
    bookmarks.updatefromremote(repo.ui, repo, csetres['bookmarks'],
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
    92
                               remote.url(), pullop.gettransaction,
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
    93
                               explicit=pullop.explicitbookmarks)
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
    94
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
    95
    manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes'])
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
    96
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
    97
    # We don't properly support shallow changeset and manifest yet. So we apply
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
    98
    # depth limiting locally.
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
    99
    if pullop.depth:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   100
        relevantcsetnodes = set()
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   101
        clnode = repo.changelog.node
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   102
41217
afa884015e66 revset: remove the last usage of "%d" for a non-revision entry
Boris Feld <boris.feld@octobus.net>
parents: 40393
diff changeset
   103
        for rev in repo.revs(b'ancestors(%ln, %s)',
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   104
                             pullheads, pullop.depth - 1):
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   105
            relevantcsetnodes.add(clnode(rev))
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   106
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   107
        csetrelevantfilter = lambda n: n in relevantcsetnodes
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   108
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   109
    else:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   110
        csetrelevantfilter = lambda n: True
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   111
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   112
    # If obtaining the raw store files, we need to scan the full repo to
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   113
    # derive all the changesets, manifests, and linkrevs.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   114
    if usingrawchangelogandmanifest:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   115
        csetsforfiles = []
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   116
        mnodesforfiles = []
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   117
        manifestlinkrevs = {}
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   118
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   119
        for rev in repo:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   120
            ctx = repo[rev]
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   121
            node = ctx.node()
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   122
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   123
            if not csetrelevantfilter(node):
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   124
                continue
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   125
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   126
            mnode = ctx.manifestnode()
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   127
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   128
            csetsforfiles.append(node)
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   129
            mnodesforfiles.append(mnode)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   130
            manifestlinkrevs[mnode] = rev
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   131
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   132
    else:
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   133
        csetsforfiles = [n for n in csetres['added'] if csetrelevantfilter(n)]
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   134
        mnodesforfiles = manres['added']
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   135
        manifestlinkrevs = manres['linkrevs']
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   136
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   137
    # Find all file nodes referenced by added manifests and fetch those
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   138
    # revisions.
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   139
    fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   140
    _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csetsforfiles,
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   141
                         manifestlinkrevs, shallow=bool(pullop.depth))
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   142
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   143
def _checkuserawstorefiledata(pullop):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   144
    """Check whether we should use rawstorefiledata command to retrieve data."""
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   145
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   146
    repo = pullop.repo
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   147
    remote = pullop.remote
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   148
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   149
    # Command to obtain raw store data isn't available.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   150
    if b'rawstorefiledata' not in remote.apidescriptor[b'commands']:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   151
        return False
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   152
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   153
    # Only honor if user requested stream clone operation.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   154
    if not pullop.streamclonerequested:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   155
        return False
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   156
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   157
    # Only works on empty repos.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   158
    if len(repo):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   159
        return False
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   160
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   161
    # TODO This is super hacky. There needs to be a storage API for this. We
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   162
    # also need to check for compatibility with the remote.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   163
    if b'revlogv1' not in repo.requirements:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   164
        return False
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   165
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   166
    return True
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   167
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   168
def _fetchrawstorefiles(repo, remote):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   169
    with remote.commandexecutor() as e:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   170
        objs = e.callcommand(b'rawstorefiledata', {
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   171
            b'files': [b'changelog', b'manifestlog'],
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   172
        }).result()
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   173
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   174
        # First object is a summary of files data that follows.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   175
        overall = next(objs)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   176
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   177
        progress = repo.ui.makeprogress(_('clone'), total=overall[b'totalsize'],
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   178
                                        unit=_('bytes'))
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   179
        with progress:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   180
            progress.update(0)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   181
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   182
            # Next are pairs of file metadata, data.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   183
            while True:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   184
                try:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   185
                    filemeta = next(objs)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   186
                except StopIteration:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   187
                    break
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   188
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   189
                for k in (b'location', b'path', b'size'):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   190
                    if k not in filemeta:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   191
                        raise error.Abort(_(b'remote file data missing key: %s')
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   192
                                          % k)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   193
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   194
                if filemeta[b'location'] == b'store':
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   195
                    vfs = repo.svfs
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   196
                else:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   197
                    raise error.Abort(_(b'invalid location for raw file data: '
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   198
                                        b'%s') % filemeta[b'location'])
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   199
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   200
                bytesremaining = filemeta[b'size']
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   201
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   202
                with vfs.open(filemeta[b'path'], b'wb') as fh:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   203
                    while True:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   204
                        try:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   205
                            chunk = next(objs)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   206
                        except StopIteration:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   207
                            break
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   208
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   209
                        bytesremaining -= len(chunk)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   210
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   211
                        if bytesremaining < 0:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   212
                            raise error.Abort(_(
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   213
                                b'received invalid number of bytes for file '
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   214
                                b'data; expected %d, got extra') %
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   215
                                              filemeta[b'size'])
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   216
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   217
                        progress.increment(step=len(chunk))
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   218
                        fh.write(chunk)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   219
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   220
                        try:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   221
                            if chunk.islast:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   222
                                break
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   223
                        except AttributeError:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   224
                            raise error.Abort(_(
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   225
                                b'did not receive indefinite length bytestring '
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   226
                                b'for file data'))
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   227
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   228
                if bytesremaining:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   229
                    raise error.Abort(_(b'received invalid number of bytes for'
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   230
                                        b'file data; expected %d got %d') %
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   231
                                      (filemeta[b'size'],
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   232
                                       filemeta[b'size'] - bytesremaining))
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   233
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   234
def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   235
    """Determine which changesets need to be pulled."""
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   236
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   237
    if heads:
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   238
        knownnode = repo.changelog.hasnode
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   239
        if all(knownnode(head) for head in heads):
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   240
            return heads, False, heads
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   241
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   242
    # TODO wire protocol version 2 is capable of more efficient discovery
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   243
    # than setdiscovery. Consider implementing something better.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   244
    common, fetch, remoteheads = setdiscovery.findcommonheads(
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   245
        repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   246
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   247
    common = set(common)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   248
    remoteheads = set(remoteheads)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   249
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   250
    # If a remote head is filtered locally, put it back in the common set.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   251
    # See the comment in exchange._pulldiscoverychangegroup() for more.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   252
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   253
    if fetch and remoteheads:
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   254
        nodemap = repo.unfiltered().changelog.nodemap
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   255
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   256
        common |= {head for head in remoteheads if head in nodemap}
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   257
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   258
        if set(remoteheads).issubset(common):
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   259
            fetch = []
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   260
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   261
    common.discard(nullid)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   262
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   263
    return common, fetch, remoteheads
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   264
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   265
def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads):
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   266
    # TODO consider adding a step here where we obtain the DAG shape first
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   267
    # (or ask the server to slice changesets into chunks for us) so that
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   268
    # we can perform multiple fetches in batches. This will facilitate
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   269
    # resuming interrupted clones, higher server-side cache hit rates due
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   270
    # to smaller segments, etc.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   271
    with remote.commandexecutor() as e:
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   272
        objs = e.callcommand(b'changesetdata', {
40176
41263df08109 wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40173
diff changeset
   273
            b'revisions': [{
41263df08109 wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40173
diff changeset
   274
                b'type': b'changesetdagrange',
41263df08109 wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40173
diff changeset
   275
                b'roots': sorted(common),
41263df08109 wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40173
diff changeset
   276
                b'heads': sorted(remoteheads),
41263df08109 wireprotov2: change how revisions are specified to changesetdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40173
diff changeset
   277
            }],
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   278
            b'fields': {b'bookmarks', b'parents', b'phase', b'revision'},
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   279
        }).result()
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   280
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   281
        # The context manager waits on all response data when exiting. So
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   282
        # we need to remain in the context manager in order to stream data.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   283
        return _processchangesetdata(repo, tr, objs)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   284
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   285
def _processchangesetdata(repo, tr, objs):
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   286
    repo.hook('prechangegroup', throw=True,
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   287
              **pycompat.strkwargs(tr.hookargs))
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   288
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   289
    urepo = repo.unfiltered()
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   290
    cl = urepo.changelog
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   291
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   292
    cl.delayupdate(tr)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   293
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   294
    # The first emitted object is a header describing the data that
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   295
    # follows.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   296
    meta = next(objs)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   297
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   298
    progress = repo.ui.makeprogress(_('changesets'),
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   299
                                    unit=_('chunks'),
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   300
                                    total=meta.get(b'totalitems'))
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   301
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   302
    manifestnodes = {}
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   303
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   304
    def linkrev(node):
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   305
        repo.ui.debug('add changeset %s\n' % short(node))
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   306
        # Linkrev for changelog is always self.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   307
        return len(cl)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   308
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   309
    def onchangeset(cl, node):
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   310
        progress.increment()
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   311
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   312
        revision = cl.changelogrevision(node)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   313
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   314
        # We need to preserve the mapping of changelog revision to node
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   315
        # so we can set the linkrev accordingly when manifests are added.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   316
        manifestnodes[cl.rev(node)] = revision.manifest
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   317
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   318
    nodesbyphase = {phase: set() for phase in phases.phasenames}
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   319
    remotebookmarks = {}
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   320
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   321
    # addgroup() expects a 7-tuple describing revisions. This normalizes
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   322
    # the wire data to that format.
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   323
    #
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   324
    # This loop also aggregates non-revision metadata, such as phase
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   325
    # data.
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   326
    def iterrevisions():
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   327
        for cset in objs:
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   328
            node = cset[b'node']
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   329
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   330
            if b'phase' in cset:
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   331
                nodesbyphase[cset[b'phase']].add(node)
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   332
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   333
            for mark in cset.get(b'bookmarks', []):
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   334
                remotebookmarks[mark] = node
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   335
39636
399ddd3227a4 wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39635
diff changeset
   336
            # TODO add mechanism for extensions to examine records so they
399ddd3227a4 wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39635
diff changeset
   337
            # can siphon off custom data fields.
399ddd3227a4 wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39635
diff changeset
   338
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   339
            extrafields = {}
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   340
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   341
            for field, size in cset.get(b'fieldsfollowing', []):
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   342
                extrafields[field] = next(objs)
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   343
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   344
            # Some entries might only be metadata only updates.
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   345
            if b'revision' not in extrafields:
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   346
                continue
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   347
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   348
            data = extrafields[b'revision']
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   349
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   350
            yield (
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   351
                node,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   352
                cset[b'parents'][0],
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   353
                cset[b'parents'][1],
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   354
                # Linknode is always itself for changesets.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   355
                cset[b'node'],
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   356
                # We always send full revisions. So delta base is not set.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   357
                nullid,
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   358
                mdiff.trivialdiffheader(len(data)) + data,
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   359
                # Flags not yet supported.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   360
                0,
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   361
            )
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   362
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   363
    added = cl.addgroup(iterrevisions(), linkrev, weakref.proxy(tr),
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   364
                        addrevisioncb=onchangeset)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   365
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   366
    progress.complete()
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   367
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   368
    return {
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   369
        'added': added,
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   370
        'nodesbyphase': nodesbyphase,
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   371
        'bookmarks': remotebookmarks,
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   372
        'manifestnodes': manifestnodes,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   373
    }
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   374
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   375
def _fetchmanifests(repo, tr, remote, manifestnodes):
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   376
    rootmanifest = repo.manifestlog.getstorage(b'')
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   377
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   378
    # Some manifests can be shared between changesets. Filter out revisions
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   379
    # we already know about.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   380
    fetchnodes = []
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   381
    linkrevs = {}
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   382
    seen = set()
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   383
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   384
    for clrev, node in sorted(manifestnodes.iteritems()):
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   385
        if node in seen:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   386
            continue
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   387
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   388
        try:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   389
            rootmanifest.rev(node)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   390
        except error.LookupError:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   391
            fetchnodes.append(node)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   392
            linkrevs[node] = clrev
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   393
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   394
        seen.add(node)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   395
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   396
    # TODO handle tree manifests
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   397
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   398
    # addgroup() expects 7-tuple describing revisions. This normalizes
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   399
    # the wire data to that format.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   400
    def iterrevisions(objs, progress):
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   401
        for manifest in objs:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   402
            node = manifest[b'node']
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   403
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   404
            extrafields = {}
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   405
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   406
            for field, size in manifest.get(b'fieldsfollowing', []):
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   407
                extrafields[field] = next(objs)
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   408
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   409
            if b'delta' in extrafields:
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   410
                basenode = manifest[b'deltabasenode']
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   411
                delta = extrafields[b'delta']
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   412
            elif b'revision' in extrafields:
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   413
                basenode = nullid
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   414
                revision = extrafields[b'revision']
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   415
                delta = mdiff.trivialdiffheader(len(revision)) + revision
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   416
            else:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   417
                continue
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   418
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   419
            yield (
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   420
                node,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   421
                manifest[b'parents'][0],
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   422
                manifest[b'parents'][1],
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   423
                # The value passed in is passed to the lookup function passed
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   424
                # to addgroup(). We already have a map of manifest node to
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   425
                # changelog revision number. So we just pass in the
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   426
                # manifest node here and use linkrevs.__getitem__ as the
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   427
                # resolution function.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   428
                node,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   429
                basenode,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   430
                delta,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   431
                # Flags not yet supported.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   432
                0
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   433
            )
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   434
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   435
            progress.increment()
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   436
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   437
    progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   438
                                    total=len(fetchnodes))
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   439
40173
b797150a1ab9 exchangev2: honor server advertised manifestdata recommended batch size
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40035
diff changeset
   440
    commandmeta = remote.apidescriptor[b'commands'][b'manifestdata']
b797150a1ab9 exchangev2: honor server advertised manifestdata recommended batch size
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40035
diff changeset
   441
    batchsize = commandmeta.get(b'recommendedbatchsize', 10000)
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   442
    # TODO make size configurable on client?
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   443
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   444
    # We send commands 1 at a time to the remote. This is not the most
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   445
    # efficient because we incur a round trip at the end of each batch.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   446
    # However, the existing frame-based reactor keeps consuming server
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   447
    # data in the background. And this results in response data buffering
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   448
    # in memory. This can consume gigabytes of memory.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   449
    # TODO send multiple commands in a request once background buffering
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   450
    # issues are resolved.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   451
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   452
    added = []
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   453
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   454
    for i in pycompat.xrange(0, len(fetchnodes), batchsize):
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   455
        batch = [node for node in fetchnodes[i:i + batchsize]]
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   456
        if not batch:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   457
            continue
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   458
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   459
        with remote.commandexecutor() as e:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   460
            objs = e.callcommand(b'manifestdata', {
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   461
                b'tree': b'',
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   462
                b'nodes': batch,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   463
                b'fields': {b'parents', b'revision'},
39641
aa7e312375cf wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39640
diff changeset
   464
                b'haveparents': True,
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   465
            }).result()
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   466
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   467
            # Chomp off header object.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   468
            next(objs)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   469
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   470
            added.extend(rootmanifest.addgroup(
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   471
                iterrevisions(objs, progress),
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   472
                linkrevs.__getitem__,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   473
                weakref.proxy(tr)))
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   474
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   475
    progress.complete()
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   476
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   477
    return {
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   478
        'added': added,
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   479
        'linkrevs': linkrevs,
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   480
    }
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   481
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   482
def _derivefilesfrommanifests(repo, matcher, manifestnodes):
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   483
    """Determine what file nodes are relevant given a set of manifest nodes.
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   484
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   485
    Returns a dict mapping file paths to dicts of file node to first manifest
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   486
    node.
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   487
    """
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   488
    ml = repo.manifestlog
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   489
    fnodes = collections.defaultdict(dict)
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   490
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   491
    progress = repo.ui.makeprogress(
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   492
        _('scanning manifests'), total=len(manifestnodes))
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   493
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   494
    with progress:
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   495
        for manifestnode in manifestnodes:
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   496
            m = ml.get(b'', manifestnode)
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   497
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   498
            # TODO this will pull in unwanted nodes because it takes the storage
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   499
            # delta into consideration. What we really want is something that
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   500
            # takes the delta between the manifest's parents. And ideally we
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   501
            # would ignore file nodes that are known locally. For now, ignore
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   502
            # both these limitations. This will result in incremental fetches
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   503
            # requesting data we already have. So this is far from ideal.
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   504
            md = m.readfast()
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   505
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   506
            for path, fnode in md.items():
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   507
                if matcher(path):
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   508
                    fnodes[path].setdefault(fnode, manifestnode)
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   509
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   510
            progress.increment()
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   511
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   512
    return fnodes
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   513
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   514
def _fetchfiles(repo, tr, remote, fnodes, linkrevs):
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   515
    """Fetch file data from explicit file revisions."""
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   516
    def iterrevisions(objs, progress):
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   517
        for filerevision in objs:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   518
            node = filerevision[b'node']
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   519
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   520
            extrafields = {}
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   521
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   522
            for field, size in filerevision.get(b'fieldsfollowing', []):
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   523
                extrafields[field] = next(objs)
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   524
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   525
            if b'delta' in extrafields:
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   526
                basenode = filerevision[b'deltabasenode']
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   527
                delta = extrafields[b'delta']
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   528
            elif b'revision' in extrafields:
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   529
                basenode = nullid
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   530
                revision = extrafields[b'revision']
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   531
                delta = mdiff.trivialdiffheader(len(revision)) + revision
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   532
            else:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   533
                continue
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   534
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   535
            yield (
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   536
                node,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   537
                filerevision[b'parents'][0],
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   538
                filerevision[b'parents'][1],
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   539
                node,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   540
                basenode,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   541
                delta,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   542
                # Flags not yet supported.
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   543
                0,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   544
            )
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   545
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   546
            progress.increment()
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   547
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   548
    progress = repo.ui.makeprogress(
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   549
        _('files'), unit=_('chunks'),
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   550
         total=sum(len(v) for v in fnodes.itervalues()))
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   551
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   552
    # TODO make batch size configurable
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   553
    batchsize = 10000
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   554
    fnodeslist = [x for x in sorted(fnodes.items())]
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   555
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   556
    for i in pycompat.xrange(0, len(fnodeslist), batchsize):
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   557
        batch = [x for x in fnodeslist[i:i + batchsize]]
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   558
        if not batch:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   559
            continue
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   560
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   561
        with remote.commandexecutor() as e:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   562
            fs = []
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   563
            locallinkrevs = {}
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   564
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   565
            for path, nodes in batch:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   566
                fs.append((path, e.callcommand(b'filedata', {
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   567
                    b'path': path,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   568
                    b'nodes': sorted(nodes),
39641
aa7e312375cf wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39640
diff changeset
   569
                    b'fields': {b'parents', b'revision'},
aa7e312375cf wireprotov2: let clients drive delta behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39640
diff changeset
   570
                    b'haveparents': True,
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   571
                })))
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   572
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   573
                locallinkrevs[path] = {
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   574
                    node: linkrevs[manifestnode]
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   575
                    for node, manifestnode in nodes.iteritems()}
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   576
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   577
            for path, f in fs:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   578
                objs = f.result()
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   579
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   580
                # Chomp off header objects.
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   581
                next(objs)
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   582
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   583
                store = repo.file(path)
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   584
                store.addgroup(
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   585
                    iterrevisions(objs, progress),
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   586
                    locallinkrevs[path].__getitem__,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   587
                    weakref.proxy(tr))
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   588
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   589
def _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csets,
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   590
                         manlinkrevs, shallow=False):
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   591
    """Fetch file data from explicit changeset revisions."""
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   592
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   593
    def iterrevisions(objs, remaining, progress):
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   594
        while remaining:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   595
            filerevision = next(objs)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   596
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   597
            node = filerevision[b'node']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   598
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   599
            extrafields = {}
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   600
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   601
            for field, size in filerevision.get(b'fieldsfollowing', []):
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   602
                extrafields[field] = next(objs)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   603
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   604
            if b'delta' in extrafields:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   605
                basenode = filerevision[b'deltabasenode']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   606
                delta = extrafields[b'delta']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   607
            elif b'revision' in extrafields:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   608
                basenode = nullid
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   609
                revision = extrafields[b'revision']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   610
                delta = mdiff.trivialdiffheader(len(revision)) + revision
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   611
            else:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   612
                continue
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   613
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   614
            if b'linknode' in filerevision:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   615
                linknode = filerevision[b'linknode']
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   616
            else:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   617
                linknode = node
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   618
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   619
            yield (
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   620
                node,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   621
                filerevision[b'parents'][0],
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   622
                filerevision[b'parents'][1],
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   623
                linknode,
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   624
                basenode,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   625
                delta,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   626
                # Flags not yet supported.
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   627
                0,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   628
            )
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   629
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   630
            progress.increment()
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   631
            remaining -= 1
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   632
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   633
    progress = repo.ui.makeprogress(
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   634
        _('files'), unit=_('chunks'),
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   635
        total=sum(len(v) for v in fnodes.itervalues()))
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   636
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   637
    commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   638
    batchsize = commandmeta.get(b'recommendedbatchsize', 50000)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   639
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   640
    shallowfiles = repository.REPO_FEATURE_SHALLOW_FILE_STORAGE in repo.features
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   641
    fields = {b'parents', b'revision'}
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   642
    clrev = repo.changelog.rev
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   643
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   644
    # There are no guarantees that we'll have ancestor revisions if
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   645
    # a) this repo has shallow file storage b) shallow data fetching is enabled.
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   646
    # Force remote to not delta against possibly unknown revisions when these
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   647
    # conditions hold.
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   648
    haveparents = not (shallowfiles or shallow)
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   649
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   650
    # Similarly, we may not have calculated linkrevs for all incoming file
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   651
    # revisions. Ask the remote to do work for us in this case.
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   652
    if not haveparents:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   653
        fields.add(b'linknode')
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   654
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   655
    for i in pycompat.xrange(0, len(csets), batchsize):
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   656
        batch = [x for x in csets[i:i + batchsize]]
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   657
        if not batch:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   658
            continue
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   659
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   660
        with remote.commandexecutor() as e:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   661
            args = {
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   662
                b'revisions': [{
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   663
                    b'type': b'changesetexplicit',
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   664
                    b'nodes': batch,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   665
                }],
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   666
                b'fields': fields,
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   667
                b'haveparents': haveparents,
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   668
            }
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   669
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   670
            if pathfilter:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   671
                args[b'pathfilter'] = pathfilter
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   672
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   673
            objs = e.callcommand(b'filesdata', args).result()
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   674
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   675
            # First object is an overall header.
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   676
            overall = next(objs)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   677
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   678
            # We have overall['totalpaths'] segments.
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   679
            for i in pycompat.xrange(overall[b'totalpaths']):
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   680
                header = next(objs)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   681
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   682
                path = header[b'path']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   683
                store = repo.file(path)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   684
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   685
                linkrevs = {
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   686
                    fnode: manlinkrevs[mnode]
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   687
                    for fnode, mnode in fnodes[path].iteritems()}
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   688
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   689
                def getlinkrev(node):
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   690
                    if node in linkrevs:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   691
                        return linkrevs[node]
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   692
                    else:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   693
                        return clrev(node)
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   694
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   695
                store.addgroup(iterrevisions(objs, header[b'totalitems'],
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   696
                                             progress),
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   697
                               getlinkrev,
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   698
                               weakref.proxy(tr),
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   699
                               maybemissingparents=shallow)