mercurial/exchangev2.py
author Raphaël Gomès <rgomes@octobus.net>
Wed, 03 Mar 2021 14:00:45 +0100
changeset 46626 ee91966aec0f
parent 46509 7a93b7b3dc2d
child 46711 a41565bef69f
permissions -rw-r--r--
requirements: add constant for revlog v1 requirement Since this series is adding a new requirement, we might as well clean up while we're here. Differential Revision: https://phab.mercurial-scm.org/D10105
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     1
# exchangev2.py - repository exchange for wire protocol version 2
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     2
#
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     3
# Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com>
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     4
#
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     5
# This software may be used and distributed according to the terms of the
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     6
# GNU General Public License version 2 or any later version.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     7
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     8
from __future__ import absolute_import
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     9
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
    10
import collections
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    11
import weakref
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    12
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    13
from .i18n import _
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    14
from .node import (
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    15
    nullid,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    16
    short,
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    17
)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    18
from . import (
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
    19
    bookmarks,
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
    20
    error,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    21
    mdiff,
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    22
    narrowspec,
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    23
    phases,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    24
    pycompat,
46626
ee91966aec0f requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents: 46509
diff changeset
    25
    requirements as requirementsmod,
42813
268662aac075 interfaces: create a new folder for interfaces and move repository.py in it
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41217
diff changeset
    26
    setdiscovery,
268662aac075 interfaces: create a new folder for interfaces and move repository.py in it
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41217
diff changeset
    27
)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    28
from .interfaces import repository
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    29
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    30
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    31
def pull(pullop):
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    32
    """Pull using wire protocol version 2."""
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    33
    repo = pullop.repo
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    34
    remote = pullop.remote
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    35
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    36
    usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    37
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    38
    # If this is a clone and it was requested to perform a "stream clone",
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    39
    # we obtain the raw files data from the remote then fall back to an
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    40
    # incremental pull. This is somewhat hacky and is not nearly robust enough
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    41
    # for long-term usage.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    42
    if usingrawchangelogandmanifest:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    43
        with repo.transaction(b'clone'):
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    44
            _fetchrawstorefiles(repo, remote)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    45
            repo.invalidate(clearfilecache=True)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
    46
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    47
    tr = pullop.trmanager.transaction()
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    48
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    49
    # We don't use the repo's narrow matcher here because the patterns passed
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    50
    # to exchange.pull() could be different.
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    51
    narrowmatcher = narrowspec.match(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    52
        repo.root,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    53
        # Empty maps to nevermatcher. So always
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    54
        # set includes if missing.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    55
        pullop.includepats or {b'path:.'},
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    56
        pullop.excludepats,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    57
    )
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    58
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    59
    if pullop.includepats or pullop.excludepats:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    60
        pathfilter = {}
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    61
        if pullop.includepats:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    62
            pathfilter[b'include'] = sorted(pullop.includepats)
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    63
        if pullop.excludepats:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    64
            pathfilter[b'exclude'] = sorted(pullop.excludepats)
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    65
    else:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    66
        pathfilter = None
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
    67
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    68
    # Figure out what needs to be fetched.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    69
    common, fetch, remoteheads = _pullchangesetdiscovery(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    70
        repo, remote, pullop.heads, abortwhenunrelated=pullop.force
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    71
    )
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    72
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    73
    # And fetch the data.
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    74
    pullheads = pullop.heads or remoteheads
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    75
    csetres = _fetchchangesets(repo, tr, remote, common, fetch, pullheads)
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    76
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    77
    # New revisions are written to the changelog. But all other updates
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    78
    # are deferred. Do those now.
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    79
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    80
    # Ensure all new changesets are draft by default. If the repo is
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    81
    # publishing, the phase will be adjusted by the loop below.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    82
    if csetres[b'added']:
45790
5d65e04b6a80 phases: convert registernew users to use revision sets
Joerg Sonnenberger <joerg@bec.de>
parents: 45788
diff changeset
    83
        phases.registernew(
5d65e04b6a80 phases: convert registernew users to use revision sets
Joerg Sonnenberger <joerg@bec.de>
parents: 45788
diff changeset
    84
            repo, tr, phases.draft, [repo[n].rev() for n in csetres[b'added']]
5d65e04b6a80 phases: convert registernew users to use revision sets
Joerg Sonnenberger <joerg@bec.de>
parents: 45788
diff changeset
    85
        )
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    86
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    87
    # And adjust the phase of all changesets accordingly.
45117
b1e51ef4e536 phases: sparsify phase lists
Joerg Sonnenberger <joerg@bec.de>
parents: 43546
diff changeset
    88
    for phasenumber, phase in phases.phasenames.items():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    89
        if phase == b'secret' or not csetres[b'nodesbyphase'][phase]:
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    90
            continue
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
    91
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    92
        phases.advanceboundary(
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
    93
            repo,
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
    94
            tr,
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
    95
            phasenumber,
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
    96
            csetres[b'nodesbyphase'][phase],
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    97
        )
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
    98
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
    99
    # Write bookmark updates.
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   100
    bookmarks.updatefromremote(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   101
        repo.ui,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   102
        repo,
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   103
        csetres[b'bookmarks'],
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   104
        remote.url(),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   105
        pullop.gettransaction,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   106
        explicit=pullop.explicitbookmarks,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   107
    )
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   108
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   109
    manres = _fetchmanifests(repo, tr, remote, csetres[b'manifestnodes'])
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   110
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   111
    # We don't properly support shallow changeset and manifest yet. So we apply
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   112
    # depth limiting locally.
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   113
    if pullop.depth:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   114
        relevantcsetnodes = set()
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   115
        clnode = repo.changelog.node
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   116
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   117
        for rev in repo.revs(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   118
            b'ancestors(%ln, %s)', pullheads, pullop.depth - 1
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   119
        ):
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   120
            relevantcsetnodes.add(clnode(rev))
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   121
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   122
        csetrelevantfilter = lambda n: n in relevantcsetnodes
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   123
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   124
    else:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   125
        csetrelevantfilter = lambda n: True
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   126
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   127
    # If obtaining the raw store files, we need to scan the full repo to
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   128
    # derive all the changesets, manifests, and linkrevs.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   129
    if usingrawchangelogandmanifest:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   130
        csetsforfiles = []
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   131
        mnodesforfiles = []
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   132
        manifestlinkrevs = {}
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   133
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   134
        for rev in repo:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   135
            ctx = repo[rev]
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   136
            node = ctx.node()
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   137
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   138
            if not csetrelevantfilter(node):
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   139
                continue
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   140
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   141
            mnode = ctx.manifestnode()
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   142
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   143
            csetsforfiles.append(node)
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   144
            mnodesforfiles.append(mnode)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   145
            manifestlinkrevs[mnode] = rev
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   146
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   147
    else:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   148
        csetsforfiles = [n for n in csetres[b'added'] if csetrelevantfilter(n)]
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   149
        mnodesforfiles = manres[b'added']
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   150
        manifestlinkrevs = manres[b'linkrevs']
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   151
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   152
    # Find all file nodes referenced by added manifests and fetch those
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   153
    # revisions.
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   154
    fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   155
    _fetchfilesfromcsets(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   156
        repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   157
        tr,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   158
        remote,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   159
        pathfilter,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   160
        fnodes,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   161
        csetsforfiles,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   162
        manifestlinkrevs,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   163
        shallow=bool(pullop.depth),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   164
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   165
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   166
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   167
def _checkuserawstorefiledata(pullop):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   168
    """Check whether we should use rawstorefiledata command to retrieve data."""
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   169
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   170
    repo = pullop.repo
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   171
    remote = pullop.remote
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   172
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   173
    # Command to obtain raw store data isn't available.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   174
    if b'rawstorefiledata' not in remote.apidescriptor[b'commands']:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   175
        return False
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   176
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   177
    # Only honor if user requested stream clone operation.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   178
    if not pullop.streamclonerequested:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   179
        return False
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   180
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   181
    # Only works on empty repos.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   182
    if len(repo):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   183
        return False
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   184
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   185
    # TODO This is super hacky. There needs to be a storage API for this. We
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   186
    # also need to check for compatibility with the remote.
46626
ee91966aec0f requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents: 46509
diff changeset
   187
    if requirementsmod.REVLOGV1_REQUIREMENT not in repo.requirements:
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   188
        return False
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   189
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   190
    return True
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   191
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   192
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   193
def _fetchrawstorefiles(repo, remote):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   194
    with remote.commandexecutor() as e:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   195
        objs = e.callcommand(
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
   196
            b'rawstorefiledata',
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
   197
            {
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
   198
                b'files': [b'changelog', b'manifestlog'],
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
   199
            },
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   200
        ).result()
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   201
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   202
        # First object is a summary of files data that follows.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   203
        overall = next(objs)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   204
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   205
        progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   206
            _(b'clone'), total=overall[b'totalsize'], unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   207
        )
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   208
        with progress:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   209
            progress.update(0)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   210
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   211
            # Next are pairs of file metadata, data.
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   212
            while True:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   213
                try:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   214
                    filemeta = next(objs)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   215
                except StopIteration:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   216
                    break
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   217
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   218
                for k in (b'location', b'path', b'size'):
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   219
                    if k not in filemeta:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   220
                        raise error.Abort(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   221
                            _(b'remote file data missing key: %s') % k
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   222
                        )
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   223
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   224
                if filemeta[b'location'] == b'store':
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   225
                    vfs = repo.svfs
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   226
                else:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   227
                    raise error.Abort(
43117
8ff1ecfadcd1 cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents: 43106
diff changeset
   228
                        _(b'invalid location for raw file data: %s')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   229
                        % filemeta[b'location']
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   230
                    )
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   231
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   232
                bytesremaining = filemeta[b'size']
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   233
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   234
                with vfs.open(filemeta[b'path'], b'wb') as fh:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   235
                    while True:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   236
                        try:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   237
                            chunk = next(objs)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   238
                        except StopIteration:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   239
                            break
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   240
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   241
                        bytesremaining -= len(chunk)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   242
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   243
                        if bytesremaining < 0:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   244
                            raise error.Abort(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   245
                                _(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   246
                                    b'received invalid number of bytes for file '
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   247
                                    b'data; expected %d, got extra'
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   248
                                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   249
                                % filemeta[b'size']
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   250
                            )
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   251
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   252
                        progress.increment(step=len(chunk))
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   253
                        fh.write(chunk)
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   254
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   255
                        try:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   256
                            if chunk.islast:
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   257
                                break
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   258
                        except AttributeError:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   259
                            raise error.Abort(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   260
                                _(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   261
                                    b'did not receive indefinite length bytestring '
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   262
                                    b'for file data'
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   263
                                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   264
                            )
40330
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   265
00a4cd368e3f exchangev2: support for calling rawstorefiledata to retrieve raw files
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40327
diff changeset
   266
                if bytesremaining:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   267
                    raise error.Abort(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   268
                        _(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   269
                            b'received invalid number of bytes for'
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   270
                            b'file data; expected %d got %d'
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   271
                        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   272
                        % (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   273
                            filemeta[b'size'],
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   274
                            filemeta[b'size'] - bytesremaining,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   275
                        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   276
                    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   277
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   278
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   279
def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   280
    """Determine which changesets need to be pulled."""
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   281
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   282
    if heads:
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   283
        knownnode = repo.changelog.hasnode
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   284
        if all(knownnode(head) for head in heads):
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   285
            return heads, False, heads
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   286
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   287
    # TODO wire protocol version 2 is capable of more efficient discovery
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   288
    # than setdiscovery. Consider implementing something better.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   289
    common, fetch, remoteheads = setdiscovery.findcommonheads(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   290
        repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   291
    )
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   292
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   293
    common = set(common)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   294
    remoteheads = set(remoteheads)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   295
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   296
    # If a remote head is filtered locally, put it back in the common set.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   297
    # See the comment in exchange._pulldiscoverychangegroup() for more.
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   298
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   299
    if fetch and remoteheads:
43546
a166fadf5c3b index: use `index.has_node` in `exchangev2._pullchangesetdiscovery`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43117
diff changeset
   300
        has_node = repo.unfiltered().changelog.index.has_node
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   301
43546
a166fadf5c3b index: use `index.has_node` in `exchangev2._pullchangesetdiscovery`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43117
diff changeset
   302
        common |= {head for head in remoteheads if has_node(head)}
39629
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   303
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   304
        if set(remoteheads).issubset(common):
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   305
            fetch = []
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   306
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   307
    common.discard(nullid)
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   308
a86d21e70b2b exchangev2: start to implement pull with wire protocol v2
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
   309
    return common, fetch, remoteheads
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   310
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   311
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   312
def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads):
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   313
    # TODO consider adding a step here where we obtain the DAG shape first
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   314
    # (or ask the server to slice changesets into chunks for us) so that
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   315
    # we can perform multiple fetches in batches. This will facilitate
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   316
    # resuming interrupted clones, higher server-side cache hit rates due
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   317
    # to smaller segments, etc.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   318
    with remote.commandexecutor() as e:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   319
        objs = e.callcommand(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   320
            b'changesetdata',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   321
            {
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   322
                b'revisions': [
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   323
                    {
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   324
                        b'type': b'changesetdagrange',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   325
                        b'roots': sorted(common),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   326
                        b'heads': sorted(remoteheads),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   327
                    }
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   328
                ],
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   329
                b'fields': {b'bookmarks', b'parents', b'phase', b'revision'},
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   330
            },
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   331
        ).result()
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   332
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   333
        # The context manager waits on all response data when exiting. So
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   334
        # we need to remain in the context manager in order to stream data.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   335
        return _processchangesetdata(repo, tr, objs)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   336
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   337
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   338
def _processchangesetdata(repo, tr, objs):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   339
    repo.hook(b'prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs))
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   340
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   341
    urepo = repo.unfiltered()
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   342
    cl = urepo.changelog
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   343
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   344
    cl.delayupdate(tr)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   345
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   346
    # The first emitted object is a header describing the data that
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   347
    # follows.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   348
    meta = next(objs)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   349
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   350
    progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   351
        _(b'changesets'), unit=_(b'chunks'), total=meta.get(b'totalitems')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   352
    )
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   353
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   354
    manifestnodes = {}
45788
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   355
    added = []
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   356
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   357
    def linkrev(node):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   358
        repo.ui.debug(b'add changeset %s\n' % short(node))
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   359
        # Linkrev for changelog is always self.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   360
        return len(cl)
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   361
46509
7a93b7b3dc2d revlog: change addgroup callbacks to take revision numbers
Joerg Sonnenberger <joerg@bec.de>
parents: 46373
diff changeset
   362
    def ondupchangeset(cl, rev):
7a93b7b3dc2d revlog: change addgroup callbacks to take revision numbers
Joerg Sonnenberger <joerg@bec.de>
parents: 46373
diff changeset
   363
        added.append(cl.node(rev))
45788
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   364
46509
7a93b7b3dc2d revlog: change addgroup callbacks to take revision numbers
Joerg Sonnenberger <joerg@bec.de>
parents: 46373
diff changeset
   365
    def onchangeset(cl, rev):
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   366
        progress.increment()
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   367
46358
009130064813 exchangev2: avoid second look-up by node
Joerg Sonnenberger <joerg@bec.de>
parents: 45942
diff changeset
   368
        revision = cl.changelogrevision(rev)
46509
7a93b7b3dc2d revlog: change addgroup callbacks to take revision numbers
Joerg Sonnenberger <joerg@bec.de>
parents: 46373
diff changeset
   369
        added.append(cl.node(rev))
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   370
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   371
        # We need to preserve the mapping of changelog revision to node
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   372
        # so we can set the linkrev accordingly when manifests are added.
46358
009130064813 exchangev2: avoid second look-up by node
Joerg Sonnenberger <joerg@bec.de>
parents: 45942
diff changeset
   373
        manifestnodes[rev] = revision.manifest
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   374
46371
0903d6b9b1df repository: introduce register_changeset callback
Joerg Sonnenberger <joerg@bec.de>
parents: 46358
diff changeset
   375
        repo.register_changeset(rev, revision)
0903d6b9b1df repository: introduce register_changeset callback
Joerg Sonnenberger <joerg@bec.de>
parents: 46358
diff changeset
   376
45117
b1e51ef4e536 phases: sparsify phase lists
Joerg Sonnenberger <joerg@bec.de>
parents: 43546
diff changeset
   377
    nodesbyphase = {phase: set() for phase in phases.phasenames.values()}
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   378
    remotebookmarks = {}
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   379
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   380
    # addgroup() expects a 7-tuple describing revisions. This normalizes
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   381
    # the wire data to that format.
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   382
    #
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   383
    # This loop also aggregates non-revision metadata, such as phase
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   384
    # data.
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   385
    def iterrevisions():
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   386
        for cset in objs:
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   387
            node = cset[b'node']
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   388
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   389
            if b'phase' in cset:
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   390
                nodesbyphase[cset[b'phase']].add(node)
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   391
39635
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   392
            for mark in cset.get(b'bookmarks', []):
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   393
                remotebookmarks[mark] = node
349482d726ee exchangev2: fetch and apply bookmarks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39633
diff changeset
   394
39636
399ddd3227a4 wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39635
diff changeset
   395
            # TODO add mechanism for extensions to examine records so they
399ddd3227a4 wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39635
diff changeset
   396
            # can siphon off custom data fields.
399ddd3227a4 wireprotov2: add TODOs around extending changesetdata fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39635
diff changeset
   397
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   398
            extrafields = {}
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   399
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   400
            for field, size in cset.get(b'fieldsfollowing', []):
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   401
                extrafields[field] = next(objs)
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   402
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   403
            # Some entries might only be metadata only updates.
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   404
            if b'revision' not in extrafields:
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   405
                continue
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   406
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   407
            data = extrafields[b'revision']
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   408
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   409
            yield (
39633
ff2de4f2eb3c exchangev2: fetch and apply phases data
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39631
diff changeset
   410
                node,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   411
                cset[b'parents'][0],
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   412
                cset[b'parents'][1],
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   413
                # Linknode is always itself for changesets.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   414
                cset[b'node'],
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   415
                # We always send full revisions. So delta base is not set.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   416
                nullid,
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   417
                mdiff.trivialdiffheader(len(data)) + data,
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   418
                # Flags not yet supported.
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   419
                0,
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   420
            )
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   421
45788
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   422
    cl.addgroup(
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   423
        iterrevisions(),
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   424
        linkrev,
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   425
        weakref.proxy(tr),
46373
711ba0f1057e revlog: decouple caching from addrevision callback for addgroup
Joerg Sonnenberger <joerg@bec.de>
parents: 46371
diff changeset
   426
        alwayscache=True,
45788
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   427
        addrevisioncb=onchangeset,
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   428
        duplicaterevisioncb=ondupchangeset,
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   429
    )
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   430
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   431
    progress.complete()
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   432
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   433
    return {
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   434
        b'added': added,
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   435
        b'nodesbyphase': nodesbyphase,
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   436
        b'bookmarks': remotebookmarks,
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   437
        b'manifestnodes': manifestnodes,
39631
b9e453d683a1 exchangev2: fetch changeset revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39629
diff changeset
   438
    }
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   439
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   440
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   441
def _fetchmanifests(repo, tr, remote, manifestnodes):
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   442
    rootmanifest = repo.manifestlog.getstorage(b'')
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   443
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   444
    # Some manifests can be shared between changesets. Filter out revisions
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   445
    # we already know about.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   446
    fetchnodes = []
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   447
    linkrevs = {}
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   448
    seen = set()
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   449
43106
d783f945a701 py3: finish porting iteritems() to pycompat and remove source transformer
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43104
diff changeset
   450
    for clrev, node in sorted(pycompat.iteritems(manifestnodes)):
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   451
        if node in seen:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   452
            continue
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   453
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   454
        try:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   455
            rootmanifest.rev(node)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   456
        except error.LookupError:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   457
            fetchnodes.append(node)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   458
            linkrevs[node] = clrev
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   459
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   460
        seen.add(node)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   461
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   462
    # TODO handle tree manifests
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   463
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   464
    # addgroup() expects 7-tuple describing revisions. This normalizes
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   465
    # the wire data to that format.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   466
    def iterrevisions(objs, progress):
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   467
        for manifest in objs:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   468
            node = manifest[b'node']
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   469
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   470
            extrafields = {}
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   471
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   472
            for field, size in manifest.get(b'fieldsfollowing', []):
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   473
                extrafields[field] = next(objs)
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   474
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   475
            if b'delta' in extrafields:
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   476
                basenode = manifest[b'deltabasenode']
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   477
                delta = extrafields[b'delta']
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   478
            elif b'revision' in extrafields:
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   479
                basenode = nullid
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   480
                revision = extrafields[b'revision']
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   481
                delta = mdiff.trivialdiffheader(len(revision)) + revision
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   482
            else:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   483
                continue
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   484
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   485
            yield (
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   486
                node,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   487
                manifest[b'parents'][0],
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   488
                manifest[b'parents'][1],
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   489
                # The value passed in is passed to the lookup function passed
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   490
                # to addgroup(). We already have a map of manifest node to
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   491
                # changelog revision number. So we just pass in the
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   492
                # manifest node here and use linkrevs.__getitem__ as the
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   493
                # resolution function.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   494
                node,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   495
                basenode,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   496
                delta,
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   497
                # Flags not yet supported.
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   498
                0,
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   499
            )
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   500
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   501
            progress.increment()
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   502
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   503
    progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   504
        _(b'manifests'), unit=_(b'chunks'), total=len(fetchnodes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   505
    )
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   506
40173
b797150a1ab9 exchangev2: honor server advertised manifestdata recommended batch size
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40035
diff changeset
   507
    commandmeta = remote.apidescriptor[b'commands'][b'manifestdata']
b797150a1ab9 exchangev2: honor server advertised manifestdata recommended batch size
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40035
diff changeset
   508
    batchsize = commandmeta.get(b'recommendedbatchsize', 10000)
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   509
    # TODO make size configurable on client?
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   510
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   511
    # We send commands 1 at a time to the remote. This is not the most
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   512
    # efficient because we incur a round trip at the end of each batch.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   513
    # However, the existing frame-based reactor keeps consuming server
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   514
    # data in the background. And this results in response data buffering
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   515
    # in memory. This can consume gigabytes of memory.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   516
    # TODO send multiple commands in a request once background buffering
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   517
    # issues are resolved.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   518
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   519
    added = []
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   520
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   521
    for i in pycompat.xrange(0, len(fetchnodes), batchsize):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   522
        batch = [node for node in fetchnodes[i : i + batchsize]]
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   523
        if not batch:
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   524
            continue
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   525
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   526
        with remote.commandexecutor() as e:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   527
            objs = e.callcommand(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   528
                b'manifestdata',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   529
                {
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   530
                    b'tree': b'',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   531
                    b'nodes': batch,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   532
                    b'fields': {b'parents', b'revision'},
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   533
                    b'haveparents': True,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   534
                },
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   535
            ).result()
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   536
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   537
            # Chomp off header object.
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   538
            next(objs)
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   539
46509
7a93b7b3dc2d revlog: change addgroup callbacks to take revision numbers
Joerg Sonnenberger <joerg@bec.de>
parents: 46373
diff changeset
   540
            def onchangeset(cl, rev):
7a93b7b3dc2d revlog: change addgroup callbacks to take revision numbers
Joerg Sonnenberger <joerg@bec.de>
parents: 46373
diff changeset
   541
                added.append(cl.node(rev))
45788
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   542
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   543
            rootmanifest.addgroup(
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   544
                iterrevisions(objs, progress),
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   545
                linkrevs.__getitem__,
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   546
                weakref.proxy(tr),
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   547
                addrevisioncb=onchangeset,
a5206e71c536 revlog: extend addgroup() with callback for duplicates
Joerg Sonnenberger <joerg@bec.de>
parents: 45117
diff changeset
   548
                duplicaterevisioncb=onchangeset,
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   549
            )
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   550
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   551
    progress.complete()
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   552
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   553
    return {
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   554
        b'added': added,
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   555
        b'linkrevs': linkrevs,
39638
d292328e0143 exchangev2: fetch manifest revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39636
diff changeset
   556
    }
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   557
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   558
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   559
def _derivefilesfrommanifests(repo, matcher, manifestnodes):
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   560
    """Determine what file nodes are relevant given a set of manifest nodes.
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   561
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   562
    Returns a dict mapping file paths to dicts of file node to first manifest
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   563
    node.
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   564
    """
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   565
    ml = repo.manifestlog
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   566
    fnodes = collections.defaultdict(dict)
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   567
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   568
    progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   569
        _(b'scanning manifests'), total=len(manifestnodes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   570
    )
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   571
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   572
    with progress:
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   573
        for manifestnode in manifestnodes:
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   574
            m = ml.get(b'', manifestnode)
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   575
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   576
            # TODO this will pull in unwanted nodes because it takes the storage
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   577
            # delta into consideration. What we really want is something that
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   578
            # takes the delta between the manifest's parents. And ideally we
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   579
            # would ignore file nodes that are known locally. For now, ignore
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   580
            # both these limitations. This will result in incremental fetches
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   581
            # requesting data we already have. So this is far from ideal.
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   582
            md = m.readfast()
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   583
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   584
            for path, fnode in md.items():
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   585
                if matcher(path):
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   586
                    fnodes[path].setdefault(fnode, manifestnode)
40035
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   587
7a347d362a45 exchangev2: add progress bar around manifest scanning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39814
diff changeset
   588
            progress.increment()
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   589
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   590
    return fnodes
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   591
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   592
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   593
def _fetchfiles(repo, tr, remote, fnodes, linkrevs):
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   594
    """Fetch file data from explicit file revisions."""
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   595
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   596
    def iterrevisions(objs, progress):
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   597
        for filerevision in objs:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   598
            node = filerevision[b'node']
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   599
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   600
            extrafields = {}
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   601
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   602
            for field, size in filerevision.get(b'fieldsfollowing', []):
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   603
                extrafields[field] = next(objs)
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   604
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   605
            if b'delta' in extrafields:
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   606
                basenode = filerevision[b'deltabasenode']
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   607
                delta = extrafields[b'delta']
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   608
            elif b'revision' in extrafields:
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   609
                basenode = nullid
39814
d059cb669632 wireprotov2: allow multiple fields to follow revision maps
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39641
diff changeset
   610
                revision = extrafields[b'revision']
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   611
                delta = mdiff.trivialdiffheader(len(revision)) + revision
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   612
            else:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   613
                continue
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   614
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   615
            yield (
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   616
                node,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   617
                filerevision[b'parents'][0],
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   618
                filerevision[b'parents'][1],
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   619
                node,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   620
                basenode,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   621
                delta,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   622
                # Flags not yet supported.
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   623
                0,
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   624
            )
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   625
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   626
            progress.increment()
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   627
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   628
    progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   629
        _(b'files'),
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   630
        unit=_(b'chunks'),
43104
74802979dd9d py3: define and use pycompat.itervalues()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
   631
        total=sum(len(v) for v in pycompat.itervalues(fnodes)),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   632
    )
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   633
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   634
    # TODO make batch size configurable
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   635
    batchsize = 10000
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   636
    fnodeslist = [x for x in sorted(fnodes.items())]
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   637
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   638
    for i in pycompat.xrange(0, len(fnodeslist), batchsize):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   639
        batch = [x for x in fnodeslist[i : i + batchsize]]
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   640
        if not batch:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   641
            continue
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   642
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   643
        with remote.commandexecutor() as e:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   644
            fs = []
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   645
            locallinkrevs = {}
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   646
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   647
            for path, nodes in batch:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   648
                fs.append(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   649
                    (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   650
                        path,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   651
                        e.callcommand(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   652
                            b'filedata',
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   653
                            {
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   654
                                b'path': path,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   655
                                b'nodes': sorted(nodes),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   656
                                b'fields': {b'parents', b'revision'},
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   657
                                b'haveparents': True,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   658
                            },
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   659
                        ),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   660
                    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   661
                )
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   662
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   663
                locallinkrevs[path] = {
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   664
                    node: linkrevs[manifestnode]
43106
d783f945a701 py3: finish porting iteritems() to pycompat and remove source transformer
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43104
diff changeset
   665
                    for node, manifestnode in pycompat.iteritems(nodes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   666
                }
39640
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   667
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   668
            for path, f in fs:
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   669
                objs = f.result()
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   670
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   671
                # Chomp off header objects.
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   672
                next(objs)
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   673
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   674
                store = repo.file(path)
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   675
                store.addgroup(
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   676
                    iterrevisions(objs, progress),
039bf1eddc2e exchangev2: fetch file revisions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39638
diff changeset
   677
                    locallinkrevs[path].__getitem__,
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   678
                    weakref.proxy(tr),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   679
                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   680
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   681
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   682
def _fetchfilesfromcsets(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   683
    repo, tr, remote, pathfilter, fnodes, csets, manlinkrevs, shallow=False
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   684
):
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   685
    """Fetch file data from explicit changeset revisions."""
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   686
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   687
    def iterrevisions(objs, remaining, progress):
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   688
        while remaining:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   689
            filerevision = next(objs)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   690
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   691
            node = filerevision[b'node']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   692
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   693
            extrafields = {}
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   694
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   695
            for field, size in filerevision.get(b'fieldsfollowing', []):
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   696
                extrafields[field] = next(objs)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   697
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   698
            if b'delta' in extrafields:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   699
                basenode = filerevision[b'deltabasenode']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   700
                delta = extrafields[b'delta']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   701
            elif b'revision' in extrafields:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   702
                basenode = nullid
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   703
                revision = extrafields[b'revision']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   704
                delta = mdiff.trivialdiffheader(len(revision)) + revision
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   705
            else:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   706
                continue
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   707
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   708
            if b'linknode' in filerevision:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   709
                linknode = filerevision[b'linknode']
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   710
            else:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   711
                linknode = node
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   712
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   713
            yield (
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   714
                node,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   715
                filerevision[b'parents'][0],
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   716
                filerevision[b'parents'][1],
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   717
                linknode,
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   718
                basenode,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   719
                delta,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   720
                # Flags not yet supported.
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   721
                0,
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   722
            )
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   723
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   724
            progress.increment()
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   725
            remaining -= 1
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   726
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   727
    progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   728
        _(b'files'),
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   729
        unit=_(b'chunks'),
43104
74802979dd9d py3: define and use pycompat.itervalues()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
   730
        total=sum(len(v) for v in pycompat.itervalues(fnodes)),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   731
    )
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   732
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   733
    commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   734
    batchsize = commandmeta.get(b'recommendedbatchsize', 50000)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   735
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   736
    shallowfiles = repository.REPO_FEATURE_SHALLOW_FILE_STORAGE in repo.features
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   737
    fields = {b'parents', b'revision'}
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   738
    clrev = repo.changelog.rev
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   739
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   740
    # There are no guarantees that we'll have ancestor revisions if
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   741
    # a) this repo has shallow file storage b) shallow data fetching is enabled.
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   742
    # Force remote to not delta against possibly unknown revisions when these
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   743
    # conditions hold.
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   744
    haveparents = not (shallowfiles or shallow)
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   745
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   746
    # Similarly, we may not have calculated linkrevs for all incoming file
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   747
    # revisions. Ask the remote to do work for us in this case.
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   748
    if not haveparents:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   749
        fields.add(b'linknode')
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   750
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   751
    for i in pycompat.xrange(0, len(csets), batchsize):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   752
        batch = [x for x in csets[i : i + batchsize]]
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   753
        if not batch:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   754
            continue
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   755
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   756
        with remote.commandexecutor() as e:
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   757
            args = {
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   758
                b'revisions': [
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
   759
                    {
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
   760
                        b'type': b'changesetexplicit',
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
   761
                        b'nodes': batch,
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45790
diff changeset
   762
                    }
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   763
                ],
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   764
                b'fields': fields,
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   765
                b'haveparents': haveparents,
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   766
            }
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   767
40327
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   768
            if pathfilter:
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   769
                args[b'pathfilter'] = pathfilter
55836a34f41b exchangev2: recognize narrow patterns when pulling
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40179
diff changeset
   770
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   771
            objs = e.callcommand(b'filesdata', args).result()
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   772
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   773
            # First object is an overall header.
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   774
            overall = next(objs)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   775
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   776
            # We have overall['totalpaths'] segments.
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   777
            for i in pycompat.xrange(overall[b'totalpaths']):
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   778
                header = next(objs)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   779
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   780
                path = header[b'path']
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   781
                store = repo.file(path)
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   782
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   783
                linkrevs = {
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   784
                    fnode: manlinkrevs[mnode]
43106
d783f945a701 py3: finish porting iteritems() to pycompat and remove source transformer
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43104
diff changeset
   785
                    for fnode, mnode in pycompat.iteritems(fnodes[path])
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   786
                }
40179
b843356d4ae1 exchangev2: use filesdata
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40176
diff changeset
   787
40393
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   788
                def getlinkrev(node):
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   789
                    if node in linkrevs:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   790
                        return linkrevs[node]
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   791
                    else:
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   792
                        return clrev(node)
229d23cdb203 exchangev2: support fetching shallow files history
Gregory Szorc <gregory.szorc@gmail.com>
parents: 40330
diff changeset
   793
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   794
                store.addgroup(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   795
                    iterrevisions(objs, header[b'totalitems'], progress),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   796
                    getlinkrev,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   797
                    weakref.proxy(tr),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   798
                    maybemissingparents=shallow,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   799
                )