mercurial/streamclone.py
author Pierre-Yves David <pierre-yves.david@octobus.net>
Mon, 29 May 2023 04:24:39 +0200
changeset 50635 3416b46320dc
parent 50634 1415e17ea20f
child 50636 c90ea9bbf327
permissions -rw-r--r--
stream-clone: directly use `_entries_walk` to generate stream-v2 This does not requires that much changes and will give us much more flexibility, like improving revlog handling to gracefully handle race situation.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
26441
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     1
# streamclone.py - producing and consuming streaming repository data
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     2
#
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     3
# Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com>
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     4
#
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     5
# This software may be used and distributed according to the terms of the
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     6
# GNU General Public License version 2 or any later version.
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     7
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     8
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
     9
import contextlib
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
    10
import os
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
    11
import struct
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
    12
26442
ef8d27f53204 streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26441
diff changeset
    13
from .i18n import _
43085
eef9a2d67051 py3: manually import pycompat.open into files that need it
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
    14
from .pycompat import open
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    15
from .interfaces import repository
26441
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    16
from . import (
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
    17
    bookmarks,
50523
a6543983b8f4 stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50522
diff changeset
    18
    bundle2 as bundle2mod,
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
    19
    cacheutil,
26442
ef8d27f53204 streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26441
diff changeset
    20
    error,
40339
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
    21
    narrowspec,
32744
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
    22
    phases,
38164
aac4be30e250 py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents: 37638
diff changeset
    23
    pycompat,
46626
ee91966aec0f requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents: 45106
diff changeset
    24
    requirements as requirementsmod,
45106
a03c177a4679 scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents: 43117
diff changeset
    25
    scmutil,
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
    26
    store,
50299
3d0b5760851c undo-files: move the undo cleanup code in the transaction module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50292
diff changeset
    27
    transaction,
26442
ef8d27f53204 streamclone: move stream_in() from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26441
diff changeset
    28
    util,
26441
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    29
)
48693
de3ac3d2c60b stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48681
diff changeset
    30
from .revlogutils import (
de3ac3d2c60b stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48681
diff changeset
    31
    nodemap,
de3ac3d2c60b stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48681
diff changeset
    32
)
26441
56527b886d1d streamclone: move applystreamclone() from localrepo.py
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    33
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    34
48601
baddab229b86 stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48600
diff changeset
    35
def new_stream_clone_requirements(default_requirements, streamed_requirements):
48596
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    36
    """determine the final set of requirement for a new stream clone
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    37
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    38
    this method combine the "default" requirements that a new repository would
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    39
    use with the constaint we get from the stream clone content. We keep local
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    40
    configuration choice when possible.
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    41
    """
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    42
    requirements = set(default_requirements)
48601
baddab229b86 stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48600
diff changeset
    43
    requirements -= requirementsmod.STREAM_FIXED_REQUIREMENTS
48596
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    44
    requirements.update(streamed_requirements)
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    45
    return requirements
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    46
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
    47
48597
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
    48
def streamed_requirements(repo):
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
    49
    """the set of requirement the new clone will have to support
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
    50
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
    51
    This is used for advertising the stream options and to generate the actual
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
    52
    stream content."""
48601
baddab229b86 stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48600
diff changeset
    53
    requiredformats = (
baddab229b86 stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48600
diff changeset
    54
        repo.requirements & requirementsmod.STREAM_FIXED_REQUIREMENTS
baddab229b86 stream-clone: add a explicit set list requirements relevant to stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48600
diff changeset
    55
    )
48597
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
    56
    return requiredformats
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
    57
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
    58
35757
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    59
def canperformstreamclone(pullop, bundle2=False):
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    60
    """Whether it is possible to perform a streaming clone as part of pull.
26445
f134fb33c906 streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26444
diff changeset
    61
35757
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    62
    ``bundle2`` will cause the function to consider stream clone through
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
    63
    bundle2 and only through bundle2.
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    64
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    65
    Returns a tuple of (supported, requirements). ``supported`` is True if
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    66
    streaming clone is supported and False otherwise. ``requirements`` is
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    67
    a set of repo requirements from the remote, or ``None`` if stream clone
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    68
    isn't supported.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
    69
    """
26466
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    70
    repo = pullop.repo
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    71
    remote = pullop.remote
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
    72
50522
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    73
    # should we consider streaming clone at all ?
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    74
    streamrequested = pullop.streamclonerequested
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    75
    # If we don't have a preference, let the server decide for us. This
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    76
    # likely only comes into play in LANs.
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    77
    if streamrequested is None:
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    78
        # The server can advertise whether to prefer streaming clone.
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    79
        streamrequested = remote.capable(b'stream-preferred')
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    80
    if not streamrequested:
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    81
        return False, None
58e4842fbfc1 stream-clone: bail-out earlier if stream clone is not requested
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50521
diff changeset
    82
50520
0558866957fa stream-clone: bail-out earlier if destination repo is not empty
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50516
diff changeset
    83
    # Streaming clone only works on an empty destination repository
0558866957fa stream-clone: bail-out earlier if destination repo is not empty
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50516
diff changeset
    84
    if len(repo):
0558866957fa stream-clone: bail-out earlier if destination repo is not empty
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50516
diff changeset
    85
        return False, None
0558866957fa stream-clone: bail-out earlier if destination repo is not empty
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50516
diff changeset
    86
50521
f697af015683 stream-clone: bail-out earlier if pull is partial
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50520
diff changeset
    87
    # Streaming clone only works if all data is being requested.
f697af015683 stream-clone: bail-out earlier if pull is partial
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50520
diff changeset
    88
    if pullop.heads:
f697af015683 stream-clone: bail-out earlier if pull is partial
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50520
diff changeset
    89
        return False, None
f697af015683 stream-clone: bail-out earlier if pull is partial
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50520
diff changeset
    90
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    91
    bundle2supported = False
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    92
    if pullop.canusebundle2:
50523
a6543983b8f4 stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50522
diff changeset
    93
        local_caps = bundle2mod.getrepocaps(repo, role=b'client')
a6543983b8f4 stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50522
diff changeset
    94
        local_supported = set(local_caps.get(b'stream', []))
a6543983b8f4 stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50522
diff changeset
    95
        remote_supported = set(pullop.remotebundle2caps.get(b'stream', []))
a6543983b8f4 stream-clone: check is a compatible protocol can be found
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50522
diff changeset
    96
        bundle2supported = bool(local_supported & remote_supported)
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
    97
        # else
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    98
        # Server doesn't support bundle2 stream clone or doesn't support
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
    99
        # the versions we support. Fall back and possibly allow legacy.
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
   100
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
   101
    # Ensures legacy code path uses available bundle2.
35757
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
   102
    if bundle2supported and not bundle2:
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
   103
        return False, None
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
   104
    # Ensures bundle2 doesn't try to do a stream clone if it isn't supported.
35757
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
   105
    elif bundle2 and not bundle2supported:
bbf7abd09ff0 streamclone: rework canperformstreamclone
Boris Feld <boris.feld@octobus.net>
parents: 35756
diff changeset
   106
        return False, None
26467
ff2c89239d49 streamclone: teach canperformstreamclone to be bundle2 aware
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26466
diff changeset
   107
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   108
    # In order for stream clone to work, the client has to support all the
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   109
    # requirements advertised by the server.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   110
    #
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   111
    # The server advertises its requirements via the "stream" and "streamreqs"
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   112
    # capability. "stream" (a value-less capability) is advertised if and only
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   113
    # if the only requirement is "revlogv1." Else, the "streamreqs" capability
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   114
    # is advertised and contains a comma-delimited list of requirements.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   115
    requirements = set()
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   116
    if remote.capable(b'stream'):
46626
ee91966aec0f requirements: add constant for revlog v1 requirement
Raphaël Gomès <rgomes@octobus.net>
parents: 45106
diff changeset
   117
        requirements.add(requirementsmod.REVLOGV1_REQUIREMENT)
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   118
    else:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   119
        streamreqs = remote.capable(b'streamreqs')
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   120
        # This is weird and shouldn't happen with modern servers.
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   121
        if not streamreqs:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   122
            pullop.repo.ui.warn(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   123
                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   124
                    b'warning: stream clone requested but server has them '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   125
                    b'disabled\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   126
                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   127
            )
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   128
            return False, None
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   129
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   130
        streamreqs = set(streamreqs.split(b','))
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   131
        # Server requires something we don't support. Bail.
48670
6d2ddea0721a stream-clone: filter possible missing requirements using all supported one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
   132
        missingreqs = streamreqs - repo.supported
32259
076f1ff43f0f clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents: 30975
diff changeset
   133
        if missingreqs:
076f1ff43f0f clone: warn when streaming was requested but couldn't be performed
Siddharth Agarwal <sid0@fb.com>
parents: 30975
diff changeset
   134
            pullop.repo.ui.warn(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   135
                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   136
                    b'warning: stream clone requested but client is missing '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   137
                    b'requirements: %s\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   138
                )
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   139
                % b', '.join(sorted(missingreqs))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   140
            )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   141
            pullop.repo.ui.warn(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   142
                _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   143
                    b'(see https://www.mercurial-scm.org/wiki/MissingRequirement '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   144
                    b'for more information)\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   145
                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   146
            )
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   147
            return False, None
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   148
        requirements = streamreqs
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   149
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   150
    return True, requirements
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   151
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   152
26462
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   153
def maybeperformlegacystreamclone(pullop):
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   154
    """Possibly perform a legacy stream clone operation.
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   155
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   156
    Legacy stream clones are performed as part of pull but before all other
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   157
    operations.
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   158
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   159
    A legacy stream clone will not be performed if a bundle2 stream clone is
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   160
    supported.
3b0ec09192ae streamclone: rename and document maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26461
diff changeset
   161
    """
39700
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   162
    from . import localrepo
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   163
26466
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
   164
    supported, requirements = canperformstreamclone(pullop)
26458
362793295640 streamclone: refactor maybeperformstreamclone to take a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26447
diff changeset
   165
26446
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   166
    if not supported:
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   167
        return
3ea10bb761ce streamclone: refactor code for deciding to stream clone
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26445
diff changeset
   168
26466
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
   169
    repo = pullop.repo
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
   170
    remote = pullop.remote
3515db5aae05 streamclone: refactor canperformstreamclone to accept a pullop
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26462
diff changeset
   171
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   172
    # Save remote branchmap. We will use it later to speed up branchcache
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   173
    # creation.
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   174
    rbranchmap = None
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   175
    if remote.capable(b'branchmap'):
37638
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   176
        with remote.commandexecutor() as e:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   177
            rbranchmap = e.callcommand(b'branchmap', {}).result()
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   178
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   179
    repo.ui.status(_(b'streaming all changes\n'))
26470
4b5647d9ee13 streamclone: move "streaming all changes" message location
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26469
diff changeset
   180
37638
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   181
    with remote.commandexecutor() as e:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   182
        fp = e.callcommand(b'stream_out', {}).result()
37638
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   183
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   184
    # TODO strictly speaking, this code should all be inside the context
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   185
    # manager because the context manager is supposed to ensure all wire state
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   186
    # is flushed when exiting. But the legacy peers don't do this, so it
65b86ee69383 streamclone: use command executor for wire protocol commands
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35804
diff changeset
   187
    # doesn't matter.
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   188
    l = fp.readline()
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   189
    try:
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   190
        resp = int(l)
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   191
    except ValueError:
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   192
        raise error.ResponseError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   193
            _(b'unexpected response from remote server:'), l
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   194
        )
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   195
    if resp == 1:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   196
        raise error.Abort(_(b'operation forbidden by server'))
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   197
    elif resp == 2:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   198
        raise error.Abort(_(b'locking the remote repository failed'))
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   199
    elif resp != 0:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   200
        raise error.Abort(_(b'the server sent an unknown error code'))
26459
3b28ffde133a streamclone: move streamin() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26458
diff changeset
   201
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   202
    l = fp.readline()
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   203
    try:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   204
        filecount, bytecount = map(int, l.split(b' ', 1))
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   205
    except (ValueError, TypeError):
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   206
        raise error.ResponseError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   207
            _(b'unexpected response from remote server:'), l
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   208
        )
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   209
27850
49cfddbf54ba with: use context manager in maybeperformlegacystreamclone
Bryan O'Sullivan <bryano@fb.com>
parents: 27845
diff changeset
   210
    with repo.lock():
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   211
        consumev1(repo, fp, filecount, bytecount)
48596
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
   212
        repo.requirements = new_stream_clone_requirements(
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
   213
            repo.requirements,
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
   214
            requirements,
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   215
        )
39700
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   216
        repo.svfs.options = localrepo.resolvestorevfsoptions(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   217
            repo.ui, repo.requirements, repo.features
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   218
        )
45106
a03c177a4679 scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents: 43117
diff changeset
   219
        scmutil.writereporequirements(repo)
48693
de3ac3d2c60b stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48681
diff changeset
   220
        nodemap.post_stream_cleanup(repo)
26461
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   221
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   222
        if rbranchmap:
41615
328ca3b9e545 branchmap: encapsulate cache updating in the map itself
Martijn Pieters <mj@octobus.net>
parents: 40494
diff changeset
   223
            repo._branchcaches.replace(repo, rbranchmap)
26461
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   224
09cc3c2e9ece streamclone: move applyremotedata() into maybeperformstreamclone()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26460
diff changeset
   225
        repo.invalidate()
26445
f134fb33c906 streamclone: move streaming clone logic from localrepo
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26444
diff changeset
   226
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   227
32744
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   228
def allowservergeneration(repo):
26444
623743010133 streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26443
diff changeset
   229
    """Whether streaming clones are allowed from the server."""
40028
51f10e6d66c7 streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39850
diff changeset
   230
    if repository.REPO_FEATURE_STREAM_CLONE not in repo.features:
51f10e6d66c7 streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39850
diff changeset
   231
        return False
51f10e6d66c7 streamclone: don't support stream clone unless repo feature present
Gregory Szorc <gregory.szorc@gmail.com>
parents: 39850
diff changeset
   232
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   233
    if not repo.ui.configbool(b'server', b'uncompressed', untrusted=True):
32744
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   234
        return False
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   235
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   236
    # The way stream clone works makes it impossible to hide secret changesets.
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   237
    # So don't allow this by default.
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   238
    secret = phases.hassecret(repo)
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   239
    if secret:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   240
        return repo.ui.configbool(b'server', b'uncompressedallowsecret')
32744
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   241
33b7283a3828 streamclone: consider secret changesets (BC) (issue5589)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 32259
diff changeset
   242
    return True
26444
623743010133 streamclone: move _allowstream() from wireproto
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26443
diff changeset
   243
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   244
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   245
# This is it's own function so extensions can override it.
50514
0925eaf09c8b store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50512
diff changeset
   246
def _walkstreamfiles(repo, matcher=None, phase=False, obsolescence=False):
0925eaf09c8b store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50512
diff changeset
   247
    return repo.store.walk(matcher, phase=phase, obsolescence=obsolescence)
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   248
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   249
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   250
def generatev1(repo):
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   251
    """Emit content for version 1 of a streaming clone.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   252
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   253
    This returns a 3-tuple of (file count, byte size, data iterator).
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   254
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   255
    The data iterator consists of N entries for each file being transferred.
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   256
    Each file entry starts as a line with the file name and integer size
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   257
    delimited by a null byte.
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   258
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   259
    The raw file data follows. Following the raw file data is the next file
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   260
    entry, or EOF.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   261
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   262
    When used on the wire protocol, an additional line indicating protocol
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   263
    success will be prepended to the stream. This function is not responsible
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   264
    for adding it.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   265
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   266
    This function will obtain a repository lock to ensure a consistent view of
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   267
    the store is captured. It therefore may raise LockError.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   268
    """
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   269
    entries = []
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   270
    total_bytes = 0
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   271
    # Get consistent snapshot of repo, lock during scan.
27845
7417e1c10253 with: use context manager in streamclone generatev1
Bryan O'Sullivan <bryano@fb.com>
parents: 27794
diff changeset
   272
    with repo.lock():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   273
        repo.ui.debug(b'scanning\n')
50471
521fec115dad store: use a StoreEntry object instead of tuple for store files
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50301
diff changeset
   274
        for entry in _walkstreamfiles(repo):
50472
9fdc28e21b68 store: introduce a EntryFile object to actually access file info
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50471
diff changeset
   275
            for f in entry.files():
50477
4cbdfab6f812 store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50472
diff changeset
   276
                file_size = f.file_size(repo.store.vfs)
4cbdfab6f812 store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50472
diff changeset
   277
                if file_size:
4cbdfab6f812 store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50472
diff changeset
   278
                    entries.append((f.unencoded_path, file_size))
4cbdfab6f812 store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50472
diff changeset
   279
                    total_bytes += file_size
46986
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   280
        _test_sync_point_walk_1(repo)
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   281
    _test_sync_point_walk_2(repo)
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   282
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   283
    repo.ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   284
        b'%d files, %d bytes to transfer\n' % (len(entries), total_bytes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   285
    )
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   286
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   287
    svfs = repo.svfs
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   288
    debugflag = repo.ui.debugflag
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   289
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   290
    def emitrevlogdata():
33258
761ccfeff8b1 streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33257
diff changeset
   291
        for name, size in entries:
761ccfeff8b1 streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33257
diff changeset
   292
            if debugflag:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   293
                repo.ui.debug(b'sending %s (%d bytes)\n' % (name, size))
33258
761ccfeff8b1 streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33257
diff changeset
   294
            # partially encode name over the wire for backwards compat
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   295
            yield b'%s\0%d\n' % (store.encodedir(name), size)
33411
50b49bb0fff3 streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33410
diff changeset
   296
            # auditing at this stage is both pointless (paths are already
50b49bb0fff3 streamclone: comment why path auditing is disabled in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33410
diff changeset
   297
            # trusted by the local repo) and expensive
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   298
            with svfs(name, b'rb', auditpath=False) as fp:
33410
c784308305c6 streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33258
diff changeset
   299
                if size <= 65536:
33258
761ccfeff8b1 streamclone: stop using 'vfs.mustaudit = False'
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 33257
diff changeset
   300
                    yield fp.read(size)
33410
c784308305c6 streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33258
diff changeset
   301
                else:
c784308305c6 streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33258
diff changeset
   302
                    for chunk in util.filechunkiter(fp, limit=size):
c784308305c6 streamclone: close large revlog files explicitly in generatev1()
Yuya Nishihara <yuya@tcha.org>
parents: 33258
diff changeset
   303
                        yield chunk
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   304
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   305
    return len(entries), total_bytes, emitrevlogdata()
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   306
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   307
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   308
def generatev1wireproto(repo):
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   309
    """Emit content for version 1 of streaming clone suitable for the wire.
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   310
35491
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   311
    This is the data output from ``generatev1()`` with 2 header lines. The
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   312
    first line indicates overall success. The 2nd contains the file count and
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   313
    byte size of payload.
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   314
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   315
    The success line contains "0" for success, "1" for stream generation not
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   316
    allowed, and "2" for error locking the repository (possibly indicating
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   317
    a permissions error for the server process).
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   318
    """
35491
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   319
    if not allowservergeneration(repo):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   320
        yield b'1\n'
35491
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   321
        return
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   322
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   323
    try:
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   324
        filecount, bytecount, it = generatev1(repo)
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   325
    except error.LockError:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   326
        yield b'2\n'
35491
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   327
        return
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   328
ded3a63f305b streamclone: move wire protocol status code from wireproto command
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33499
diff changeset
   329
    # Indicates successful response.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   330
    yield b'0\n'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   331
    yield b'%d %d\n' % (filecount, bytecount)
26469
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   332
    for chunk in it:
fb743268510e streamclone: move payload header generation into own function
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26468
diff changeset
   333
        yield chunk
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   334
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   335
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   336
def generatebundlev1(repo, compression=b'UN'):
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   337
    """Emit content for version 1 of a stream clone bundle.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   338
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   339
    The first 4 bytes of the output ("HGS1") denote this as stream clone
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   340
    bundle version 1.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   341
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   342
    The next 2 bytes indicate the compression type. Only "UN" is currently
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   343
    supported.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   344
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   345
    The next 16 bytes are two 64-bit big endian unsigned integers indicating
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   346
    file count and byte count, respectively.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   347
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   348
    The next 2 bytes is a 16-bit big endian unsigned short declaring the length
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   349
    of the requirements string, including a trailing \0. The following N bytes
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   350
    are the requirements string, which is ASCII containing a comma-delimited
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   351
    list of repo requirements that are needed to support the data.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   352
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   353
    The remaining content is the output of ``generatev1()`` (which may be
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   354
    compressed in the future).
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   355
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   356
    Returns a tuple of (requirements, data generator).
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   357
    """
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   358
    if compression != b'UN':
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   359
        raise ValueError(b'we do not support the compression argument yet')
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   360
48597
8475a1364909 stream-clone: factor computation of requirement of a stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48596
diff changeset
   361
    requirements = streamed_requirements(repo)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   362
    requires = b','.join(sorted(requirements))
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   363
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   364
    def gen():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   365
        yield b'HGS1'
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   366
        yield compression
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   367
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   368
        filecount, bytecount, it = generatev1(repo)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   369
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   370
            _(b'writing %d bytes for %d files\n') % (bytecount, filecount)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   371
        )
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   372
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   373
        yield struct.pack(b'>QQ', filecount, bytecount)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   374
        yield struct.pack(b'>H', len(requires) + 1)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   375
        yield requires + b'\0'
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   376
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   377
        # This is where we'll add compression in the future.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   378
        assert compression == b'UN'
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   379
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   380
        progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   381
            _(b'bundle'), total=bytecount, unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   382
        )
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   383
        progress.update(0)
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   384
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   385
        for chunk in it:
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   386
            progress.increment(step=len(chunk))
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   387
            yield chunk
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   388
38373
ef692614e601 progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents: 38360
diff changeset
   389
        progress.complete()
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   390
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   391
    return requirements, gen()
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   392
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   393
26468
19bbd53af46d streamclone: move payload header line consumption
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26467
diff changeset
   394
def consumev1(repo, fp, filecount, bytecount):
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   395
    """Apply the contents from version 1 of a streaming clone file handle.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   396
30332
318a24b52eeb spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents: 29919
diff changeset
   397
    This takes the output from "stream_out" and applies it to the specified
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   398
    repository.
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   399
30332
318a24b52eeb spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents: 29919
diff changeset
   400
    Like "stream_out," the status line added by the wire protocol is not
318a24b52eeb spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents: 29919
diff changeset
   401
    handled by this function.
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   402
    """
27859
f55a5ace8e69 with: use context manager in streamclone consumev1
Bryan O'Sullivan <bryano@fb.com>
parents: 27850
diff changeset
   403
    with repo.lock():
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   404
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   405
            _(b'%d files to transfer, %s of data\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   406
            % (filecount, util.bytecount(bytecount))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   407
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   408
        progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   409
            _(b'clone'), total=bytecount, unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   410
        )
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   411
        progress.update(0)
30975
22fbca1d11ed mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents: 30332
diff changeset
   412
        start = util.timer()
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   413
29919
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   414
        # TODO: get rid of (potential) inconsistency
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   415
        #
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   416
        # If transaction is started and any @filecache property is
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   417
        # changed at this point, it causes inconsistency between
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   418
        # in-memory cached property and streamclone-ed file on the
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   419
        # disk. Nested transaction prevents transaction scope "clone"
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   420
        # below from writing in-memory changes out at the end of it,
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   421
        # even though in-memory changes are discarded at the end of it
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   422
        # regardless of transaction nesting.
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   423
        #
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   424
        # But transaction nesting can't be simply prohibited, because
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   425
        # nesting occurs also in ordinary case (e.g. enabling
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   426
        # clonebundles).
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   427
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   428
        with repo.transaction(b'clone'):
27897
2fdbf22a1b63 streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27896
diff changeset
   429
            with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount):
49284
d44e3c45f0e4 py3: replace `pycompat.xrange` by `range`
Manuel Jacob <me@manueljacob.de>
parents: 49190
diff changeset
   430
                for i in range(filecount):
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   431
                    # XXX doesn't support '\n' or '\r' in filenames
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   432
                    l = fp.readline()
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   433
                    try:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   434
                        name, size = l.split(b'\0', 1)
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   435
                        size = int(size)
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   436
                    except (ValueError, TypeError):
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   437
                        raise error.ResponseError(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   438
                            _(b'unexpected response from remote server:'), l
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   439
                        )
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   440
                    if repo.ui.debugflag:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   441
                        repo.ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   442
                            b'adding %s (%s)\n' % (name, util.bytecount(size))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   443
                        )
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   444
                    # for backwards compat, name was partially encoded
27897
2fdbf22a1b63 streamclone: use backgroundfilecloser (issue4889)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27896
diff changeset
   445
                    path = store.decodedir(name)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   446
                    with repo.svfs(path, b'w', backgroundclose=True) as ofp:
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   447
                        for chunk in util.filechunkiter(fp, limit=size):
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   448
                            progress.increment(step=len(chunk))
27896
1d29893240cc streamclone: indent code
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27882
diff changeset
   449
                            ofp.write(chunk)
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   450
29919
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   451
            # force @filecache properties to be reloaded from
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   452
            # streamclone-ed file at next access
519a02267f90 streamclone: clear caches after writing changes into files for visibility
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29917
diff changeset
   453
            repo.invalidate(clearfilecache=True)
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   454
30975
22fbca1d11ed mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents: 30332
diff changeset
   455
        elapsed = util.timer() - start
26443
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   456
        if elapsed <= 0:
d947086d8973 streamclone: move code out of exchange.py
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26442
diff changeset
   457
            elapsed = 0.001
38373
ef692614e601 progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents: 38360
diff changeset
   458
        progress.complete()
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   459
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   460
            _(b'transferred %s in %.1f seconds (%s/sec)\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   461
            % (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   462
                util.bytecount(bytecount),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   463
                elapsed,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   464
                util.bytecount(bytecount / elapsed),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   465
            )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   466
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   467
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   468
27882
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   469
def readbundle1header(fp):
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   470
    compression = fp.read(2)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   471
    if compression != b'UN':
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   472
        raise error.Abort(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   473
            _(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   474
                b'only uncompressed stream clone bundles are '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   475
                b'supported; got %s'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   476
            )
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   477
            % compression
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   478
        )
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   479
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   480
    filecount, bytecount = struct.unpack(b'>QQ', fp.read(16))
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   481
    requireslen = struct.unpack(b'>H', fp.read(2))[0]
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   482
    requires = fp.read(requireslen)
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   483
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   484
    if not requires.endswith(b'\0'):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   485
        raise error.Abort(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   486
            _(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   487
                b'malformed stream clone bundle: '
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   488
                b'requirements not properly encoded'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   489
            )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   490
        )
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   491
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   492
    requirements = set(requires.rstrip(b'\0').split(b','))
27882
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   493
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   494
    return filecount, bytecount, requirements
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   495
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   496
27882
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   497
def applybundlev1(repo, fp):
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   498
    """Apply the content from a stream clone bundle version 1.
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   499
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   500
    We assume the 4 byte header has been read and validated and the file handle
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   501
    is at the 2 byte compression identifier.
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   502
    """
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   503
    if len(repo):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   504
        raise error.Abort(
43117
8ff1ecfadcd1 cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents: 43085
diff changeset
   505
            _(b'cannot apply stream clone bundle on non-empty repo')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   506
        )
27882
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   507
319b0bf6ecc9 streamclone: extract code for reading header fields
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27870
diff changeset
   508
    filecount, bytecount, requirements = readbundle1header(fp)
48670
6d2ddea0721a stream-clone: filter possible missing requirements using all supported one
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
   509
    missingreqs = requirements - repo.supported
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   510
    if missingreqs:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   511
        raise error.Abort(
43117
8ff1ecfadcd1 cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents: 43085
diff changeset
   512
            _(b'unable to apply stream clone: unsupported format: %s')
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   513
            % b', '.join(sorted(missingreqs))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   514
        )
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   515
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   516
    consumev1(repo, fp, filecount, bytecount)
48693
de3ac3d2c60b stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48681
diff changeset
   517
    nodemap.post_stream_cleanup(repo)
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   518
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   519
48946
642e31cb55f0 py3: use class X: instead of class X(object):
Gregory Szorc <gregory.szorc@gmail.com>
parents: 48875
diff changeset
   520
class streamcloneapplier:
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   521
    """Class to manage applying streaming clone bundles.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   522
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   523
    We need to wrap ``applybundlev1()`` in a dedicated type to enable bundle
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   524
    readers to perform bundle type-specific functionality.
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   525
    """
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   526
26755
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   527
    def __init__(self, fh):
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   528
        self._fh = fh
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   529
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   530
    def apply(self, repo):
bb0b955d050d streamclone: support for producing and consuming stream clone bundles
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26587
diff changeset
   531
        return applybundlev1(repo, self._fh)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   532
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   533
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   534
# type of file to stream
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   535
_fileappend = 0  # append only file
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   536
_filefull = 1  # full snapshot file
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   537
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   538
# Source of the file
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   539
_srcstore = b's'  # store (svfs)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   540
_srccache = b'c'  # cache (cache)
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   541
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   542
# This is it's own function so extensions can override it.
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   543
def _walkstreamfullstorefiles(repo):
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   544
    """list snapshot file from the store"""
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   545
    fnames = []
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   546
    if not repo.publishing():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   547
        fnames.append(b'phaseroots')
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   548
    return fnames
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   549
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   550
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   551
def _filterfull(entry, copy, vfsmap):
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   552
    """actually copy the snapshot files"""
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   553
    src, name, ftype, data = entry
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   554
    if ftype != _filefull:
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   555
        return entry
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   556
    return (src, name, ftype, copy(vfsmap[src].join(name)))
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   557
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   558
50630
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   559
class TempCopyManager:
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   560
    """Manage temporary backup of volatile file during stream clone
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   561
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   562
    This should be used as a Python context, the copies will be discarded when
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   563
    exiting the context.
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   564
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   565
    A copy can be done by calling the object on the real path (encoded full
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   566
    path)
49190
4ff4e23de7df clone: use better names for temp files
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents: 48946
diff changeset
   567
50630
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   568
    The backup path can be retrieved using the __getitem__ protocol, obj[path].
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   569
    On file without backup, it will return the unmodified path. (equivalent to
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   570
    `dict.get(x, x)`)
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   571
    """
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   572
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   573
    def __init__(self):
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   574
        self._copies = None
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   575
        self._dst_dir = None
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   576
50630
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   577
    def __enter__(self):
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   578
        if self._copies is not None:
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   579
            msg = "Copies context already open"
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   580
            raise error.ProgrammingError(msg)
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   581
        self._copies = {}
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   582
        self._dst_dir = pycompat.mkdtemp(prefix=b'hg-clone-')
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   583
        return self
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   584
50630
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   585
    def __call__(self, src):
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   586
        """create a backup of the file at src"""
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   587
        prefix = os.path.basename(src)
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   588
        fd, dst = pycompat.mkstemp(prefix=prefix, dir=self._dst_dir)
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   589
        os.close(fd)
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   590
        self._copies[src] = dst
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   591
        util.copyfiles(src, dst, hardlink=True)
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   592
        return dst
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   593
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   594
    def __getitem__(self, src):
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   595
        """return the path to a valid version of `src`
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   596
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   597
        If the file has no backup, the path of the file is returned
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   598
        unmodified."""
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   599
        return self._copies.get(src, src)
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   600
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   601
    def __exit__(self, *args, **kwars):
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   602
        """discard all backups"""
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   603
        for tmp in self._copies.values():
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   604
            util.tryunlink(tmp)
50630
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   605
        util.tryrmdir(self._dst_dir)
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   606
        self._copies = None
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   607
        self._dst_dir = None
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   608
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   609
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   610
def _makemap(repo):
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   611
    """make a (src -> vfs) map for the repo"""
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   612
    vfsmap = {
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   613
        _srcstore: repo.svfs,
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   614
        _srccache: repo.cachevfs,
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   615
    }
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   616
    # we keep repo.vfs out of the on purpose, ther are too many danger there
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   617
    # (eg: .hg/hgrc)
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   618
    assert repo.vfs not in vfsmap.values()
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   619
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   620
    return vfsmap
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   621
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   622
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   623
def _emit2(repo, entries):
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   624
    """actually emit the stream bundle"""
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   625
    vfsmap = _makemap(repo)
47502
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   626
    # we keep repo.vfs out of the on purpose, ther are too many danger there
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   627
    # (eg: .hg/hgrc),
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   628
    #
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   629
    # this assert is duplicated (from _makemap) as author might think this is
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   630
    # fine, while this is really not fine.
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   631
    if repo.vfs in vfsmap.values():
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   632
        raise error.ProgrammingError(
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   633
            b'repo.vfs must not be added to vfsmap for security reasons'
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   634
        )
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   635
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   636
    # translate the vfs one
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   637
    entries = [(vfs_key, vfsmap[vfs_key], e) for (vfs_key, e) in entries]
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   638
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   639
    file_count = totalfilesize = 0
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   640
    # record the expected size of every file
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   641
    for k, vfs, e in entries:
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   642
        for f in e.files():
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   643
            file_count += 1
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   644
            totalfilesize += f.file_size(vfs)
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   645
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   646
    progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   647
        _(b'bundle'), total=totalfilesize, unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   648
    )
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   649
    progress.update(0)
50630
8c7b04e69894 stream-clone: introduce a richer TempCopyManager object
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50524
diff changeset
   650
    with TempCopyManager() as copy, progress:
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   651
        # create a copy of volatile files
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   652
        for k, vfs, e in entries:
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   653
            for f in e.files():
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   654
                if f.is_volatile:
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   655
                    copy(vfs.join(f.unencoded_path))
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   656
        # the first yield release the lock on the repository
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   657
        yield file_count, totalfilesize
47788
48f07adbda98 streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents: 47759
diff changeset
   658
        totalbytecount = 0
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   659
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   660
        for src, vfs, e in entries:
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   661
            for f in e.files():
50634
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   662
                yield src
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   663
                name = f.unencoded_path
50634
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   664
                yield util.uvarintencode(len(name))
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   665
                actual_path = copy[vfs.join(name)]
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   666
                fp = open(actual_path, b'rb')
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   667
                size = f.file_size(vfs)
50634
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   668
                bytecount = 0
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   669
                try:
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   670
                    yield util.uvarintencode(size)
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   671
                    yield name
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   672
                    if size <= 65536:
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   673
                        chunks = (fp.read(size),)
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   674
                    else:
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   675
                        chunks = util.filechunkiter(fp, limit=size)
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   676
                    for chunk in chunks:
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   677
                        bytecount += len(chunk)
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   678
                        totalbytecount += len(chunk)
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   679
                        progress.update(totalbytecount)
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   680
                        yield chunk
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   681
                    if bytecount != size:
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   682
                        # Would most likely be caused by a race due to `hg
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   683
                        # strip` or a revlog split
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   684
                        msg = _(
47788
48f07adbda98 streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents: 47759
diff changeset
   685
                            b'clone could only read %d bytes from %s, but '
48f07adbda98 streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents: 47759
diff changeset
   686
                            b'expected %d bytes'
48f07adbda98 streamclone: ensure the server sends the right amount of data
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents: 47759
diff changeset
   687
                        )
50634
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   688
                        raise error.Abort(msg % (bytecount, name, size))
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   689
                finally:
1415e17ea20f stream-clone: pre-indent some code
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50633
diff changeset
   690
                    fp.close()
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   691
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   692
46986
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   693
def _test_sync_point_walk_1(repo):
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   694
    """a function for synchronisation during tests"""
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   695
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   696
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   697
def _test_sync_point_walk_2(repo):
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   698
    """a function for synchronisation during tests"""
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   699
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   700
50515
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   701
def _entries_walk(repo, includes, excludes, includeobsmarkers):
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   702
    """emit a seris of files information useful to clone a repo
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   703
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   704
    return (vfs-key, entry) iterator
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   705
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   706
    Where `entry` is StoreEntry. (used even for cache entries)
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   707
    """
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   708
    assert repo._currentlock(repo._lockref) is not None
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   709
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   710
    matcher = None
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   711
    if includes or excludes:
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   712
        matcher = narrowspec.match(repo.root, includes, excludes)
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   713
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   714
    phase = not repo.publishing()
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   715
    entries = _walkstreamfiles(
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   716
        repo,
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   717
        matcher,
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   718
        phase=phase,
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   719
        obsolescence=includeobsmarkers,
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   720
    )
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   721
    for entry in entries:
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   722
        yield (_srcstore, entry)
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   723
50516
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   724
    for name in cacheutil.cachetocopy(repo):
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   725
        if repo.cachevfs.exists(name):
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   726
            # not really a StoreEntry, but close enough
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   727
            entry = store.SimpleStoreEntry(
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   728
                entry_path=name,
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   729
                is_volatile=True,
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   730
            )
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   731
            yield (_srccache, entry)
43ed1f12b00a stream-clone: yield cache entry in `_entries_walk` too
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50515
diff changeset
   732
50515
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   733
47444
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   734
def _v2_walk(repo, includes, excludes, includeobsmarkers):
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   735
    """emit a seris of files information useful to clone a repo
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   736
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   737
    return (entries, totalfilesize)
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   738
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   739
    entries is a list of tuple (vfs-key, file-path, file-type, size)
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   740
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   741
    - `vfs-key`: is a key to the right vfs to write the file (see _makemap)
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   742
    - `name`: file path of the file to copy (to be feed to the vfss)
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   743
    - `file-type`: do this file need to be copied with the source lock ?
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   744
    - `size`: the size of the file (or None)
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   745
    """
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   746
    assert repo._currentlock(repo._lockref) is not None
50514
0925eaf09c8b store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50512
diff changeset
   747
    files = []
47444
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   748
    totalfilesize = 0
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   749
50515
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   750
    vfsmap = _makemap(repo)
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   751
    entries = _entries_walk(repo, includes, excludes, includeobsmarkers)
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   752
    for vfs_key, entry in entries:
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   753
        vfs = vfsmap[vfs_key]
50472
9fdc28e21b68 store: introduce a EntryFile object to actually access file info
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50471
diff changeset
   754
        for f in entry.files():
50515
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   755
            file_size = f.file_size(vfs)
50477
4cbdfab6f812 store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50472
diff changeset
   756
            if file_size:
50472
9fdc28e21b68 store: introduce a EntryFile object to actually access file info
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50471
diff changeset
   757
                ft = _fileappend
9fdc28e21b68 store: introduce a EntryFile object to actually access file info
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50471
diff changeset
   758
                if f.is_volatile:
9fdc28e21b68 store: introduce a EntryFile object to actually access file info
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50471
diff changeset
   759
                    ft = _filefull
50515
06d580b8f432 stream-clone: introduce a _entries_walk
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50514
diff changeset
   760
                files.append((vfs_key, f.unencoded_path, ft, file_size))
50477
4cbdfab6f812 store: lazily get file size on demand for the fncache case
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50472
diff changeset
   761
                totalfilesize += file_size
50514
0925eaf09c8b store: make `walk` return an entry for obsolescence if requested so
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50512
diff changeset
   762
    return files, totalfilesize
47444
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   763
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   764
40398
0ac794e0e285 streamclone: include obsstore file into stream bundle if client can read it
Anton Shestakov <av6@dwimlabs.net>
parents: 40339
diff changeset
   765
def generatev2(repo, includes, excludes, includeobsmarkers):
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   766
    """Emit content for version 2 of a streaming clone.
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   767
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   768
    the data stream consists the following entries:
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   769
    1) A char representing the file destination (eg: store or cache)
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   770
    2) A varint containing the length of the filename
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   771
    3) A varint containing the length of file data
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   772
    4) N bytes containing the filename (the internal, store-agnostic form)
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   773
    5) N bytes containing the file data
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   774
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   775
    Returns a 3-tuple of (file count, file size, data iterator).
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   776
    """
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   777
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   778
    with repo.lock():
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   779
47444
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   780
        repo.ui.debug(b'scanning\n')
40339
f0e8f27768eb streamclone: pass narrowing related info in _walkstreamfiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40338
diff changeset
   781
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   782
        entries = _entries_walk(
47444
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   783
            repo,
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   784
            includes=includes,
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   785
            excludes=excludes,
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   786
            includeobsmarkers=includeobsmarkers,
2f4ca4807033 streamingclone: extract the scanning part from the generation part
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   787
        )
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   788
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   789
        chunks = _emit2(repo, entries)
35765
56c30b31afbe streamclone: add support for cloning non append-only file
Boris Feld <boris.feld@octobus.net>
parents: 35757
diff changeset
   790
        first = next(chunks)
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   791
        file_count, total_file_size = first
46986
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   792
        _test_sync_point_walk_1(repo)
faa43f09ad98 streamclone: remove sleep based "synchronisation" in tests
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46895
diff changeset
   793
    _test_sync_point_walk_2(repo)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   794
50635
3416b46320dc stream-clone: directly use `_entries_walk` to generate stream-v2
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50634
diff changeset
   795
    return file_count, total_file_size, chunks
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   796
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   797
50524
58adcabc295f stream-clone: introduce the notion of an experimental "v3" version
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50523
diff changeset
   798
def generatev3(repo, includes, excludes, includeobsmarkers):
58adcabc295f stream-clone: introduce the notion of an experimental "v3" version
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50523
diff changeset
   799
    return generatev2(repo, includes, excludes, includeobsmarkers)
58adcabc295f stream-clone: introduce the notion of an experimental "v3" version
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50523
diff changeset
   800
58adcabc295f stream-clone: introduce the notion of an experimental "v3" version
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50523
diff changeset
   801
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   802
@contextlib.contextmanager
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   803
def nested(*ctxs):
39757
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   804
    this = ctxs[0]
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   805
    rest = ctxs[1:]
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   806
    with this:
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   807
        if rest:
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   808
            with nested(*rest):
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   809
                yield
97f2992c26f6 streamclone: reimplement nested context manager
Augie Fackler <augie@google.com>
parents: 39700
diff changeset
   810
        else:
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   811
            yield
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   812
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   813
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   814
def consumev2(repo, fp, filecount, filesize):
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   815
    """Apply the contents from a version 2 streaming clone.
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   816
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   817
    Data is read from an object that only needs to provide a ``read(size)``
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   818
    method.
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   819
    """
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   820
    with repo.lock():
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   821
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   822
            _(b'%d files to transfer, %s of data\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   823
            % (filecount, util.bytecount(filesize))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   824
        )
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   825
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   826
        start = util.timer()
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   827
        progress = repo.ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   828
            _(b'clone'), total=filesize, unit=_(b'bytes')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   829
        )
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   830
        progress.update(0)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   831
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   832
        vfsmap = _makemap(repo)
47502
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   833
        # we keep repo.vfs out of the on purpose, ther are too many danger
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   834
        # there (eg: .hg/hgrc),
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   835
        #
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   836
        # this assert is duplicated (from _makemap) as author might think this
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   837
        # is fine, while this is really not fine.
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   838
        if repo.vfs in vfsmap.values():
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   839
            raise error.ProgrammingError(
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   840
                b'repo.vfs must not be added to vfsmap for security reasons'
65c519661991 stream: double check that self.vfs is *not* in the vfsmap
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46989
diff changeset
   841
            )
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   842
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   843
        with repo.transaction(b'clone'):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   844
            ctxs = (vfs.backgroundclosing(repo.ui) for vfs in vfsmap.values())
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   845
            with nested(*ctxs):
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   846
                for i in range(filecount):
35803
3ad3aaeb1134 streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents: 35802
diff changeset
   847
                    src = util.readexactly(fp, 1)
35767
5f5fb279fd39 streamclone: also stream caches to the client
Boris Feld <boris.feld@octobus.net>
parents: 35765
diff changeset
   848
                    vfs = vfsmap[src]
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   849
                    namelen = util.uvarintdecodestream(fp)
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   850
                    datalen = util.uvarintdecodestream(fp)
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   851
35803
3ad3aaeb1134 streamclone: use readexactly when reading stream v2
Boris Feld <boris.feld@octobus.net>
parents: 35802
diff changeset
   852
                    name = util.readexactly(fp, namelen)
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   853
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   854
                    if repo.ui.debugflag:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   855
                        repo.ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   856
                            b'adding [%s] %s (%s)\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   857
                            % (src, name, util.bytecount(datalen))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   858
                        )
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   859
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   860
                    with vfs(name, b'w') as ofp:
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   861
                        for chunk in util.filechunkiter(fp, limit=datalen):
38349
e59eaf51cc0d streamclone: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents: 38164
diff changeset
   862
                            progress.increment(step=len(chunk))
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   863
                            ofp.write(chunk)
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   864
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   865
            # force @filecache properties to be reloaded from
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   866
            # streamclone-ed file at next access
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   867
            repo.invalidate(clearfilecache=True)
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   868
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   869
        elapsed = util.timer() - start
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   870
        if elapsed <= 0:
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   871
            elapsed = 0.001
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   872
        repo.ui.status(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   873
            _(b'transferred %s in %.1f seconds (%s/sec)\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   874
            % (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   875
                util.bytecount(progress.pos),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   876
                elapsed,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   877
                util.bytecount(progress.pos / elapsed),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   878
            )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   879
        )
38373
ef692614e601 progress: hide update(None) in a new complete() method
Martin von Zweigbergk <martinvonz@google.com>
parents: 38360
diff changeset
   880
        progress.complete()
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   881
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   882
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   883
def applybundlev2(repo, fp, filecount, filesize, requirements):
39700
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   884
    from . import localrepo
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   885
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   886
    missingreqs = [r for r in requirements if r not in repo.supported]
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   887
    if missingreqs:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   888
        raise error.Abort(
43117
8ff1ecfadcd1 cleanup: join string literals that are already on one line
Martin von Zweigbergk <martinvonz@google.com>
parents: 43085
diff changeset
   889
            _(b'unable to apply stream clone: unsupported format: %s')
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   890
            % b', '.join(sorted(missingreqs))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   891
        )
35756
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   892
cfdccd560b66 streamclone: define first iteration of version 2 of stream format
Boris Feld <boris.feld@octobus.net>
parents: 35491
diff changeset
   893
    consumev2(repo, fp, filecount, filesize)
35804
2d3e486d09d0 streamclone: move requirement update into consumev2
Boris Feld <boris.feld@octobus.net>
parents: 35803
diff changeset
   894
48596
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
   895
    repo.requirements = new_stream_clone_requirements(
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
   896
        repo.requirements,
739f2ca3aa3f stream-clone: factor computation of new clone requirement out
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47878
diff changeset
   897
        requirements,
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   898
    )
39700
b10d145837bc localrepo: extract resolving of opener options to standalone functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38783
diff changeset
   899
    repo.svfs.options = localrepo.resolvestorevfsoptions(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   900
        repo.ui, repo.requirements, repo.features
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42813
diff changeset
   901
    )
45106
a03c177a4679 scmutil: add writereporequirements() and route requires writing through it
Pulkit Goyal <7895pulkit@gmail.com>
parents: 43117
diff changeset
   902
    scmutil.writereporequirements(repo)
48693
de3ac3d2c60b stream-clone: allow to change persistent-nodemap format during stream clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 48681
diff changeset
   903
    nodemap.post_stream_cleanup(repo)
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   904
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   905
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   906
def _copy_files(src_vfs_map, dst_vfs_map, entries, progress):
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   907
    hardlink = [True]
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   908
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   909
    def copy_used():
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   910
        hardlink[0] = False
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   911
        progress.topic = _(b'copying')
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   912
50633
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   913
    for k, path in entries:
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   914
        src_vfs = src_vfs_map[k]
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   915
        dst_vfs = dst_vfs_map[k]
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   916
        src_path = src_vfs.join(path)
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   917
        dst_path = dst_vfs.join(path)
47871
132525ead0db clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47788
diff changeset
   918
        # We cannot use dirname and makedirs of dst_vfs here because the store
132525ead0db clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47788
diff changeset
   919
        # encoding confuses them. See issue 6581 for details.
132525ead0db clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47788
diff changeset
   920
        dirname = os.path.dirname(dst_path)
132525ead0db clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47788
diff changeset
   921
        if not os.path.exists(dirname):
132525ead0db clone: properly create target directories during local clone (issue6581)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47788
diff changeset
   922
            util.makedirs(dirname)
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   923
        dst_vfs.register_file(path)
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   924
        # XXX we could use the #nb_bytes argument.
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   925
        util.copyfile(
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   926
            src_path,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   927
            dst_path,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   928
            hardlink=hardlink[0],
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   929
            no_hardlink_cb=copy_used,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   930
            check_fs_hardlink=False,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   931
        )
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   932
        progress.increment()
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   933
    return hardlink[0]
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   934
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   935
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   936
def local_copy(src_repo, dest_repo):
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   937
    """copy all content from one local repository to another
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   938
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   939
    This is useful for local clone"""
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   940
    src_store_requirements = {
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   941
        r
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   942
        for r in src_repo.requirements
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   943
        if r not in requirementsmod.WORKING_DIR_REQUIREMENTS
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   944
    }
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   945
    dest_store_requirements = {
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   946
        r
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   947
        for r in dest_repo.requirements
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   948
        if r not in requirementsmod.WORKING_DIR_REQUIREMENTS
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   949
    }
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   950
    assert src_store_requirements == dest_store_requirements
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   951
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   952
    with dest_repo.lock():
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   953
        with src_repo.lock():
47448
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   954
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   955
            # bookmark is not integrated to the streaming as it might use the
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   956
            # `repo.vfs` and they are too many sentitive data accessible
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   957
            # through `repo.vfs` to expose it to streaming clone.
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   958
            src_book_vfs = bookmarks.bookmarksvfs(src_repo)
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   959
            srcbookmarks = src_book_vfs.join(b'bookmarks')
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   960
            bm_count = 0
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   961
            if os.path.exists(srcbookmarks):
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   962
                bm_count = 1
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   963
50633
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   964
            entries = _entries_walk(
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   965
                src_repo,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   966
                includes=None,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   967
                excludes=None,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   968
                includeobsmarkers=True,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   969
            )
50633
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   970
            entries = list(entries)
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   971
            src_vfs_map = _makemap(src_repo)
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   972
            dest_vfs_map = _makemap(dest_repo)
50633
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   973
            total_files = sum(len(e[1].files()) for e in entries) + bm_count
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   974
            progress = src_repo.ui.makeprogress(
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   975
                topic=_(b'linking'),
50633
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   976
                total=total_files,
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   977
                unit=_(b'files'),
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   978
            )
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   979
            # copy  files
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   980
            #
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   981
            # We could copy the full file while the source repository is locked
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   982
            # and the other one without the lock. However, in the linking case,
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   983
            # this would also requires checks that nobody is appending any data
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   984
            # to the files while we do the clone, so this is not done yet. We
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   985
            # could do this blindly when copying files.
50633
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   986
            files = [
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   987
                (vfs_key, f.unencoded_path)
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   988
                for vfs_key, e in entries
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   989
                for f in e.files()
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
   990
            ]
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   991
            hardlink = _copy_files(src_vfs_map, dest_vfs_map, files, progress)
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   992
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   993
            # copy bookmarks over
47448
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   994
            if bm_count:
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   995
                dst_book_vfs = bookmarks.bookmarksvfs(dest_repo)
d370256636fe clone: also report the bookmark file as copied
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47447
diff changeset
   996
                dstbookmarks = dst_book_vfs.join(b'bookmarks')
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   997
                util.copyfile(srcbookmarks, dstbookmarks)
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   998
        progress.complete()
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
   999
        if hardlink:
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
  1000
            msg = b'linked %d files\n'
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
  1001
        else:
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
  1002
            msg = b'copied %d files\n'
50633
f2ae815ae34c local-clone: perform the hardlink/copy based from _entries_walk returns
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50630
diff changeset
  1003
        src_repo.ui.debug(msg % total_files)
47447
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
  1004
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
  1005
        with dest_repo.transaction(b"localclone") as tr:
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
  1006
            dest_repo.store.write(tr)
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
  1007
377d8fc20e34 clone: reuse the stream clone logic for local clone
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 47444
diff changeset
  1008
        # clean up transaction file as they do not make sense
50301
d89eecf9605e undo-files: no longer pass the `repo` to `cleanup_undo_files`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 50299
diff changeset
  1009
        transaction.cleanup_undo_files(dest_repo.ui.warn, dest_repo.vfs_map)