hgext/remotefilelog/repack.py
author Martin von Zweigbergk <martinvonz@google.com>
Fri, 13 Dec 2019 09:43:43 -0800
changeset 43862 5606e1cb4685
parent 43506 9f70512ae2cf
parent 43854 612b4b63fb22
child 44452 9d2b2df2c2ba
permissions -rw-r--r--
merge with stable
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     1
from __future__ import absolute_import
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     2
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     3
import os
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     4
import time
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     5
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     6
from mercurial.i18n import _
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     7
from mercurial.node import (
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     8
    nullid,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
     9
    short,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    10
)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    11
from mercurial import (
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    12
    encoding,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    13
    error,
42943
5fadf6103790 remotefilelog: replace repack lock to solve race condition
Boris Feld <boris.feld@octobus.net>
parents: 42842
diff changeset
    14
    lock as lockmod,
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    15
    mdiff,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    16
    policy,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    17
    pycompat,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    18
    scmutil,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    19
    util,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    20
    vfs,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    21
)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    22
from mercurial.utils import procutil
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    23
from . import (
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    24
    constants,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    25
    contentstore,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    26
    datapack,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    27
    historypack,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    28
    metadatastore,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    29
    shallowutil,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    30
)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    31
43506
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43115
diff changeset
    32
osutil = policy.importmod('osutil')
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    33
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    34
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    35
class RepackAlreadyRunning(error.Abort):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    36
    pass
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    37
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    38
43854
612b4b63fb22 remotefilelog: remove the `ensurestart` usage
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43849
diff changeset
    39
def backgroundrepack(repo, incremental=True, packsonly=False):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    40
    cmd = [procutil.hgexecutable(), b'-R', repo.origroot, b'repack']
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    41
    msg = _(b"(running background repack)\n")
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    42
    if incremental:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    43
        cmd.append(b'--incremental')
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    44
        msg = _(b"(running background incremental repack)\n")
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    45
    if packsonly:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    46
        cmd.append(b'--packsonly')
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    47
    repo.ui.warn(msg)
42497
df1419c5756a remotefilelog: tell runbgcommand to not block on child process startup
Augie Fackler <augie@google.com>
parents: 42057
diff changeset
    48
    # We know this command will find a binary, so don't block on it starting.
43849
63bb6dc62f24 remotefilelog: add a developer option to wait for background processes
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43115
diff changeset
    49
    kwargs = {}
63bb6dc62f24 remotefilelog: add a developer option to wait for background processes
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43115
diff changeset
    50
    if repo.ui.configbool(b'devel', b'remotefilelog.bg-wait'):
63bb6dc62f24 remotefilelog: add a developer option to wait for background processes
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43115
diff changeset
    51
        kwargs['record_wait'] = repo.ui.atexit
63bb6dc62f24 remotefilelog: add a developer option to wait for background processes
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43115
diff changeset
    52
43854
612b4b63fb22 remotefilelog: remove the `ensurestart` usage
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43849
diff changeset
    53
    procutil.runbgcommand(cmd, encoding.environ, ensurestart=False, **kwargs)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    54
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    55
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    56
def fullrepack(repo, options=None):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    57
    """If ``packsonly`` is True, stores creating only loose objects are skipped.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    58
    """
43115
4aa72cdf616f py3: delete b'' prefix from safehasattr arguments
Martin von Zweigbergk <martinvonz@google.com>
parents: 43105
diff changeset
    59
    if util.safehasattr(repo, 'shareddatastores'):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    60
        datasource = contentstore.unioncontentstore(*repo.shareddatastores)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    61
        historysource = metadatastore.unionmetadatastore(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    62
            *repo.sharedhistorystores, allowincomplete=True
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    63
        )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    64
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    65
        packpath = shallowutil.getcachepackpath(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    66
            repo, constants.FILEPACK_CATEGORY
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    67
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    68
        _runrepack(
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    69
            repo,
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    70
            datasource,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    71
            historysource,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    72
            packpath,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    73
            constants.FILEPACK_CATEGORY,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    74
            options=options,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    75
        )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    76
43115
4aa72cdf616f py3: delete b'' prefix from safehasattr arguments
Martin von Zweigbergk <martinvonz@google.com>
parents: 43105
diff changeset
    77
    if util.safehasattr(repo.manifestlog, 'datastore'):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    78
        localdata, shareddata = _getmanifeststores(repo)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    79
        lpackpath, ldstores, lhstores = localdata
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    80
        spackpath, sdstores, shstores = shareddata
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    81
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    82
        # Repack the shared manifest store
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    83
        datasource = contentstore.unioncontentstore(*sdstores)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    84
        historysource = metadatastore.unionmetadatastore(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    85
            *shstores, allowincomplete=True
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    86
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    87
        _runrepack(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    88
            repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    89
            datasource,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    90
            historysource,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    91
            spackpath,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    92
            constants.TREEPACK_CATEGORY,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    93
            options=options,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    94
        )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    95
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    96
        # Repack the local manifest store
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
    97
        datasource = contentstore.unioncontentstore(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    98
            *ldstores, allowincomplete=True
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
    99
        )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   100
        historysource = metadatastore.unionmetadatastore(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   101
            *lhstores, allowincomplete=True
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   102
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   103
        _runrepack(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   104
            repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   105
            datasource,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   106
            historysource,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   107
            lpackpath,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   108
            constants.TREEPACK_CATEGORY,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   109
            options=options,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   110
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   111
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   112
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   113
def incrementalrepack(repo, options=None):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   114
    """This repacks the repo by looking at the distribution of pack files in the
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   115
    repo and performing the most minimal repack to keep the repo in good shape.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   116
    """
43115
4aa72cdf616f py3: delete b'' prefix from safehasattr arguments
Martin von Zweigbergk <martinvonz@google.com>
parents: 43105
diff changeset
   117
    if util.safehasattr(repo, 'shareddatastores'):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   118
        packpath = shallowutil.getcachepackpath(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   119
            repo, constants.FILEPACK_CATEGORY
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   120
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   121
        _incrementalrepack(
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   122
            repo,
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   123
            repo.shareddatastores,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   124
            repo.sharedhistorystores,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   125
            packpath,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   126
            constants.FILEPACK_CATEGORY,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   127
            options=options,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   128
        )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   129
43115
4aa72cdf616f py3: delete b'' prefix from safehasattr arguments
Martin von Zweigbergk <martinvonz@google.com>
parents: 43105
diff changeset
   130
    if util.safehasattr(repo.manifestlog, 'datastore'):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   131
        localdata, shareddata = _getmanifeststores(repo)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   132
        lpackpath, ldstores, lhstores = localdata
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   133
        spackpath, sdstores, shstores = shareddata
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   134
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   135
        # Repack the shared manifest store
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   136
        _incrementalrepack(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   137
            repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   138
            sdstores,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   139
            shstores,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   140
            spackpath,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   141
            constants.TREEPACK_CATEGORY,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   142
            options=options,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   143
        )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   144
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   145
        # Repack the local manifest store
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   146
        _incrementalrepack(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   147
            repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   148
            ldstores,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   149
            lhstores,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   150
            lpackpath,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   151
            constants.TREEPACK_CATEGORY,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   152
            allowincompletedata=True,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   153
            options=options,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   154
        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   155
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   156
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   157
def _getmanifeststores(repo):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   158
    shareddatastores = repo.manifestlog.shareddatastores
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   159
    localdatastores = repo.manifestlog.localdatastores
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   160
    sharedhistorystores = repo.manifestlog.sharedhistorystores
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   161
    localhistorystores = repo.manifestlog.localhistorystores
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   162
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   163
    sharedpackpath = shallowutil.getcachepackpath(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   164
        repo, constants.TREEPACK_CATEGORY
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   165
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   166
    localpackpath = shallowutil.getlocalpackpath(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   167
        repo.svfs.vfs.base, constants.TREEPACK_CATEGORY
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   168
    )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   169
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   170
    return (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   171
        (localpackpath, localdatastores, localhistorystores),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   172
        (sharedpackpath, shareddatastores, sharedhistorystores),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   173
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   174
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   175
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   176
def _topacks(packpath, files, constructor):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   177
    paths = list(os.path.join(packpath, p) for p in files)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   178
    packs = list(constructor(p) for p in paths)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   179
    return packs
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   180
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   181
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   182
def _deletebigpacks(repo, folder, files):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   183
    """Deletes packfiles that are bigger than ``packs.maxpacksize``.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   184
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   185
    Returns ``files` with the removed files omitted."""
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   186
    maxsize = repo.ui.configbytes(b"packs", b"maxpacksize")
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   187
    if maxsize <= 0:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   188
        return files
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   189
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   190
    # This only considers datapacks today, but we could broaden it to include
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   191
    # historypacks.
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   192
    VALIDEXTS = [b".datapack", b".dataidx"]
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   193
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   194
    # Either an oversize index or datapack will trigger cleanup of the whole
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   195
    # pack:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   196
    oversized = {
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   197
        os.path.splitext(path)[0]
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   198
        for path, ftype, stat in files
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   199
        if (stat.st_size > maxsize and (os.path.splitext(path)[1] in VALIDEXTS))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   200
    }
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   201
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   202
    for rootfname in oversized:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   203
        rootpath = os.path.join(folder, rootfname)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   204
        for ext in VALIDEXTS:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   205
            path = rootpath + ext
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   206
            repo.ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   207
                b'removing oversize packfile %s (%s)\n'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   208
                % (path, util.bytecount(os.stat(path).st_size))
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   209
            )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   210
            os.unlink(path)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   211
    return [row for row in files if os.path.basename(row[0]) not in oversized]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   212
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   213
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   214
def _incrementalrepack(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   215
    repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   216
    datastore,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   217
    historystore,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   218
    packpath,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   219
    category,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   220
    allowincompletedata=False,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   221
    options=None,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   222
):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   223
    shallowutil.mkstickygroupdir(repo.ui, packpath)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   224
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   225
    files = osutil.listdir(packpath, stat=True)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   226
    files = _deletebigpacks(repo, packpath, files)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   227
    datapacks = _topacks(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   228
        packpath, _computeincrementaldatapack(repo.ui, files), datapack.datapack
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   229
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   230
    datapacks.extend(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   231
        s for s in datastore if not isinstance(s, datapack.datapackstore)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   232
    )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   233
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   234
    historypacks = _topacks(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   235
        packpath,
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   236
        _computeincrementalhistorypack(repo.ui, files),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   237
        historypack.historypack,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   238
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   239
    historypacks.extend(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   240
        s
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   241
        for s in historystore
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   242
        if not isinstance(s, historypack.historypackstore)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   243
    )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   244
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   245
    # ``allhistory{files,packs}`` contains all known history packs, even ones we
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   246
    # don't plan to repack. They are used during the datapack repack to ensure
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   247
    # good ordering of nodes.
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   248
    allhistoryfiles = _allpackfileswithsuffix(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   249
        files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   250
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   251
    allhistorypacks = _topacks(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   252
        packpath,
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   253
        (f for f, mode, stat in allhistoryfiles),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   254
        historypack.historypack,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   255
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   256
    allhistorypacks.extend(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   257
        s
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   258
        for s in historystore
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   259
        if not isinstance(s, historypack.historypackstore)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   260
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   261
    _runrepack(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   262
        repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   263
        contentstore.unioncontentstore(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   264
            *datapacks, allowincomplete=allowincompletedata
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   265
        ),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   266
        metadatastore.unionmetadatastore(*historypacks, allowincomplete=True),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   267
        packpath,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   268
        category,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   269
        fullhistory=metadatastore.unionmetadatastore(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   270
            *allhistorypacks, allowincomplete=True
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   271
        ),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   272
        options=options,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   273
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   274
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   275
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   276
def _computeincrementaldatapack(ui, files):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   277
    opts = {
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   278
        b'gencountlimit': ui.configint(b'remotefilelog', b'data.gencountlimit'),
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   279
        b'generations': ui.configlist(b'remotefilelog', b'data.generations'),
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   280
        b'maxrepackpacks': ui.configint(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   281
            b'remotefilelog', b'data.maxrepackpacks'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   282
        ),
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   283
        b'repackmaxpacksize': ui.configbytes(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   284
            b'remotefilelog', b'data.repackmaxpacksize'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   285
        ),
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   286
        b'repacksizelimit': ui.configbytes(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   287
            b'remotefilelog', b'data.repacksizelimit'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   288
        ),
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   289
    }
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   290
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   291
    packfiles = _allpackfileswithsuffix(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   292
        files, datapack.PACKSUFFIX, datapack.INDEXSUFFIX
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   293
    )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   294
    return _computeincrementalpack(packfiles, opts)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   295
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   296
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   297
def _computeincrementalhistorypack(ui, files):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   298
    opts = {
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   299
        b'gencountlimit': ui.configint(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   300
            b'remotefilelog', b'history.gencountlimit'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   301
        ),
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   302
        b'generations': ui.configlist(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   303
            b'remotefilelog', b'history.generations', [b'100MB']
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   304
        ),
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   305
        b'maxrepackpacks': ui.configint(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   306
            b'remotefilelog', b'history.maxrepackpacks'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   307
        ),
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   308
        b'repackmaxpacksize': ui.configbytes(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   309
            b'remotefilelog', b'history.repackmaxpacksize', b'400MB'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   310
        ),
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   311
        b'repacksizelimit': ui.configbytes(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   312
            b'remotefilelog', b'history.repacksizelimit'
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   313
        ),
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   314
    }
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   315
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   316
    packfiles = _allpackfileswithsuffix(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   317
        files, historypack.PACKSUFFIX, historypack.INDEXSUFFIX
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   318
    )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   319
    return _computeincrementalpack(packfiles, opts)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   320
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   321
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   322
def _allpackfileswithsuffix(files, packsuffix, indexsuffix):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   323
    result = []
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   324
    fileset = set(fn for fn, mode, stat in files)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   325
    for filename, mode, stat in files:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   326
        if not filename.endswith(packsuffix):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   327
            continue
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   328
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   329
        prefix = filename[: -len(packsuffix)]
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   330
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   331
        # Don't process a pack if it doesn't have an index.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   332
        if (prefix + indexsuffix) not in fileset:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   333
            continue
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   334
        result.append((prefix, mode, stat))
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   335
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   336
    return result
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   337
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   338
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   339
def _computeincrementalpack(files, opts):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   340
    """Given a set of pack files along with the configuration options, this
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   341
    function computes the list of files that should be packed as part of an
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   342
    incremental repack.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   343
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   344
    It tries to strike a balance between keeping incremental repacks cheap (i.e.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   345
    packing small things when possible, and rolling the packs up to the big ones
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   346
    over time).
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   347
    """
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   348
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   349
    limits = list(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   350
        sorted((util.sizetoint(s) for s in opts[b'generations']), reverse=True)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   351
    )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   352
    limits.append(0)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   353
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   354
    # Group the packs by generation (i.e. by size)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   355
    generations = []
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   356
    for i in pycompat.xrange(len(limits)):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   357
        generations.append([])
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   358
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   359
    sizes = {}
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   360
    for prefix, mode, stat in files:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   361
        size = stat.st_size
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   362
        if size > opts[b'repackmaxpacksize']:
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   363
            continue
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   364
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   365
        sizes[prefix] = size
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   366
        for i, limit in enumerate(limits):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   367
            if size > limit:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   368
                generations[i].append(prefix)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   369
                break
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   370
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   371
    # Steps for picking what packs to repack:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   372
    # 1. Pick the largest generation with > gencountlimit pack files.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   373
    # 2. Take the smallest three packs.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   374
    # 3. While total-size-of-packs < repacksizelimit: add another pack
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   375
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   376
    # Find the largest generation with more than gencountlimit packs
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   377
    genpacks = []
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   378
    for i, limit in enumerate(limits):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   379
        if len(generations[i]) > opts[b'gencountlimit']:
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   380
            # Sort to be smallest last, for easy popping later
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   381
            genpacks.extend(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   382
                sorted(generations[i], reverse=True, key=lambda x: sizes[x])
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   383
            )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   384
            break
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   385
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   386
    # Take as many packs from the generation as we can
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   387
    chosenpacks = genpacks[-3:]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   388
    genpacks = genpacks[:-3]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   389
    repacksize = sum(sizes[n] for n in chosenpacks)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   390
    while (
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   391
        repacksize < opts[b'repacksizelimit']
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   392
        and genpacks
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   393
        and len(chosenpacks) < opts[b'maxrepackpacks']
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   394
    ):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   395
        chosenpacks.append(genpacks.pop())
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   396
        repacksize += sizes[chosenpacks[-1]]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   397
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   398
    return chosenpacks
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   399
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   400
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   401
def _runrepack(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   402
    repo, data, history, packpath, category, fullhistory=None, options=None
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   403
):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   404
    shallowutil.mkstickygroupdir(repo.ui, packpath)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   405
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   406
    def isold(repo, filename, node):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   407
        """Check if the file node is older than a limit.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   408
        Unless a limit is specified in the config the default limit is taken.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   409
        """
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   410
        filectx = repo.filectx(filename, fileid=node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   411
        filetime = repo[filectx.linkrev()].date()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   412
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   413
        ttl = repo.ui.configint(b'remotefilelog', b'nodettl')
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   414
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   415
        limit = time.time() - ttl
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   416
        return filetime[0] < limit
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   417
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   418
    garbagecollect = repo.ui.configbool(b'remotefilelog', b'gcrepack')
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   419
    if not fullhistory:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   420
        fullhistory = history
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   421
    packer = repacker(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   422
        repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   423
        data,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   424
        history,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   425
        fullhistory,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   426
        category,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   427
        gc=garbagecollect,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   428
        isold=isold,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   429
        options=options,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   430
    )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   431
41806
118c1ec4f31b remotefilelog: do not specify an explicit version for repack
Kyle Lippincott <spectral@google.com>
parents: 41365
diff changeset
   432
    with datapack.mutabledatapack(repo.ui, packpath) as dpack:
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   433
        with historypack.mutablehistorypack(repo.ui, packpath) as hpack:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   434
            try:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   435
                packer.run(dpack, hpack)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   436
            except error.LockHeld:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   437
                raise RepackAlreadyRunning(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   438
                    _(
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   439
                        b"skipping repack - another repack "
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   440
                        b"is already running"
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   441
                    )
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   442
                )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   443
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   444
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   445
def keepset(repo, keyfn, lastkeepkeys=None):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   446
    """Computes a keepset which is not garbage collected.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   447
    'keyfn' is a function that maps filename, node to a unique key.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   448
    'lastkeepkeys' is an optional argument and if provided the keepset
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   449
    function updates lastkeepkeys with more keys and returns the result.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   450
    """
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   451
    if not lastkeepkeys:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   452
        keepkeys = set()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   453
    else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   454
        keepkeys = lastkeepkeys
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   455
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   456
    # We want to keep:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   457
    # 1. Working copy parent
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   458
    # 2. Draft commits
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   459
    # 3. Parents of draft commits
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   460
    # 4. Pullprefetch and bgprefetchrevs revsets if specified
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   461
    revs = [b'.', b'draft()', b'parents(draft())']
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   462
    prefetchrevs = repo.ui.config(b'remotefilelog', b'pullprefetch', None)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   463
    if prefetchrevs:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   464
        revs.append(b'(%s)' % prefetchrevs)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   465
    prefetchrevs = repo.ui.config(b'remotefilelog', b'bgprefetchrevs', None)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   466
    if prefetchrevs:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   467
        revs.append(b'(%s)' % prefetchrevs)
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   468
    revs = b'+'.join(revs)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   469
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   470
    revs = [b'sort((%s), "topo")' % revs]
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   471
    keep = scmutil.revrange(repo, revs)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   472
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   473
    processed = set()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   474
    lastmanifest = None
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   475
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   476
    # process the commits in toposorted order starting from the oldest
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   477
    for r in reversed(keep._list):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   478
        if repo[r].p1().rev() in processed:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   479
            # if the direct parent has already been processed
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   480
            # then we only need to process the delta
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   481
            m = repo[r].manifestctx().readdelta()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   482
        else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   483
            # otherwise take the manifest and diff it
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   484
            # with the previous manifest if one exists
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   485
            if lastmanifest:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   486
                m = repo[r].manifest().diff(lastmanifest)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   487
            else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   488
                m = repo[r].manifest()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   489
        lastmanifest = repo[r].manifest()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   490
        processed.add(r)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   491
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   492
        # populate keepkeys with keys from the current manifest
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   493
        if type(m) is dict:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   494
            # m is a result of diff of two manifests and is a dictionary that
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   495
            # maps filename to ((newnode, newflag), (oldnode, oldflag)) tuple
43105
649d3ac37a12 py3: define and use pycompat.iteritems() for hgext/
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43104
diff changeset
   496
            for filename, diff in pycompat.iteritems(m):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   497
                if diff[0][0] is not None:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   498
                    keepkeys.add(keyfn(filename, diff[0][0]))
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   499
        else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   500
            # m is a manifest object
43105
649d3ac37a12 py3: define and use pycompat.iteritems() for hgext/
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43104
diff changeset
   501
            for filename, filenode in pycompat.iteritems(m):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   502
                keepkeys.add(keyfn(filename, filenode))
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   503
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   504
    return keepkeys
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   505
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   506
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   507
class repacker(object):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   508
    """Class for orchestrating the repack of data and history information into a
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   509
    new format.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   510
    """
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   511
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   512
    def __init__(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   513
        self,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   514
        repo,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   515
        data,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   516
        history,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   517
        fullhistory,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   518
        category,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   519
        gc=False,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   520
        isold=None,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   521
        options=None,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   522
    ):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   523
        self.repo = repo
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   524
        self.data = data
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   525
        self.history = history
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   526
        self.fullhistory = fullhistory
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   527
        self.unit = constants.getunits(category)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   528
        self.garbagecollect = gc
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   529
        self.options = options
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   530
        if self.garbagecollect:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   531
            if not isold:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   532
                raise ValueError(b"Function 'isold' is not properly specified")
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   533
            # use (filename, node) tuple as a keepset key
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   534
            self.keepkeys = keepset(repo, lambda f, n: (f, n))
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   535
            self.isold = isold
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   536
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   537
    def run(self, targetdata, targethistory):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   538
        ledger = repackledger()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   539
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   540
        with lockmod.lock(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   541
            repacklockvfs(self.repo), b"repacklock", desc=None, timeout=0
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   542
        ):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   543
            self.repo.hook(b'prerepack')
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   544
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   545
            # Populate ledger from source
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   546
            self.data.markledger(ledger, options=self.options)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   547
            self.history.markledger(ledger, options=self.options)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   548
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   549
            # Run repack
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   550
            self.repackdata(ledger, targetdata)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   551
            self.repackhistory(ledger, targethistory)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   552
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   553
            # Call cleanup on each source
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   554
            for source in ledger.sources:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   555
                source.cleanup(ledger)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   556
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   557
    def _chainorphans(self, ui, filename, nodes, orphans, deltabases):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   558
        """Reorderes ``orphans`` into a single chain inside ``nodes`` and
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   559
        ``deltabases``.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   560
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   561
        We often have orphan entries (nodes without a base that aren't
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   562
        referenced by other nodes -- i.e., part of a chain) due to gaps in
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   563
        history. Rather than store them as individual fulltexts, we prefer to
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   564
        insert them as one chain sorted by size.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   565
        """
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   566
        if not orphans:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   567
            return nodes
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   568
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   569
        def getsize(node, default=0):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   570
            meta = self.data.getmeta(filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   571
            if constants.METAKEYSIZE in meta:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   572
                return meta[constants.METAKEYSIZE]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   573
            else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   574
                return default
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   575
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   576
        # Sort orphans by size; biggest first is preferred, since it's more
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   577
        # likely to be the newest version assuming files grow over time.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   578
        # (Sort by node first to ensure the sort is stable.)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   579
        orphans = sorted(orphans)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   580
        orphans = list(sorted(orphans, key=getsize, reverse=True))
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   581
        if ui.debugflag:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   582
            ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   583
                b"%s: orphan chain: %s\n"
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   584
                % (filename, b", ".join([short(s) for s in orphans]))
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   585
            )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   586
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   587
        # Create one contiguous chain and reassign deltabases.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   588
        for i, node in enumerate(orphans):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   589
            if i == 0:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   590
                deltabases[node] = (nullid, 0)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   591
            else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   592
                parent = orphans[i - 1]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   593
                deltabases[node] = (parent, deltabases[parent][1] + 1)
41255
60b3edccf2f7 remotefilelog: use list comprehension instead of filter for py3 portability
Augie Fackler <augie@google.com>
parents: 40843
diff changeset
   594
        nodes = [n for n in nodes if n not in orphans]
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   595
        nodes += orphans
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   596
        return nodes
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   597
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   598
    def repackdata(self, ledger, target):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   599
        ui = self.repo.ui
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   600
        maxchainlen = ui.configint(b'packs', b'maxchainlen', 1000)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   601
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   602
        byfile = {}
43104
74802979dd9d py3: define and use pycompat.itervalues()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
   603
        for entry in pycompat.itervalues(ledger.entries):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   604
            if entry.datasource:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   605
                byfile.setdefault(entry.filename, {})[entry.node] = entry
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   606
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   607
        count = 0
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   608
        repackprogress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   609
            _(b"repacking data"), unit=self.unit, total=len(byfile)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   610
        )
43105
649d3ac37a12 py3: define and use pycompat.iteritems() for hgext/
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43104
diff changeset
   611
        for filename, entries in sorted(pycompat.iteritems(byfile)):
40843
b80915b52476 remotefilelog: use progress helper in repack
Martin von Zweigbergk <martinvonz@google.com>
parents: 40804
diff changeset
   612
            repackprogress.update(count)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   613
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   614
            ancestors = {}
40613
9769e0f6ffe0 py3: don't use dict.iterkeys()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40526
diff changeset
   615
            nodes = list(node for node in entries)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   616
            nohistory = []
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   617
            buildprogress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   618
                _(b"building history"), unit=b'nodes', total=len(nodes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   619
            )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   620
            for i, node in enumerate(nodes):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   621
                if node in ancestors:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   622
                    continue
40843
b80915b52476 remotefilelog: use progress helper in repack
Martin von Zweigbergk <martinvonz@google.com>
parents: 40804
diff changeset
   623
                buildprogress.update(i)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   624
                try:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   625
                    ancestors.update(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   626
                        self.fullhistory.getancestors(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   627
                            filename, node, known=ancestors
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   628
                        )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   629
                    )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   630
                except KeyError:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   631
                    # Since we're packing data entries, we may not have the
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   632
                    # corresponding history entries for them. It's not a big
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   633
                    # deal, but the entries won't be delta'd perfectly.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   634
                    nohistory.append(node)
40843
b80915b52476 remotefilelog: use progress helper in repack
Martin von Zweigbergk <martinvonz@google.com>
parents: 40804
diff changeset
   635
            buildprogress.complete()
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   636
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   637
            # Order the nodes children first, so we can produce reverse deltas
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   638
            orderednodes = list(reversed(self._toposort(ancestors)))
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   639
            if len(nohistory) > 0:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   640
                ui.debug(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   641
                    b'repackdata: %d nodes without history\n' % len(nohistory)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   642
                )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   643
            orderednodes.extend(sorted(nohistory))
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   644
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   645
            # Filter orderednodes to just the nodes we want to serialize (it
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   646
            # currently also has the edge nodes' ancestors).
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   647
            orderednodes = list(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   648
                filter(lambda node: node in nodes, orderednodes)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   649
            )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   650
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   651
            # Garbage collect old nodes:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   652
            if self.garbagecollect:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   653
                neworderednodes = []
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   654
                for node in orderednodes:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   655
                    # If the node is old and is not in the keepset, we skip it,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   656
                    # and mark as garbage collected
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   657
                    if (filename, node) not in self.keepkeys and self.isold(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   658
                        self.repo, filename, node
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   659
                    ):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   660
                        entries[node].gced = True
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   661
                        continue
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   662
                    neworderednodes.append(node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   663
                orderednodes = neworderednodes
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   664
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   665
            # Compute delta bases for nodes:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   666
            deltabases = {}
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   667
            nobase = set()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   668
            referenced = set()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   669
            nodes = set(nodes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   670
            processprogress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   671
                _(b"processing nodes"), unit=b'nodes', total=len(orderednodes)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   672
            )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   673
            for i, node in enumerate(orderednodes):
40843
b80915b52476 remotefilelog: use progress helper in repack
Martin von Zweigbergk <martinvonz@google.com>
parents: 40804
diff changeset
   674
                processprogress.update(i)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   675
                # Find delta base
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   676
                # TODO: allow delta'ing against most recent descendant instead
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   677
                # of immediate child
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   678
                deltatuple = deltabases.get(node, None)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   679
                if deltatuple is None:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   680
                    deltabase, chainlen = nullid, 0
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   681
                    deltabases[node] = (nullid, 0)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   682
                    nobase.add(node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   683
                else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   684
                    deltabase, chainlen = deltatuple
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   685
                    referenced.add(deltabase)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   686
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   687
                # Use available ancestor information to inform our delta choices
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   688
                ancestorinfo = ancestors.get(node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   689
                if ancestorinfo:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   690
                    p1, p2, linknode, copyfrom = ancestorinfo
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   691
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   692
                    # The presence of copyfrom means we're at a point where the
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   693
                    # file was copied from elsewhere. So don't attempt to do any
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   694
                    # deltas with the other file.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   695
                    if copyfrom:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   696
                        p1 = nullid
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   697
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   698
                    if chainlen < maxchainlen:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   699
                        # Record this child as the delta base for its parents.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   700
                        # This may be non optimal, since the parents may have
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   701
                        # many children, and this will only choose the last one.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   702
                        # TODO: record all children and try all deltas to find
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   703
                        # best
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   704
                        if p1 != nullid:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   705
                            deltabases[p1] = (node, chainlen + 1)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   706
                        if p2 != nullid:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   707
                            deltabases[p2] = (node, chainlen + 1)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   708
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   709
            # experimental config: repack.chainorphansbysize
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   710
            if ui.configbool(b'repack', b'chainorphansbysize'):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   711
                orphans = nobase - referenced
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   712
                orderednodes = self._chainorphans(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   713
                    ui, filename, orderednodes, orphans, deltabases
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   714
                )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   715
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   716
            # Compute deltas and write to the pack
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   717
            for i, node in enumerate(orderednodes):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   718
                deltabase, chainlen = deltabases[node]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   719
                # Compute delta
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   720
                # TODO: Optimize the deltachain fetching. Since we're
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   721
                # iterating over the different version of the file, we may
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   722
                # be fetching the same deltachain over and over again.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   723
                if deltabase != nullid:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   724
                    deltaentry = self.data.getdelta(filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   725
                    delta, deltabasename, origdeltabase, meta = deltaentry
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   726
                    size = meta.get(constants.METAKEYSIZE)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   727
                    if (
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   728
                        deltabasename != filename
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   729
                        or origdeltabase != deltabase
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   730
                        or size is None
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   731
                    ):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   732
                        deltabasetext = self.data.get(filename, deltabase)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   733
                        original = self.data.get(filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   734
                        size = len(original)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   735
                        delta = mdiff.textdiff(deltabasetext, original)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   736
                else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   737
                    delta = self.data.get(filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   738
                    size = len(delta)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   739
                    meta = self.data.getmeta(filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   740
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   741
                # TODO: don't use the delta if it's larger than the fulltext
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   742
                if constants.METAKEYSIZE not in meta:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   743
                    meta[constants.METAKEYSIZE] = size
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   744
                target.add(filename, node, deltabase, delta, meta)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   745
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   746
                entries[node].datarepacked = True
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   747
40843
b80915b52476 remotefilelog: use progress helper in repack
Martin von Zweigbergk <martinvonz@google.com>
parents: 40804
diff changeset
   748
            processprogress.complete()
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   749
            count += 1
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   750
40843
b80915b52476 remotefilelog: use progress helper in repack
Martin von Zweigbergk <martinvonz@google.com>
parents: 40804
diff changeset
   751
        repackprogress.complete()
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   752
        target.close(ledger=ledger)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   753
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   754
    def repackhistory(self, ledger, target):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   755
        ui = self.repo.ui
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   756
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   757
        byfile = {}
43104
74802979dd9d py3: define and use pycompat.itervalues()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
   758
        for entry in pycompat.itervalues(ledger.entries):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   759
            if entry.historysource:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   760
                byfile.setdefault(entry.filename, {})[entry.node] = entry
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   761
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   762
        progress = ui.makeprogress(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   763
            _(b"repacking history"), unit=self.unit, total=len(byfile)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   764
        )
43105
649d3ac37a12 py3: define and use pycompat.iteritems() for hgext/
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43104
diff changeset
   765
        for filename, entries in sorted(pycompat.iteritems(byfile)):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   766
            ancestors = {}
40613
9769e0f6ffe0 py3: don't use dict.iterkeys()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40526
diff changeset
   767
            nodes = list(node for node in entries)
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   768
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   769
            for node in nodes:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   770
                if node in ancestors:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   771
                    continue
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   772
                ancestors.update(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   773
                    self.history.getancestors(filename, node, known=ancestors)
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   774
                )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   775
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   776
            # Order the nodes children first
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   777
            orderednodes = reversed(self._toposort(ancestors))
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   778
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   779
            # Write to the pack
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   780
            dontprocess = set()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   781
            for node in orderednodes:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   782
                p1, p2, linknode, copyfrom = ancestors[node]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   783
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   784
                # If the node is marked dontprocess, but it's also in the
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   785
                # explicit entries set, that means the node exists both in this
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   786
                # file and in another file that was copied to this file.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   787
                # Usually this happens if the file was copied to another file,
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   788
                # then the copy was deleted, then reintroduced without copy
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   789
                # metadata. The original add and the new add have the same hash
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   790
                # since the content is identical and the parents are null.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   791
                if node in dontprocess and node not in entries:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   792
                    # If copyfrom == filename, it means the copy history
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   793
                    # went to come other file, then came back to this one, so we
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   794
                    # should continue processing it.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   795
                    if p1 != nullid and copyfrom != filename:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   796
                        dontprocess.add(p1)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   797
                    if p2 != nullid:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   798
                        dontprocess.add(p2)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   799
                    continue
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   800
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   801
                if copyfrom:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   802
                    dontprocess.add(p1)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   803
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   804
                target.add(filename, node, p1, p2, linknode, copyfrom)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   805
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   806
                if node in entries:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   807
                    entries[node].historyrepacked = True
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   808
40843
b80915b52476 remotefilelog: use progress helper in repack
Martin von Zweigbergk <martinvonz@google.com>
parents: 40804
diff changeset
   809
            progress.increment()
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   810
40843
b80915b52476 remotefilelog: use progress helper in repack
Martin von Zweigbergk <martinvonz@google.com>
parents: 40804
diff changeset
   811
        progress.complete()
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   812
        target.close(ledger=ledger)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   813
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   814
    def _toposort(self, ancestors):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   815
        def parentfunc(node):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   816
            p1, p2, linknode, copyfrom = ancestors[node]
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   817
            parents = []
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   818
            if p1 != nullid:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   819
                parents.append(p1)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   820
            if p2 != nullid:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   821
                parents.append(p2)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   822
            return parents
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   823
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   824
        sortednodes = shallowutil.sortnodes(ancestors.keys(), parentfunc)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   825
        return sortednodes
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   826
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   827
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   828
class repackledger(object):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   829
    """Storage for all the bookkeeping that happens during a repack. It contains
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   830
    the list of revisions being repacked, what happened to each revision, and
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   831
    which source store contained which revision originally (for later cleanup).
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   832
    """
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   833
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   834
    def __init__(self):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   835
        self.entries = {}
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   836
        self.sources = {}
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   837
        self.created = set()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   838
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   839
    def markdataentry(self, source, filename, node):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   840
        """Mark the given filename+node revision as having a data rev in the
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   841
        given source.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   842
        """
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   843
        entry = self._getorcreateentry(filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   844
        entry.datasource = True
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   845
        entries = self.sources.get(source)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   846
        if not entries:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   847
            entries = set()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   848
            self.sources[source] = entries
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   849
        entries.add(entry)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   850
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   851
    def markhistoryentry(self, source, filename, node):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   852
        """Mark the given filename+node revision as having a history rev in the
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   853
        given source.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   854
        """
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   855
        entry = self._getorcreateentry(filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   856
        entry.historysource = True
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   857
        entries = self.sources.get(source)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   858
        if not entries:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   859
            entries = set()
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   860
            self.sources[source] = entries
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   861
        entries.add(entry)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   862
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   863
    def _getorcreateentry(self, filename, node):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   864
        key = (filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   865
        value = self.entries.get(key)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   866
        if not value:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   867
            value = repackentry(filename, node)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   868
            self.entries[key] = value
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   869
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   870
        return value
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   871
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   872
    def addcreated(self, value):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   873
        self.created.add(value)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   874
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   875
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   876
class repackentry(object):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   877
    """Simple class representing a single revision entry in the repackledger.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   878
    """
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   879
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   880
    __slots__ = (
43506
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43115
diff changeset
   881
        'filename',
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43115
diff changeset
   882
        'node',
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43115
diff changeset
   883
        'datasource',
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43115
diff changeset
   884
        'historysource',
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43115
diff changeset
   885
        'datarepacked',
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43115
diff changeset
   886
        'historyrepacked',
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43115
diff changeset
   887
        'gced',
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   888
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   889
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   890
    def __init__(self, filename, node):
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   891
        self.filename = filename
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   892
        self.node = node
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   893
        # If the revision has a data entry in the source
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   894
        self.datasource = False
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   895
        # If the revision has a history entry in the source
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   896
        self.historysource = False
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   897
        # If the revision's data entry was repacked into the repack target
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   898
        self.datarepacked = False
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   899
        # If the revision's history entry was repacked into the repack target
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   900
        self.historyrepacked = False
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   901
        # If garbage collected
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   902
        self.gced = False
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   903
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   904
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   905
def repacklockvfs(repo):
43115
4aa72cdf616f py3: delete b'' prefix from safehasattr arguments
Martin von Zweigbergk <martinvonz@google.com>
parents: 43105
diff changeset
   906
    if util.safehasattr(repo, 'name'):
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   907
        # Lock in the shared cache so repacks across multiple copies of the same
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   908
        # repo are coordinated.
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   909
        sharedcachepath = shallowutil.getcachepackpath(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   910
            repo, constants.FILEPACK_CATEGORY
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42943
diff changeset
   911
        )
40495
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   912
        return vfs.vfs(sharedcachepath)
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   913
    else:
3a333a582d7b remotefilelog: import pruned-down remotefilelog extension from hg-experimental
Augie Fackler <augie@google.com>
parents:
diff changeset
   914
        return repo.svfs