mercurial/sparse.py
author Gregory Szorc <gregory.szorc@gmail.com>
Thu, 06 Jul 2017 12:20:53 -0700
changeset 33302 36a415b5a4b2
parent 33301 ca4b78eb11e7
child 33303 8b571495d811
permissions -rw-r--r--
localrepo: add sparse caches The sparse extension maintains caches for the sparse files to a signature and a signature to a matcher. This allows the sparse matchers to be resolved quickly, which is apparently something that can occur in loops. This patch ports the sparse caches to the localrepo class pretty much as-is. There is potentially room to improve the caching mechanism. But that can be done as a follow-up. The default invalidatecaches() now clears the relevant sparse cache. invalidatesignaturecache() has been moved to sparse.py.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
33297
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     1
# sparse.py - functionality for sparse checkouts
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     2
#
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     3
# Copyright 2014 Facebook, Inc.
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     4
#
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     5
# This software may be used and distributed according to the terms of the
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     6
# GNU General Public License version 2 or any later version.
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     7
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     8
from __future__ import absolute_import
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
     9
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    10
from .i18n import _
33301
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
    11
from .node import nullid
33297
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    12
from . import (
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    13
    error,
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    14
)
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    15
33299
41448fc51510 sparse: variable to track if sparse is enabled
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33298
diff changeset
    16
# Whether sparse features are enabled. This variable is intended to be
41448fc51510 sparse: variable to track if sparse is enabled
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33298
diff changeset
    17
# temporary to facilitate porting sparse to core. It should eventually be
41448fc51510 sparse: variable to track if sparse is enabled
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33298
diff changeset
    18
# a per-repo option, possibly a repo requirement.
41448fc51510 sparse: variable to track if sparse is enabled
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33298
diff changeset
    19
enabled = False
41448fc51510 sparse: variable to track if sparse is enabled
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33298
diff changeset
    20
33297
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    21
def parseconfig(ui, raw):
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    22
    """Parse sparse config file content.
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    23
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    24
    Returns a tuple of includes, excludes, and profiles.
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    25
    """
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    26
    includes = set()
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    27
    excludes = set()
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    28
    current = includes
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    29
    profiles = []
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    30
    for line in raw.split('\n'):
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    31
        line = line.strip()
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    32
        if not line or line.startswith('#'):
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    33
            # empty or comment line, skip
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    34
            continue
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    35
        elif line.startswith('%include '):
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    36
            line = line[9:].strip()
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    37
            if line:
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    38
                profiles.append(line)
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    39
        elif line == '[include]':
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    40
            if current != includes:
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    41
                # TODO pass filename into this API so we can report it.
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    42
                raise error.Abort(_('sparse config cannot have includes ' +
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    43
                                    'after excludes'))
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    44
            continue
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    45
        elif line == '[exclude]':
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    46
            current = excludes
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    47
        elif line:
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    48
            if line.strip().startswith('/'):
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    49
                ui.warn(_('warning: sparse profile cannot use' +
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    50
                          ' paths starting with /, ignoring %s\n') % line)
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    51
                continue
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    52
            current.add(line)
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    53
ba5d89774db6 sparse: move config parsing into core
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff changeset
    54
    return includes, excludes, profiles
33298
f41a99c45956 sparse: move profile reading into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33297
diff changeset
    55
f41a99c45956 sparse: move profile reading into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33297
diff changeset
    56
# Exists as separate function to facilitate monkeypatching.
f41a99c45956 sparse: move profile reading into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33297
diff changeset
    57
def readprofile(repo, profile, changeid):
f41a99c45956 sparse: move profile reading into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33297
diff changeset
    58
    """Resolve the raw content of a sparse profile file."""
f41a99c45956 sparse: move profile reading into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33297
diff changeset
    59
    # TODO add some kind of cache here because this incurs a manifest
f41a99c45956 sparse: move profile reading into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33297
diff changeset
    60
    # resolve and can be slow.
f41a99c45956 sparse: move profile reading into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33297
diff changeset
    61
    return repo.filectx(profile, changeid=changeid).data()
33300
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    62
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    63
def patternsforrev(repo, rev):
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    64
    """Obtain sparse checkout patterns for the given rev.
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    65
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    66
    Returns a tuple of iterables representing includes, excludes, and
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    67
    patterns.
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    68
    """
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    69
    # Feature isn't enabled. No-op.
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    70
    if not enabled:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    71
        return set(), set(), []
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    72
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    73
    raw = repo.vfs.tryread('sparse')
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    74
    if not raw:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    75
        return set(), set(), []
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    76
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    77
    if rev is None:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    78
        raise error.Abort(_('cannot parse sparse patterns from working '
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    79
                            'directory'))
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    80
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    81
    includes, excludes, profiles = parseconfig(repo.ui, raw)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    82
    ctx = repo[rev]
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    83
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    84
    if profiles:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    85
        visited = set()
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    86
        while profiles:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    87
            profile = profiles.pop()
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    88
            if profile in visited:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    89
                continue
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    90
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    91
            visited.add(profile)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    92
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    93
            try:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    94
                raw = readprofile(repo, profile, rev)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    95
            except error.ManifestLookupError:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    96
                msg = (
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    97
                    "warning: sparse profile '%s' not found "
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    98
                    "in rev %s - ignoring it\n" % (profile, ctx))
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
    99
                # experimental config: sparse.missingwarning
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   100
                if repo.ui.configbool(
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   101
                        'sparse', 'missingwarning', True):
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   102
                    repo.ui.warn(msg)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   103
                else:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   104
                    repo.ui.debug(msg)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   105
                continue
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   106
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   107
            pincludes, pexcludes, subprofs = parseconfig(repo.ui, raw)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   108
            includes.update(pincludes)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   109
            excludes.update(pexcludes)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   110
            for subprofile in subprofs:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   111
                profiles.append(subprofile)
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   112
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   113
        profiles = visited
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   114
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   115
    if includes:
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   116
        includes.add('.hg*')
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   117
f7a106b3f089 sparse: move resolving of sparse patterns for rev into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33299
diff changeset
   118
    return includes, excludes, profiles
33301
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   119
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   120
def activeprofiles(repo):
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   121
    revs = [repo.changelog.rev(node) for node in
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   122
            repo.dirstate.parents() if node != nullid]
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   123
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   124
    profiles = set()
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   125
    for rev in revs:
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   126
        profiles.update(patternsforrev(repo, rev)[2])
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   127
ca4b78eb11e7 sparse: move active profiles function into core
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33300
diff changeset
   128
    return profiles
33302
36a415b5a4b2 localrepo: add sparse caches
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33301
diff changeset
   129
36a415b5a4b2 localrepo: add sparse caches
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33301
diff changeset
   130
def invalidatesignaturecache(repo):
36a415b5a4b2 localrepo: add sparse caches
Gregory Szorc <gregory.szorc@gmail.com>
parents: 33301
diff changeset
   131
    repo._sparsesignaturecache.clear()