mercurial/store.py
author Augie Fackler <raf@durin42.com>
Fri, 27 Nov 2020 17:03:29 -0500
changeset 45942 89a2afe31e82
parent 45483 d252f51ab032
child 46113 59fa3890d40a
permissions -rw-r--r--
formating: upgrade to black 20.8b1 This required a couple of small tweaks to un-confuse black, but now it works. Big formatting changes come from: * Dramatically improved collection-splitting logic upstream * Black having a strong (correct IMO) opinion that """ is better than ''' Differential Revision: https://phab.mercurial-scm.org/D9430
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
     1
# store.py - repository store handling for Mercurial
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
     2
#
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
     3
# Copyright 2008 Matt Mackall <mpm@selenic.com>
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
     4
#
8225
46293a0c7e9f updated license to be explicit about GPL version 2
Martin Geisler <mg@lazybytes.net>
parents: 8210
diff changeset
     5
# This software may be used and distributed according to the terms of the
10263
25e572394f5c Update license to GPLv2+
Matt Mackall <mpm@selenic.com>
parents: 9133
diff changeset
     6
# GNU General Public License version 2 or any later version.
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
     7
27480
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
     8
from __future__ import absolute_import
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
     9
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    10
import errno
41978
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
    11
import functools
27480
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    12
import os
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    13
import stat
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    14
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    15
from .i18n import _
43089
c59eb1560c44 py3: manually import getattr where it is needed
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43077
diff changeset
    16
from .pycompat import getattr
27480
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    17
from . import (
42905
3df3b139a43d localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents: 42715
diff changeset
    18
    changelog,
27480
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    19
    error,
42905
3df3b139a43d localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents: 42715
diff changeset
    20
    manifest,
35582
72b91f905065 py3: use node.hex(h.digest()) instead of h.hexdigest()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 34215
diff changeset
    21
    node,
32372
df448de7cf3b parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents: 31362
diff changeset
    22
    policy,
30077
8f42d8c412c8 py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents: 30076
diff changeset
    23
    pycompat,
27480
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    24
    util,
31234
9b7a2ef4f27c vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31219
diff changeset
    25
    vfs as vfsmod,
27480
509159675cdb store: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26778
diff changeset
    26
)
44060
a61287a95dc3 core: migrate uses of hashlib.sha1 to hashutil.sha1
Augie Fackler <augie@google.com>
parents: 43506
diff changeset
    27
from .utils import hashutil
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
    28
43506
9f70512ae2cf cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents: 43106
diff changeset
    29
parsers = policy.importmod('parsers')
41978
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
    30
# how much bytes should be read from fncache in one read
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
    31
# It is done to prevent loading large fncache files into memory
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
    32
fncache_chunksize = 10 ** 6
32372
df448de7cf3b parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents: 31362
diff changeset
    33
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    34
40494
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    35
def _matchtrackedpath(path, matcher):
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    36
    """parses a fncache entry and returns whether the entry is tracking a path
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    37
    matched by matcher or not.
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    38
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    39
    If matcher is None, returns True"""
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    40
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    41
    if matcher is None:
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    42
        return True
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    43
    path = decodedir(path)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    44
    if path.startswith(b'data/'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    45
        return matcher(path[len(b'data/') : -len(b'.i')])
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    46
    elif path.startswith(b'meta/'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    47
        return matcher.visitdir(path[len(b'meta/') : -len(b'/00manifest.i')])
40494
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
    48
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    49
    raise error.ProgrammingError(b"cannot decode path %s" % path)
40624
66adfd58cb77 store: raise ProgrammingError if unable to decode a storage path
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40584
diff changeset
    50
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    51
8531
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
    52
# This avoids a collision between a file named foo and a dir named
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
    53
# foo.i or foo.d
17607
cc58dc47cb5e store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents: 17605
diff changeset
    54
def _encodedir(path):
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
    55
    """
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
    56
    >>> _encodedir(b'data/foo.i')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
    57
    'data/foo.i'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
    58
    >>> _encodedir(b'data/foo.i/bla.i')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
    59
    'data/foo.i.hg/bla.i'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
    60
    >>> _encodedir(b'data/foo.i.hg/bla.i')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
    61
    'data/foo.i.hg.hg/bla.i'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
    62
    >>> _encodedir(b'data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n')
17605
e9cc29be3305 store: add multiline doctest case for encodedir()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17604
diff changeset
    63
    'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n'
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
    64
    """
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    65
    return (
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    66
        path.replace(b".hg/", b".hg.hg/")
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    67
        .replace(b".i/", b".i.hg/")
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    68
        .replace(b".d/", b".d.hg/")
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    69
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    70
8531
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
    71
17607
cc58dc47cb5e store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents: 17605
diff changeset
    72
encodedir = getattr(parsers, 'encodedir', _encodedir)
cc58dc47cb5e store: use fast C implementation of encodedir() if it's available
Adrian Buehlmann <adrian@cadifra.com>
parents: 17605
diff changeset
    73
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    74
8531
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
    75
def decodedir(path):
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
    76
    """
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
    77
    >>> decodedir(b'data/foo.i')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
    78
    'data/foo.i'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
    79
    >>> decodedir(b'data/foo.i.hg/bla.i')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
    80
    'data/foo.i/bla.i'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
    81
    >>> decodedir(b'data/foo.i.hg.hg/bla.i')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
    82
    'data/foo.i.hg/bla.i'
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
    83
    """
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    84
    if b".hg/" not in path:
8531
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
    85
        return path
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    86
    return (
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    87
        path.replace(b".d.hg/", b".d/")
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    88
        .replace(b".i.hg/", b".i/")
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
    89
        .replace(b".hg.hg/", b".hg/")
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    90
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
    91
8531
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
    92
29071
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
    93
def _reserved():
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
    94
    """characters that are problematic for filesystems
29071
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
    95
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
    96
    * ascii escapes (0..31)
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
    97
    * ascii hi (126..255)
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
    98
    * windows specials
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
    99
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   100
    these characters will be escaped by encodefunctions
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   101
    """
30076
400dfded8a29 py3: make the string unicode so its iterable in py3k
Mateusz Kwapich <mitrandir@fb.com>
parents: 29338
diff changeset
   102
    winreserved = [ord(x) for x in u'\\:*?"<>|']
29071
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   103
    for x in range(32):
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   104
        yield x
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   105
    for x in range(126, 256):
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   106
        yield x
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   107
    for x in winreserved:
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   108
        yield x
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   109
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   110
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   111
def _buildencodefun():
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   112
    """
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   113
    >>> enc, dec = _buildencodefun()
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   114
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   115
    >>> enc(b'nothing/special.txt')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   116
    'nothing/special.txt'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   117
    >>> dec(b'nothing/special.txt')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   118
    'nothing/special.txt'
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   119
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   120
    >>> enc(b'HELLO')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   121
    '_h_e_l_l_o'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   122
    >>> dec(b'_h_e_l_l_o')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   123
    'HELLO'
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   124
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   125
    >>> enc(b'hello:world?')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   126
    'hello~3aworld~3f'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   127
    >>> dec(b'hello~3aworld~3f')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   128
    'hello:world?'
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   129
34136
414a3513c2bd doctest: do not embed non-ascii characters in docstring
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
   130
    >>> enc(b'the\\x07quick\\xADshot')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   131
    'the~07quick~adshot'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   132
    >>> dec(b'the~07quick~adshot')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   133
    'the\\x07quick\\xadshot'
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   134
    """
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   135
    e = b'_'
31253
64596338ba10 py3: factor out bytechr() function
Yuya Nishihara <yuya@tcha.org>
parents: 31234
diff changeset
   136
    xchr = pycompat.bytechr
64596338ba10 py3: factor out bytechr() function
Yuya Nishihara <yuya@tcha.org>
parents: 31234
diff changeset
   137
    asciistr = list(map(xchr, range(127)))
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   138
    capitals = list(range(ord(b"A"), ord(b"Z") + 1))
30077
8f42d8c412c8 py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents: 30076
diff changeset
   139
44452
9d2b2df2c2ba cleanup: run pyupgrade on our source tree to clean up varying things
Augie Fackler <augie@google.com>
parents: 44060
diff changeset
   140
    cmap = {x: x for x in asciistr}
29071
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   141
    for x in _reserved():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   142
        cmap[xchr(x)] = b"~%02x" % x
30077
8f42d8c412c8 py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents: 30076
diff changeset
   143
    for x in capitals + [ord(e)]:
8f42d8c412c8 py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents: 30076
diff changeset
   144
        cmap[xchr(x)] = e + xchr(x).lower()
8f42d8c412c8 py3: make encodefun in store.py compatible with py3k
Mateusz Kwapich <mitrandir@fb.com>
parents: 30076
diff changeset
   145
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   146
    dmap = {}
43106
d783f945a701 py3: finish porting iteritems() to pycompat and remove source transformer
Gregory Szorc <gregory.szorc@gmail.com>
parents: 43089
diff changeset
   147
    for k, v in pycompat.iteritems(cmap):
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   148
        dmap[v] = k
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   149
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   150
    def decode(s):
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   151
        i = 0
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   152
        while i < len(s):
38783
e7aa113b14f7 global: use pycompat.xrange()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 38696
diff changeset
   153
            for l in pycompat.xrange(1, 4):
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   154
                try:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   155
                    yield dmap[s[i : i + l]]
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   156
                    i += l
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   157
                    break
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   158
                except KeyError:
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   159
                    pass
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   160
            else:
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   161
                raise KeyError
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   162
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   163
    return (
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   164
        lambda s: b''.join(
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   165
            [cmap[s[c : c + 1]] for c in pycompat.xrange(len(s))]
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   166
        ),
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   167
        lambda s: b''.join(list(decode(s))),
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   168
    )
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   169
17608
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   170
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   171
_encodefname, _decodefname = _buildencodefun()
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   172
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   173
17608
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   174
def encodefilename(s):
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   175
    """
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   176
    >>> encodefilename(b'foo.i/bar.d/bla.hg/hi:world?/HELLO')
17608
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   177
    'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o'
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   178
    """
17608
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   179
    return _encodefname(encodedir(s))
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   180
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   181
17608
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   182
def decodefilename(s):
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   183
    """
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   184
    >>> decodefilename(b'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o')
17608
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   185
    'foo.i/bar.d/bla.hg/hi:world?/HELLO'
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   186
    """
17608
776240123525 store: extract functions _encodefname and _decodefname
Adrian Buehlmann <adrian@cadifra.com>
parents: 17607
diff changeset
   187
    return decodedir(_decodefname(s))
6839
01db3e101362 move filename encoding functions from util.py to new store.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
diff changeset
   188
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   189
14288
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   190
def _buildlowerencodefun():
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   191
    """
14288
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   192
    >>> f = _buildlowerencodefun()
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   193
    >>> f(b'nothing/special.txt')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   194
    'nothing/special.txt'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   195
    >>> f(b'HELLO')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   196
    'hello'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   197
    >>> f(b'hello:world?')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   198
    'hello~3aworld~3f'
34136
414a3513c2bd doctest: do not embed non-ascii characters in docstring
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
   199
    >>> f(b'the\\x07quick\\xADshot')
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   200
    'the~07quick~adshot'
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   201
    """
34214
7e3f078b6f31 py3: use bytechr() in store._buildlowerencodefun()
Yuya Nishihara <yuya@tcha.org>
parents: 34213
diff changeset
   202
    xchr = pycompat.bytechr
44452
9d2b2df2c2ba cleanup: run pyupgrade on our source tree to clean up varying things
Augie Fackler <augie@google.com>
parents: 44060
diff changeset
   203
    cmap = {xchr(x): xchr(x) for x in pycompat.xrange(127)}
29071
2f58975eb2cb store: treat range as a generator instead of a list for py3 compat
timeless <timeless@mozdev.org>
parents: 28007
diff changeset
   204
    for x in _reserved():
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   205
        cmap[xchr(x)] = b"~%02x" % x
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   206
    for x in range(ord(b"A"), ord(b"Z") + 1):
34214
7e3f078b6f31 py3: use bytechr() in store._buildlowerencodefun()
Yuya Nishihara <yuya@tcha.org>
parents: 34213
diff changeset
   207
        cmap[xchr(x)] = xchr(x).lower()
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   208
34213
96808804b68f store: give name to lowerencode function
Yuya Nishihara <yuya@tcha.org>
parents: 34136
diff changeset
   209
    def lowerencode(s):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   210
        return b"".join([cmap[c] for c in pycompat.iterbytestr(s)])
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   211
34213
96808804b68f store: give name to lowerencode function
Yuya Nishihara <yuya@tcha.org>
parents: 34136
diff changeset
   212
    return lowerencode
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   213
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   214
18430
0459c6555f69 store: implement lowerencode in C
Bryan O'Sullivan <bryano@fb.com>
parents: 18054
diff changeset
   215
lowerencode = getattr(parsers, 'lowerencode', None) or _buildlowerencodefun()
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   216
17570
f53a7b256ca6 store: optimze _auxencode() a bit by grouping the reserved names by length
Adrian Buehlmann <adrian@cadifra.com>
parents: 17569
diff changeset
   217
# Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   218
_winres3 = (b'aux', b'con', b'prn', b'nul')  # length 3
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   219
_winres4 = (b'com', b'lpt')  # length 4 (with trailing 1..9)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   220
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   221
12687
34d8247a4595 store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12171
diff changeset
   222
def _auxencode(path, dotencode):
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   223
    """
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   224
    Encodes filenames containing names reserved by Windows or which end in
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   225
    period or space. Does not touch other single reserved characters c.
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   226
    Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here.
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   227
    Additionally encodes space or period at the beginning, if dotencode is
17569
e9af2134825c store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17568
diff changeset
   228
    True. Parameter path is assumed to be all lowercase.
e9af2134825c store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17568
diff changeset
   229
    A segment only needs encoding if a reserved name appears as a
e9af2134825c store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17568
diff changeset
   230
    basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux"
e9af2134825c store: explain "aux.foo" versus "foo.aux" in doc of _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17568
diff changeset
   231
    doesn't need encoding.
13949
ba43aa1e173c store: add some doctests
Adrian Buehlmann <adrian@cadifra.com>
parents: 13426
diff changeset
   232
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   233
    >>> s = b'.foo/aux.txt/txt.aux/con/prn/nul/foo.'
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   234
    >>> _auxencode(s.split(b'/'), True)
17574
81a033bb29bc store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents: 17573
diff changeset
   235
    ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e']
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   236
    >>> s = b'.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.'
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   237
    >>> _auxencode(s.split(b'/'), False)
17574
81a033bb29bc store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents: 17573
diff changeset
   238
    ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e']
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   239
    >>> _auxencode([b'foo. '], True)
17574
81a033bb29bc store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents: 17573
diff changeset
   240
    ['foo.~20']
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 33412
diff changeset
   241
    >>> _auxencode([b' .foo'], True)
17574
81a033bb29bc store: let _auxencode() return the list of path segments
Adrian Buehlmann <adrian@cadifra.com>
parents: 17573
diff changeset
   242
    ['~20.foo']
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   243
    """
17589
b11024849db6 store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents: 17588
diff changeset
   244
    for i, n in enumerate(path):
17572
b644287e79a8 store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17571
diff changeset
   245
        if not n:
b644287e79a8 store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17571
diff changeset
   246
            continue
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   247
        if dotencode and n[0] in b'. ':
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   248
            n = b"~%02x" % ord(n[0:1]) + n[1:]
17589
b11024849db6 store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents: 17588
diff changeset
   249
            path[i] = n
17572
b644287e79a8 store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17571
diff changeset
   250
        else:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   251
            l = n.find(b'.')
17572
b644287e79a8 store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17571
diff changeset
   252
            if l == -1:
b644287e79a8 store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17571
diff changeset
   253
                l = len(n)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   254
            if (l == 3 and n[:3] in _winres3) or (
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   255
                l == 4
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   256
                and n[3:4] <= b'9'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   257
                and n[3:4] >= b'1'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   258
                and n[:3] in _winres4
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   259
            ):
17572
b644287e79a8 store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17571
diff changeset
   260
                # encode third letter ('aux' -> 'au~78')
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   261
                ec = b"~%02x" % ord(n[2:3])
17572
b644287e79a8 store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17571
diff changeset
   262
                n = n[0:2] + ec + n[3:]
17589
b11024849db6 store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents: 17588
diff changeset
   263
                path[i] = n
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   264
        if n[-1] in b'. ':
17572
b644287e79a8 store: unindent most of the contents of the for loop in _auxencode()
Adrian Buehlmann <adrian@cadifra.com>
parents: 17571
diff changeset
   265
            # encode last period or space ('foo...' -> 'foo..~2e')
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   266
            path[i] = n[:-1] + b"~%02x" % ord(n[-1:])
17589
b11024849db6 store: parameter path of _auxencode is now a list of strings
Adrian Buehlmann <adrian@cadifra.com>
parents: 17588
diff changeset
   267
    return path
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   268
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   269
14288
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   270
_maxstorepathlen = 120
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   271
_dirprefixlen = 8
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   272
_maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   273
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   274
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   275
def _hashencode(path, dotencode):
44060
a61287a95dc3 core: migrate uses of hashlib.sha1 to hashutil.sha1
Augie Fackler <augie@google.com>
parents: 43506
diff changeset
   276
    digest = node.hex(hashutil.sha1(path).digest())
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   277
    le = lowerencode(path[5:]).split(b'/')  # skips prefix 'data/' or 'meta/'
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   278
    parts = _auxencode(le, dotencode)
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   279
    basename = parts[-1]
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   280
    _root, ext = os.path.splitext(basename)
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   281
    sdirs = []
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   282
    sdirslen = 0
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   283
    for p in parts[:-1]:
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   284
        d = p[:_dirprefixlen]
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   285
        if d[-1] in b'. ':
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   286
            # Windows can't access dirs ending in period or space
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   287
            d = d[:-1] + b'_'
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   288
        if sdirslen == 0:
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   289
            t = len(d)
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   290
        else:
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   291
            t = sdirslen + 1 + len(d)
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   292
            if t > _maxshortdirslen:
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   293
                break
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   294
        sdirs.append(d)
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   295
        sdirslen = t
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   296
    dirs = b'/'.join(sdirs)
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   297
    if len(dirs) > 0:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   298
        dirs += b'/'
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   299
    res = b'dh/' + dirs + digest + ext
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   300
    spaceleft = _maxstorepathlen - len(res)
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   301
    if spaceleft > 0:
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   302
        filler = basename[:spaceleft]
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   303
        res = b'dh/' + dirs + filler + digest + ext
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   304
    return res
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   305
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   306
17590
eb0884680f5c store: eliminate one level of lambda functions on _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents: 17589
diff changeset
   307
def _hybridencode(path, dotencode):
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   308
    """encodes path with a length limit
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   309
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   310
    Encodes all paths that begin with 'data/', according to the following.
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   311
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   312
    Default encoding (reversible):
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   313
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   314
    Encodes all uppercase letters 'X' as '_x'. All reserved or illegal
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   315
    characters are encoded as '~xx', where xx is the two digit hex code
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   316
    of the character (see encodefilename).
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   317
    Relevant path components consisting of Windows reserved filenames are
17738
b8424c92ba2b spelling: fix minor spell checker issues
Mads Kiilerich <mads@kiilerich.com>
parents: 17731
diff changeset
   318
    masked by encoding the third character ('aux' -> 'au~78', see _auxencode).
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   319
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   320
    Hashed encoding (not reversible):
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   321
14288
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   322
    If the default-encoded path is longer than _maxstorepathlen, a
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   323
    non-reversible hybrid hashing of the path is done instead.
14288
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   324
    This encoding uses up to _dirprefixlen characters of all directory
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   325
    levels of the lowerencoded path, but not more levels than can fit into
14288
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   326
    _maxshortdirslen.
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   327
    Then follows the filler followed by the sha digest of the full path.
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   328
    The filler is the beginning of the basename of the lowerencoded path
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   329
    (the basename is everything after the last path separator). The filler
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   330
    is as long as possible, filling in characters from the basename until
14288
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   331
    the encoded path has _maxstorepathlen characters (or all chars of the
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   332
    basename have been taken).
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   333
    The extension (e.g. '.i' or '.d') is preserved.
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   334
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   335
    The string 'data/' at the beginning is replaced with 'dh/', if the hashed
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   336
    encoding was used.
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   337
    """
17609
cbc180cfd60b store: reuse direncoded path in _hybridencode
Adrian Buehlmann <adrian@cadifra.com>
parents: 17608
diff changeset
   338
    path = encodedir(path)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   339
    ef = _encodefname(path).split(b'/')
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   340
    res = b'/'.join(_auxencode(ef, dotencode))
14288
00a0ab08f986 store: change names to comply with project coding standards
Adrian Buehlmann <adrian@cadifra.com>
parents: 14194
diff changeset
   341
    if len(res) > _maxstorepathlen:
17610
d0afa149e059 store: refactor hashed encoding into its own function
Bryan O'Sullivan <bryano@fb.com>
parents: 17609
diff changeset
   342
        res = _hashencode(path, dotencode)
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   343
    return res
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   344
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   345
17624
ae103510f6aa store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents: 17623
diff changeset
   346
def _pathencode(path):
18435
8c019d2fd7c0 store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents: 18430
diff changeset
   347
    de = encodedir(path)
17693
0c6de45e1212 store: optimize _pathencode by checking the length of the unencoded path
Adrian Buehlmann <adrian@cadifra.com>
parents: 17653
diff changeset
   348
    if len(path) > _maxstorepathlen:
18435
8c019d2fd7c0 store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents: 18430
diff changeset
   349
        return _hashencode(de, True)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   350
    ef = _encodefname(de).split(b'/')
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   351
    res = b'/'.join(_auxencode(ef, True))
17624
ae103510f6aa store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents: 17623
diff changeset
   352
    if len(res) > _maxstorepathlen:
18435
8c019d2fd7c0 store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents: 18430
diff changeset
   353
        return _hashencode(de, True)
17624
ae103510f6aa store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents: 17623
diff changeset
   354
    return res
ae103510f6aa store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents: 17623
diff changeset
   355
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   356
17624
ae103510f6aa store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents: 17623
diff changeset
   357
_pathencode = getattr(parsers, 'pathencode', _pathencode)
ae103510f6aa store: add a fallback _pathencode Python function
Adrian Buehlmann <adrian@cadifra.com>
parents: 17623
diff changeset
   358
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   359
17623
448e6ed7c557 store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents: 17621
diff changeset
   360
def _plainhybridencode(f):
448e6ed7c557 store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents: 17621
diff changeset
   361
    return _hybridencode(f, False)
448e6ed7c557 store: move _plainhybridencode and _dothybridencode higher up in the file
Adrian Buehlmann <adrian@cadifra.com>
parents: 17621
diff changeset
   362
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   363
17726
7cb7e17c23b2 store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17725
diff changeset
   364
def _calcmode(vfs):
6898
69aeaaaf6e07 store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents: 6897
diff changeset
   365
    try:
69aeaaaf6e07 store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents: 6897
diff changeset
   366
        # files in .hg/ will be created using this mode
17726
7cb7e17c23b2 store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17725
diff changeset
   367
        mode = vfs.stat().st_mode
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   368
        # avoid some useless chmods
25658
e93036747902 global: mass rewrite to use modern octal syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25091
diff changeset
   369
        if (0o777 & ~util.umask) == (0o777 & mode):
6898
69aeaaaf6e07 store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents: 6897
diff changeset
   370
            mode = None
69aeaaaf6e07 store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents: 6897
diff changeset
   371
    except OSError:
69aeaaaf6e07 store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents: 6897
diff changeset
   372
        mode = None
69aeaaaf6e07 store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents: 6897
diff changeset
   373
    return mode
69aeaaaf6e07 store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents: 6897
diff changeset
   374
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   375
45351
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   376
_data = [
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   377
    b'bookmarks',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   378
    b'narrowspec',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   379
    b'data',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   380
    b'meta',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   381
    b'00manifest.d',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   382
    b'00manifest.i',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   383
    b'00changelog.d',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   384
    b'00changelog.i',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   385
    b'phaseroots',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   386
    b'obsstore',
45483
d252f51ab032 share: introduce config option to store requires in .hg/store
Pulkit Goyal <7895pulkit@gmail.com>
parents: 45351
diff changeset
   387
    b'requires',
45351
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   388
]
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   389
6903
0642d9d7ec80 clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents: 6902
diff changeset
   390
37409
4c15bee42e9c store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35582
diff changeset
   391
def isrevlog(f, kind, st):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   392
    return kind == stat.S_IFREG and f[-2:] in (b'.i', b'.d')
37409
4c15bee42e9c store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35582
diff changeset
   393
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   394
8778
c5f36402daad use new style classes
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8633
diff changeset
   395
class basicstore(object):
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   396
    '''base class for local repository stores'''
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   397
17651
3b49c28658f6 store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17649
diff changeset
   398
    def __init__(self, path, vfstype):
17724
bf4b72d8dd4d store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17722
diff changeset
   399
        vfs = vfstype(path)
bf4b72d8dd4d store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17722
diff changeset
   400
        self.path = vfs.base
17726
7cb7e17c23b2 store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17725
diff changeset
   401
        self.createmode = _calcmode(vfs)
17652
2c6f7231becc store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17651
diff changeset
   402
        vfs.createmode = self.createmode
17728
004bd533880d store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17727
diff changeset
   403
        self.rawvfs = vfs
31234
9b7a2ef4f27c vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31219
diff changeset
   404
        self.vfs = vfsmod.filtervfs(vfs, encodedir)
17653
dacb50696b75 store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17652
diff changeset
   405
        self.opener = self.vfs
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   406
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   407
    def join(self, f):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   408
        return self.path + b'/' + encodedir(f)
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   409
37409
4c15bee42e9c store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35582
diff changeset
   410
    def _walk(self, relpath, recurse, filefilter=isrevlog):
6900
def492d1b592 store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents: 6899
diff changeset
   411
        '''yields (unencoded, encoded, size)'''
13426
643b8212813e store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents: 13391
diff changeset
   412
        path = self.path
643b8212813e store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents: 13391
diff changeset
   413
        if relpath:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   414
            path += b'/' + relpath
13426
643b8212813e store: remove pointless pathjoiner parameter
Adrian Buehlmann <adrian@cadifra.com>
parents: 13391
diff changeset
   415
        striplen = len(self.path) + 1
6899
56a7a54e074f store: simplify walking
Matt Mackall <mpm@selenic.com>
parents: 6898
diff changeset
   416
        l = []
17728
004bd533880d store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17727
diff changeset
   417
        if self.rawvfs.isdir(path):
6899
56a7a54e074f store: simplify walking
Matt Mackall <mpm@selenic.com>
parents: 6898
diff changeset
   418
            visit = [path]
17747
aad3bce98f76 store: invoke "osutil.listdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17745
diff changeset
   419
            readdir = self.rawvfs.readdir
6899
56a7a54e074f store: simplify walking
Matt Mackall <mpm@selenic.com>
parents: 6898
diff changeset
   420
            while visit:
56a7a54e074f store: simplify walking
Matt Mackall <mpm@selenic.com>
parents: 6898
diff changeset
   421
                p = visit.pop()
17747
aad3bce98f76 store: invoke "osutil.listdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17745
diff changeset
   422
                for f, kind, st in readdir(p, stat=True):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   423
                    fp = p + b'/' + f
37409
4c15bee42e9c store: make file filtering during walk configurable
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35582
diff changeset
   424
                    if filefilter(f, kind, st):
6900
def492d1b592 store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents: 6899
diff changeset
   425
                        n = util.pconvert(fp[striplen:])
8531
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
   426
                        l.append((decodedir(n), n, st.st_size))
6899
56a7a54e074f store: simplify walking
Matt Mackall <mpm@selenic.com>
parents: 6898
diff changeset
   427
                    elif kind == stat.S_IFDIR and recurse:
56a7a54e074f store: simplify walking
Matt Mackall <mpm@selenic.com>
parents: 6898
diff changeset
   428
                        visit.append(fp)
17054
125ff5654b72 store: sort filenames in place
Bryan O'Sullivan <bryano@fb.com>
parents: 16404
diff changeset
   429
        l.sort()
125ff5654b72 store: sort filenames in place
Bryan O'Sullivan <bryano@fb.com>
parents: 16404
diff changeset
   430
        return l
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   431
42905
3df3b139a43d localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents: 42715
diff changeset
   432
    def changelog(self, trypending):
3df3b139a43d localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents: 42715
diff changeset
   433
        return changelog.changelog(self.vfs, trypending=trypending)
3df3b139a43d localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents: 42715
diff changeset
   434
3df3b139a43d localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents: 42715
diff changeset
   435
    def manifestlog(self, repo, storenarrowmatch):
3df3b139a43d localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents: 42715
diff changeset
   436
        rootstore = manifest.manifestrevlog(self.vfs)
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   437
        return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch)
42905
3df3b139a43d localrepo: push manifestlog and changelog construction code into store
Augie Fackler <augie@google.com>
parents: 42715
diff changeset
   438
40340
2d45b549392f store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 39698
diff changeset
   439
    def datafiles(self, matcher=None):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   440
        return self._walk(b'data', True) + self._walk(b'meta', True)
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   441
19177
1e104aaa4c44 store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents: 18435
diff changeset
   442
    def topfiles(self):
1e104aaa4c44 store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents: 18435
diff changeset
   443
        # yield manifest before changelog
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   444
        return reversed(self._walk(b'', False))
19177
1e104aaa4c44 store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents: 18435
diff changeset
   445
40340
2d45b549392f store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 39698
diff changeset
   446
    def walk(self, matcher=None):
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   447
        """yields (unencoded, encoded, size)
40340
2d45b549392f store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 39698
diff changeset
   448
2d45b549392f store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 39698
diff changeset
   449
        if a matcher is passed, storage files of only those tracked paths
2d45b549392f store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 39698
diff changeset
   450
        are passed with matches the matcher
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   451
        """
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   452
        # yield data files first
40340
2d45b549392f store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 39698
diff changeset
   453
        for x in self.datafiles(matcher):
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   454
            yield x
19177
1e104aaa4c44 store: move top file walk to a separate function
Durham Goode <durham@fb.com>
parents: 18435
diff changeset
   455
        for x in self.topfiles():
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   456
            yield x
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   457
6903
0642d9d7ec80 clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents: 6902
diff changeset
   458
    def copylist(self):
45483
d252f51ab032 share: introduce config option to store requires in .hg/store
Pulkit Goyal <7895pulkit@gmail.com>
parents: 45351
diff changeset
   459
        return _data
6903
0642d9d7ec80 clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents: 6902
diff changeset
   460
20883
cd443c7589cc fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents: 20879
diff changeset
   461
    def write(self, tr):
13391
d00bbff8600e fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents: 13169
diff changeset
   462
        pass
d00bbff8600e fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents: 13169
diff changeset
   463
20884
2efdd186925d caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents: 20883
diff changeset
   464
    def invalidatecaches(self):
2efdd186925d caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents: 20883
diff changeset
   465
        pass
2efdd186925d caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents: 20883
diff changeset
   466
20885
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   467
    def markremoved(self, fn):
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   468
        pass
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   469
17744
09d5b2055295 store: add a contains method to basicstore
smuralid
parents: 17738
diff changeset
   470
    def __contains__(self, path):
09d5b2055295 store: add a contains method to basicstore
smuralid
parents: 17738
diff changeset
   471
        '''Checks if the store contains path'''
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   472
        path = b"/".join((b"data", path))
17744
09d5b2055295 store: add a contains method to basicstore
smuralid
parents: 17738
diff changeset
   473
        # file?
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   474
        if self.vfs.exists(path + b".i"):
17744
09d5b2055295 store: add a contains method to basicstore
smuralid
parents: 17738
diff changeset
   475
            return True
09d5b2055295 store: add a contains method to basicstore
smuralid
parents: 17738
diff changeset
   476
        # dir?
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   477
        if not path.endswith(b"/"):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   478
            path = path + b"/"
19903
ca875b271ac3 store: use "vfs.exists()" instead of "os.path.exists()"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 19177
diff changeset
   479
        return self.vfs.exists(path)
17744
09d5b2055295 store: add a contains method to basicstore
smuralid
parents: 17738
diff changeset
   480
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   481
6898
69aeaaaf6e07 store: simplify class hierarchy
Matt Mackall <mpm@selenic.com>
parents: 6897
diff changeset
   482
class encodedstore(basicstore):
17651
3b49c28658f6 store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17649
diff changeset
   483
    def __init__(self, path, vfstype):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   484
        vfs = vfstype(path + b'/store')
17724
bf4b72d8dd4d store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17722
diff changeset
   485
        self.path = vfs.base
17726
7cb7e17c23b2 store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17725
diff changeset
   486
        self.createmode = _calcmode(vfs)
17652
2c6f7231becc store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17651
diff changeset
   487
        vfs.createmode = self.createmode
17728
004bd533880d store: invoke "os.path.isdir()" via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17727
diff changeset
   488
        self.rawvfs = vfs
31234
9b7a2ef4f27c vfs: use 'vfs' module directly in 'mercurial.store'
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31219
diff changeset
   489
        self.vfs = vfsmod.filtervfs(vfs, encodefilename)
17653
dacb50696b75 store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17652
diff changeset
   490
        self.opener = self.vfs
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   491
40340
2d45b549392f store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 39698
diff changeset
   492
    def datafiles(self, matcher=None):
28007
fb92927f9775 treemanifests: fix streaming clone
Martin von Zweigbergk <martinvonz@google.com>
parents: 28006
diff changeset
   493
        for a, b, size in super(encodedstore, self).datafiles():
6892
dab95717058d verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents: 6890
diff changeset
   494
            try:
6900
def492d1b592 store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents: 6899
diff changeset
   495
                a = decodefilename(a)
6892
dab95717058d verify: check repo.store
Adrian Buehlmann <adrian@cadifra.com>
parents: 6890
diff changeset
   496
            except KeyError:
6900
def492d1b592 store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents: 6899
diff changeset
   497
                a = None
40584
a694a7159125 store: pass in decoded filename to narrow matcher
Yuya Nishihara <yuya@tcha.org>
parents: 40494
diff changeset
   498
            if a is not None and not _matchtrackedpath(a, matcher):
a694a7159125 store: pass in decoded filename to narrow matcher
Yuya Nishihara <yuya@tcha.org>
parents: 40494
diff changeset
   499
                continue
6900
def492d1b592 store: change handling of decoding errors
Matt Mackall <mpm@selenic.com>
parents: 6899
diff changeset
   500
            yield a, b, size
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   501
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   502
    def join(self, f):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   503
        return self.path + b'/' + encodefilename(f)
6840
80e51429cb9a introduce store classes
Adrian Buehlmann <adrian@cadifra.com>
parents: 6839
diff changeset
   504
6903
0642d9d7ec80 clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents: 6902
diff changeset
   505
    def copylist(self):
45351
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   506
        return [b'requires', b'00changelog.i'] + [b'store/' + f for f in _data]
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   507
6903
0642d9d7ec80 clone: get a list of files to clone from store
Matt Mackall <mpm@selenic.com>
parents: 6902
diff changeset
   508
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   509
class fncache(object):
8531
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
   510
    # the filename used to be partially encoded
810387f59696 filelog encoding: move the encoding/decoding into store
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8530
diff changeset
   511
    # hence the encodedir/decodedir dance
17722
3b976051034d store: rename field name from "opener" to "vfs" in internal classes for fncache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17721
diff changeset
   512
    def __init__(self, vfs):
3b976051034d store: rename field name from "opener" to "vfs" in internal classes for fncache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17721
diff changeset
   513
        self.vfs = vfs
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   514
        self.entries = None
13391
d00bbff8600e fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents: 13169
diff changeset
   515
        self._dirty = False
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   516
        # set of new additions to fncache
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   517
        self.addls = set()
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   518
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   519
    def ensureloaded(self, warn=None):
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   520
        """read the fncache file if not already read.
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   521
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   522
        If the file on disk is corrupted, raise. If warn is provided,
45942
89a2afe31e82 formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents: 45483
diff changeset
   523
        warn and keep going instead."""
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   524
        if self.entries is None:
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   525
            self._load(warn)
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   526
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   527
    def _load(self, warn=None):
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   528
        '''fill the entries from the fncache file'''
13391
d00bbff8600e fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents: 13169
diff changeset
   529
        self._dirty = False
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   530
        try:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   531
            fp = self.vfs(b'fncache', mode=b'rb')
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   532
        except IOError:
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   533
            # skip nonexistent file
16404
9fca5b056c0a store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents: 15742
diff changeset
   534
            self.entries = set()
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   535
            return
41978
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   536
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   537
        self.entries = set()
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   538
        chunk = b''
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   539
        for c in iter(functools.partial(fp.read, fncache_chunksize), b''):
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   540
            chunk += c
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   541
            try:
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   542
                p = chunk.rindex(b'\n')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   543
                self.entries.update(decodedir(chunk[: p + 1]).splitlines())
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   544
                chunk = chunk[p + 1 :]
41978
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   545
            except ValueError:
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   546
                # substring '\n' not found, maybe the entry is bigger than the
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   547
                # chunksize, so let's keep iterating
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   548
                pass
a56487081109 store: don't read the whole fncache in memory
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41973
diff changeset
   549
41981
a920a9e1795a store: error out if fncache does not ends with a newline
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41978
diff changeset
   550
        if chunk:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   551
            msg = _(b"fncache does not ends with a newline")
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   552
            if warn:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   553
                warn(msg + b'\n')
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   554
            else:
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   555
                raise error.Abort(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   556
                    msg,
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   557
                    hint=_(
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   558
                        b"use 'hg debugrebuildfncache' to "
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   559
                        b"rebuild the fncache"
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   560
                    ),
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   561
                )
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   562
        self._checkentries(fp, warn)
41973
d7ef84e595f8 store: move logic to check for invalid entry in fncache to own function
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41093
diff changeset
   563
        fp.close()
d7ef84e595f8 store: move logic to check for invalid entry in fncache to own function
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41093
diff changeset
   564
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   565
    def _checkentries(self, fp, warn):
41973
d7ef84e595f8 store: move logic to check for invalid entry in fncache to own function
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 41093
diff changeset
   566
        """ make sure there is no empty string in entries """
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   567
        if b'' in self.entries:
16404
9fca5b056c0a store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents: 15742
diff changeset
   568
            fp.seek(0)
30398
b63bef41a7b6 store: migrate to util.iterfile
Jun Wu <quark@fb.com>
parents: 30108
diff changeset
   569
            for n, line in enumerate(util.iterfile(fp)):
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   570
                if not line.rstrip(b'\n'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   571
                    t = _(b'invalid entry in fncache, line %d') % (n + 1)
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   572
                    if warn:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   573
                        warn(t + b'\n')
42715
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   574
                    else:
f59f8a5e9096 fncache: make debugrebuildfncache not fail on broken fncache
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents: 42341
diff changeset
   575
                        raise error.Abort(t)
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   576
20883
cd443c7589cc fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents: 20879
diff changeset
   577
    def write(self, tr):
16404
9fca5b056c0a store: speed up read and write of large fncache files
Bryan O'Sullivan <bryano@fb.com>
parents: 15742
diff changeset
   578
        if self._dirty:
38696
89d93dd1a222 store: assert the fncache have been loaded if dirty
Boris Feld <boris.feld@octobus.net>
parents: 38661
diff changeset
   579
            assert self.entries is not None
40748
df8ed31a8ad8 store: write fncache only once if there are both adds and removes
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40736
diff changeset
   580
            self.entries = self.entries | self.addls
df8ed31a8ad8 store: write fncache only once if there are both adds and removes
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40736
diff changeset
   581
            self.addls = set()
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   582
            tr.addbackup(b'fncache')
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   583
            fp = self.vfs(b'fncache', mode=b'wb', atomictemp=True)
20879
cd03854a2e06 fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents: 19903
diff changeset
   584
            if self.entries:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   585
                fp.write(encodedir(b'\n'.join(self.entries) + b'\n'))
20879
cd03854a2e06 fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents: 19903
diff changeset
   586
            fp.close()
cd03854a2e06 fncache: remove the rewriting logic
Durham Goode <durham@fb.com>
parents: 19903
diff changeset
   587
            self._dirty = False
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   588
        if self.addls:
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   589
            # if we have just new entries, let's append them to the fncache
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   590
            tr.addbackup(b'fncache')
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   591
            fp = self.vfs(b'fncache', mode=b'ab', atomictemp=True)
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   592
            if self.addls:
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   593
                fp.write(encodedir(b'\n'.join(self.addls) + b'\n'))
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   594
            fp.close()
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   595
            self.entries = None
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   596
            self.addls = set()
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   597
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   598
    def add(self, fn):
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   599
        if self.entries is None:
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   600
            self._load()
10577
d5bd1beff794 store: only add new entries to the fncache file
Adrian Buehlmann <adrian@cadifra.com>
parents: 10339
diff changeset
   601
        if fn not in self.entries:
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   602
            self.addls.add(fn)
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   603
20885
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   604
    def remove(self, fn):
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   605
        if self.entries is None:
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   606
            self._load()
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   607
        if fn in self.addls:
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   608
            self.addls.remove(fn)
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   609
            return
20885
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   610
        try:
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   611
            self.entries.remove(fn)
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   612
            self._dirty = True
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   613
        except KeyError:
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   614
            pass
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   615
17782
8095306c1fb2 store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17747
diff changeset
   616
    def __contains__(self, fn):
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   617
        if fn in self.addls:
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   618
            return True
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   619
        if self.entries is None:
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   620
            self._load()
17782
8095306c1fb2 store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17747
diff changeset
   621
        return fn in self.entries
8530
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   622
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   623
    def __iter__(self):
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   624
        if self.entries is None:
03196ac9a8b9 store: refactor the fncache handling
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8480
diff changeset
   625
            self._load()
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   626
        return iter(self.entries | self.addls)
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   627
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   628
41093
6498f0e03526 vfs: fix proxyvfs inheritance
Boris Feld <boris.feld@octobus.net>
parents: 40748
diff changeset
   629
class _fncachevfs(vfsmod.proxyvfs):
17721
cf236e3501c3 store: rename argument name from "op"(ener) to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17693
diff changeset
   630
    def __init__(self, vfs, fnc, encode):
33412
a42369e04aee vfs: rename auditvfs to proxyvfs
Yuya Nishihara <yuya@tcha.org>
parents: 32372
diff changeset
   631
        vfsmod.proxyvfs.__init__(self, vfs)
14194
3a90fb3addc1 store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents: 14166
diff changeset
   632
        self.fncache = fnc
3a90fb3addc1 store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents: 14166
diff changeset
   633
        self.encode = encode
3a90fb3addc1 store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents: 14166
diff changeset
   634
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   635
    def __call__(self, path, mode=b'r', *args, **kw):
38661
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   636
        encoded = self.encode(path)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   637
        if mode not in (b'r', b'rb') and (
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   638
            path.startswith(b'data/') or path.startswith(b'meta/')
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   639
        ):
38661
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   640
            # do not trigger a fncache load when adding a file that already is
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   641
            # known to exist.
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   642
            notload = self.fncache.entries is None and self.vfs.exists(encoded)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   643
            if notload and b'a' in mode and not self.vfs.stat(encoded).st_size:
38661
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   644
                # when appending to an existing file, if the file has size zero,
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   645
                # it should be considered as missing. Such zero-size files are
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   646
                # the result of truncation when a transaction is aborted.
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   647
                notload = False
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   648
            if not notload:
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   649
                self.fncache.add(path)
8ac0c9cd4c48 fncache: avoid loading the filename cache when not actually modifying it
Martijn Pieters <mj@zopatista.com>
parents: 37409
diff changeset
   650
        return self.vfs(encoded, mode, *args, **kw)
14194
3a90fb3addc1 store: break up reference cycle introduced in 9cbff8a39a2a
Adrian Buehlmann <adrian@cadifra.com>
parents: 14166
diff changeset
   651
17725
ffd589d4b785 vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17724
diff changeset
   652
    def join(self, path):
ffd589d4b785 vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17724
diff changeset
   653
        if path:
ffd589d4b785 vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17724
diff changeset
   654
            return self.vfs.join(self.encode(path))
ffd589d4b785 vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17724
diff changeset
   655
        else:
ffd589d4b785 vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17724
diff changeset
   656
            return self.vfs.join(path)
ffd589d4b785 vfs: define "join()" in each classes derived from "abstractvfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17724
diff changeset
   657
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   658
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   659
class fncachestore(basicstore):
17651
3b49c28658f6 store: rename "openertype" argument to "vfstype"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17649
diff changeset
   660
    def __init__(self, path, vfstype, dotencode):
17591
9a5c2ecd1158 store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17590
diff changeset
   661
        if dotencode:
18435
8c019d2fd7c0 store: switch to C-based hashed path encoding
Bryan O'Sullivan <bryano@fb.com>
parents: 18430
diff changeset
   662
            encode = _pathencode
17591
9a5c2ecd1158 store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17590
diff changeset
   663
        else:
9a5c2ecd1158 store: move encode lambda logic into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17590
diff changeset
   664
            encode = _plainhybridencode
12687
34d8247a4595 store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12171
diff changeset
   665
        self.encode = encode
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   666
        vfs = vfstype(path + b'/store')
17724
bf4b72d8dd4d store: initialize vfs field first to use it for initialization of others
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17722
diff changeset
   667
        self.path = vfs.base
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   668
        self.pathsep = self.path + b'/'
17726
7cb7e17c23b2 store: invoke "os.stat()" for "createmode" initialization via vfs
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17725
diff changeset
   669
        self.createmode = _calcmode(vfs)
17652
2c6f7231becc store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17651
diff changeset
   670
        vfs.createmode = self.createmode
17727
6492b39a44d5 store: replace invocation of "getsize()" by "vfs.stat()"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17726
diff changeset
   671
        self.rawvfs = vfs
17652
2c6f7231becc store: rename "op" variables to "vfs"
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17651
diff changeset
   672
        fnc = fncache(vfs)
9133
996c1cd8f530 store: eliminate reference cycle in fncachestore
Simon Heimberg <simohe@besonet.ch>
parents: 8778
diff changeset
   673
        self.fncache = fnc
17653
dacb50696b75 store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17652
diff changeset
   674
        self.vfs = _fncachevfs(vfs, fnc, encode)
dacb50696b75 store: initialize "vfs" fields by "vfs" constructors
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 17652
diff changeset
   675
        self.opener = self.vfs
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   676
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   677
    def join(self, f):
17562
b42b0729744d store: reduce string concatenation when joining
Bryan O'Sullivan <bryano@fb.com>
parents: 17555
diff changeset
   678
        return self.pathsep + self.encode(f)
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   679
17731
c85dbae29684 store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents: 17728
diff changeset
   680
    def getsize(self, path):
c85dbae29684 store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents: 17728
diff changeset
   681
        return self.rawvfs.stat(path).st_size
c85dbae29684 store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents: 17728
diff changeset
   682
40340
2d45b549392f store: pass matcher to store.datafiles()
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 39698
diff changeset
   683
    def datafiles(self, matcher=None):
17373
4cbb1137941d store: sort the results of fncachestore.datafiles()
Bryan O'Sullivan <bryano@fb.com>
parents: 17249
diff changeset
   684
        for f in sorted(self.fncache):
40494
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
   685
            if not _matchtrackedpath(f, matcher):
9aeb9e2d28a7 store: introduce _matchtrackedpath() and use it to filter store files
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40340
diff changeset
   686
                continue
12687
34d8247a4595 store: encode first period or space in filenames (issue1713)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12171
diff changeset
   687
            ef = self.encode(f)
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   688
            try:
17731
c85dbae29684 store: restore getsize method
Matt Mackall <mpm@selenic.com>
parents: 17728
diff changeset
   689
                yield f, ef, self.getsize(ef)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25658
diff changeset
   690
            except OSError as err:
17374
0cec762790ed store: only one kind of OSError means "nonexistent entry"
Bryan O'Sullivan <bryano@fb.com>
parents: 17373
diff changeset
   691
                if err.errno != errno.ENOENT:
0cec762790ed store: only one kind of OSError means "nonexistent entry"
Bryan O'Sullivan <bryano@fb.com>
parents: 17373
diff changeset
   692
                    raise
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   693
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   694
    def copylist(self):
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   695
        d = (
45351
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   696
            b'bookmarks',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   697
            b'narrowspec',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   698
            b'data',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   699
            b'meta',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   700
            b'dh',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   701
            b'fncache',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   702
            b'phaseroots',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   703
            b'obsstore',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   704
            b'00manifest.d',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   705
            b'00manifest.i',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   706
            b'00changelog.d',
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   707
            b'00changelog.i',
45483
d252f51ab032 share: introduce config option to store requires in .hg/store
Pulkit Goyal <7895pulkit@gmail.com>
parents: 45351
diff changeset
   708
            b'requires',
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 42905
diff changeset
   709
        )
45351
909dafff6a78 store: refactor space delimited list to proper data structure
Pulkit Goyal <7895pulkit@gmail.com>
parents: 44452
diff changeset
   710
        return [b'requires', b'00changelog.i'] + [b'store/' + f for f in d]
7229
7946503ec76e introduce fncache repository layout
Adrian Buehlmann <adrian@cadifra.com>
parents: 6989
diff changeset
   711
20883
cd443c7589cc fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents: 20879
diff changeset
   712
    def write(self, tr):
cd443c7589cc fncache: move fncache writing to be in a transaction
Durham Goode <durham@fb.com>
parents: 20879
diff changeset
   713
        self.fncache.write(tr)
13391
d00bbff8600e fncachestore: defer updating the fncache file to a single file open
Adrian Buehlmann <adrian@cadifra.com>
parents: 13169
diff changeset
   714
20884
2efdd186925d caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents: 20883
diff changeset
   715
    def invalidatecaches(self):
2efdd186925d caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents: 20883
diff changeset
   716
        self.fncache.entries = None
40736
0728d87a8631 store: append to fncache if there are only new files to write
Pulkit Goyal <pulkit@yandex-team.ru>
parents: 40624
diff changeset
   717
        self.fncache.addls = set()
20884
2efdd186925d caches: invalidate store caches when lock is taken
Durham Goode <durham@fb.com>
parents: 20883
diff changeset
   718
20885
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   719
    def markremoved(self, fn):
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   720
        self.fncache.remove(fn)
f49d60fa40a5 fncache: clean up fncache during strips
Durham Goode <durham@fb.com>
parents: 20884
diff changeset
   721
17783
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   722
    def _exists(self, f):
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   723
        ef = self.encode(f)
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   724
        try:
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   725
            self.getsize(ef)
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   726
            return True
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25658
diff changeset
   727
        except OSError as err:
17783
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   728
            if err.errno != errno.ENOENT:
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   729
                raise
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   730
            # nonexistent entry
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   731
            return False
df55ce6854c3 store: add new _exists helper function on fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17782
diff changeset
   732
17745
b9a56b816ff2 store: add a contains method to fncachestore
smuralid
parents: 17744
diff changeset
   733
    def __contains__(self, path):
b9a56b816ff2 store: add a contains method to fncachestore
smuralid
parents: 17744
diff changeset
   734
        '''Checks if the store contains path'''
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   735
        path = b"/".join((b"data", path))
17782
8095306c1fb2 store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17747
diff changeset
   736
        # check for files (exact match)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   737
        e = path + b'.i'
17784
73e1ab39792c store: fncache may contain non-existent entries (fixes b9a56b816ff2)
Adrian Buehlmann <adrian@cadifra.com>
parents: 17783
diff changeset
   738
        if e in self.fncache and self._exists(e):
17782
8095306c1fb2 store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17747
diff changeset
   739
            return True
8095306c1fb2 store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17747
diff changeset
   740
        # now check for directories (prefix match)
43077
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   741
        if not path.endswith(b'/'):
687b865b95ad formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents: 43076
diff changeset
   742
            path += b'/'
17782
8095306c1fb2 store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17747
diff changeset
   743
        for e in self.fncache:
17784
73e1ab39792c store: fncache may contain non-existent entries (fixes b9a56b816ff2)
Adrian Buehlmann <adrian@cadifra.com>
parents: 17783
diff changeset
   744
            if e.startswith(path) and self._exists(e):
17782
8095306c1fb2 store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17747
diff changeset
   745
                return True
8095306c1fb2 store: move __contains__() implementation from class fncache into fncachestore
Adrian Buehlmann <adrian@cadifra.com>
parents: 17747
diff changeset
   746
        return False