tests/flagprocessorext.py
author Remi Chaintron <remi@fb.com>
Tue, 10 Jan 2017 16:15:21 +0000
changeset 30745 c1b7b2285522
child 31832 77f746e5383a
permissions -rw-r--r--
revlog: flag processor Add the ability for revlog objects to process revision flags and apply registered transforms on read/write operations. This patch introduces: - the 'revlog._processflags()' method that looks at revision flags and applies flag processors registered on them. Due to the need to handle non-commutative operations, flag transforms are applied in stable order but the order in which the transforms are applied is reversed between read and write operations. - the 'addflagprocessor()' method allowing to register processors on flags. Flag processors are defined as a 3-tuple of (read, write, raw) functions to be applied depending on the operation being performed. - an update on 'revlog.addrevision()' behavior. The current flagprocessor design relies on extensions to wrap around 'addrevision()' to set flags on revision data, and on the flagprocessor to perform the actual transformation of its contents. In the lfs case, this means we need to process flags before we meet the 2GB size check, leading to performing some operations before it happens: - if flags are set on the revision data, we assume some extensions might be modifying the contents using the flag processor next, and we compute the node for the original revision data (still allowing extension to override the node by wrapping around 'addrevision()'). - we then invoke the flag processor to apply registered transforms (in lfs's case, drastically reducing the size of large blobs). - finally, we proceed with the 2GB size check. Note: In the case a cachedelta is passed to 'addrevision()' and we detect the flag processor modified the revision data, we chose to trust the flag processor and drop the cachedelta.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
30745
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     1
# coding=UTF-8
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     2
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     3
from __future__ import absolute_import
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     4
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     5
import base64
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     6
import zlib
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     7
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     8
from mercurial import (
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
     9
    changegroup,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    10
    extensions,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    11
    filelog,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    12
    revlog,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    13
    util,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    14
)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    15
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    16
# Test only: These flags are defined here only in the context of testing the
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    17
# behavior of the flag processor. The canonical way to add flags is to get in
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    18
# touch with the community and make them known in revlog.
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    19
REVIDX_NOOP = (1 << 3)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    20
REVIDX_BASE64 = (1 << 2)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    21
REVIDX_GZIP = (1 << 1)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    22
REVIDX_FAIL = 1
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    23
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    24
def validatehash(self, text):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    25
    return True
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    26
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    27
def bypass(self, text):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    28
    return False
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    29
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    30
def noopdonothing(self, text):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    31
    return (text, True)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    32
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    33
def b64encode(self, text):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    34
    return (base64.b64encode(text), False)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    35
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    36
def b64decode(self, text):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    37
    return (base64.b64decode(text), True)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    38
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    39
def gzipcompress(self, text):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    40
    return (zlib.compress(text), False)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    41
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    42
def gzipdecompress(self, text):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    43
    return (zlib.decompress(text), True)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    44
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    45
def supportedoutgoingversions(orig, repo):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    46
    versions = orig(repo)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    47
    versions.discard('01')
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    48
    versions.discard('02')
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    49
    versions.add('03')
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    50
    return versions
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    51
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    52
def allsupportedversions(orig, ui):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    53
    versions = orig(ui)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    54
    versions.add('03')
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    55
    return versions
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    56
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    57
def noopaddrevision(orig, self, text, transaction, link, p1, p2,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    58
                    cachedelta=None, node=None,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    59
                    flags=revlog.REVIDX_DEFAULT_FLAGS):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    60
    if '[NOOP]' in text:
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    61
        flags |= REVIDX_NOOP
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    62
    return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    63
                node=node, flags=flags)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    64
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    65
def b64addrevision(orig, self, text, transaction, link, p1, p2,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    66
                   cachedelta=None, node=None,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    67
                   flags=revlog.REVIDX_DEFAULT_FLAGS):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    68
    if '[BASE64]' in text:
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    69
        flags |= REVIDX_BASE64
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    70
    return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    71
                node=node, flags=flags)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    72
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    73
def gzipaddrevision(orig, self, text, transaction, link, p1, p2,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    74
                    cachedelta=None, node=None,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    75
                    flags=revlog.REVIDX_DEFAULT_FLAGS):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    76
    if '[GZIP]' in text:
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    77
        flags |= REVIDX_GZIP
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    78
    return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    79
                node=node, flags=flags)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    80
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    81
def failaddrevision(orig, self, text, transaction, link, p1, p2,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    82
                    cachedelta=None, node=None,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    83
                    flags=revlog.REVIDX_DEFAULT_FLAGS):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    84
    # This addrevision wrapper is meant to add a flag we will not have
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    85
    # transforms registered for, ensuring we handle this error case.
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    86
    if '[FAIL]' in text:
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    87
        flags |= REVIDX_FAIL
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    88
    return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    89
                node=node, flags=flags)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    90
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    91
def extsetup(ui):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    92
    # Enable changegroup3 for flags to be sent over the wire
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    93
    wrapfunction = extensions.wrapfunction
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    94
    wrapfunction(changegroup,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    95
                 'supportedoutgoingversions',
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    96
                 supportedoutgoingversions)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    97
    wrapfunction(changegroup,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    98
                 'allsupportedversions',
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
    99
                 allsupportedversions)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   100
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   101
    # Teach revlog about our test flags
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   102
    flags = [REVIDX_NOOP, REVIDX_BASE64, REVIDX_GZIP, REVIDX_FAIL]
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   103
    revlog.REVIDX_KNOWN_FLAGS |= util.bitsfrom(flags)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   104
    revlog.REVIDX_FLAGS_ORDER.extend(flags)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   105
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   106
    # Add wrappers for addrevision, responsible to set flags depending on the
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   107
    # revision data contents.
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   108
    wrapfunction(filelog.filelog, 'addrevision', noopaddrevision)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   109
    wrapfunction(filelog.filelog, 'addrevision', b64addrevision)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   110
    wrapfunction(filelog.filelog, 'addrevision', gzipaddrevision)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   111
    wrapfunction(filelog.filelog, 'addrevision', failaddrevision)
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   112
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   113
    # Register flag processors for each extension
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   114
    revlog.addflagprocessor(
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   115
        REVIDX_NOOP,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   116
        (
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   117
            noopdonothing,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   118
            noopdonothing,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   119
            validatehash,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   120
        )
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   121
    )
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   122
    revlog.addflagprocessor(
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   123
        REVIDX_BASE64,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   124
        (
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   125
            b64decode,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   126
            b64encode,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   127
            bypass,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   128
        ),
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   129
    )
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   130
    revlog.addflagprocessor(
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   131
        REVIDX_GZIP,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   132
        (
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   133
            gzipdecompress,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   134
            gzipcompress,
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   135
            bypass
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   136
        )
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents:
diff changeset
   137
    )