hgext/largefiles/remotestore.py
author liscju <piotr.listkiewicz@gmail.com>
Tue, 03 May 2016 23:48:31 +0200
changeset 29068 305f9c36a0f5
parent 29067 207c0db08953
child 29218 fd288d118074
permissions -rw-r--r--
largefiles: makes verify batching stat calls to remote Instead of sending stat calls for each files separately, it sends one batch call with stat invocations for all files.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     1
# Copyright 2010-2011 Fog Creek Software
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     2
# Copyright 2010-2011 Unity Technologies
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     3
#
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     4
# This software may be used and distributed according to the terms of the
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     5
# GNU General Public License version 2 or any later version.
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     6
17425
e95ec38f86b0 fix wording and not-completely-trivial spelling errors and bad docstrings
Mads Kiilerich <mads@kiilerich.com>
parents: 17127
diff changeset
     7
'''remote largefile store; the base class for wirestore'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
     8
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
     9
from mercurial import util, wireproto, error
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    10
from mercurial.i18n import _
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    11
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
    12
urlerr = util.urlerr
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
    13
urlreq = util.urlreq
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
    14
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    15
import lfutil
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    16
import basestore
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    17
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    18
class remotestore(basestore.basestore):
15252
6e809bb4f969 largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents: 15188
diff changeset
    19
    '''a largefile store accessed over a network'''
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    20
    def __init__(self, ui, repo, url):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    21
        super(remotestore, self).__init__(ui, repo, url)
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    22
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    23
    def put(self, source, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    24
        if self.sendfile(source, hash):
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
    25
            raise error.Abort(
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    26
                _('remotestore: could not put %s to remote store %s')
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    27
                % (source, util.hidepassword(self.url)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    28
        self.ui.debug(
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    29
            _('remotestore: put %s to remote store %s\n')
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    30
            % (source, util.hidepassword(self.url)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    31
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    32
    def exists(self, hashes):
20688
a61ed1c2d7a7 check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents: 19950
diff changeset
    33
        return dict((h, s == 0) for (h, s) in # dict-from-generator
a61ed1c2d7a7 check-code: disallow use of dict(key=value) construction
Augie Fackler <raf@durin42.com>
parents: 19950
diff changeset
    34
                    self._stat(hashes).iteritems())
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    35
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    36
    def sendfile(self, filename, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    37
        self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    38
        fd = None
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    39
        try:
25079
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
    40
            fd = lfutil.httpsendfile(self.ui, filename)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    41
            return self._put(hash, fd)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25079
diff changeset
    42
        except IOError as e:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
    43
            raise error.Abort(
25079
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
    44
                _('remotestore: could not open file %s: %s')
bee00e0c2e45 largefiles: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 21084
diff changeset
    45
                % (filename, str(e)))
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    46
        finally:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    47
            if fd:
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    48
                fd.close()
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    49
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    50
    def _getfile(self, tmpfile, filename, hash):
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    51
        try:
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 19003
diff changeset
    52
            chunks = self._get(hash)
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
    53
        except urlerr.httperror as e:
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
    54
            # 401s get converted to error.Aborts; everything else is fine being
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    55
            # turned into a StoreError
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    56
            raise basestore.StoreError(filename, hash, self.url, str(e))
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28442
diff changeset
    57
        except urlerr.urlerror as e:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    58
            # This usually indicates a connection problem, so don't
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    59
            # keep trying with the other files... they will probably
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    60
            # all fail too.
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25660
diff changeset
    61
            raise error.Abort('%s: %s' %
19950
cce7ab960312 largefiles: hide passwords in URLs in ui messages
Mads Kiilerich <madski@unity3d.com>
parents: 19948
diff changeset
    62
                             (util.hidepassword(self.url), e.reason))
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25079
diff changeset
    63
        except IOError as e:
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    64
            raise basestore.StoreError(filename, hash, self.url, str(e))
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    65
19004
6614e5e24e66 largefiles: move protocol conversion into getlfile and make it an iterable
Mads Kiilerich <madski@unity3d.com>
parents: 19003
diff changeset
    66
        return lfutil.copyandhash(chunks, tmpfile)
15168
cfccd3bee7b3 hgext: add largefiles extension
various
parents:
diff changeset
    67
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    68
    def _verifyfiles(self, contents, filestocheck):
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    69
        failed = False
29068
305f9c36a0f5 largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    70
        expectedhashes = [expectedhash
305f9c36a0f5 largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    71
                          for cset, filename, expectedhash in filestocheck]
305f9c36a0f5 largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    72
        stats = self._stat(expectedhashes)
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    73
        for cset, filename, expectedhash in filestocheck:
29068
305f9c36a0f5 largefiles: makes verify batching stat calls to remote
liscju <piotr.listkiewicz@gmail.com>
parents: 29067
diff changeset
    74
            stat = stats[expectedhash]
29067
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    75
            if stat:
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    76
                if stat == 1:
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    77
                    self.ui.warn(
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    78
                        _('changeset %s: %s: contents differ\n')
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    79
                        % (cset, filename))
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    80
                    failed = True
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    81
                elif stat == 2:
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    82
                    self.ui.warn(
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    83
                        _('changeset %s: %s missing\n')
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    84
                        % (cset, filename))
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    85
                    failed = True
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    86
                else:
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    87
                    raise RuntimeError('verify failed: unexpected response '
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    88
                                       'from statlfile (%r)' % stat)
207c0db08953 largefiles: change basestore._verifyfile to take list of files to check
liscju <piotr.listkiewicz@gmail.com>
parents: 28883
diff changeset
    89
        return failed
17127
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    90
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    91
    def batch(self):
9e1616307c4c largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents: 15253
diff changeset
    92
        '''Support for remote batching.'''
21084
70252bdfd39c largefiles: import whole modules instead of importing parts of them
Mads Kiilerich <madski@unity3d.com>
parents: 20688
diff changeset
    93
        return wireproto.remotebatch(self)
28442
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
    94
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
    95
    def _put(self, hash, fd):
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
    96
        '''Put file with the given hash in the remote store.'''
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
    97
        raise NotImplementedError('abstract method')
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
    98
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
    99
    def _get(self, hash):
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   100
        '''Get file with the given hash from the remote store.'''
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   101
        raise NotImplementedError('abstract method')
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   102
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   103
    def _stat(self, hashes):
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   104
        '''Get information about availability of files specified by
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   105
        hashes in the remote store. Return dictionary mapping hashes
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   106
        to return code where 0 means that file is available, other
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   107
        values if not.'''
3be2e89c5d9f largefiles: add abstract methods in remotestore class
liscju <piotr.listkiewicz@gmail.com>
parents: 26587
diff changeset
   108
        raise NotImplementedError('abstract method')